From 3797fb67137a9d9ce659ae0841b64f174d27473b Mon Sep 17 00:00:00 2001 From: hanhotfox Date: Tue, 5 Nov 2024 19:51:29 +0800 Subject: [PATCH 01/92] feat(functions): add map_insert function (#15567) * feat(functions): add map_insert function * feat(functions): check ci * feat(functions): domain fix * feat(functions): add new params format in map_insert * feat(functions): cargo fmt * feat(functions): fix NULL mapType in insert * fix * fix tests * fix typos --------- Co-authored-by: baishen --- src/query/functions/src/scalars/map.rs | 88 +++++++++++++++++ src/query/functions/tests/it/scalars/map.rs | 49 +++++++++ .../it/scalars/testdata/function_list.txt | 2 + .../tests/it/scalars/testdata/map.txt | 99 +++++++++++++++++++ .../query/functions/02_0074_function_map.test | 37 +++++++ 5 files changed, 275 insertions(+) diff --git a/src/query/functions/src/scalars/map.rs b/src/query/functions/src/scalars/map.rs index a71e2aadda59..449cb7efbc29 100644 --- a/src/query/functions/src/scalars/map.rs +++ b/src/query/functions/src/scalars/map.rs @@ -34,6 +34,8 @@ use databend_common_expression::types::SimpleDomain; use databend_common_expression::types::ValueType; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; +use databend_common_expression::vectorize_with_builder_3_arg; +use databend_common_expression::vectorize_with_builder_4_arg; use databend_common_expression::ColumnBuilder; use databend_common_expression::Function; use databend_common_expression::FunctionDomain; @@ -348,6 +350,92 @@ pub fn register(registry: &mut FunctionRegistry) { }, ); + registry.register_3_arg_core::, GenericType<1>>>, GenericType<0>, GenericType<1>, MapType, GenericType<1>>, _, _>( + "map_insert", + |_, map_domain, key_domain, value_domain| { + let domain = map_domain + .value + .as_ref() + .map(|box inner_domain| { + inner_domain + .as_ref() + .map(|(inner_key_domain, inner_value_domain)| (inner_key_domain.merge(key_domain), inner_value_domain.merge(value_domain))) + .unwrap_or((key_domain.clone(), value_domain.clone())) + }); + FunctionDomain::Domain(domain) + }, + vectorize_with_builder_3_arg::, GenericType<1>>>, GenericType<0>, GenericType<1>, MapType, GenericType<1>>>( + |map, key, val, output, ctx| { + let key_type = &ctx.generics[0]; + if !check_valid_map_key_type(key_type) { + ctx.set_error(output.len(), format!("map keys can not be {}", key_type)); + output.commit_row(); + return; + } + if let Some(map) = map { + for (k, _) in map.iter() { + if k == key { + ctx.set_error(output.len(), format!("map key `{}` duplicate", key)); + output.commit_row(); + return; + } + } + for (k, v) in map.iter() { + output.put_item((k, v)); + } + } + output.put_item((key, val)); + output.commit_row(); + }) + ); + + registry.register_4_arg_core::, GenericType<1>>>, GenericType<0>, GenericType<1>, BooleanType, MapType, GenericType<1>>, _, _>( + "map_insert", + |_, map_domain, key_domain, value_domain, _| { + let domain = map_domain + .value + .as_ref() + .map(|box inner_domain| { + inner_domain + .as_ref() + .map(|(inner_key_domain, inner_value_domain)| (inner_key_domain.merge(key_domain), inner_value_domain.merge(value_domain))) + .unwrap_or((key_domain.clone(), value_domain.clone())) + }); + FunctionDomain::Domain(domain) + }, + vectorize_with_builder_4_arg::, GenericType<1>>>, GenericType<0>, GenericType<1>, BooleanType, MapType, GenericType<1>>>( + |map, key, val, update_flag, output, ctx| { + let key_type = &ctx.generics[0]; + if !check_valid_map_key_type(key_type) { + ctx.set_error(output.len(), format!("map keys can not be {}", key_type)); + output.commit_row(); + return; + } + let mut is_duplicate = false; + if let Some(map) = map { + for (k, _) in map.iter() { + if k == key && !update_flag { + ctx.set_error(output.len(), format!("map key `{}` duplicate", key)); + output.commit_row(); + return; + } + } + for (k, v) in map.iter() { + if k == key { + is_duplicate = true; + output.put_item((k, val.clone())); + } else { + output.put_item((k, v)); + } + } + } + if !is_duplicate { + output.put_item((key, val)); + } + output.commit_row(); + }) + ); + registry.register_function_factory("map_pick", |_, args_type: &[DataType]| { let return_type = check_map_arg_types(args_type)?; Some(Arc::new(Function { diff --git a/src/query/functions/tests/it/scalars/map.rs b/src/query/functions/tests/it/scalars/map.rs index 7b2f0f0fd6d4..b7714227249d 100644 --- a/src/query/functions/tests/it/scalars/map.rs +++ b/src/query/functions/tests/it/scalars/map.rs @@ -34,6 +34,7 @@ fn test_map() { test_map_delete(file); test_map_contains_key(file); test_map_pick(file); + test_map_insert(file) } fn test_map_cat(file: &mut impl Write) { @@ -419,3 +420,51 @@ fn test_map_pick(file: &mut impl Write) { &columns, ); } + +fn test_map_insert(file: &mut impl Write) { + run_ast(file, "map_insert({}, 'k1', 'v1')", &[]); + run_ast(file, "map_insert({'k1': 'v1'}, 'k2', 'v2')", &[]); + run_ast( + file, + "map_insert({'k1': 'v1', 'k2': 'v2'}, 'k1', 'v10', false)", + &[], + ); + run_ast( + file, + "map_insert({'k1': 'v1', 'k2': 'v2'}, 'k1', 'v10', true)", + &[], + ); + + let columns = [ + ("a_col", StringType::from_data(vec!["a", "b", "c"])), + ("b_col", StringType::from_data(vec!["d", "e", "f"])), + ("c_col", StringType::from_data(vec!["x", "y", "z"])), + ( + "d_col", + StringType::from_data_with_validity(vec!["v1", "v2", "v3"], vec![true, true, true]), + ), + ( + "e_col", + StringType::from_data_with_validity(vec!["v4", "v5", ""], vec![true, true, false]), + ), + ( + "f_col", + StringType::from_data_with_validity(vec!["v6", "", "v7"], vec![true, false, true]), + ), + ]; + run_ast( + file, + "map_insert(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'k1', 'v10')", + &columns, + ); + run_ast( + file, + "map_insert(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'a', 'v10', true)", + &columns, + ); + run_ast( + file, + "map_insert(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'a', 'v10', false)", + &columns, + ); +} diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index d8b6c7dfd055..89e237d43f5d 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -2546,6 +2546,8 @@ Functions overloads: 1 map_contains_key(Map(T0, T1), T0) :: Boolean 2 map_contains_key(Map(T0, T1) NULL, T0 NULL) :: Boolean NULL 0 map_delete FACTORY +0 map_insert(Map(T0, T1) NULL, T0, T1) :: Map(T0, T1) +1 map_insert(Map(T0, T1) NULL, T0, T1, Boolean) :: Map(T0, T1) 0 map_keys(Map(Nothing)) :: Array(Nothing) 1 map_keys(Map(T0, T1)) :: Array(T0) 2 map_keys(Map(T0, T1) NULL) :: Array(T0) NULL diff --git a/src/query/functions/tests/it/scalars/testdata/map.txt b/src/query/functions/tests/it/scalars/testdata/map.txt index e8c3d262cae0..f8618c82aa66 100644 --- a/src/query/functions/tests/it/scalars/testdata/map.txt +++ b/src/query/functions/tests/it/scalars/testdata/map.txt @@ -874,3 +874,102 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +ast : map_insert({}, 'k1', 'v1') +raw expr : map_insert(map(array(), array()), 'k1', 'v1') +checked expr : map_insert(CAST(map(array<>(), array<>()) AS Map(String, String) NULL), "k1", "v1") +optimized expr : {"k1":"v1"} +output type : Map(String, String) +output domain : {[{"k1"..="k1"}], [{"v1"..="v1"}]} +output : {'k1':'v1'} + + +ast : map_insert({'k1': 'v1'}, 'k2', 'v2') +raw expr : map_insert(map(array('k1'), array('v1')), 'k2', 'v2') +checked expr : map_insert(CAST(map(array("k1"), array("v1")) AS Map(String, String) NULL), "k2", "v2") +optimized expr : {"k1":"v1", "k2":"v2"} +output type : Map(String, String) +output domain : {[{"k1"..="k2"}], [{"v1"..="v2"}]} +output : {'k1':'v1', 'k2':'v2'} + + +error: + --> SQL:1:1 + | +1 | map_insert({'k1': 'v1', 'k2': 'v2'}, 'k1', 'v10', false) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ map key `'k1'` duplicate while evaluating function `map_insert({'k1':'v1', 'k2':'v2'}, 'k1', 'v10', false)` in expr `map_insert(CAST(map(array('k1', 'k2'), array('v1', 'v2')) AS Map(String, String) NULL), 'k1', 'v10', false)` + + + +ast : map_insert({'k1': 'v1', 'k2': 'v2'}, 'k1', 'v10', true) +raw expr : map_insert(map(array('k1', 'k2'), array('v1', 'v2')), 'k1', 'v10', true) +checked expr : map_insert(CAST(map(array("k1", "k2"), array("v1", "v2")) AS Map(String, String) NULL), "k1", "v10", true) +optimized expr : {"k1":"v10", "k2":"v2"} +output type : Map(String, String) +output domain : {[{"k1"..="k2"}], [{"v10"..="v2"}]} +output : {'k1':'v10', 'k2':'v2'} + + +ast : map_insert(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'k1', 'v10') +raw expr : map_insert(map(array(a_col::String, b_col::String, c_col::String), array(d_col::String NULL, e_col::String NULL, f_col::String NULL)), 'k1', 'v10') +checked expr : map_insert(CAST(map(array(a_col, b_col, c_col), array(d_col, e_col, f_col)) AS Map(String, String NULL) NULL), "k1", CAST("v10" AS String NULL)) +optimized expr : map_insert(CAST(map(array(a_col, b_col, c_col), array(d_col, e_col, f_col)) AS Map(String, String NULL) NULL), "k1", "v10") +evaluation: ++--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------------------------------+ +| | a_col | b_col | c_col | d_col | e_col | f_col | Output | ++--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------------------------------+ +| Type | String | String | String | String NULL | String NULL | String NULL | Map(String, String NULL) | +| Domain | {"a"..="c"} | {"d"..="f"} | {"x"..="z"} | {"v1"..="v3"} | {""..="v5"} ∪ {NULL} | {""..="v7"} ∪ {NULL} | Unknown | +| Row 0 | 'a' | 'd' | 'x' | 'v1' | 'v4' | 'v6' | {'a':'v1', 'd':'v4', 'x':'v6', 'k1':'v10'} | +| Row 1 | 'b' | 'e' | 'y' | 'v2' | 'v5' | NULL | {'b':'v2', 'e':'v5', 'y':NULL, 'k1':'v10'} | +| Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | {'c':'v3', 'f':NULL, 'z':'v7', 'k1':'v10'} | ++--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------------------------------+ +evaluation (internal): ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | +| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | +| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | +| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x6164786b316265796b3163667a6b31, offsets: [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 15] }, NullableColumn { column: StringColumn { data: 0x7631763476367631307632763576313076337637763130, offsets: [0, 2, 4, 6, 9, 11, 13, 13, 16, 18, 18, 20, 23] }, validity: [0b10111111, 0b____1101] }]), offsets: [0, 4, 8, 12] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +ast : map_insert(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'a', 'v10', true) +raw expr : map_insert(map(array(a_col::String, b_col::String, c_col::String), array(d_col::String NULL, e_col::String NULL, f_col::String NULL)), 'a', 'v10', true) +checked expr : map_insert(CAST(map(array(a_col, b_col, c_col), array(d_col, e_col, f_col)) AS Map(String, String NULL) NULL), "a", CAST("v10" AS String NULL), true) +optimized expr : map_insert(CAST(map(array(a_col, b_col, c_col), array(d_col, e_col, f_col)) AS Map(String, String NULL) NULL), "a", "v10", true) +evaluation: ++--------+-------------+-------------+-------------+---------------+----------------------+----------------------+-------------------------------------------+ +| | a_col | b_col | c_col | d_col | e_col | f_col | Output | ++--------+-------------+-------------+-------------+---------------+----------------------+----------------------+-------------------------------------------+ +| Type | String | String | String | String NULL | String NULL | String NULL | Map(String, String NULL) | +| Domain | {"a"..="c"} | {"d"..="f"} | {"x"..="z"} | {"v1"..="v3"} | {""..="v5"} ∪ {NULL} | {""..="v7"} ∪ {NULL} | Unknown | +| Row 0 | 'a' | 'd' | 'x' | 'v1' | 'v4' | 'v6' | {'a':'v10', 'd':'v4', 'x':'v6'} | +| Row 1 | 'b' | 'e' | 'y' | 'v2' | 'v5' | NULL | {'b':'v2', 'e':'v5', 'y':NULL, 'a':'v10'} | +| Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | {'c':'v3', 'f':NULL, 'z':'v7', 'a':'v10'} | ++--------+-------------+-------------+-------------+---------------+----------------------+----------------------+-------------------------------------------+ +evaluation (internal): ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | +| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | +| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | +| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x6164786265796163667a61, offsets: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] }, NullableColumn { column: StringColumn { data: 0x763130763476367632763576313076337637763130, offsets: [0, 3, 5, 7, 9, 11, 11, 14, 16, 16, 18, 21] }, validity: [0b11011111, 0b_____110] }]), offsets: [0, 3, 7, 11] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +error: + --> SQL:1:1 + | +1 | map_insert(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'a', 'v10', false) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ map key `'a'` duplicate while evaluating function `map_insert({'a':'v1', 'd':'v4', 'x':'v6'}, 'a', 'v10', false)` in expr `map_insert(CAST(map(array(a_col, b_col, c_col), array(d_col, e_col, f_col)) AS Map(String, String NULL) NULL), 'a', CAST('v10' AS String NULL), false)` + + + diff --git a/tests/sqllogictests/suites/query/functions/02_0074_function_map.test b/tests/sqllogictests/suites/query/functions/02_0074_function_map.test index 5fc29b96f1f8..e29e03a18fc7 100644 --- a/tests/sqllogictests/suites/query/functions/02_0074_function_map.test +++ b/tests/sqllogictests/suites/query/functions/02_0074_function_map.test @@ -274,6 +274,43 @@ SELECT map_pick(col_str, 'k1', 'k3'), map_pick(col_int, ['a', 'e', 'x']) FROM ma {'k1':'v1','k3':NULL} {'a':10} {} {'e':NULL} +query +SELECT map_insert({'k1': 'v1', 'k2': 'v2'}, 'k3', NULL) +---- +{'k1':'v1','k2':'v2','k3':NULL} + +statement error 1006 +SELECT map_insert({'k1': 'v1', 'k2': 'v2'}, 'k1', 'new_v1') + +query +SELECT map_insert({}, 'k1', 'v1') +---- +{'k1':'v1'} + +query +SELECT map_insert({'k1': 'v1', 'k2': 'v2'}, 'k2', 'v3', true) +---- +{'k1':'v1','k2':'v3'} + +statement error 1006 +SELECT map_insert({'k1': 'v1', 'k2': 'v2'}, 'k2', 'v3', false) + +statement ok +drop table if exists map_insert_test all + +statement ok +CREATE TABLE map_insert_test(col_str Map(String, String Null) Not Null, col_int Map(String, Int Null) Null); + +statement ok +INSERT INTO map_insert_test VALUES ({'k1':'v1','k2':'v2','k3':null},{'a':10,'b':20}), ({'k5':'v5','k6':'v6'}, {'d':40,'e':null,'f':50}), ({}, NULL); + +query +SELECT map_insert(col_str, 'k1', 'k100', true), map_insert(col_int, 'c', 100, false) FROM map_insert_test; +---- +{'k1':'k100','k2':'v2','k3':NULL} {'a':10,'b':20,'c':100} +{'k5':'v5','k6':'v6','k1':'k100'} {'d':40,'e':NULL,'f':50,'c':100} +{'k1':'k100'} {'c':100} + # Test map_filter query T SELECT map_filter({1:0,2:2,3:-1}, (k, v) -> k > v); From 8b59fb096c1c684816c70a4c56ad4677384d499f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Wed, 6 Nov 2024 12:55:02 +0800 Subject: [PATCH 02/92] chore: refine raft-types import path (#16764) --- src/meta/binaries/metactl/import.rs | 14 +++---- src/meta/process/src/kv_processor.rs | 2 +- src/meta/raft-store/src/applier.rs | 6 +-- src/meta/raft-store/src/config.rs | 2 +- src/meta/raft-store/src/key_spaces.rs | 8 ++-- .../raft-store/src/leveled_store/sys_data.rs | 6 +-- .../src/leveled_store/sys_data_api.rs | 6 +-- src/meta/raft-store/src/log/raft_log.rs | 6 +-- src/meta/raft-store/src/sm_v003/adapter.rs | 4 +- .../sm_v003/compact_immutable_levels_test.rs | 4 +- .../src/sm_v003/compact_with_db_test.rs | 4 +- src/meta/raft-store/src/sm_v003/received.rs | 4 +- .../raft-store/src/sm_v003/receiver_v003.rs | 4 +- src/meta/raft-store/src/sm_v003/sm_v003.rs | 4 +- .../src/sm_v003/snapshot_store_v002.rs | 4 +- src/meta/raft-store/src/state/raft_state.rs | 6 +-- .../raft-store/src/state/raft_state_kv.rs | 6 +-- .../raft-store/src/state_machine/log_meta.rs | 2 +- src/meta/raft-store/src/state_machine/sm.rs | 10 ++--- .../src/state_machine/snapshot_id.rs | 6 +-- .../src/state_machine/state_machine_meta.rs | 4 +- .../raft-store/src/state_machine/testing.rs | 6 +-- src/meta/raft-store/tests/it/log.rs | 6 +-- src/meta/raft-store/tests/it/state.rs | 2 +- .../tests/it/state_machine/expire.rs | 6 +-- .../raft-store/tests/it/state_machine/mod.rs | 6 +-- src/meta/raft-store/tests/it/types.rs | 2 +- src/meta/service/src/api/http/v1/ctrl.rs | 2 +- src/meta/service/src/message.rs | 2 +- .../service/src/meta_service/forwarder.rs | 2 +- .../service/src/meta_service/meta_leader.rs | 8 ++-- .../service/src/meta_service/meta_node.rs | 14 +++---- .../src/meta_service/meta_node_status.rs | 4 +- .../src/meta_service/raft_service_impl.rs | 20 +++++----- .../src/meta_service/snapshot_receiver_v1.rs | 10 ++--- src/meta/service/src/metrics/meta_metrics.rs | 4 +- src/meta/service/src/network.rs | 38 +++++++++---------- src/meta/service/src/raft_client.rs | 2 +- src/meta/service/src/request_handling.rs | 2 +- .../src/store/raft_log_storage_impl.rs | 12 +++--- .../src/store/raft_state_machine_impl.rs | 14 +++---- src/meta/service/src/store/store_inner.rs | 14 +++---- .../service/src/store/to_storage_error.rs | 4 +- .../tests/it/meta_node/meta_node_lifecycle.rs | 4 +- .../it/meta_node/meta_node_replication.rs | 14 +++---- .../meta_node/meta_node_request_forwarding.rs | 4 +- src/meta/service/tests/it/store.rs | 14 +++---- src/meta/service/tests/it/tests/meta_node.rs | 2 +- src/meta/service/tests/it/tests/service.rs | 2 +- src/meta/sled-store/src/sled_serde.rs | 2 +- src/meta/sled-store/src/sled_serde_impl.rs | 14 +++---- src/meta/sled-store/tests/it/sled_iter.rs | 6 +-- src/meta/sled-store/tests/it/sled_tree.rs | 8 ++-- .../tests/it/testing/fake_key_spaces.rs | 6 +-- .../it/testing/fake_state_machine_meta.rs | 4 +- src/meta/types/src/cmd/mod.rs | 2 +- src/meta/types/src/errors/meta_api_errors.rs | 4 +- src/meta/types/src/errors/meta_raft_errors.rs | 2 +- .../types/src/errors/meta_startup_errors.rs | 2 +- src/meta/types/src/grpc_helper.rs | 2 +- src/meta/types/src/lib.rs | 36 ------------------ .../types/src/proto_ext/raft_types_ext.rs | 2 +- .../proto_ext/snapshot_chunk_request_ext.rs | 8 ++-- .../proto_ext/transfer_leader_request_ext.rs | 2 +- src/meta/types/src/snapshot_db.rs | 2 +- src/meta/types/src/sys_data.rs | 6 +-- 66 files changed, 202 insertions(+), 238 deletions(-) diff --git a/src/meta/binaries/metactl/import.rs b/src/meta/binaries/metactl/import.rs index 6f3178ee73de..c8a552b8cbaa 100644 --- a/src/meta/binaries/metactl/import.rs +++ b/src/meta/binaries/metactl/import.rs @@ -38,18 +38,18 @@ use databend_common_meta_sled_store::get_sled_db; use databend_common_meta_sled_store::init_sled_db; use databend_common_meta_sled_store::openraft::storage::RaftLogStorageExt; use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; +use databend_common_meta_types::raft_types::CommittedLeaderId; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::Membership; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::sys_data::SysData; use databend_common_meta_types::Cmd; -use databend_common_meta_types::CommittedLeaderId; use databend_common_meta_types::Endpoint; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::LogEntry; -use databend_common_meta_types::LogId; -use databend_common_meta_types::Membership; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::StoredMembership; use databend_meta::store::RaftStore; use url::Url; diff --git a/src/meta/process/src/kv_processor.rs b/src/meta/process/src/kv_processor.rs index 47e7c2e80b90..c86fb75934d5 100644 --- a/src/meta/process/src/kv_processor.rs +++ b/src/meta/process/src/kv_processor.rs @@ -14,11 +14,11 @@ use anyhow::Error; use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; +use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::txn_condition::Target; use databend_common_meta_types::txn_op::Request; use databend_common_meta_types::Cmd; -use databend_common_meta_types::Entry; use databend_common_meta_types::LogEntry; use databend_common_meta_types::Operation; use databend_common_meta_types::TxnCondition; diff --git a/src/meta/raft-store/src/applier.rs b/src/meta/raft-store/src/applier.rs index 5a985d99115b..be70fd45f460 100644 --- a/src/meta/raft-store/src/applier.rs +++ b/src/meta/raft-store/src/applier.rs @@ -17,6 +17,9 @@ use std::time::Duration; use databend_common_base::display::display_unix_epoch::DisplayUnixTimeStampExt; use databend_common_meta_types::protobuf as pb; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; +use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::seq_value::SeqValue; use databend_common_meta_types::txn_condition; @@ -27,13 +30,10 @@ use databend_common_meta_types::Change; use databend_common_meta_types::Cmd; use databend_common_meta_types::CmdContext; use databend_common_meta_types::ConditionResult; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::Interval; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaSpec; use databend_common_meta_types::Node; -use databend_common_meta_types::StoredMembership; use databend_common_meta_types::TxnCondition; use databend_common_meta_types::TxnDeleteByPrefixRequest; use databend_common_meta_types::TxnDeleteByPrefixResponse; diff --git a/src/meta/raft-store/src/config.rs b/src/meta/raft-store/src/config.rs index 9c4f383cfa46..24a7ef8dfe14 100644 --- a/src/meta/raft-store/src/config.rs +++ b/src/meta/raft-store/src/config.rs @@ -17,9 +17,9 @@ use std::sync::LazyLock; use databend_common_exception::Result; use databend_common_grpc::DNSResolver; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::MetaStartupError; -use databend_common_meta_types::NodeId; pub static DATABEND_COMMIT_VERSION: LazyLock = LazyLock::new(|| { let build_semver = option_env!("VERGEN_BUILD_SEMVER"); diff --git a/src/meta/raft-store/src/key_spaces.rs b/src/meta/raft-store/src/key_spaces.rs index eb957026f466..d4e6c4ffa9b2 100644 --- a/src/meta/raft-store/src/key_spaces.rs +++ b/src/meta/raft-store/src/key_spaces.rs @@ -20,12 +20,12 @@ use databend_common_meta_sled_store::SledKeySpace; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; use databend_common_meta_stoerr::MetaStorageError; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::LogIndex; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::Entry; -use databend_common_meta_types::LogId; -use databend_common_meta_types::LogIndex; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; use databend_common_meta_types::SeqNum; use serde::Deserialize; use serde::Serialize; diff --git a/src/meta/raft-store/src/leveled_store/sys_data.rs b/src/meta/raft-store/src/leveled_store/sys_data.rs index ff19ae246809..3afb198c6e72 100644 --- a/src/meta/raft-store/src/leveled_store/sys_data.rs +++ b/src/meta/raft-store/src/leveled_store/sys_data.rs @@ -14,11 +14,11 @@ use std::collections::BTreeMap; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::LogId; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::StoredMembership; use crate::leveled_store::sys_data_api::SysDataApiRO; diff --git a/src/meta/raft-store/src/leveled_store/sys_data_api.rs b/src/meta/raft-store/src/leveled_store/sys_data_api.rs index 8262721e7410..aa1a82fa9620 100644 --- a/src/meta/raft-store/src/leveled_store/sys_data_api.rs +++ b/src/meta/raft-store/src/leveled_store/sys_data_api.rs @@ -14,10 +14,10 @@ use std::collections::BTreeMap; -use databend_common_meta_types::LogId; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::StoredMembership; /// APIs to access the non-user-data of the state machine(leveled map). pub trait SysDataApiRO { diff --git a/src/meta/raft-store/src/log/raft_log.rs b/src/meta/raft-store/src/log/raft_log.rs index 9cc99953ceaa..2e7461895bf8 100644 --- a/src/meta/raft-store/src/log/raft_log.rs +++ b/src/meta/raft-store/src/log/raft_log.rs @@ -18,9 +18,9 @@ use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::AsKeySpace; use databend_common_meta_sled_store::SledTree; use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::Entry; -use databend_common_meta_types::LogId; -use databend_common_meta_types::LogIndex; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::LogIndex; use log::info; use crate::config::RaftConfig; diff --git a/src/meta/raft-store/src/sm_v003/adapter.rs b/src/meta/raft-store/src/sm_v003/adapter.rs index 2789e09f1dbf..883cae089fdb 100644 --- a/src/meta/raft-store/src/sm_v003/adapter.rs +++ b/src/meta/raft-store/src/sm_v003/adapter.rs @@ -17,11 +17,11 @@ use std::iter::repeat_with; use std::sync::Arc; use std::sync::Mutex; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::LogId; use databend_common_meta_types::SnapshotData; -use databend_common_meta_types::StoredMembership; use itertools::Itertools; use log::info; use openraft::SnapshotId; diff --git a/src/meta/raft-store/src/sm_v003/compact_immutable_levels_test.rs b/src/meta/raft-store/src/sm_v003/compact_immutable_levels_test.rs index a911829bd5e4..48c602a637aa 100644 --- a/src/meta/raft-store/src/sm_v003/compact_immutable_levels_test.rs +++ b/src/meta/raft-store/src/sm_v003/compact_immutable_levels_test.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_meta_types::raft_types::Membership; +use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::seq_value::KVMeta; use databend_common_meta_types::Endpoint; -use databend_common_meta_types::Membership; use databend_common_meta_types::Node; -use databend_common_meta_types::StoredMembership; use databend_common_meta_types::UpsertKV; use futures_util::TryStreamExt; use maplit::btreemap; diff --git a/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs b/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs index 2468b9cef06d..aa868c338f70 100644 --- a/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs +++ b/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs @@ -14,11 +14,11 @@ use std::io; +use databend_common_meta_types::raft_types::Membership; +use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::seq_value::KVMeta; use databend_common_meta_types::Endpoint; -use databend_common_meta_types::Membership; use databend_common_meta_types::Node; -use databend_common_meta_types::StoredMembership; use databend_common_meta_types::UpsertKV; use futures_util::TryStreamExt; use maplit::btreemap; diff --git a/src/meta/raft-store/src/sm_v003/received.rs b/src/meta/raft-store/src/sm_v003/received.rs index 4449186cc011..9a8aadf6ee2f 100644 --- a/src/meta/raft-store/src/sm_v003/received.rs +++ b/src/meta/raft-store/src/sm_v003/received.rs @@ -14,8 +14,8 @@ use std::fmt; -use databend_common_meta_types::SnapshotMeta; -use databend_common_meta_types::Vote; +use databend_common_meta_types::raft_types::SnapshotMeta; +use databend_common_meta_types::raft_types::Vote; /// Contains information about a received snapshot data. #[derive(Debug)] diff --git a/src/meta/raft-store/src/sm_v003/receiver_v003.rs b/src/meta/raft-store/src/sm_v003/receiver_v003.rs index 10386d32df46..e0e2b149aab0 100644 --- a/src/meta/raft-store/src/sm_v003/receiver_v003.rs +++ b/src/meta/raft-store/src/sm_v003/receiver_v003.rs @@ -20,8 +20,8 @@ use std::io::BufWriter; use std::io::Write; use databend_common_meta_types::protobuf::SnapshotChunkRequestV003; -use databend_common_meta_types::SnapshotMeta; -use databend_common_meta_types::Vote; +use databend_common_meta_types::raft_types::SnapshotMeta; +use databend_common_meta_types::raft_types::Vote; use log::debug; use log::error; use log::info; diff --git a/src/meta/raft-store/src/sm_v003/sm_v003.rs b/src/meta/raft-store/src/sm_v003/sm_v003.rs index 9b30d7ca72fa..b310a69df595 100644 --- a/src/meta/raft-store/src/sm_v003/sm_v003.rs +++ b/src/meta/raft-store/src/sm_v003/sm_v003.rs @@ -21,17 +21,17 @@ use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::UpsertKVReply; use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::StorageError; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::seq_value::SeqValue; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::sys_data::SysData; use databend_common_meta_types::AppliedState; use databend_common_meta_types::CmdContext; -use databend_common_meta_types::Entry; use databend_common_meta_types::EvalExpireTime; use databend_common_meta_types::MatchSeqExt; use databend_common_meta_types::Operation; -use databend_common_meta_types::StorageError; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; diff --git a/src/meta/raft-store/src/sm_v003/snapshot_store_v002.rs b/src/meta/raft-store/src/sm_v003/snapshot_store_v002.rs index 6c0a9c110351..ff8a58f33ead 100644 --- a/src/meta/raft-store/src/sm_v003/snapshot_store_v002.rs +++ b/src/meta/raft-store/src/sm_v003/snapshot_store_v002.rs @@ -17,9 +17,9 @@ use std::io; use std::str::FromStr; use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::ErrorSubject; +use databend_common_meta_types::raft_types::ErrorSubject; +use databend_common_meta_types::raft_types::StorageError; use databend_common_meta_types::SnapshotData; -use databend_common_meta_types::StorageError; use log::error; use log::info; use log::warn; diff --git a/src/meta/raft-store/src/state/raft_state.rs b/src/meta/raft-store/src/state/raft_state.rs index 5a60748be710..bdb9dfb41907 100644 --- a/src/meta/raft-store/src/state/raft_state.rs +++ b/src/meta/raft-store/src/state/raft_state.rs @@ -18,10 +18,10 @@ use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::AsKeySpace; use databend_common_meta_sled_store::SledTree; use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::LogId; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::MetaStartupError; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::Vote; use log::debug; use log::info; diff --git a/src/meta/raft-store/src/state/raft_state_kv.rs b/src/meta/raft-store/src/state/raft_state_kv.rs index bc0c808b4b2a..eb4c4fbc776c 100644 --- a/src/meta/raft-store/src/state/raft_state_kv.rs +++ b/src/meta/raft-store/src/state/raft_state_kv.rs @@ -19,9 +19,9 @@ use databend_common_meta_sled_store::SledBytesError; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; use databend_common_meta_types::anyerror::AnyError; -use databend_common_meta_types::LogId; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::Vote; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::Vote; use serde::Deserialize; use serde::Serialize; use sled::IVec; diff --git a/src/meta/raft-store/src/state_machine/log_meta.rs b/src/meta/raft-store/src/state_machine/log_meta.rs index fac80d8ccef7..4de3977f6fe6 100644 --- a/src/meta/raft-store/src/state_machine/log_meta.rs +++ b/src/meta/raft-store/src/state_machine/log_meta.rs @@ -19,7 +19,7 @@ use databend_common_meta_sled_store::SledBytesError; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; use databend_common_meta_types::anyerror::AnyError; -use databend_common_meta_types::LogId; +use databend_common_meta_types::raft_types::LogId; use serde::Deserialize; use serde::Serialize; use sled::IVec; diff --git a/src/meta/raft-store/src/state_machine/sm.rs b/src/meta/raft-store/src/state_machine/sm.rs index 58e25cd5fa68..287d50039482 100644 --- a/src/meta/raft-store/src/state_machine/sm.rs +++ b/src/meta/raft-store/src/state_machine/sm.rs @@ -28,6 +28,11 @@ use databend_common_meta_sled_store::Store; use databend_common_meta_sled_store::TransactionSledTree; use databend_common_meta_stoerr::MetaStorageError; use databend_common_meta_types::protobuf as pb; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::seq_value::SeqValue; use databend_common_meta_types::txn_condition; @@ -38,17 +43,12 @@ use databend_common_meta_types::Change; use databend_common_meta_types::Cmd; use databend_common_meta_types::CmdContext; use databend_common_meta_types::ConditionResult; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::Interval; -use databend_common_meta_types::LogId; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MatchSeqExt; use databend_common_meta_types::MetaSpec; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; use databend_common_meta_types::Operation; -use databend_common_meta_types::StoredMembership; use databend_common_meta_types::TxnCondition; use databend_common_meta_types::TxnDeleteByPrefixRequest; use databend_common_meta_types::TxnDeleteByPrefixResponse; diff --git a/src/meta/raft-store/src/state_machine/snapshot_id.rs b/src/meta/raft-store/src/state_machine/snapshot_id.rs index 8d604ca32e31..c9c8ae2e332d 100644 --- a/src/meta/raft-store/src/state_machine/snapshot_id.rs +++ b/src/meta/raft-store/src/state_machine/snapshot_id.rs @@ -17,8 +17,8 @@ use std::str::FromStr; use std::time::SystemTime; use std::time::UNIX_EPOCH; -use databend_common_meta_types::new_log_id; -use databend_common_meta_types::LogId; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::LogId; /// Structured snapshot id used by meta service #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] @@ -153,7 +153,7 @@ impl Display for MetaSnapshotId { mod tests { use std::str::FromStr; - use databend_common_meta_types::new_log_id; + use databend_common_meta_types::raft_types::new_log_id; use crate::state_machine::snapshot_id::MetaSnapshotId; diff --git a/src/meta/raft-store/src/state_machine/state_machine_meta.rs b/src/meta/raft-store/src/state_machine/state_machine_meta.rs index 32b18d0777ea..448da6a3775a 100644 --- a/src/meta/raft-store/src/state_machine/state_machine_meta.rs +++ b/src/meta/raft-store/src/state_machine/state_machine_meta.rs @@ -19,8 +19,8 @@ use databend_common_meta_sled_store::SledBytesError; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; use databend_common_meta_types::anyerror::AnyError; -use databend_common_meta_types::LogId; -use databend_common_meta_types::StoredMembership; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::StoredMembership; use serde::Deserialize; use serde::Serialize; use sled::IVec; diff --git a/src/meta/raft-store/src/state_machine/testing.rs b/src/meta/raft-store/src/state_machine/testing.rs index 75c3790d9e37..32c20a7a1870 100644 --- a/src/meta/raft-store/src/state_machine/testing.rs +++ b/src/meta/raft-store/src/state_machine/testing.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_meta_sled_store::openraft; -use databend_common_meta_types::new_log_id; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; use databend_common_meta_types::Cmd; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::LogEntry; use databend_common_meta_types::RaftTxId; use databend_common_meta_types::UpsertKV; diff --git a/src/meta/raft-store/tests/it/log.rs b/src/meta/raft-store/tests/it/log.rs index ec2c6150c687..4aa3141390ef 100644 --- a/src/meta/raft-store/tests/it/log.rs +++ b/src/meta/raft-store/tests/it/log.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_meta_raft_store::log::RaftLog; -use databend_common_meta_types::new_log_id; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; use databend_common_meta_types::Cmd; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::LogEntry; use databend_common_meta_types::UpsertKV; use test_harness::test; diff --git a/src/meta/raft-store/tests/it/state.rs b/src/meta/raft-store/tests/it/state.rs index ce8153460f47..f9577500f34f 100644 --- a/src/meta/raft-store/tests/it/state.rs +++ b/src/meta/raft-store/tests/it/state.rs @@ -13,7 +13,7 @@ // limitations under the License. use databend_common_meta_raft_store::state::RaftState; -use databend_common_meta_types::Vote; +use databend_common_meta_types::raft_types::Vote; use test_harness::test; use crate::testing::new_raft_test_context; diff --git a/src/meta/raft-store/tests/it/state_machine/expire.rs b/src/meta/raft-store/tests/it/state_machine/expire.rs index b13140663a56..72dd93853531 100644 --- a/src/meta/raft-store/tests/it/state_machine/expire.rs +++ b/src/meta/raft-store/tests/it/state_machine/expire.rs @@ -20,10 +20,10 @@ use databend_common_meta_raft_store::key_spaces::GenericKV; use databend_common_meta_raft_store::state_machine::ExpireKey; use databend_common_meta_raft_store::state_machine::StateMachine; use databend_common_meta_sled_store::AsKeySpace; -use databend_common_meta_types::new_log_id; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; use databend_common_meta_types::Cmd; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::LogEntry; use databend_common_meta_types::MetaSpec; use databend_common_meta_types::UpsertKV; diff --git a/src/meta/raft-store/tests/it/state_machine/mod.rs b/src/meta/raft-store/tests/it/state_machine/mod.rs index 797dbdd5f250..27a07649a2ee 100644 --- a/src/meta/raft-store/tests/it/state_machine/mod.rs +++ b/src/meta/raft-store/tests/it/state_machine/mod.rs @@ -18,7 +18,9 @@ use std::time::UNIX_EPOCH; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_raft_store::state_machine::StateMachine; -use databend_common_meta_types::new_log_id; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; use databend_common_meta_types::seq_value::KVMeta; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::seq_value::SeqValue; @@ -27,8 +29,6 @@ use databend_common_meta_types::Change; use databend_common_meta_types::Cmd; use databend_common_meta_types::CmdContext; use databend_common_meta_types::Endpoint; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::LogEntry; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaSpec; diff --git a/src/meta/raft-store/tests/it/types.rs b/src/meta/raft-store/tests/it/types.rs index 6d0dfb6ff701..c24a8b6b5469 100644 --- a/src/meta/raft-store/tests/it/types.rs +++ b/src/meta/raft-store/tests/it/types.rs @@ -17,7 +17,7 @@ use std::ops::Bound; use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledRangeSerde; -use databend_common_meta_types::NodeId; +use databend_common_meta_types::raft_types::NodeId; #[test] fn test_node_id_serde_ser() -> anyhow::Result<()> { diff --git a/src/meta/service/src/api/http/v1/ctrl.rs b/src/meta/service/src/api/http/v1/ctrl.rs index 2578db12c70b..2af9f368d3df 100644 --- a/src/meta/service/src/api/http/v1/ctrl.rs +++ b/src/meta/service/src/api/http/v1/ctrl.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use databend_common_meta_sled_store::openraft::async_runtime::watch::WatchReceiver; -use databend_common_meta_types::NodeId; +use databend_common_meta_types::raft_types::NodeId; use http::StatusCode; use log::info; use log::warn; diff --git a/src/meta/service/src/message.rs b/src/meta/service/src/message.rs index 2b917e9ffd35..e75fb544d8d5 100644 --- a/src/meta/service/src/message.rs +++ b/src/meta/service/src/message.rs @@ -21,11 +21,11 @@ use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReply; use databend_common_meta_kvapi::kvapi::MGetKVReq; use databend_common_meta_types::protobuf::RaftRequest; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::AppliedState; use databend_common_meta_types::Endpoint; use databend_common_meta_types::LogEntry; use databend_common_meta_types::MetaAPIError; -use databend_common_meta_types::NodeId; #[derive(serde::Serialize, serde::Deserialize, Debug, Default, Clone, PartialEq, Eq)] pub struct JoinRequest { diff --git a/src/meta/service/src/meta_service/forwarder.rs b/src/meta/service/src/meta_service/forwarder.rs index 435d5ad672e7..705f58ebf510 100644 --- a/src/meta/service/src/meta_service/forwarder.rs +++ b/src/meta/service/src/meta_service/forwarder.rs @@ -18,13 +18,13 @@ use databend_common_meta_api::reply::reply_to_api_result; use databend_common_meta_client::MetaGrpcReadReq; use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::ConnectionError; use databend_common_meta_types::Endpoint; use databend_common_meta_types::ForwardRPCError; use databend_common_meta_types::GrpcConfig; use databend_common_meta_types::MetaAPIError; use databend_common_meta_types::MetaNetworkError; -use databend_common_meta_types::NodeId; use log::debug; use tonic::codegen::BoxStream; use tonic::transport::Channel; diff --git a/src/meta/service/src/meta_service/meta_leader.rs b/src/meta/service/src/meta_service/meta_leader.rs index 39350e2c9016..9dfce821c98c 100644 --- a/src/meta/service/src/meta_service/meta_leader.rs +++ b/src/meta/service/src/meta_service/meta_leader.rs @@ -22,18 +22,18 @@ use databend_common_meta_raft_store::sm_v003::SMV003; use databend_common_meta_sled_store::openraft::ChangeMembers; use databend_common_meta_stoerr::MetaStorageError; use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::raft_types::ClientWriteError; +use databend_common_meta_types::raft_types::MembershipNode; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::RaftError; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::AppliedState; -use databend_common_meta_types::ClientWriteError; use databend_common_meta_types::Cmd; use databend_common_meta_types::LogEntry; -use databend_common_meta_types::MembershipNode; use databend_common_meta_types::MetaDataError; use databend_common_meta_types::MetaDataReadError; use databend_common_meta_types::MetaOperationError; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::RaftError; use databend_common_metrics::count::Count; use futures::StreamExt; use log::debug; diff --git a/src/meta/service/src/meta_service/meta_node.rs b/src/meta/service/src/meta_service/meta_node.rs index ba777977309c..326d4e715758 100644 --- a/src/meta/service/src/meta_service/meta_node.rs +++ b/src/meta/service/src/meta_service/meta_node.rs @@ -38,16 +38,19 @@ use databend_common_meta_stoerr::MetaStorageError; use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; use databend_common_meta_types::protobuf::raft_service_server::RaftServiceServer; use databend_common_meta_types::protobuf::WatchRequest; +use databend_common_meta_types::raft_types::CommittedLeaderId; +use databend_common_meta_types::raft_types::ForwardToLeader; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::MembershipNode; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::RaftMetrics; +use databend_common_meta_types::raft_types::TypeConfig; use databend_common_meta_types::AppliedState; use databend_common_meta_types::Cmd; -use databend_common_meta_types::CommittedLeaderId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::ForwardRPCError; -use databend_common_meta_types::ForwardToLeader; use databend_common_meta_types::GrpcConfig; use databend_common_meta_types::LogEntry; -use databend_common_meta_types::LogId; -use databend_common_meta_types::MembershipNode; use databend_common_meta_types::MetaAPIError; use databend_common_meta_types::MetaError; use databend_common_meta_types::MetaManagementError; @@ -55,9 +58,6 @@ use databend_common_meta_types::MetaNetworkError; use databend_common_meta_types::MetaOperationError; use databend_common_meta_types::MetaStartupError; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::RaftMetrics; -use databend_common_meta_types::TypeConfig; use fastrace::func_name; use fastrace::prelude::*; use futures::channel::oneshot; diff --git a/src/meta/service/src/meta_service/meta_node_status.rs b/src/meta/service/src/meta_service/meta_node_status.rs index abcf0a0a1b78..73c084fb77b5 100644 --- a/src/meta/service/src/meta_service/meta_node_status.rs +++ b/src/meta/service/src/meta_service/meta_node_status.rs @@ -15,9 +15,9 @@ use std::collections::BTreeMap; use databend_common_meta_raft_store::ondisk::DataVersion; -use databend_common_meta_types::LogId; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; #[derive(serde::Serialize)] pub struct MetaNodeStatus { diff --git a/src/meta/service/src/meta_service/raft_service_impl.rs b/src/meta/service/src/meta_service/raft_service_impl.rs index bd805dbdab59..208e0f3c27d7 100644 --- a/src/meta/service/src/meta_service/raft_service_impl.rs +++ b/src/meta/service/src/meta_service/raft_service_impl.rs @@ -35,20 +35,20 @@ use databend_common_meta_types::protobuf::SnapshotChunkRequest; use databend_common_meta_types::protobuf::SnapshotChunkRequestV003; use databend_common_meta_types::protobuf::SnapshotResponseV003; use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::raft_types::AppendEntriesRequest; +use databend_common_meta_types::raft_types::InstallSnapshotError; +use databend_common_meta_types::raft_types::InstallSnapshotRequest; +use databend_common_meta_types::raft_types::InstallSnapshotResponse; +use databend_common_meta_types::raft_types::RaftError; +use databend_common_meta_types::raft_types::Snapshot; +use databend_common_meta_types::raft_types::SnapshotMeta; +use databend_common_meta_types::raft_types::StorageError; use databend_common_meta_types::raft_types::TransferLeaderRequest; +use databend_common_meta_types::raft_types::Vote; +use databend_common_meta_types::raft_types::VoteRequest; use databend_common_meta_types::snapshot_db::DB; -use databend_common_meta_types::AppendEntriesRequest; use databend_common_meta_types::GrpcHelper; -use databend_common_meta_types::InstallSnapshotError; -use databend_common_meta_types::InstallSnapshotRequest; -use databend_common_meta_types::InstallSnapshotResponse; -use databend_common_meta_types::RaftError; -use databend_common_meta_types::Snapshot; use databend_common_meta_types::SnapshotData; -use databend_common_meta_types::SnapshotMeta; -use databend_common_meta_types::StorageError; -use databend_common_meta_types::Vote; -use databend_common_meta_types::VoteRequest; use databend_common_metrics::count::Count; use fastrace::func_path; use fastrace::prelude::*; diff --git a/src/meta/service/src/meta_service/snapshot_receiver_v1.rs b/src/meta/service/src/meta_service/snapshot_receiver_v1.rs index 8a338b950b40..a137df539d99 100644 --- a/src/meta/service/src/meta_service/snapshot_receiver_v1.rs +++ b/src/meta/service/src/meta_service/snapshot_receiver_v1.rs @@ -24,11 +24,11 @@ use databend_common_meta_sled_store::openraft::ErrorSubject; use databend_common_meta_sled_store::openraft::ErrorVerb; use databend_common_meta_sled_store::openraft::SnapshotId; use databend_common_meta_sled_store::openraft::SnapshotSegmentId; -use databend_common_meta_types::InstallSnapshotError; -use databend_common_meta_types::InstallSnapshotRequest; -use databend_common_meta_types::RaftError; -use databend_common_meta_types::SnapshotMismatch; -use databend_common_meta_types::StorageError; +use databend_common_meta_types::raft_types::InstallSnapshotError; +use databend_common_meta_types::raft_types::InstallSnapshotRequest; +use databend_common_meta_types::raft_types::RaftError; +use databend_common_meta_types::raft_types::SnapshotMismatch; +use databend_common_meta_types::raft_types::StorageError; use fastrace::func_name; use log::info; diff --git a/src/meta/service/src/metrics/meta_metrics.rs b/src/meta/service/src/metrics/meta_metrics.rs index 7b98a9c3a16e..6698b0385657 100644 --- a/src/meta/service/src/metrics/meta_metrics.rs +++ b/src/meta/service/src/metrics/meta_metrics.rs @@ -33,7 +33,7 @@ use prometheus_client::encoding::text::encode as prometheus_encode; pub mod server_metrics { use std::sync::LazyLock; - use databend_common_meta_types::NodeId; + use databend_common_meta_types::raft_types::NodeId; use prometheus_client::metrics::counter::Counter; use prometheus_client::metrics::family::Family; use prometheus_client::metrics::gauge::Gauge; @@ -239,7 +239,7 @@ pub mod raft_metrics { pub mod network { use std::sync::LazyLock; - use databend_common_meta_types::NodeId; + use databend_common_meta_types::raft_types::NodeId; use prometheus_client; use prometheus_client::encoding::EncodeLabelSet; use prometheus_client::metrics::counter::Counter; diff --git a/src/meta/service/src/network.rs b/src/meta/service/src/network.rs index b03c9c56d93d..5739276c2bca 100644 --- a/src/meta/service/src/network.rs +++ b/src/meta/service/src/network.rs @@ -43,30 +43,30 @@ use databend_common_meta_types::protobuf::RaftReply; use databend_common_meta_types::protobuf::RaftRequest; use databend_common_meta_types::protobuf::SnapshotChunkRequest; use databend_common_meta_types::protobuf::SnapshotChunkRequestV003; +use databend_common_meta_types::raft_types::AppendEntriesRequest; +use databend_common_meta_types::raft_types::AppendEntriesResponse; +use databend_common_meta_types::raft_types::ErrorSubject; +use databend_common_meta_types::raft_types::InstallSnapshotError; +use databend_common_meta_types::raft_types::InstallSnapshotRequest; +use databend_common_meta_types::raft_types::InstallSnapshotResponse; +use databend_common_meta_types::raft_types::MembershipNode; +use databend_common_meta_types::raft_types::NetworkError; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::RPCError; +use databend_common_meta_types::raft_types::RaftError; +use databend_common_meta_types::raft_types::Snapshot; +use databend_common_meta_types::raft_types::SnapshotResponse; +use databend_common_meta_types::raft_types::StorageError; +use databend_common_meta_types::raft_types::StreamingError; use databend_common_meta_types::raft_types::TransferLeaderRequest; -use databend_common_meta_types::AppendEntriesRequest; -use databend_common_meta_types::AppendEntriesResponse; +use databend_common_meta_types::raft_types::TypeConfig; +use databend_common_meta_types::raft_types::Vote; +use databend_common_meta_types::raft_types::VoteRequest; +use databend_common_meta_types::raft_types::VoteResponse; use databend_common_meta_types::Endpoint; -use databend_common_meta_types::ErrorSubject; use databend_common_meta_types::GrpcConfig; use databend_common_meta_types::GrpcHelper; -use databend_common_meta_types::InstallSnapshotError; -use databend_common_meta_types::InstallSnapshotRequest; -use databend_common_meta_types::InstallSnapshotResponse; -use databend_common_meta_types::MembershipNode; use databend_common_meta_types::MetaNetworkError; -use databend_common_meta_types::NetworkError; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::RPCError; -use databend_common_meta_types::RaftError; -use databend_common_meta_types::Snapshot; -use databend_common_meta_types::SnapshotResponse; -use databend_common_meta_types::StorageError; -use databend_common_meta_types::StreamingError; -use databend_common_meta_types::TypeConfig; -use databend_common_meta_types::Vote; -use databend_common_meta_types::VoteRequest; -use databend_common_meta_types::VoteResponse; use databend_common_metrics::count::Count; use fastrace::func_name; use futures::FutureExt; diff --git a/src/meta/service/src/raft_client.rs b/src/meta/service/src/raft_client.rs index 5428893fbe21..90cc3865a79d 100644 --- a/src/meta/service/src/raft_client.rs +++ b/src/meta/service/src/raft_client.rs @@ -13,9 +13,9 @@ // limitations under the License. use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::GrpcConfig; -use databend_common_meta_types::NodeId; use databend_common_metrics::count; use log::debug; use tonic::transport::channel::Channel; diff --git a/src/meta/service/src/request_handling.rs b/src/meta/service/src/request_handling.rs index 0328b2f00e7a..88b625755ac8 100644 --- a/src/meta/service/src/request_handling.rs +++ b/src/meta/service/src/request_handling.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_meta_client::RequestFor; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::ForwardRPCError; use databend_common_meta_types::MetaOperationError; -use databend_common_meta_types::NodeId; use crate::message::ForwardRequest; use crate::message::ForwardRequestBody; diff --git a/src/meta/service/src/store/raft_log_storage_impl.rs b/src/meta/service/src/store/raft_log_storage_impl.rs index e59307ac85ec..743a699400ae 100644 --- a/src/meta/service/src/store/raft_log_storage_impl.rs +++ b/src/meta/service/src/store/raft_log_storage_impl.rs @@ -30,12 +30,12 @@ use databend_common_meta_sled_store::openraft::LogState; use databend_common_meta_sled_store::openraft::OptionalSend; use databend_common_meta_sled_store::openraft::RaftLogId; use databend_common_meta_sled_store::openraft::RaftLogReader; -use databend_common_meta_types::Entry; -use databend_common_meta_types::LogId; -use databend_common_meta_types::Membership; -use databend_common_meta_types::StorageError; -use databend_common_meta_types::TypeConfig; -use databend_common_meta_types::Vote; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::Membership; +use databend_common_meta_types::raft_types::StorageError; +use databend_common_meta_types::raft_types::TypeConfig; +use databend_common_meta_types::raft_types::Vote; use deepsize::DeepSizeOf; use log::debug; use log::error; diff --git a/src/meta/service/src/store/raft_state_machine_impl.rs b/src/meta/service/src/store/raft_state_machine_impl.rs index 0df9f575dad9..3e3dd2e43d2c 100644 --- a/src/meta/service/src/store/raft_state_machine_impl.rs +++ b/src/meta/service/src/store/raft_state_machine_impl.rs @@ -17,15 +17,15 @@ use databend_common_meta_raft_store::sm_v003::SnapshotStoreV003; use databend_common_meta_sled_store::openraft::storage::RaftStateMachine; use databend_common_meta_sled_store::openraft::OptionalSend; use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::Snapshot; +use databend_common_meta_types::raft_types::SnapshotMeta; +use databend_common_meta_types::raft_types::StorageError; +use databend_common_meta_types::raft_types::StoredMembership; +use databend_common_meta_types::raft_types::TypeConfig; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::AppliedState; -use databend_common_meta_types::Entry; -use databend_common_meta_types::LogId; -use databend_common_meta_types::Snapshot; -use databend_common_meta_types::SnapshotMeta; -use databend_common_meta_types::StorageError; -use databend_common_meta_types::StoredMembership; -use databend_common_meta_types::TypeConfig; use log::debug; use log::error; use log::info; diff --git a/src/meta/service/src/store/store_inner.rs b/src/meta/service/src/store/store_inner.rs index f3f0fece7f87..86a84fb6a6a2 100644 --- a/src/meta/service/src/store/store_inner.rs +++ b/src/meta/service/src/store/store_inner.rs @@ -37,18 +37,18 @@ use databend_common_meta_raft_store::state_machine::MetaSnapshotId; use databend_common_meta_sled_store::get_sled_db; use databend_common_meta_sled_store::SledTree; use databend_common_meta_stoerr::MetaStorageError; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::Membership; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::Snapshot; +use databend_common_meta_types::raft_types::SnapshotMeta; +use databend_common_meta_types::raft_types::StorageError; +use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::Endpoint; -use databend_common_meta_types::LogId; -use databend_common_meta_types::Membership; use databend_common_meta_types::MetaNetworkError; use databend_common_meta_types::MetaStartupError; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; -use databend_common_meta_types::Snapshot; -use databend_common_meta_types::SnapshotMeta; -use databend_common_meta_types::StorageError; -use databend_common_meta_types::Vote; use futures::TryStreamExt; use log::debug; use log::error; diff --git a/src/meta/service/src/store/to_storage_error.rs b/src/meta/service/src/store/to_storage_error.rs index 1792c34456d0..ea72405bab16 100644 --- a/src/meta/service/src/store/to_storage_error.rs +++ b/src/meta/service/src/store/to_storage_error.rs @@ -14,8 +14,8 @@ use databend_common_meta_sled_store::openraft; use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::ErrorSubject; -use databend_common_meta_types::StorageError; +use databend_common_meta_types::raft_types::ErrorSubject; +use databend_common_meta_types::raft_types::StorageError; use openraft::ErrorVerb; /// Convert MetaStorageError to openraft::StorageError; diff --git a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs index dfafa3b78313..0dcebfe27872 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs @@ -19,12 +19,12 @@ use databend_common_base::base::tokio::time::sleep; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_sled_store::openraft::LogIdOptionExt; use databend_common_meta_sled_store::openraft::ServerState; -use databend_common_meta_types::new_log_id; use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Cmd; use databend_common_meta_types::Endpoint; use databend_common_meta_types::LogEntry; -use databend_common_meta_types::NodeId; use databend_common_meta_types::UpsertKV; use databend_meta::configs; use databend_meta::message::ForwardRequest; diff --git a/src/meta/service/tests/it/meta_node/meta_node_replication.rs b/src/meta/service/tests/it/meta_node/meta_node_replication.rs index 4dac4d55b083..73d8e45bd94f 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_replication.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_replication.rs @@ -24,18 +24,18 @@ use databend_common_meta_sled_store::openraft::LogIdOptionExt; use databend_common_meta_sled_store::openraft::ServerState; use databend_common_meta_types::protobuf::SnapshotChunkRequest; use databend_common_meta_types::protobuf::SnapshotChunkRequestV003; +use databend_common_meta_types::raft_types::InstallSnapshotError; +use databend_common_meta_types::raft_types::InstallSnapshotRequest; +use databend_common_meta_types::raft_types::RaftError; +use databend_common_meta_types::raft_types::SnapshotMeta; +use databend_common_meta_types::raft_types::SnapshotResponse; +use databend_common_meta_types::raft_types::StoredMembership; +use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::sys_data::SysData; use databend_common_meta_types::Cmd; -use databend_common_meta_types::InstallSnapshotError; -use databend_common_meta_types::InstallSnapshotRequest; use databend_common_meta_types::LogEntry; -use databend_common_meta_types::RaftError; -use databend_common_meta_types::SnapshotMeta; -use databend_common_meta_types::SnapshotResponse; -use databend_common_meta_types::StoredMembership; use databend_common_meta_types::UpsertKV; -use databend_common_meta_types::Vote; use databend_meta::meta_service::MetaNode; use futures::stream; use itertools::Itertools; diff --git a/src/meta/service/tests/it/meta_node/meta_node_request_forwarding.rs b/src/meta/service/tests/it/meta_node/meta_node_request_forwarding.rs index a9ace2a51c25..68caaa6f48da 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_request_forwarding.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_request_forwarding.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_meta_sled_store::openraft::error::RaftError; -use databend_common_meta_types::ClientWriteError; +use databend_common_meta_types::raft_types::ClientWriteError; +use databend_common_meta_types::raft_types::ForwardToLeader; use databend_common_meta_types::Cmd; -use databend_common_meta_types::ForwardToLeader; use databend_common_meta_types::LogEntry; use databend_common_meta_types::UpsertKV; use databend_meta::meta_service::meta_leader::MetaLeader; diff --git a/src/meta/service/tests/it/store.rs b/src/meta/service/tests/it/store.rs index cd9654d127b3..92565d0df8ec 100644 --- a/src/meta/service/tests/it/store.rs +++ b/src/meta/service/tests/it/store.rs @@ -25,14 +25,14 @@ use databend_common_meta_sled_store::openraft::testing::log::StoreBuilder; use databend_common_meta_sled_store::openraft::testing::log_id; use databend_common_meta_sled_store::openraft::RaftLogReader; use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; -use databend_common_meta_types::new_log_id; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::Membership; +use databend_common_meta_types::raft_types::StorageError; +use databend_common_meta_types::raft_types::StoredMembership; +use databend_common_meta_types::raft_types::TypeConfig; +use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::snapshot_db::DB; -use databend_common_meta_types::Entry; -use databend_common_meta_types::Membership; -use databend_common_meta_types::StorageError; -use databend_common_meta_types::StoredMembership; -use databend_common_meta_types::TypeConfig; -use databend_common_meta_types::Vote; use databend_meta::meta_service::meta_node::LogStore; use databend_meta::meta_service::meta_node::SMStore; use databend_meta::store::RaftStore; diff --git a/src/meta/service/tests/it/tests/meta_node.rs b/src/meta/service/tests/it/tests/meta_node.rs index cd32b7ffbb1d..27fd4bc4c139 100644 --- a/src/meta/service/tests/it/tests/meta_node.rs +++ b/src/meta/service/tests/it/tests/meta_node.rs @@ -19,9 +19,9 @@ use std::sync::Arc; use std::time::Duration; use databend_common_meta_sled_store::openraft::ServerState; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::AppliedState; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; use databend_meta::meta_service::MetaNode; use databend_meta::Opened; use log::info; diff --git a/src/meta/service/tests/it/tests/service.rs b/src/meta/service/tests/it/tests/service.rs index c8a43f53f587..e145626fef3b 100644 --- a/src/meta/service/tests/it/tests/service.rs +++ b/src/meta/service/tests/it/tests/service.rs @@ -27,8 +27,8 @@ use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_kvapi::kvapi; use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::MetaClientError; -use databend_common_meta_types::NodeId; use databend_meta::api::GrpcServer; use databend_meta::configs; use databend_meta::message::ForwardRequest; diff --git a/src/meta/sled-store/src/sled_serde.rs b/src/meta/sled-store/src/sled_serde.rs index f06fe025ec2b..d12141373069 100644 --- a/src/meta/sled-store/src/sled_serde.rs +++ b/src/meta/sled-store/src/sled_serde.rs @@ -18,7 +18,7 @@ use std::ops::RangeBounds; use byteorder::BigEndian; use byteorder::ByteOrder; -use databend_common_meta_types::Entry; +use databend_common_meta_types::raft_types::Entry; use serde::de::DeserializeOwned; use serde::Serialize; use sled::IVec; diff --git a/src/meta/sled-store/src/sled_serde_impl.rs b/src/meta/sled-store/src/sled_serde_impl.rs index 08825465223b..d28407dc58e3 100644 --- a/src/meta/sled-store/src/sled_serde_impl.rs +++ b/src/meta/sled-store/src/sled_serde_impl.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::Membership; +use databend_common_meta_types::raft_types::SnapshotMeta; +use databend_common_meta_types::raft_types::StoredMembership; +use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; -use databend_common_meta_types::LogId; -use databend_common_meta_types::Membership; use databend_common_meta_types::Node; use databend_common_meta_types::SeqNum; -use databend_common_meta_types::SnapshotMeta; -use databend_common_meta_types::StoredMembership; -use databend_common_meta_types::Vote; use crate::SledBytesError; use crate::SledSerde; diff --git a/src/meta/sled-store/tests/it/sled_iter.rs b/src/meta/sled-store/tests/it/sled_iter.rs index 4d6a1f605b83..dd6804f5bffd 100644 --- a/src/meta/sled-store/tests/it/sled_iter.rs +++ b/src/meta/sled-store/tests/it/sled_iter.rs @@ -14,10 +14,10 @@ use databend_common_meta_sled_store::SledItem; use databend_common_meta_sled_store::SledTree; -use databend_common_meta_types::new_log_id; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; use databend_common_meta_types::Cmd; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::LogEntry; use databend_common_meta_types::UpsertKV; use log::info; diff --git a/src/meta/sled-store/tests/it/sled_tree.rs b/src/meta/sled-store/tests/it/sled_tree.rs index 11ee7a6a6ed1..b2ed93a298b1 100644 --- a/src/meta/sled-store/tests/it/sled_tree.rs +++ b/src/meta/sled-store/tests/it/sled_tree.rs @@ -13,13 +13,13 @@ // limitations under the License. use databend_common_meta_sled_store::SledTree; -use databend_common_meta_types::new_log_id; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; +use databend_common_meta_types::raft_types::LogIndex; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::Cmd; -use databend_common_meta_types::Entry; -use databend_common_meta_types::EntryPayload; use databend_common_meta_types::LogEntry; -use databend_common_meta_types::LogIndex; use databend_common_meta_types::UpsertKV; use test_harness::test; diff --git a/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs b/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs index 6d69d02f8524..f9c80e47c2e6 100644 --- a/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs +++ b/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs @@ -13,11 +13,11 @@ // limitations under the License. use databend_common_meta_sled_store::SledKeySpace; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::LogIndex; +use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::Entry; -use databend_common_meta_types::LogIndex; use databend_common_meta_types::Node; -use databend_common_meta_types::NodeId; use crate::testing::fake_state_machine_meta::StateMachineMetaKey; use crate::testing::fake_state_machine_meta::StateMachineMetaValue; diff --git a/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs b/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs index a917408f0a70..0d3f2e0f03c1 100644 --- a/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs +++ b/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs @@ -18,8 +18,8 @@ use databend_common_meta_sled_store::SledBytesError; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; use databend_common_meta_types::anyerror::AnyError; -use databend_common_meta_types::LogId; -use databend_common_meta_types::Membership; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::Membership; use serde::Deserialize; use serde::Serialize; use sled::IVec; diff --git a/src/meta/types/src/cmd/mod.rs b/src/meta/types/src/cmd/mod.rs index 22ec14dbbaac..ffb46d54ffbd 100644 --- a/src/meta/types/src/cmd/mod.rs +++ b/src/meta/types/src/cmd/mod.rs @@ -17,8 +17,8 @@ use std::fmt; use serde::Deserialize; use serde::Serialize; +use crate::raft_types::NodeId; use crate::Node; -use crate::NodeId; use crate::TxnRequest; mod cmd_context; diff --git a/src/meta/types/src/errors/meta_api_errors.rs b/src/meta/types/src/errors/meta_api_errors.rs index e6e85a20c314..c34b8d4e7f6e 100644 --- a/src/meta/types/src/errors/meta_api_errors.rs +++ b/src/meta/types/src/errors/meta_api_errors.rs @@ -20,13 +20,13 @@ use tonic::Status; use crate::errors; use crate::raft_types::ChangeMembershipError; +use crate::raft_types::ClientWriteError; use crate::raft_types::Fatal; use crate::raft_types::ForwardToLeader; -use crate::ClientWriteError; +use crate::raft_types::RaftError; use crate::InvalidArgument; use crate::InvalidReply; use crate::MetaNetworkError; -use crate::RaftError; /// Errors raised when meta-service handling a request. #[derive(thiserror::Error, serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] diff --git a/src/meta/types/src/errors/meta_raft_errors.rs b/src/meta/types/src/errors/meta_raft_errors.rs index 356288e177d1..2af33338aeb9 100644 --- a/src/meta/types/src/errors/meta_raft_errors.rs +++ b/src/meta/types/src/errors/meta_raft_errors.rs @@ -18,9 +18,9 @@ pub use openraft::error::InProgress; pub use openraft::error::InitializeError; use crate::raft_types::ClientWriteError; +use crate::raft_types::RaftError; use crate::MetaDataError; use crate::MetaOperationError; -use crate::RaftError; // Collection of errors that occur when change membership on local raft node. pub type RaftChangeMembershipError = ClientWriteError; diff --git a/src/meta/types/src/errors/meta_startup_errors.rs b/src/meta/types/src/errors/meta_startup_errors.rs index dd4a0e91587b..f4836f2a3065 100644 --- a/src/meta/types/src/errors/meta_startup_errors.rs +++ b/src/meta/types/src/errors/meta_startup_errors.rs @@ -16,8 +16,8 @@ use anyerror::AnyError; use databend_common_meta_stoerr::MetaStorageError; use crate::raft_types::InitializeError; +use crate::raft_types::RaftError; use crate::MetaNetworkError; -use crate::RaftError; /// Error raised when meta-server startup. #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] diff --git a/src/meta/types/src/grpc_helper.rs b/src/meta/types/src/grpc_helper.rs index 6a1b5503df9a..3eba3851d84c 100644 --- a/src/meta/types/src/grpc_helper.rs +++ b/src/meta/types/src/grpc_helper.rs @@ -22,8 +22,8 @@ use tonic::metadata::MetadataValue; use crate::protobuf::RaftReply; use crate::protobuf::RaftRequest; +use crate::raft_types::RaftError; use crate::Endpoint; -use crate::RaftError; const HEADER_LEADER: &str = "x-databend-meta-leader-grpc-endpoint"; // const HEADER_LEADER_BIN: &str = "x-databend-meta-leader-grpc-endpoint-bin"; diff --git a/src/meta/types/src/lib.rs b/src/meta/types/src/lib.rs index 11af3e85e5f3..cead64922b6c 100644 --- a/src/meta/types/src/lib.rs +++ b/src/meta/types/src/lib.rs @@ -121,39 +121,3 @@ pub use crate::grpc_helper::GrpcHelper; pub use crate::non_empty::NonEmptyStr; pub use crate::non_empty::NonEmptyString; pub use crate::raft_snapshot_data::SnapshotData; -pub use crate::raft_types::new_log_id; -pub use crate::raft_types::AppendEntriesRequest; -pub use crate::raft_types::AppendEntriesResponse; -pub use crate::raft_types::ChangeMembershipError; -pub use crate::raft_types::ClientWriteError; -pub use crate::raft_types::CommittedLeaderId; -pub use crate::raft_types::Entry; -pub use crate::raft_types::EntryPayload; -pub use crate::raft_types::ErrorSubject; -pub use crate::raft_types::Fatal; -pub use crate::raft_types::ForwardToLeader; -pub use crate::raft_types::InstallSnapshotError; -pub use crate::raft_types::InstallSnapshotRequest; -pub use crate::raft_types::InstallSnapshotResponse; -pub use crate::raft_types::LogId; -pub use crate::raft_types::LogIndex; -pub use crate::raft_types::Membership; -pub use crate::raft_types::MembershipNode; -pub use crate::raft_types::NetworkError; -pub use crate::raft_types::NodeId; -pub use crate::raft_types::RPCError; -pub use crate::raft_types::RaftError; -pub use crate::raft_types::RaftMetrics; -pub use crate::raft_types::RemoteError; -pub use crate::raft_types::Snapshot; -pub use crate::raft_types::SnapshotMeta; -pub use crate::raft_types::SnapshotMismatch; -pub use crate::raft_types::SnapshotResponse; -pub use crate::raft_types::StorageError; -pub use crate::raft_types::StoredMembership; -pub use crate::raft_types::StreamingError; -pub use crate::raft_types::Term; -pub use crate::raft_types::TypeConfig; -pub use crate::raft_types::Vote; -pub use crate::raft_types::VoteRequest; -pub use crate::raft_types::VoteResponse; diff --git a/src/meta/types/src/proto_ext/raft_types_ext.rs b/src/meta/types/src/proto_ext/raft_types_ext.rs index 3588dca6645d..6c9bc72a398e 100644 --- a/src/meta/types/src/proto_ext/raft_types_ext.rs +++ b/src/meta/types/src/proto_ext/raft_types_ext.rs @@ -42,7 +42,7 @@ mod log_id_impls { use crate::protobuf as pb; use crate::raft_types; - use crate::CommittedLeaderId; + use crate::raft_types::CommittedLeaderId; impl From for pb::LogId { fn from(log_id: raft_types::LogId) -> Self { diff --git a/src/meta/types/src/proto_ext/snapshot_chunk_request_ext.rs b/src/meta/types/src/proto_ext/snapshot_chunk_request_ext.rs index 3fcd4331f566..c438de3191bc 100644 --- a/src/meta/types/src/proto_ext/snapshot_chunk_request_ext.rs +++ b/src/meta/types/src/proto_ext/snapshot_chunk_request_ext.rs @@ -18,10 +18,10 @@ use crate::protobuf::SnapshotChunkRequest; use crate::protobuf::SnapshotChunkRequestV003; use crate::protobuf::SnapshotChunkV1; use crate::protobuf::SnapshotResponseV003; -use crate::InstallSnapshotRequest; -use crate::NetworkError; -use crate::SnapshotMeta; -use crate::Vote; +use crate::raft_types::InstallSnapshotRequest; +use crate::raft_types::NetworkError; +use crate::raft_types::SnapshotMeta; +use crate::raft_types::Vote; impl SnapshotChunkRequest { /// Return the length of the data in the chunk. diff --git a/src/meta/types/src/proto_ext/transfer_leader_request_ext.rs b/src/meta/types/src/proto_ext/transfer_leader_request_ext.rs index 2a448d4af665..69f763f3e543 100644 --- a/src/meta/types/src/proto_ext/transfer_leader_request_ext.rs +++ b/src/meta/types/src/proto_ext/transfer_leader_request_ext.rs @@ -16,7 +16,7 @@ use tonic::Status; use crate::protobuf as pb; use crate::raft_types::TransferLeaderRequest; -use crate::Vote; +use crate::raft_types::Vote; impl From for pb::TransferLeaderRequest { fn from(req: TransferLeaderRequest) -> Self { diff --git a/src/meta/types/src/snapshot_db.rs b/src/meta/types/src/snapshot_db.rs index dcf17e6e34ab..5c79a97bdc74 100644 --- a/src/meta/types/src/snapshot_db.rs +++ b/src/meta/types/src/snapshot_db.rs @@ -23,8 +23,8 @@ use rotbl::v001::stat::RotblStat; use rotbl::v001::Rotbl; use rotbl::v001::SeqMarked; +use crate::raft_types::SnapshotMeta; use crate::sys_data::SysData; -use crate::SnapshotMeta; /// A readonly leveled map that owns the data. #[derive(Debug, Clone)] diff --git a/src/meta/types/src/sys_data.rs b/src/meta/types/src/sys_data.rs index 338287e1dcc8..fd1e82ee3bb1 100644 --- a/src/meta/types/src/sys_data.rs +++ b/src/meta/types/src/sys_data.rs @@ -16,10 +16,10 @@ use std::collections::BTreeMap; use log::debug; -use crate::LogId; +use crate::raft_types::LogId; +use crate::raft_types::NodeId; +use crate::raft_types::StoredMembership; use crate::Node; -use crate::NodeId; -use crate::StoredMembership; /// Snapshot System data(non-user data). /// From 4f987bae7f3c41af47a852261bb82ebe989f6653 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Wed, 6 Nov 2024 13:56:45 +0800 Subject: [PATCH 03/92] feat: filter null value before join (#16722) * feat: filter null value before join * fix lint * add annotations and process possible crash * dedup filters and fix tests (also need to fix native explain test) * fix test * support semi join * fix test for semi join * adjust threshold and enable only distribution --- .../src/planner/optimizer/cascades/cascade.rs | 43 ++- .../optimizer/cascades/tasks/apply_rule.rs | 2 +- .../optimizer/cascades/tasks/optimize_expr.rs | 4 +- .../cascades/tasks/optimize_group.rs | 2 +- .../src/planner/optimizer/hyper_dp/dphyp.rs | 71 ++--- .../sql/src/planner/optimizer/optimizer.rs | 51 +--- .../sql/src/planner/optimizer/rule/factory.rs | 28 +- .../src/planner/optimizer/rule/rewrite/mod.rs | 2 + .../rule/rewrite/rule_filter_nulls.rs | 197 ++++++++++++++ .../rewrite/rule_push_down_filter_scan.rs | 11 +- .../sql/src/planner/optimizer/rule/rule.rs | 5 +- .../suites/mode/cluster/filter_nulls.test | 253 ++++++++++++++++++ .../mode/standalone/explain/explain.test | 2 +- .../mode/standalone/explain/subquery.test | 2 +- .../standalone/explain_native/subquery.test | 2 +- 15 files changed, 560 insertions(+), 115 deletions(-) create mode 100644 src/query/sql/src/planner/optimizer/rule/rewrite/rule_filter_nulls.rs create mode 100644 tests/sqllogictests/suites/mode/cluster/filter_nulls.test diff --git a/src/query/sql/src/planner/optimizer/cascades/cascade.rs b/src/query/sql/src/planner/optimizer/cascades/cascade.rs index ccf959eb3c06..77e9102e1b98 100644 --- a/src/query/sql/src/planner/optimizer/cascades/cascade.rs +++ b/src/query/sql/src/planner/optimizer/cascades/cascade.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use log::debug; @@ -30,32 +29,30 @@ use crate::optimizer::format::display_memo; use crate::optimizer::memo::Memo; use crate::optimizer::rule::TransformResult; use crate::optimizer::Distribution; +use crate::optimizer::OptimizerContext; use crate::optimizer::RequiredProperty; use crate::optimizer::RuleSet; use crate::optimizer::SExpr; use crate::IndexType; -use crate::MetadataRef; /// A cascades-style search engine to enumerate possible alternations of a relational expression and /// find the optimal one. pub struct CascadesOptimizer { - pub(crate) ctx: Arc, + pub(crate) opt_ctx: OptimizerContext, pub(crate) memo: Memo, pub(crate) cost_model: Box, pub(crate) explore_rule_set: RuleSet, - pub(crate) metadata: MetadataRef, - pub(crate) enforce_distribution: bool, } impl CascadesOptimizer { - pub fn new( - ctx: Arc, - metadata: MetadataRef, - mut optimized: bool, - enforce_distribution: bool, - ) -> Result { - let explore_rule_set = if ctx.get_settings().get_enable_cbo()? { - if unsafe { ctx.get_settings().get_disable_join_reorder()? } { + pub fn new(opt_ctx: OptimizerContext, mut optimized: bool) -> Result { + let explore_rule_set = if opt_ctx.table_ctx.get_settings().get_enable_cbo()? { + if unsafe { + opt_ctx + .table_ctx + .get_settings() + .get_disable_join_reorder()? + } { optimized = true; } get_explore_rule_set(optimized) @@ -63,23 +60,25 @@ impl CascadesOptimizer { RuleSet::create() }; - let cluster_peers = ctx.get_cluster().nodes.len(); - let dop = ctx.get_settings().get_max_threads()? as usize; + let cluster_peers = opt_ctx.table_ctx.get_cluster().nodes.len(); + let dop = opt_ctx.table_ctx.get_settings().get_max_threads()? as usize; let cost_model = Box::new( - DefaultCostModel::new(ctx.clone())? + DefaultCostModel::new(opt_ctx.table_ctx.clone())? .with_cluster_peers(cluster_peers) .with_degree_of_parallelism(dop), ); Ok(CascadesOptimizer { - ctx, + opt_ctx, memo: Memo::create(), cost_model, explore_rule_set, - metadata, - enforce_distribution, }) } + pub(crate) fn enforce_distribution(&self) -> bool { + self.opt_ctx.enable_distributed_optimization + } + fn init(&mut self, expression: SExpr) -> Result<()> { self.memo.init(expression)?; @@ -97,7 +96,7 @@ impl CascadesOptimizer { .ok_or_else(|| ErrorCode::Internal("Root group cannot be None after initialization"))? .group_index; - let root_required_prop = if self.enforce_distribution { + let root_required_prop = if self.enforce_distribution() { RequiredProperty { distribution: Distribution::Serial, } @@ -106,13 +105,13 @@ impl CascadesOptimizer { }; let root_task = OptimizeGroupTask::new( - self.ctx.clone(), + self.opt_ctx.table_ctx.clone(), None, root_index, root_required_prop.clone(), ); - let task_limit = if self.ctx.get_settings().get_enable_cbo()? { + let task_limit = if self.opt_ctx.table_ctx.get_settings().get_enable_cbo()? { DEFAULT_TASK_LIMIT } else { 0 diff --git a/src/query/sql/src/planner/optimizer/cascades/tasks/apply_rule.rs b/src/query/sql/src/planner/optimizer/cascades/tasks/apply_rule.rs index 975bb1530a36..45f63e607283 100644 --- a/src/query/sql/src/planner/optimizer/cascades/tasks/apply_rule.rs +++ b/src/query/sql/src/planner/optimizer/cascades/tasks/apply_rule.rs @@ -52,7 +52,7 @@ impl ApplyRuleTask { let group = optimizer.memo.group(self.target_group_index)?; let m_expr = group.m_expr(self.m_expr_index)?; let mut state = TransformResult::new(); - let rule = RuleFactory::create_rule(self.rule_id, optimizer.metadata.clone())?; + let rule = RuleFactory::create_rule(self.rule_id, optimizer.opt_ctx.clone())?; m_expr.apply_rule(&optimizer.memo, &rule, &mut state)?; optimizer.insert_from_transform_state(self.target_group_index, state)?; diff --git a/src/query/sql/src/planner/optimizer/cascades/tasks/optimize_expr.rs b/src/query/sql/src/planner/optimizer/cascades/tasks/optimize_expr.rs index 4cb8645be82e..5cd674e8802f 100644 --- a/src/query/sql/src/planner/optimizer/cascades/tasks/optimize_expr.rs +++ b/src/query/sql/src/planner/optimizer/cascades/tasks/optimize_expr.rs @@ -357,7 +357,7 @@ impl OptimizeExprTask { let should_enforce = { let mut should_enforce = true; - if optimizer.enforce_distribution + if optimizer.enforce_distribution() && physical_prop.distribution == Distribution::Serial && !matches!( self.required_prop.distribution, @@ -367,7 +367,7 @@ impl OptimizeExprTask { should_enforce = false; } - if optimizer.enforce_distribution + if optimizer.enforce_distribution() && children_best_props .iter() .any(|prop| prop.distribution == Distribution::Serial) diff --git a/src/query/sql/src/planner/optimizer/cascades/tasks/optimize_group.rs b/src/query/sql/src/planner/optimizer/cascades/tasks/optimize_group.rs index 7abfa8d04980..f541ee3b53e8 100644 --- a/src/query/sql/src/planner/optimizer/cascades/tasks/optimize_group.rs +++ b/src/query/sql/src/planner/optimizer/cascades/tasks/optimize_group.rs @@ -174,7 +174,7 @@ impl OptimizeGroupTask { } let rel_expr = RelExpr::with_m_expr(m_expr, &optimizer.memo); - let children_required_props = if optimizer.enforce_distribution { + let children_required_props = if optimizer.enforce_distribution() { rel_expr.compute_required_prop_children(self.ctx.clone(), &self.required_prop)? } else { vec![vec![RequiredProperty::default(); m_expr.plan.arity()]] diff --git a/src/query/sql/src/planner/optimizer/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/hyper_dp/dphyp.rs index db2e0d8699b4..bd968b999508 100644 --- a/src/query/sql/src/planner/optimizer/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/hyper_dp/dphyp.rs @@ -28,6 +28,7 @@ use crate::optimizer::hyper_dp::query_graph::QueryGraph; use crate::optimizer::hyper_dp::util::intersect; use crate::optimizer::hyper_dp::util::union; use crate::optimizer::rule::TransformResult; +use crate::optimizer::OptimizerContext; use crate::optimizer::RuleFactory; use crate::optimizer::RuleID; use crate::optimizer::SExpr; @@ -45,9 +46,7 @@ const RELATION_THRESHOLD: usize = 10; // The join reorder algorithm follows the paper: Dynamic Programming Strikes Back // See the paper for more details. pub struct DPhpy { - ctx: Arc, - sample_executor: Option>, - metadata: MetadataRef, + opt_ctx: OptimizerContext, join_relations: Vec, // base table index -> index of join_relations table_index_map: HashMap, @@ -61,15 +60,9 @@ pub struct DPhpy { } impl DPhpy { - pub fn new( - ctx: Arc, - metadata: MetadataRef, - sample_executor: Option>, - ) -> Self { + pub fn new(opt_ctx: OptimizerContext) -> Self { Self { - ctx, - sample_executor, - metadata, + opt_ctx, join_relations: vec![], table_index_map: Default::default(), dp_table: Default::default(), @@ -80,22 +73,30 @@ impl DPhpy { } } + fn table_ctx(&self) -> Arc { + self.opt_ctx.table_ctx.clone() + } + + fn metadata(&self) -> MetadataRef { + self.opt_ctx.metadata.clone() + } + + fn sample_executor(&self) -> Option> { + self.opt_ctx.sample_executor.clone() + } + async fn new_children(&mut self, s_expr: &SExpr) -> Result { // Parallel process children: start a new dphyp for each child. - let ctx = self.ctx.clone(); - let metadata = self.metadata.clone(); - let sample_executor = self.sample_executor.clone(); let left_expr = s_expr.children[0].clone(); + let opt_ctx = self.opt_ctx.clone(); let left_res = spawn(async move { - let mut dphyp = DPhpy::new(ctx, metadata, sample_executor); + let mut dphyp = DPhpy::new(opt_ctx.clone()); (dphyp.optimize(&left_expr).await, dphyp.table_index_map) }); - let ctx = self.ctx.clone(); - let metadata = self.metadata.clone(); - let sample_executor = self.sample_executor.clone(); let right_expr = s_expr.children[1].clone(); + let opt_ctx = self.opt_ctx.clone(); let right_res = spawn(async move { - let mut dphyp = DPhpy::new(ctx, metadata, sample_executor); + let mut dphyp = DPhpy::new(opt_ctx.clone()); (dphyp.optimize(&right_expr).await, dphyp.table_index_map) }); let left_res = left_res @@ -130,19 +131,17 @@ impl DPhpy { ) -> Result<(Arc, bool)> { if is_subquery { // If it's a subquery, start a new dphyp - let mut dphyp = DPhpy::new( - self.ctx.clone(), - self.metadata.clone(), - self.sample_executor.clone(), - ); + let mut dphyp = DPhpy::new(self.opt_ctx.clone()); let (new_s_expr, _) = dphyp.optimize(s_expr).await?; // Merge `table_index_map` of subquery into current `table_index_map`. let relation_idx = self.join_relations.len() as IndexType; for table_index in dphyp.table_index_map.keys() { self.table_index_map.insert(*table_index, relation_idx); } - self.join_relations - .push(JoinRelation::new(&new_s_expr, self.sample_executor.clone())); + self.join_relations.push(JoinRelation::new( + &new_s_expr, + self.sample_executor().clone(), + )); return Ok((new_s_expr, true)); } @@ -152,9 +151,9 @@ impl DPhpy { // Check if relation contains filter, if exists, check if the filter in `filters` // If exists, remove it from `filters` self.check_filter(relation); - JoinRelation::new(relation, self.sample_executor.clone()) + JoinRelation::new(relation, self.sample_executor().clone()) } else { - JoinRelation::new(s_expr, self.sample_executor.clone()) + JoinRelation::new(s_expr, self.sample_executor().clone()) }; self.table_index_map .insert(op.table_index, self.join_relations.len() as IndexType); @@ -217,8 +216,10 @@ impl DPhpy { } if !is_inner_join { let new_s_expr = self.new_children(s_expr).await?; - self.join_relations - .push(JoinRelation::new(&new_s_expr, self.sample_executor.clone())); + self.join_relations.push(JoinRelation::new( + &new_s_expr, + self.sample_executor().clone(), + )); Ok((Arc::new(new_s_expr), true)) } else { let left_res = self @@ -278,8 +279,10 @@ impl DPhpy { } RelOperator::UnionAll(_) => { let new_s_expr = self.new_children(s_expr).await?; - self.join_relations - .push(JoinRelation::new(&new_s_expr, self.sample_executor.clone())); + self.join_relations.push(JoinRelation::new( + &new_s_expr, + self.sample_executor().clone(), + )); Ok((Arc::new(new_s_expr), true)) } RelOperator::Exchange(_) => { @@ -383,7 +386,7 @@ impl DPhpy { // Get nodes in `relation_set_tree` let nodes = self.relation_set_tree.get_relation_set_by_index(idx)?; let ce = relation - .cardinality(self.ctx.clone(), self.metadata.clone()) + .cardinality(self.table_ctx().clone(), self.metadata().clone()) .await?; let join = JoinNode { join_type: JoinType::Inner, @@ -822,7 +825,7 @@ impl DPhpy { fn apply_rule(&self, s_expr: &SExpr) -> Result { let mut s_expr = s_expr.clone(); - let rule = RuleFactory::create_rule(RuleID::PushDownFilterJoin, self.metadata.clone())?; + let rule = RuleFactory::create_rule(RuleID::PushDownFilterJoin, self.opt_ctx.clone())?; let mut state = TransformResult::new(); if rule .matchers() diff --git a/src/query/sql/src/planner/optimizer/optimizer.rs b/src/query/sql/src/planner/optimizer/optimizer.rs index cca17e8a9983..de1ea299a3fd 100644 --- a/src/query/sql/src/planner/optimizer/optimizer.rs +++ b/src/query/sql/src/planner/optimizer/optimizer.rs @@ -63,16 +63,16 @@ use crate::MetadataRef; #[educe(Debug)] pub struct OptimizerContext { #[educe(Debug(ignore))] - table_ctx: Arc, - metadata: MetadataRef, + pub(crate) table_ctx: Arc, + pub(crate) metadata: MetadataRef, // Optimizer configurations - enable_distributed_optimization: bool, + pub(crate) enable_distributed_optimization: bool, enable_join_reorder: bool, enable_dphyp: bool, planning_agg_index: bool, #[educe(Debug(ignore))] - sample_executor: Option>, + pub(crate) sample_executor: Option>, } impl OptimizerContext { @@ -158,7 +158,7 @@ impl<'a> RecursiveOptimizer<'a> { fn apply_transform_rules(&self, s_expr: &SExpr, rules: &[RuleID]) -> Result { let mut s_expr = s_expr.clone(); for rule_id in rules { - let rule = RuleFactory::create_rule(*rule_id, self.ctx.metadata.clone())?; + let rule = RuleFactory::create_rule(*rule_id, self.ctx.clone())?; let mut state = TransformResult::new(); if rule .matchers() @@ -241,7 +241,7 @@ pub async fn optimize(mut opt_ctx: OptimizerContext, plan: Plan) -> Result } ExplainKind::Memo(_) => { if let box Plan::Query { ref s_expr, .. } = plan { - let memo = get_optimized_memo(opt_ctx, *s_expr.clone()).await?; + let memo = get_optimized_memo(&mut opt_ctx, *s_expr.clone()).await?; Ok(Plan::Explain { config, kind: ExplainKind::Memo(display_memo(&memo)?), @@ -405,13 +405,7 @@ pub async fn optimize_query(opt_ctx: &mut OptimizerContext, mut s_expr: SExpr) - // Cost based optimization let mut dphyp_optimized = false; if opt_ctx.enable_dphyp && opt_ctx.enable_join_reorder { - let (dp_res, optimized) = DPhpy::new( - opt_ctx.table_ctx.clone(), - opt_ctx.metadata.clone(), - opt_ctx.sample_executor.clone(), - ) - .optimize(&s_expr) - .await?; + let (dp_res, optimized) = DPhpy::new(opt_ctx.clone()).optimize(&s_expr).await?; if optimized { s_expr = (*dp_res).clone(); dphyp_optimized = true; @@ -423,12 +417,7 @@ pub async fn optimize_query(opt_ctx: &mut OptimizerContext, mut s_expr: SExpr) - // Deduplicate join conditions. s_expr = DeduplicateJoinConditionOptimizer::new().run(&s_expr)?; - let mut cascades = CascadesOptimizer::new( - opt_ctx.table_ctx.clone(), - opt_ctx.metadata.clone(), - dphyp_optimized, - opt_ctx.enable_distributed_optimization, - )?; + let mut cascades = CascadesOptimizer::new(opt_ctx.clone(), dphyp_optimized)?; if opt_ctx.enable_join_reorder { s_expr = RecursiveOptimizer::new([RuleID::CommuteJoin].as_slice(), opt_ctx).run(&s_expr)?; @@ -468,10 +457,9 @@ pub async fn optimize_query(opt_ctx: &mut OptimizerContext, mut s_expr: SExpr) - } // TODO(leiysky): reuse the optimization logic with `optimize_query` -async fn get_optimized_memo(opt_ctx: OptimizerContext, mut s_expr: SExpr) -> Result { - let mut enable_distributed_query = opt_ctx.enable_distributed_optimization; +async fn get_optimized_memo(opt_ctx: &mut OptimizerContext, mut s_expr: SExpr) -> Result { if contains_local_table_scan(&s_expr, &opt_ctx.metadata) { - enable_distributed_query = false; + opt_ctx.enable_distributed_optimization = false; info!("Disable distributed optimization due to local table scan."); } @@ -492,31 +480,20 @@ async fn get_optimized_memo(opt_ctx: OptimizerContext, mut s_expr: SExpr) -> Res // Pull up and infer filter. s_expr = PullUpFilterOptimizer::new(opt_ctx.metadata.clone()).run(&s_expr)?; // Run default rewrite rules - s_expr = RecursiveOptimizer::new(&DEFAULT_REWRITE_RULES, &opt_ctx).run(&s_expr)?; + s_expr = RecursiveOptimizer::new(&DEFAULT_REWRITE_RULES, opt_ctx).run(&s_expr)?; // Run post rewrite rules - s_expr = RecursiveOptimizer::new(&[RuleID::SplitAggregate], &opt_ctx).run(&s_expr)?; + s_expr = RecursiveOptimizer::new(&[RuleID::SplitAggregate], opt_ctx).run(&s_expr)?; // Cost based optimization let mut dphyp_optimized = false; if opt_ctx.enable_dphyp && opt_ctx.enable_join_reorder { - let (dp_res, optimized) = DPhpy::new( - opt_ctx.table_ctx.clone(), - opt_ctx.metadata.clone(), - opt_ctx.sample_executor.clone(), - ) - .optimize(&s_expr) - .await?; + let (dp_res, optimized) = DPhpy::new(opt_ctx.clone()).optimize(&s_expr).await?; if optimized { s_expr = (*dp_res).clone(); dphyp_optimized = true; } } - let mut cascades = CascadesOptimizer::new( - opt_ctx.table_ctx.clone(), - opt_ctx.metadata.clone(), - dphyp_optimized, - enable_distributed_query, - )?; + let mut cascades = CascadesOptimizer::new(opt_ctx.clone(), dphyp_optimized)?; cascades.optimize(s_expr)?; Ok(cascades.memo) diff --git a/src/query/sql/src/planner/optimizer/rule/factory.rs b/src/query/sql/src/planner/optimizer/rule/factory.rs index dfd6e5c5147e..98ece251cf6b 100644 --- a/src/query/sql/src/planner/optimizer/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/rule/factory.rs @@ -30,6 +30,7 @@ use super::rewrite::RulePushDownSortEvalScalar; use super::rewrite::RuleTryApplyAggIndex; use crate::optimizer::rule::rewrite::RuleEliminateFilter; use crate::optimizer::rule::rewrite::RuleEliminateSort; +use crate::optimizer::rule::rewrite::RuleFilterNulls; use crate::optimizer::rule::rewrite::RuleMergeEvalScalar; use crate::optimizer::rule::rewrite::RuleMergeFilter; use crate::optimizer::rule::rewrite::RulePushDownFilterProjectSet; @@ -50,29 +51,32 @@ use crate::optimizer::rule::transform::RuleEagerAggregation; use crate::optimizer::rule::transform::RuleLeftExchangeJoin; use crate::optimizer::rule::RuleID; use crate::optimizer::rule::RulePtr; -use crate::MetadataRef; +use crate::optimizer::OptimizerContext; pub struct RuleFactory; pub const MAX_PUSH_DOWN_LIMIT: usize = 10000; impl RuleFactory { - pub fn create_rule(id: RuleID, metadata: MetadataRef) -> Result { + pub fn create_rule(id: RuleID, ctx: OptimizerContext) -> Result { match id { - RuleID::EliminateUnion => Ok(Box::new(RuleEliminateUnion::new(metadata))), - RuleID::EliminateEvalScalar => Ok(Box::new(RuleEliminateEvalScalar::new(metadata))), + RuleID::EliminateUnion => Ok(Box::new(RuleEliminateUnion::new(ctx.metadata))), + RuleID::EliminateEvalScalar => Ok(Box::new(RuleEliminateEvalScalar::new(ctx.metadata))), + RuleID::FilterNulls => Ok(Box::new(RuleFilterNulls::new( + ctx.enable_distributed_optimization, + ))), RuleID::PushDownFilterUnion => Ok(Box::new(RulePushDownFilterUnion::new())), RuleID::PushDownFilterEvalScalar => Ok(Box::new(RulePushDownFilterEvalScalar::new())), - RuleID::PushDownFilterJoin => Ok(Box::new(RulePushDownFilterJoin::new(metadata))), - RuleID::PushDownFilterScan => Ok(Box::new(RulePushDownFilterScan::new(metadata))), + RuleID::PushDownFilterJoin => Ok(Box::new(RulePushDownFilterJoin::new(ctx.metadata))), + RuleID::PushDownFilterScan => Ok(Box::new(RulePushDownFilterScan::new(ctx.metadata))), RuleID::PushDownFilterSort => Ok(Box::new(RulePushDownFilterSort::new())), RuleID::PushDownFilterProjectSet => Ok(Box::new(RulePushDownFilterProjectSet::new())), - RuleID::PushDownLimit => Ok(Box::new(RulePushDownLimit::new(metadata))), + RuleID::PushDownLimit => Ok(Box::new(RulePushDownLimit::new(ctx.metadata))), RuleID::PushDownLimitUnion => Ok(Box::new(RulePushDownLimitUnion::new())), RuleID::PushDownLimitScan => Ok(Box::new(RulePushDownLimitScan::new())), RuleID::PushDownSortScan => Ok(Box::new(RulePushDownSortScan::new())), RuleID::PushDownSortEvalScalar => { - Ok(Box::new(RulePushDownSortEvalScalar::new(metadata))) + Ok(Box::new(RulePushDownSortEvalScalar::new(ctx.metadata))) } RuleID::PushDownLimitOuterJoin => Ok(Box::new(RulePushDownLimitOuterJoin::new())), RuleID::PushDownLimitEvalScalar => Ok(Box::new(RulePushDownLimitEvalScalar::new())), @@ -87,7 +91,7 @@ impl RuleFactory { } RuleID::PushDownFilterAggregate => Ok(Box::new(RulePushDownFilterAggregate::new())), RuleID::PushDownFilterWindow => Ok(Box::new(RulePushDownFilterWindow::new())), - RuleID::EliminateFilter => Ok(Box::new(RuleEliminateFilter::new(metadata))), + RuleID::EliminateFilter => Ok(Box::new(RuleEliminateFilter::new(ctx.metadata))), RuleID::MergeEvalScalar => Ok(Box::new(RuleMergeEvalScalar::new())), RuleID::MergeFilter => Ok(Box::new(RuleMergeFilter::new())), RuleID::NormalizeScalarFilter => Ok(Box::new(RuleNormalizeScalarFilter::new())), @@ -96,9 +100,9 @@ impl RuleFactory { RuleID::CommuteJoin => Ok(Box::new(RuleCommuteJoin::new())), RuleID::CommuteJoinBaseTable => Ok(Box::new(RuleCommuteJoinBaseTable::new())), RuleID::LeftExchangeJoin => Ok(Box::new(RuleLeftExchangeJoin::new())), - RuleID::EagerAggregation => Ok(Box::new(RuleEagerAggregation::new(metadata))), - RuleID::PushDownPrewhere => Ok(Box::new(RulePushDownPrewhere::new(metadata))), - RuleID::TryApplyAggIndex => Ok(Box::new(RuleTryApplyAggIndex::new(metadata))), + RuleID::EagerAggregation => Ok(Box::new(RuleEagerAggregation::new(ctx.metadata))), + RuleID::PushDownPrewhere => Ok(Box::new(RulePushDownPrewhere::new(ctx.metadata))), + RuleID::TryApplyAggIndex => Ok(Box::new(RuleTryApplyAggIndex::new(ctx.metadata))), RuleID::EliminateSort => Ok(Box::new(RuleEliminateSort::new())), RuleID::SemiToInnerJoin => Ok(Box::new(RuleSemiToInnerJoin::new())), } diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs index 5191e4f5f74d..d8c306a88bfa 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs @@ -19,6 +19,7 @@ mod rule_eliminate_eval_scalar; mod rule_eliminate_filter; mod rule_eliminate_sort; mod rule_eliminate_union; +mod rule_filter_nulls; mod rule_fold_count_aggregate; mod rule_merge_eval_scalar; mod rule_merge_filter; @@ -51,6 +52,7 @@ pub use rule_eliminate_eval_scalar::RuleEliminateEvalScalar; pub use rule_eliminate_filter::RuleEliminateFilter; pub use rule_eliminate_sort::RuleEliminateSort; pub use rule_eliminate_union::RuleEliminateUnion; +pub use rule_filter_nulls::RuleFilterNulls; pub use rule_fold_count_aggregate::RuleFoldCountAggregate; pub use rule_merge_eval_scalar::RuleMergeEvalScalar; pub use rule_merge_filter::RuleMergeFilter; diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_filter_nulls.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_filter_nulls.rs new file mode 100644 index 000000000000..dd18b058b1f1 --- /dev/null +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_filter_nulls.rs @@ -0,0 +1,197 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::Result; + +use crate::optimizer::extract::Matcher; +use crate::optimizer::rule::Rule; +use crate::optimizer::rule::TransformResult; +use crate::optimizer::RelExpr; +use crate::optimizer::RuleID; +use crate::optimizer::SExpr; +use crate::optimizer::Statistics; +use crate::plans::Filter; +use crate::plans::FunctionCall; +use crate::plans::Join; +use crate::plans::JoinType; +use crate::plans::RelOp; +use crate::plans::RelOperator; +use crate::ScalarExpr; + +const NULL_THRESHOLD_RATIO: f64 = 0.2; + +/// A rule that filters out NULL values from join keys when they exceed a certain threshold. +/// This optimization can improve join performance by reducing the amount of data processed. +pub struct RuleFilterNulls { + id: RuleID, + matchers: Vec, + is_distributed: bool, +} + +impl RuleFilterNulls { + pub fn new(is_distributed: bool) -> Self { + Self { + id: RuleID::FilterNulls, + // Join + // / \ + //... ... + matchers: vec![Matcher::MatchOp { + op_type: RelOp::Join, + children: vec![Matcher::Leaf, Matcher::Leaf], + }], + is_distributed, + } + } + + /// Checks if null filtering should be applied to a join key based on statistics. + /// + /// # Arguments + /// * `key_expr` - The join key expression to check + /// * `stats` - Statistics for the relation containing the key + /// * `cardinality` - Total cardinality of the relation + /// + /// # Returns + /// Some(filter) if null filtering should be applied, None otherwise + fn should_filter_nulls( + key_expr: &ScalarExpr, + stats: &Statistics, + cardinality: f64, + ) -> Option { + if !key_expr.has_one_column_ref() { + return None; + } + + let column_id = *key_expr.used_columns().iter().next()?; + let column_stats = stats.column_stats.get(&column_id)?; + + if cardinality == 0.0 { + return None; + } + + if (column_stats.null_count as f64 / cardinality) >= NULL_THRESHOLD_RATIO { + Some(join_key_null_filter(key_expr)) + } else { + None + } + } +} + +impl Rule for RuleFilterNulls { + fn id(&self) -> RuleID { + self.id + } + + fn apply(&self, s_expr: &SExpr, state: &mut TransformResult) -> Result<()> { + if !self.is_distributed { + state.add_result(s_expr.clone()); + return Ok(()); + } + let join: Join = s_expr.plan().clone().try_into()?; + if !matches!( + join.join_type, + JoinType::Inner | JoinType::LeftSemi | JoinType::RightSemi + ) || join.is_lateral + { + state.add_result(s_expr.clone()); + return Ok(()); + } + let mut left_child = s_expr.child(0)?.clone(); + let mut right_child = s_expr.child(1)?.clone(); + + let left_stat = RelExpr::with_s_expr(&left_child).derive_cardinality()?; + let right_stat = RelExpr::with_s_expr(&right_child).derive_cardinality()?; + let mut left_null_predicates = vec![]; + let mut right_null_predicates = vec![]; + for join_key in join.equi_conditions.iter() { + let left_key = &join_key.left; + let right_key = &join_key.right; + if single_plan(&left_child) { + if let Some(filter) = Self::should_filter_nulls( + left_key, + &left_stat.statistics, + left_stat.cardinality, + ) { + left_null_predicates.push(filter); + } + } + if single_plan(&right_child) { + if let Some(filter) = Self::should_filter_nulls( + right_key, + &right_stat.statistics, + right_stat.cardinality, + ) { + right_null_predicates.push(filter); + } + } + } + if !left_null_predicates.is_empty() { + let left_null_filter = Filter { + predicates: left_null_predicates, + }; + left_child = SExpr::create_unary( + Arc::new(RelOperator::Filter(left_null_filter)), + Arc::new(left_child.clone()), + ); + } + + if !right_null_predicates.is_empty() { + let right_null_filter = Filter { + predicates: right_null_predicates, + }; + right_child = SExpr::create_unary( + Arc::new(RelOperator::Filter(right_null_filter)), + Arc::new(right_child.clone()), + ); + } + + let mut res = s_expr.replace_children(vec![Arc::new(left_child), Arc::new(right_child)]); + res.set_applied_rule(&self.id()); + state.add_result(res); + + Ok(()) + } + + fn matchers(&self) -> &[Matcher] { + &self.matchers + } +} + +/// Creates a NOT NULL filter predicate for a given join key. +fn join_key_null_filter(key: &ScalarExpr) -> ScalarExpr { + ScalarExpr::FunctionCall(FunctionCall { + span: None, + func_name: "is_not_null".to_string(), + params: vec![], + arguments: vec![key.clone()], + }) +} + +/// Checks if an expression represents a single-path plan. +/// +/// A single-path plan is one that follows a linear chain of operations, +/// meaning each node has at most one child. Examples include: +/// - Leaf nodes (table scans) +/// - Linear chains of unary operators (such as: scan -> filter -> project -> limit) +fn single_plan(expr: &SExpr) -> bool { + let children = &expr.children; + if children.len() > 1 { + return false; + } + if children.is_empty() { + return true; + } + single_plan(&children[0]) +} diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_scan.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_scan.rs index 3043b51f07d5..0e159989974a 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_scan.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_scan.rs @@ -182,9 +182,16 @@ impl Rule for RulePushDownFilterScan { let mut scan: Scan = s_expr.child(0)?.plan().clone().try_into()?; let add_filters = self.find_push_down_predicates(&filter.predicates, &scan)?; - match scan.push_down_predicates.as_mut() { - Some(vs) => vs.extend(add_filters), + Some(vs) => { + // Add `add_filters` to vs if there's not already there. + // Keep the order of `vs` to ensure the tests are stable. + for filter in add_filters { + if !vs.contains(&filter) { + vs.push(filter); + } + } + } None => scan.push_down_predicates = Some(add_filters), } diff --git a/src/query/sql/src/planner/optimizer/rule/rule.rs b/src/query/sql/src/planner/optimizer/rule/rule.rs index 9ff6a17c29c1..b765aba22c82 100644 --- a/src/query/sql/src/planner/optimizer/rule/rule.rs +++ b/src/query/sql/src/planner/optimizer/rule/rule.rs @@ -30,6 +30,7 @@ pub static DEFAULT_REWRITE_RULES: LazyLock> = LazyLock::new(|| { RuleID::EliminateUnion, RuleID::MergeEvalScalar, // Filter + RuleID::FilterNulls, RuleID::EliminateFilter, RuleID::MergeFilter, RuleID::NormalizeScalarFilter, @@ -38,9 +39,9 @@ pub static DEFAULT_REWRITE_RULES: LazyLock> = LazyLock::new(|| { RuleID::PushDownFilterWindow, RuleID::PushDownFilterSort, RuleID::PushDownFilterEvalScalar, - // Limit RuleID::PushDownFilterJoin, RuleID::PushDownFilterProjectSet, + // Limit RuleID::PushDownLimit, RuleID::PushDownLimitUnion, RuleID::PushDownSortEvalScalar, @@ -82,6 +83,7 @@ pub enum RuleID { NormalizeScalarFilter, PushDownFilterAggregate, PushDownFilterEvalScalar, + FilterNulls, PushDownFilterUnion, PushDownFilterJoin, PushDownFilterScan, @@ -119,6 +121,7 @@ pub enum RuleID { impl Display for RuleID { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { match self { + RuleID::FilterNulls => write!(f, "FilterNulls"), RuleID::PushDownFilterUnion => write!(f, "PushDownFilterUnion"), RuleID::PushDownFilterEvalScalar => write!(f, "PushDownFilterEvalScalar"), RuleID::PushDownFilterJoin => write!(f, "PushDownFilterJoin"), diff --git a/tests/sqllogictests/suites/mode/cluster/filter_nulls.test b/tests/sqllogictests/suites/mode/cluster/filter_nulls.test new file mode 100644 index 000000000000..96e20e8bf451 --- /dev/null +++ b/tests/sqllogictests/suites/mode/cluster/filter_nulls.test @@ -0,0 +1,253 @@ +statement ok +CREATE OR REPLACE TABLE table1 ( + value INT +); + +statement ok +INSERT INTO table1 (value) SELECT number from numbers(2000); + +statement ok +INSERT INTO table1 (value) SELECT NULL FROM numbers(2000); + +statement ok +CREATE OR REPLACE TABLE table2 ( + value INT +); + +statement ok +INSERT INTO table2 (value) SELECT number from numbers(1000); + +statement ok +INSERT INTO table2 (value) SELECT NULL FROM numbers(3000); + +statement ok +CREATE OR REPLACE TABLE table3 ( + value INT +); + +statement ok +INSERT INTO table3 (value) SELECT number from numbers(2000); + +statement ok +INSERT INTO table3 (value) SELECT NULL FROM numbers(2000); + + +query T +explain SELECT * FROM table1 INNER JOIN table2 ON table1.value = table2.value; +---- +Exchange +├── output columns: [table1.value (#0), table2.value (#1)] +├── exchange type: Merge +└── HashJoin + ├── output columns: [table1.value (#0), table2.value (#1)] + ├── join type: INNER + ├── build keys: [table2.value (#1)] + ├── probe keys: [table1.value (#0)] + ├── filters: [] + ├── estimated rows: 2000.00 + ├── Exchange(Build) + │ ├── output columns: [table2.value (#1)] + │ ├── exchange type: Broadcast + │ └── Filter + │ ├── output columns: [table2.value (#1)] + │ ├── filters: [is_not_null(table2.value (#1))] + │ ├── estimated rows: 1000.00 + │ └── TableScan + │ ├── table: default.default.table2 + │ ├── output columns: [value (#1)] + │ ├── read rows: 1000 + │ ├── read size: 2.30 KiB + │ ├── partitions total: 6 + │ ├── partitions scanned: 3 + │ ├── pruning stats: [segments: , blocks: ] + │ ├── push downs: [filters: [is_not_null(table2.value (#1))], limit: NONE] + │ └── estimated rows: 4000.00 + └── Filter(Probe) + ├── output columns: [table1.value (#0)] + ├── filters: [is_not_null(table1.value (#0))] + ├── estimated rows: 2000.00 + └── TableScan + ├── table: default.default.table1 + ├── output columns: [value (#0)] + ├── read rows: 2000 + ├── read size: 3.94 KiB + ├── partitions total: 6 + ├── partitions scanned: 3 + ├── pruning stats: [segments: , blocks: ] + ├── push downs: [filters: [is_not_null(table1.value (#0))], limit: NONE] + └── estimated rows: 4000.00 + +query T +explain SELECT * FROM table1 INNER JOIN table2 INNER JOIN table3 ON table1.value = table2.value and table2.value = table3.value; +---- +Exchange +├── output columns: [table1.value (#0), table2.value (#1), table3.value (#2)] +├── exchange type: Merge +└── HashJoin + ├── output columns: [table1.value (#0), table2.value (#1), table3.value (#2)] + ├── join type: INNER + ├── build keys: [table3.value (#2)] + ├── probe keys: [table1.value (#0)] + ├── filters: [] + ├── estimated rows: 4000.00 + ├── HashJoin(Build) + │ ├── output columns: [table3.value (#2), table2.value (#1)] + │ ├── join type: INNER + │ ├── build keys: [table2.value (#1)] + │ ├── probe keys: [table3.value (#2)] + │ ├── filters: [] + │ ├── estimated rows: 2000.00 + │ ├── Exchange(Build) + │ │ ├── output columns: [table2.value (#1)] + │ │ ├── exchange type: Hash(table2.value (#1)) + │ │ └── Filter + │ │ ├── output columns: [table2.value (#1)] + │ │ ├── filters: [is_not_null(table2.value (#1))] + │ │ ├── estimated rows: 1000.00 + │ │ └── TableScan + │ │ ├── table: default.default.table2 + │ │ ├── output columns: [value (#1)] + │ │ ├── read rows: 1000 + │ │ ├── read size: 2.30 KiB + │ │ ├── partitions total: 6 + │ │ ├── partitions scanned: 3 + │ │ ├── pruning stats: [segments: , blocks: ] + │ │ ├── push downs: [filters: [is_not_null(table2.value (#1))], limit: NONE] + │ │ └── estimated rows: 4000.00 + │ └── Exchange(Probe) + │ ├── output columns: [table3.value (#2)] + │ ├── exchange type: Hash(table3.value (#2)) + │ └── Filter + │ ├── output columns: [table3.value (#2)] + │ ├── filters: [is_not_null(table3.value (#2))] + │ ├── estimated rows: 2000.00 + │ └── TableScan + │ ├── table: default.default.table3 + │ ├── output columns: [value (#2)] + │ ├── read rows: 2000 + │ ├── read size: 3.94 KiB + │ ├── partitions total: 6 + │ ├── partitions scanned: 3 + │ ├── pruning stats: [segments: , blocks: ] + │ ├── push downs: [filters: [is_not_null(table3.value (#2))], limit: NONE] + │ └── estimated rows: 4000.00 + └── Exchange(Probe) + ├── output columns: [table1.value (#0)] + ├── exchange type: Hash(table1.value (#0)) + └── Filter + ├── output columns: [table1.value (#0)] + ├── filters: [is_not_null(table1.value (#0))] + ├── estimated rows: 2000.00 + └── TableScan + ├── table: default.default.table1 + ├── output columns: [value (#0)] + ├── read rows: 2000 + ├── read size: 3.94 KiB + ├── partitions total: 6 + ├── partitions scanned: 3 + ├── pruning stats: [segments: , blocks: ] + ├── push downs: [filters: [is_not_null(table1.value (#0))], limit: NONE] + └── estimated rows: 4000.00 + + +query T +explain SELECT * FROM table1 LEFT SEMI JOIN table2 ON table1.value = table2.value; +---- +Exchange +├── output columns: [table1.value (#0)] +├── exchange type: Merge +└── HashJoin + ├── output columns: [table1.value (#0)] + ├── join type: LEFT SEMI + ├── build keys: [table2.value (#1)] + ├── probe keys: [table1.value (#0)] + ├── filters: [] + ├── estimated rows: 2000.00 + ├── Exchange(Build) + │ ├── output columns: [table2.value (#1)] + │ ├── exchange type: Broadcast + │ └── Filter + │ ├── output columns: [table2.value (#1)] + │ ├── filters: [is_not_null(table2.value (#1))] + │ ├── estimated rows: 1000.00 + │ └── TableScan + │ ├── table: default.default.table2 + │ ├── output columns: [value (#1)] + │ ├── read rows: 1000 + │ ├── read size: 2.30 KiB + │ ├── partitions total: 6 + │ ├── partitions scanned: 3 + │ ├── pruning stats: [segments: , blocks: ] + │ ├── push downs: [filters: [is_not_null(table2.value (#1))], limit: NONE] + │ └── estimated rows: 4000.00 + └── Filter(Probe) + ├── output columns: [table1.value (#0)] + ├── filters: [is_not_null(table1.value (#0))] + ├── estimated rows: 2000.00 + └── TableScan + ├── table: default.default.table1 + ├── output columns: [value (#0)] + ├── read rows: 2000 + ├── read size: 3.94 KiB + ├── partitions total: 6 + ├── partitions scanned: 3 + ├── pruning stats: [segments: , blocks: ] + ├── push downs: [filters: [is_not_null(table1.value (#0))], limit: NONE] + └── estimated rows: 4000.00 + +query T +explain SELECT * FROM table1 RIGHT SEMI JOIN table2 ON table1.value = table2.value; +---- +Exchange +├── output columns: [table2.value (#1)] +├── exchange type: Merge +└── HashJoin + ├── output columns: [table2.value (#1)] + ├── join type: RIGHT SEMI + ├── build keys: [table2.value (#1)] + ├── probe keys: [table1.value (#0)] + ├── filters: [] + ├── estimated rows: 1000.00 + ├── Exchange(Build) + │ ├── output columns: [table2.value (#1)] + │ ├── exchange type: Hash(table2.value (#1)) + │ └── Filter + │ ├── output columns: [table2.value (#1)] + │ ├── filters: [is_not_null(table2.value (#1))] + │ ├── estimated rows: 1000.00 + │ └── TableScan + │ ├── table: default.default.table2 + │ ├── output columns: [value (#1)] + │ ├── read rows: 1000 + │ ├── read size: 2.30 KiB + │ ├── partitions total: 6 + │ ├── partitions scanned: 3 + │ ├── pruning stats: [segments: , blocks: ] + │ ├── push downs: [filters: [is_not_null(table2.value (#1))], limit: NONE] + │ └── estimated rows: 4000.00 + └── Exchange(Probe) + ├── output columns: [table1.value (#0)] + ├── exchange type: Hash(table1.value (#0)) + └── Filter + ├── output columns: [table1.value (#0)] + ├── filters: [is_not_null(table1.value (#0))] + ├── estimated rows: 2000.00 + └── TableScan + ├── table: default.default.table1 + ├── output columns: [value (#0)] + ├── read rows: 2000 + ├── read size: 3.94 KiB + ├── partitions total: 6 + ├── partitions scanned: 3 + ├── pruning stats: [segments: , blocks: ] + ├── push downs: [filters: [is_not_null(table1.value (#0))], limit: NONE] + └── estimated rows: 4000.00 + +statement ok +drop table table1; + +statement ok +drop table table2; + +statement ok +drop table table3; diff --git a/tests/sqllogictests/suites/mode/standalone/explain/explain.test b/tests/sqllogictests/suites/mode/standalone/explain/explain.test index bff989763ab1..d29b67e5dd6f 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/explain.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/explain.test @@ -1129,7 +1129,7 @@ HashJoin │ ├── partitions total: 1 │ ├── partitions scanned: 1 │ ├── pruning stats: [segments: , blocks: ] -│ ├── push downs: [filters: [and_filters(b.id (#2) = b.id (#2), b.id (#2) = b.id (#2))], limit: NONE] +│ ├── push downs: [filters: [is_true(b.id (#2) = b.id (#2))], limit: NONE] │ └── estimated rows: 2.00 └── TableScan(Probe) ├── table: default.default.a diff --git a/tests/sqllogictests/suites/mode/standalone/explain/subquery.test b/tests/sqllogictests/suites/mode/standalone/explain/subquery.test index a4ccf6732461..e4ee23dd4b9c 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/subquery.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/subquery.test @@ -293,7 +293,7 @@ HashJoin │ ├── read size: < 1 KiB │ ├── partitions total: 1 │ ├── partitions scanned: 1 -│ ├── push downs: [filters: [and_filters(and_filters(numbers.number (#1) = 0, numbers.number (#1) < 10), numbers.number (#1) = 0)], limit: NONE] +│ ├── push downs: [filters: [and_filters(numbers.number (#1) = 0, numbers.number (#1) < 10)], limit: NONE] │ └── estimated rows: 1.00 └── Filter(Probe) ├── output columns: [t.number (#0)] diff --git a/tests/sqllogictests/suites/mode/standalone/explain_native/subquery.test b/tests/sqllogictests/suites/mode/standalone/explain_native/subquery.test index 3be20bbd5cc0..24742372f4f2 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain_native/subquery.test +++ b/tests/sqllogictests/suites/mode/standalone/explain_native/subquery.test @@ -293,7 +293,7 @@ HashJoin │ ├── read size: < 1 KiB │ ├── partitions total: 1 │ ├── partitions scanned: 1 -│ ├── push downs: [filters: [and_filters(and_filters(numbers.number (#1) = 0, numbers.number (#1) < 10), numbers.number (#1) = 0)], limit: NONE] +│ ├── push downs: [filters: [and_filters(numbers.number (#1) = 0, numbers.number (#1) < 10)], limit: NONE] │ └── estimated rows: 1.00 └── Filter(Probe) ├── output columns: [t.number (#0)] From bb56f16f0de687e999cf4440f03cb7799cb14f8a Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 6 Nov 2024 20:28:46 +0800 Subject: [PATCH 04/92] fix(query): add tls config for udf client (#16782) --- src/query/expression/src/utils/udf_client.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/query/expression/src/utils/udf_client.rs b/src/query/expression/src/utils/udf_client.rs index be77dd37ef19..66cb3b061c5c 100644 --- a/src/query/expression/src/utils/udf_client.rs +++ b/src/query/expression/src/utils/udf_client.rs @@ -37,6 +37,7 @@ use tonic::metadata::MetadataKey; use tonic::metadata::MetadataMap; use tonic::metadata::MetadataValue; use tonic::transport::channel::Channel; +use tonic::transport::ClientTlsConfig; use tonic::transport::Endpoint; use tonic::Request; @@ -65,6 +66,7 @@ impl UDFFlightClient { request_timeout: u64, batch_rows: u64, ) -> Result { + let tls_config = ClientTlsConfig::new().with_native_roots(); let endpoint = Endpoint::from_shared(addr.to_string()) .map_err(|err| { ErrorCode::UDFServerConnectError(format!("Invalid UDF Server address: {err}")) @@ -78,7 +80,11 @@ impl UDFFlightClient { .tcp_keepalive(Some(Duration::from_secs(UDF_TCP_KEEP_ALIVE_SEC))) .http2_keep_alive_interval(Duration::from_secs(UDF_HTTP2_KEEP_ALIVE_INTERVAL_SEC)) .keep_alive_timeout(Duration::from_secs(UDF_KEEP_ALIVE_TIMEOUT_SEC)) - .keep_alive_while_idle(true); + .keep_alive_while_idle(true) + .tls_config(tls_config) + .map_err(|err| { + ErrorCode::UDFServerConnectError(format!("Invalid UDF Client TLS Config: {err}")) + })?; let mut connector = HttpConnector::new_with_resolver(DNSService); connector.enforce_http(false); From 9778d23eab410502e88e77860612712a71976d86 Mon Sep 17 00:00:00 2001 From: Winnie-Hong0927 <20226679@stu.neu.edu.cn> Date: Thu, 7 Nov 2024 10:03:43 +0800 Subject: [PATCH 05/92] feat(query): Support rename dictionary ddl. (#16754) * feat: show dictionaries stmt,planner. * feat: add storages dictionaries * feat: dictionaries_table. * update: disctionaries_table and binder. * fix * fix * fix and test. * fmt * update. * update binder and test. * update privilege_access and testdata. * feat: rename dictionary. * fix: schema_api_impl and fmt * fix * fix * feat: stmt * update:stmt * feat interpreters. * fix. * update test. * fix: api & interface * fix --------- Co-authored-by: winnie-wenli <136137323+winnie-wenli@users.noreply.github.com> --- src/meta/api/src/schema_api.rs | 3 + src/meta/api/src/schema_api_impl.rs | 68 +++++++++++++++++ src/meta/api/src/schema_api_test_suite.rs | 35 ++++++++- src/meta/app/src/schema/dictionary.rs | 44 +++++++++++ .../ast/src/ast/statements/dictionary.rs | 35 +++++++++ src/query/ast/src/ast/statements/statement.rs | 2 + src/query/ast/src/parser/statement.rs | 24 ++++++ src/query/catalog/src/catalog/interface.rs | 3 + .../src/catalogs/default/database_catalog.rs | 5 ++ .../src/catalogs/default/immutable_catalog.rs | 5 ++ .../src/catalogs/default/mutable_catalog.rs | 7 ++ .../src/catalogs/default/session_catalog.rs | 5 ++ .../interpreters/access/privilege_access.rs | 3 +- .../interpreter_dictionary_rename.rs | 75 +++++++++++++++++++ .../src/interpreters/interpreter_factory.rs | 3 + src/query/service/src/interpreters/mod.rs | 2 + .../tests/it/sql/exec/get_table_bind_test.rs | 5 ++ .../it/storages/fuse/operations/commit.rs | 5 ++ src/query/sql/src/planner/binder/binder.rs | 1 + .../sql/src/planner/binder/ddl/dictionary.rs | 48 ++++++++++++ .../sql/src/planner/format/display_plan.rs | 1 + .../sql/src/planner/plans/ddl/dictionary.rs | 21 ++++++ src/query/sql/src/planner/plans/plan.rs | 2 + .../storages/hive/hive/src/hive_catalog.rs | 5 ++ src/query/storages/iceberg/src/catalog.rs | 5 ++ .../base/05_ddl/05_0037_ddl_dictionary.test | 40 ++++++++++ 26 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 src/query/service/src/interpreters/interpreter_dictionary_rename.rs diff --git a/src/meta/api/src/schema_api.rs b/src/meta/api/src/schema_api.rs index 4bbd22bf39c5..5a4992b6302b 100644 --- a/src/meta/api/src/schema_api.rs +++ b/src/meta/api/src/schema_api.rs @@ -72,6 +72,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -372,6 +373,8 @@ pub trait SchemaApi: Send + Sync { req: ListDictionaryReq, ) -> Result, KVAppError>; + async fn rename_dictionary(&self, req: RenameDictionaryReq) -> Result<(), KVAppError>; + /// Generic get() implementation for any kvapi::Key. /// /// This method just return an `Option` of the value without seq number. diff --git a/src/meta/api/src/schema_api_impl.rs b/src/meta/api/src/schema_api_impl.rs index c886a9174ae3..814ddc85d2ca 100644 --- a/src/meta/api/src/schema_api_impl.rs +++ b/src/meta/api/src/schema_api_impl.rs @@ -62,6 +62,7 @@ use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdentRaw; use databend_common_meta_app::schema::dictionary_id_ident::DictionaryId; use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameRsc; use databend_common_meta_app::schema::index_id_ident::IndexId; use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; use databend_common_meta_app::schema::index_id_to_name_ident::IndexIdToNameIdent; @@ -130,6 +131,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyAction; @@ -162,6 +164,7 @@ use databend_common_meta_app::schema::UpsertTableOptionReq; use databend_common_meta_app::schema::VirtualColumnIdent; use databend_common_meta_app::schema::VirtualColumnMeta; use databend_common_meta_app::tenant::Tenant; +use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_app::tenant_key::errors::UnknownError; use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; @@ -3031,6 +3034,55 @@ impl + ?Sized> SchemaApi for KV { .map(|(name, _seq_id, seq_meta)| (name.dict_name(), seq_meta.data)) .collect()) } + + #[logcall::logcall] + #[fastrace::trace] + async fn rename_dictionary(&self, req: RenameDictionaryReq) -> Result<(), KVAppError> { + debug!(req :? =(&req); "SchemaApi: {}", func_name!()); + + let mut trials = txn_backoff(None, func_name!()); + loop { + trials.next().unwrap()?.await; + + let dict_id = self + .get_pb(&req.name_ident) + .await? + .ok_or_else(|| AppError::from(req.name_ident.unknown_error(func_name!())))?; + + let new_name_ident = DictionaryNameIdent::new(req.tenant(), req.new_dict_ident.clone()); + let new_dict_id_seq = self.get_seq(&new_name_ident).await?; + let _ = dict_has_to_not_exist(new_dict_id_seq, &new_name_ident, "rename_dictionary") + .map_err(|_| AppError::from(new_name_ident.exist_error(func_name!())))?; + + let condition = vec![ + txn_cond_seq(&req.name_ident, Eq, dict_id.seq), + txn_cond_seq(&new_name_ident, Eq, 0), + ]; + let if_then = vec![ + txn_op_del(&req.name_ident), // del old dict name + txn_op_put_pb(&new_name_ident, &dict_id.data, None)?, // put new dict name + ]; + + let txn_req = TxnRequest { + condition, + if_then, + else_then: vec![], + }; + + let (succ, _responses) = send_txn(self, txn_req).await?; + + debug!( + name :? =(req.name_ident), + to :? =(&new_name_ident), + succ = succ; + "rename_dictionary" + ); + + if succ { + return Ok(()); + } + } + } } async fn get_retainable_table_metas( @@ -3448,6 +3500,22 @@ fn table_has_to_not_exist( } } +/// Return OK if a dictionary_id or dictionary_meta does not exist by checking the seq. +/// +/// Otherwise returns DictionaryAlreadyExists error +fn dict_has_to_not_exist( + seq: u64, + name_ident: &DictionaryNameIdent, + _ctx: impl Display, +) -> Result<(), ExistError> { + if seq == 0 { + Ok(()) + } else { + debug!(seq = seq, name_ident :? =(name_ident); "exist"); + Err(name_ident.exist_error(func_name!())) + } +} + fn build_upsert_table_deduplicated_label(deduplicated_label: String) -> TxnOp { TxnOp::put_with_ttl( deduplicated_label, diff --git a/src/meta/api/src/schema_api_test_suite.rs b/src/meta/api/src/schema_api_test_suite.rs index cf54d311f853..daa218786f87 100644 --- a/src/meta/api/src/schema_api_test_suite.rs +++ b/src/meta/api/src/schema_api_test_suite.rs @@ -99,6 +99,7 @@ use databend_common_meta_app::schema::ListTableReq; use databend_common_meta_app::schema::ListVirtualColumnsReq; use databend_common_meta_app::schema::LockKey; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SequenceIdent; use databend_common_meta_app::schema::SetTableColumnMaskPolicyAction; @@ -349,7 +350,6 @@ impl SchemaApiTestSuite { suite.test_sequence(&b.build().await).await?; suite.dictionary_create_list_drop(&b.build().await).await?; - Ok(()) } @@ -7304,6 +7304,7 @@ impl SchemaApiTestSuite { let tbl_name = "tb2"; let dict_name1 = "dict1"; let dict_name2 = "dict2"; + let dict_name3 = "dict3"; let dict_tenant = Tenant::new_or_err(tenant_name.to_string(), func_name!())?; let mut util = Util::new(mt, tenant_name, db_name, tbl_name, "eng1"); @@ -7361,6 +7362,8 @@ impl SchemaApiTestSuite { dict_tenant.clone(), DictionaryIdentity::new(db_id, dict_name2.to_string()), ); + let dict3 = DictionaryIdentity::new(db_id, dict_name3.to_string()); + let dict_ident3 = DictionaryNameIdent::new(dict_tenant.clone(), dict3.clone()); { info!("--- create dictionary"); @@ -7459,8 +7462,36 @@ impl SchemaApiTestSuite { } { - info!("--- drop dictionary"); + info!("--- rename dictionary"); + let rename_req = RenameDictionaryReq { + name_ident: dict_ident1.clone(), + new_dict_ident: dict3.clone(), + }; + mt.rename_dictionary(rename_req).await?; let req = dict_ident1.clone(); + let res = mt.get_dictionary(req).await?; + assert!(res.is_none()); + + let req = dict_ident3.clone(); + let res = mt.get_dictionary(req).await?; + assert!(res.is_some()); + + let dict_reply = res.unwrap(); + assert_eq!(*dict_reply.0.data, dict_id); + assert_eq!(dict_reply.1.source, "postgresql".to_string()); + + info!("--- rename unknown dictionary"); + let rename_req = RenameDictionaryReq { + name_ident: dict_ident1.clone(), + new_dict_ident: dict3.clone(), + }; + let res = mt.rename_dictionary(rename_req).await; + assert!(res.is_err()); + } + + { + info!("--- drop dictionary"); + let req = dict_ident3.clone(); let res = mt.drop_dictionary(req).await?; assert!(res.is_some()); } diff --git a/src/meta/app/src/schema/dictionary.rs b/src/meta/app/src/schema/dictionary.rs index 99679b2a8c99..15be7c77019d 100644 --- a/src/meta/app/src/schema/dictionary.rs +++ b/src/meta/app/src/schema/dictionary.rs @@ -25,8 +25,10 @@ use databend_common_exception::Result; use databend_common_expression::TableSchema; use super::dictionary_name_ident::DictionaryNameIdent; +use crate::schema::DictionaryIdentity; use crate::tenant::Tenant; use crate::tenant::ToTenant; +use crate::KeyWithTenant; /// Represents the metadata of a dictionary within the system. #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq)] @@ -172,3 +174,45 @@ pub struct UpdateDictionaryReq { pub struct UpdateDictionaryReply { pub dictionary_id: u64, } + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RenameDictionaryReq { + pub name_ident: DictionaryNameIdent, + pub new_dict_ident: DictionaryIdentity, +} + +impl RenameDictionaryReq { + pub fn tenant(&self) -> &Tenant { + self.name_ident.tenant() + } + + pub fn db_id(&self) -> u64 { + self.name_ident.db_id() + } + + pub fn dictionary_name(&self) -> String { + self.name_ident.dict_name().clone() + } + + pub fn new_db_id(&self) -> u64 { + self.new_dict_ident.db_id + } + + pub fn new_dictionary_name(&self) -> String { + self.new_dict_ident.dict_name.clone() + } +} + +impl Display for RenameDictionaryReq { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "rename_dictionary:{}/{}-{}=>{}-{}", + self.tenant().tenant_name(), + self.db_id(), + self.dictionary_name(), + self.new_db_id(), + self.new_dictionary_name(), + ) + } +} diff --git a/src/query/ast/src/ast/statements/dictionary.rs b/src/query/ast/src/ast/statements/dictionary.rs index 2ddf086c7e11..8bcbc09f3085 100644 --- a/src/query/ast/src/ast/statements/dictionary.rs +++ b/src/query/ast/src/ast/statements/dictionary.rs @@ -143,3 +143,38 @@ impl Display for ShowDictionariesStmt { Ok(()) } } + +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub struct RenameDictionaryStmt { + pub if_exists: bool, + pub catalog: Option, + pub database: Option, + pub dictionary: Identifier, + pub new_catalog: Option, + pub new_database: Option, + pub new_dictionary: Identifier, +} + +impl Display for RenameDictionaryStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "RENAME DICTIONARY ")?; + if self.if_exists { + write!(f, "IF EXISTS ")?; + } + write_dot_separated_list( + f, + self.catalog + .iter() + .chain(&self.database) + .chain(Some(&self.dictionary)), + )?; + write!(f, " TO ")?; + write_dot_separated_list( + f, + self.new_catalog + .iter() + .chain(&self.new_database) + .chain(Some(&self.new_dictionary)), + ) + } +} diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index e6d875aaca46..6ccf682ddc0f 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -161,6 +161,7 @@ pub enum Statement { DropDictionary(DropDictionaryStmt), ShowCreateDictionary(ShowCreateDictionaryStmt), ShowDictionaries(ShowDictionariesStmt), + RenameDictionary(RenameDictionaryStmt), // Columns ShowColumns(ShowColumnsStmt), @@ -576,6 +577,7 @@ impl Display for Statement { Statement::DropDictionary(stmt) => write!(f, "{stmt}")?, Statement::ShowCreateDictionary(stmt) => write!(f, "{stmt}")?, Statement::ShowDictionaries(stmt) => write!(f, "{stmt}")?, + Statement::RenameDictionary(stmt) => write!(f, "{stmt}")?, Statement::CreateView(stmt) => write!(f, "{stmt}")?, Statement::AlterView(stmt) => write!(f, "{stmt}")?, Statement::DropView(stmt) => write!(f, "{stmt}")?, diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 011508d9c0fa..0c581bfe5046 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -1018,6 +1018,29 @@ pub fn statement_body(i: Input) -> IResult { }) }, ); + let rename_dictionary = map( + rule! { + RENAME ~ DICTIONARY ~ ( IF ~ ^EXISTS )? ~ #dot_separated_idents_1_to_3 ~ TO ~ #dot_separated_idents_1_to_3 + }, + |( + _, + _, + opt_if_exists, + (catalog, database, dictionary), + _, + (new_catalog, new_database, new_dictionary), + )| { + Statement::RenameDictionary(RenameDictionaryStmt { + if_exists: opt_if_exists.is_some(), + catalog, + database, + dictionary, + new_catalog, + new_database, + new_dictionary, + }) + }, + ); let create_view = map_res( rule! { @@ -2336,6 +2359,7 @@ pub fn statement_body(i: Input) -> IResult { | #drop_dictionary : "`DROP DICTIONARY [IF EXISTS] `" | #show_create_dictionary : "`SHOW CREATE DICTIONARY `" | #show_dictionaries : "`SHOW DICTIONARIES [, ...]`" + | #rename_dictionary: "`RENAME DICTIONARY [.] TO `" ), // view,index rule!( diff --git a/src/query/catalog/src/catalog/interface.rs b/src/query/catalog/src/catalog/interface.rs index 98e52720967c..935c3706a9fc 100644 --- a/src/query/catalog/src/catalog/interface.rs +++ b/src/query/catalog/src/catalog/interface.rs @@ -74,6 +74,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -526,4 +527,6 @@ pub trait Catalog: DynClone + Send + Sync + Debug { &self, req: ListDictionaryReq, ) -> Result>; + + async fn rename_dictionary(&self, req: RenameDictionaryReq) -> Result<()>; } diff --git a/src/query/service/src/catalogs/default/database_catalog.rs b/src/query/service/src/catalogs/default/database_catalog.rs index e0b89124a4c3..b1ecc39360e4 100644 --- a/src/query/service/src/catalogs/default/database_catalog.rs +++ b/src/query/service/src/catalogs/default/database_catalog.rs @@ -79,6 +79,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -846,4 +847,8 @@ impl Catalog for DatabaseCatalog { ) -> Result> { self.mutable_catalog.list_dictionaries(req).await } + + async fn rename_dictionary(&self, req: RenameDictionaryReq) -> Result<()> { + self.mutable_catalog.rename_dictionary(req).await + } } diff --git a/src/query/service/src/catalogs/default/immutable_catalog.rs b/src/query/service/src/catalogs/default/immutable_catalog.rs index dc6e02d5aa63..e2a37915e0cf 100644 --- a/src/query/service/src/catalogs/default/immutable_catalog.rs +++ b/src/query/service/src/catalogs/default/immutable_catalog.rs @@ -72,6 +72,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -566,4 +567,8 @@ impl Catalog for ImmutableCatalog { ) -> Result> { unimplemented!() } + + async fn rename_dictionary(&self, _req: RenameDictionaryReq) -> Result<()> { + unimplemented!() + } } diff --git a/src/query/service/src/catalogs/default/mutable_catalog.rs b/src/query/service/src/catalogs/default/mutable_catalog.rs index 0e89fe51887a..07ae212bbdc2 100644 --- a/src/query/service/src/catalogs/default/mutable_catalog.rs +++ b/src/query/service/src/catalogs/default/mutable_catalog.rs @@ -88,6 +88,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -764,4 +765,10 @@ impl Catalog for MutableCatalog { ) -> Result> { Ok(self.ctx.meta.list_dictionaries(req).await?) } + + #[async_backtrace::framed] + async fn rename_dictionary(&self, req: RenameDictionaryReq) -> Result<()> { + let res = self.ctx.meta.rename_dictionary(req).await?; + Ok(res) + } } diff --git a/src/query/service/src/catalogs/default/session_catalog.rs b/src/query/service/src/catalogs/default/session_catalog.rs index 9cfcd7ad21da..7565b800c2f5 100644 --- a/src/query/service/src/catalogs/default/session_catalog.rs +++ b/src/query/service/src/catalogs/default/session_catalog.rs @@ -77,6 +77,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -700,6 +701,10 @@ impl Catalog for SessionCatalog { ) -> Result> { self.inner.list_dictionaries(req).await } + + async fn rename_dictionary(&self, req: RenameDictionaryReq) -> Result<()> { + self.inner.rename_dictionary(req).await + } } impl SessionCatalog { diff --git a/src/query/service/src/interpreters/access/privilege_access.rs b/src/query/service/src/interpreters/access/privilege_access.rs index 0eaa433bc235..e5ed82af8dc8 100644 --- a/src/query/service/src/interpreters/access/privilege_access.rs +++ b/src/query/service/src/interpreters/access/privilege_access.rs @@ -1021,7 +1021,8 @@ impl AccessChecker for PrivilegeAccess { // Dictionary Plan::ShowCreateDictionary(_) | Plan::CreateDictionary(_) - | Plan::DropDictionary(_) => { + | Plan::DropDictionary(_) + | Plan::RenameDictionary(_) => { self.validate_access(&GrantObject::Global, UserPrivilegeType::Super, false, false) .await?; } diff --git a/src/query/service/src/interpreters/interpreter_dictionary_rename.rs b/src/query/service/src/interpreters/interpreter_dictionary_rename.rs new file mode 100644 index 000000000000..d1303faf6f13 --- /dev/null +++ b/src/query/service/src/interpreters/interpreter_dictionary_rename.rs @@ -0,0 +1,75 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_meta_app::schema::DictionaryIdentity; +use databend_common_meta_app::schema::RenameDictionaryReq; +use databend_common_sql::plans::RenameDictionaryPlan; + +use crate::interpreters::Interpreter; +use crate::pipelines::PipelineBuildResult; +use crate::sessions::QueryContext; +use crate::sessions::TableContext; + +pub struct RenameDictionaryInterpreter { + ctx: Arc, + plan: RenameDictionaryPlan, +} + +impl RenameDictionaryInterpreter { + pub fn try_create(ctx: Arc, plan: RenameDictionaryPlan) -> Result { + Ok(RenameDictionaryInterpreter { ctx, plan }) + } +} + +#[async_trait::async_trait] +impl Interpreter for RenameDictionaryInterpreter { + fn name(&self) -> &str { + "RenameDictionaryInterpreter" + } + + fn is_ddl(&self) -> bool { + true + } + + #[async_backtrace::framed] + async fn execute2(&self) -> Result { + let tenant = &self.plan.tenant; + let catalog = self.ctx.get_catalog(&self.plan.catalog).await?; + + let dict_ident = + DictionaryIdentity::new(self.plan.database_id, self.plan.dictionary.clone()); + let name_ident = DictionaryNameIdent::new(tenant, dict_ident); + + let new_dict_ident = + DictionaryIdentity::new(self.plan.new_database_id, self.plan.new_dictionary.clone()); + + let req = RenameDictionaryReq { + name_ident, + new_dict_ident, + }; + + let reply = catalog.rename_dictionary(req).await; + if let Err(err) = reply { + if !(self.plan.if_exists && err.code() == ErrorCode::UNKNOWN_DICTIONARY) { + return Err(err); + } + } + Ok(PipelineBuildResult::create()) + } +} diff --git a/src/query/service/src/interpreters/interpreter_factory.rs b/src/query/service/src/interpreters/interpreter_factory.rs index 773f82788c82..aecf00f6c941 100644 --- a/src/query/service/src/interpreters/interpreter_factory.rs +++ b/src/query/service/src/interpreters/interpreter_factory.rs @@ -607,6 +607,9 @@ impl InterpreterFactory { ctx, *drop_dict.clone(), )?)), + Plan::RenameDictionary(rename_dictionary) => Ok(Arc::new( + RenameDictionaryInterpreter::try_create(ctx, *rename_dictionary.clone())?, + )), Plan::CreateProcedure(p) => Ok(Arc::new(CreateProcedureInterpreter::try_create( ctx, *p.clone(), diff --git a/src/query/service/src/interpreters/mod.rs b/src/query/service/src/interpreters/mod.rs index 2142577b4a9e..9362b7092121 100644 --- a/src/query/service/src/interpreters/mod.rs +++ b/src/query/service/src/interpreters/mod.rs @@ -38,6 +38,7 @@ mod interpreter_database_show_create; mod interpreter_database_undrop; mod interpreter_dictionary_create; mod interpreter_dictionary_drop; +mod interpreter_dictionary_rename; mod interpreter_dictionary_show_create; mod interpreter_execute_immediate; mod interpreter_explain; @@ -163,6 +164,7 @@ pub use interpreter_database_drop::DropDatabaseInterpreter; pub use interpreter_database_rename::RenameDatabaseInterpreter; pub use interpreter_database_show_create::ShowCreateDatabaseInterpreter; pub use interpreter_database_undrop::UndropDatabaseInterpreter; +pub use interpreter_dictionary_rename::RenameDictionaryInterpreter; pub use interpreter_execute_immediate::ExecuteImmediateInterpreter; pub use interpreter_explain::ExplainInterpreter; pub use interpreter_factory::InterpreterFactory; diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 6b4aaf93c707..a1cc8fce6f38 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -107,6 +107,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -457,6 +458,10 @@ impl Catalog for FakedCatalog { ) -> Result> { todo!() } + + async fn rename_dictionary(&self, _req: RenameDictionaryReq) -> Result<()> { + todo!() + } } struct CtxDelegation { diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 181e42e6ca76..fe5b5e69a207 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -106,6 +106,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -1210,4 +1211,8 @@ impl Catalog for FakedCatalog { ) -> Result> { todo!() } + + async fn rename_dictionary(&self, _req: RenameDictionaryReq) -> Result<()> { + todo!() + } } diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index 0646164cbf2f..c38bdc067f1c 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -309,6 +309,7 @@ impl<'a> Binder { Statement::DropDictionary(stmt) => self.bind_drop_dictionary(stmt).await?, Statement::ShowCreateDictionary(stmt) => self.bind_show_create_dictionary(stmt).await?, Statement::ShowDictionaries(stmt) => self.bind_show_dictionaries(bind_context, stmt).await?, + Statement::RenameDictionary(stmt) => self.bind_rename_dictionary(stmt).await?, // Views Statement::CreateView(stmt) => self.bind_create_view(stmt).await?, Statement::AlterView(stmt) => self.bind_alter_view(stmt).await?, diff --git a/src/query/sql/src/planner/binder/ddl/dictionary.rs b/src/query/sql/src/planner/binder/ddl/dictionary.rs index 800b16ef5bff..94099664073d 100644 --- a/src/query/sql/src/planner/binder/ddl/dictionary.rs +++ b/src/query/sql/src/planner/binder/ddl/dictionary.rs @@ -18,6 +18,7 @@ use std::sync::LazyLock; use databend_common_ast::ast::CreateDictionaryStmt; use databend_common_ast::ast::DropDictionaryStmt; +use databend_common_ast::ast::RenameDictionaryStmt; use databend_common_ast::ast::ShowCreateDictionaryStmt; use databend_common_ast::ast::ShowDictionariesStmt; use databend_common_ast::ast::ShowLimit; @@ -35,6 +36,7 @@ use log::debug; use crate::plans::CreateDictionaryPlan; use crate::plans::DropDictionaryPlan; use crate::plans::Plan; +use crate::plans::RenameDictionaryPlan; use crate::plans::RewriteKind; use crate::plans::ShowCreateDictionaryPlan; use crate::BindContext; @@ -435,4 +437,50 @@ impl Binder { ) .await } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_rename_dictionary( + &mut self, + stmt: &RenameDictionaryStmt, + ) -> Result { + let RenameDictionaryStmt { + if_exists, + catalog, + database, + dictionary, + new_catalog, + new_database, + new_dictionary, + } = stmt; + + let tenant = self.ctx.get_tenant(); + let (catalog, database, dictionary) = + self.normalize_object_identifier_triple(catalog, database, dictionary); + + let (new_catalog, new_database, new_dictionary) = + self.normalize_object_identifier_triple(new_catalog, new_database, new_dictionary); + + if new_catalog != catalog { + return Err(ErrorCode::BadArguments( + "Rename dictionary not allow modify catalog", + )); + } + + let catalog_info = self.ctx.get_catalog(&catalog).await?; + let db = catalog_info.get_database(&tenant, &database).await?; + let database_id = db.get_db_info().database_id.db_id; + + let new_db = catalog_info.get_database(&tenant, &new_database).await?; + let new_database_id = new_db.get_db_info().database_id.db_id; + + Ok(Plan::RenameDictionary(Box::new(RenameDictionaryPlan { + tenant, + if_exists: *if_exists, + catalog, + database_id, + dictionary, + new_database_id, + new_dictionary, + }))) + } } diff --git a/src/query/sql/src/planner/format/display_plan.rs b/src/query/sql/src/planner/format/display_plan.rs index 543942bab07e..31cc20f78735 100644 --- a/src/query/sql/src/planner/format/display_plan.rs +++ b/src/query/sql/src/planner/format/display_plan.rs @@ -211,6 +211,7 @@ impl Plan { Plan::CreateDictionary(_) => Ok("CreateDictionary".to_string()), Plan::DropDictionary(_) => Ok("DropDictionary".to_string()), Plan::ShowCreateDictionary(_) => Ok("ShowCreateDictionary".to_string()), + Plan::RenameDictionary(_) => Ok("RenameDictionary".to_string()), } } } diff --git a/src/query/sql/src/planner/plans/ddl/dictionary.rs b/src/query/sql/src/planner/plans/ddl/dictionary.rs index 845d66a6155a..1a0ce7bd4cd8 100644 --- a/src/query/sql/src/planner/plans/ddl/dictionary.rs +++ b/src/query/sql/src/planner/plans/ddl/dictionary.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use databend_common_ast::ast::CreateOption; +use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::DictionaryMeta; use databend_common_meta_app::tenant::Tenant; @@ -49,3 +52,21 @@ impl ShowCreateDictionaryPlan { self.schema.clone() } } + +/// Rename. +#[derive(Clone, Debug)] +pub struct RenameDictionaryPlan { + pub tenant: Tenant, + pub if_exists: bool, + pub catalog: String, + pub database_id: u64, + pub dictionary: String, + pub new_database_id: u64, + pub new_dictionary: String, +} + +impl RenameDictionaryPlan { + pub fn schema(&self) -> DataSchemaRef { + Arc::new(DataSchema::empty()) + } +} diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index 0a3efacdc753..80556dbbcdd6 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -26,6 +26,7 @@ use databend_common_expression::DataSchemaRefExt; use super::CreateDictionaryPlan; use super::DropDictionaryPlan; +use super::RenameDictionaryPlan; use super::ShowCreateDictionaryPlan; use crate::binder::ExplainConfig; use crate::optimizer::SExpr; @@ -372,6 +373,7 @@ pub enum Plan { CreateDictionary(Box), DropDictionary(Box), ShowCreateDictionary(Box), + RenameDictionary(Box), } #[derive(Clone, Debug)] diff --git a/src/query/storages/hive/hive/src/hive_catalog.rs b/src/query/storages/hive/hive/src/hive_catalog.rs index 2e40396ee602..990dd0607937 100644 --- a/src/query/storages/hive/hive/src/hive_catalog.rs +++ b/src/query/storages/hive/hive/src/hive_catalog.rs @@ -80,6 +80,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -750,4 +751,8 @@ impl Catalog for HiveCatalog { ) -> Result> { unimplemented!() } + + async fn rename_dictionary(&self, _req: RenameDictionaryReq) -> Result<()> { + unimplemented!() + } } diff --git a/src/query/storages/iceberg/src/catalog.rs b/src/query/storages/iceberg/src/catalog.rs index db95af3e7498..d627c95f4cc2 100644 --- a/src/query/storages/iceberg/src/catalog.rs +++ b/src/query/storages/iceberg/src/catalog.rs @@ -79,6 +79,7 @@ use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::RenameTableReply; use databend_common_meta_app::schema::RenameTableReq; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; @@ -602,4 +603,8 @@ impl Catalog for IcebergCatalog { ) -> Result> { unimplemented!() } + + async fn rename_dictionary(&self, _req: RenameDictionaryReq) -> Result<()> { + unimplemented!() + } } diff --git a/tests/sqllogictests/suites/base/05_ddl/05_0037_ddl_dictionary.test b/tests/sqllogictests/suites/base/05_ddl/05_0037_ddl_dictionary.test index 09e8bc224c4c..5826d8313d45 100644 --- a/tests/sqllogictests/suites/base/05_ddl/05_0037_ddl_dictionary.test +++ b/tests/sqllogictests/suites/base/05_ddl/05_0037_ddl_dictionary.test @@ -103,5 +103,45 @@ CREATE DATABASE db1 statement ok CREATE DICTIONARY db1.test1(a int NOT NULL, b int NOT NULL) PRIMARY KEY a SOURCE(mysql(host='localhost' port='3306' username='root' password='1234' db='db1' table='test_table')) +statement ok +DROP DICTIONARY IF EXISTS db1.d + +statement ok +DROP DICTIONARY IF EXISTS db1.dd + +statement ok +CREATE DICTIONARY IF NOT EXISTS db1.d(c1 int NOT NULL, c2 Varchar NOT NULL) PRIMARY KEY c1 SOURCE(mysql(host='localhost' port='3306' username='root' password='1234' db='db1' table='test_table')) + +statement error 3113 +RENAME DICTIONARY db1.d TO db1.test1 + +statement error 3114 +RENAME DICTIONARY db1.d0 TO db1.dd + +statement ok +RENAME DICTIONARY db1.d TO db1.dd + +statement error 3114 +SHOW CREATE DICTIONARY db1.d + +query TT +SHOW CREATE DICTIONARY db1.dd +---- +dd CREATE DICTIONARY dd ( c1 INT NOT NULL, c2 VARCHAR NOT NULL ) PRIMARY KEY c1 SOURCE(mysql(db='db1' host='localhost' password='[HIDDEN]' port='3306' table='test_table' username='root')) + +statement ok +DROP DICTIONARY IF EXISTS default.dd2 + +statement ok +rename dictionary db1.dd to default.dd2 + +statement error 3114 +show create dictionary db1.dd + +query TT +show create dictionary default.dd2 +---- +dd2 CREATE DICTIONARY dd2 ( c1 INT NOT NULL, c2 VARCHAR NOT NULL ) PRIMARY KEY c1 SOURCE(mysql(db='db1' host='localhost' password='[HIDDEN]' port='3306' table='test_table' username='root')) + statement ok DROP DATABASE db1 From 806210c0a6ba8adfda68a2a96b715db16a96dcac Mon Sep 17 00:00:00 2001 From: Freejww <103876282+Freejww@users.noreply.github.com> Date: Thu, 7 Nov 2024 21:29:25 +0800 Subject: [PATCH 06/92] feat(query): add last_day, previous_day, next_day function (#16740) * add date function * add tests * add tests --- src/query/ast/src/ast/expr.rs | 55 ++++++ src/query/ast/src/ast/format/syntax/expr.rs | 18 ++ src/query/ast/src/parser/expr.rs | 118 ++++++++++++- src/query/ast/src/parser/token.rs | 23 +++ .../ast/tests/it/testdata/expr-error.txt | 4 +- .../ast/tests/it/testdata/stmt-error.txt | 4 +- src/query/expression/src/utils/date_helper.rs | 146 ++++++++++++++++ src/query/functions/src/scalars/datetime.rs | 162 ++++------------- .../functions/tests/it/scalars/datetime.rs | 27 +++ .../functions/tests/it/scalars/parser.rs | 46 +++++ .../tests/it/scalars/testdata/datetime.txt | 162 +++++++++++++++++ .../it/scalars/testdata/function_list.txt | 72 ++++++++ .../sql/src/planner/semantic/type_check.rs | 63 +++++++ .../functions/02_0012_function_datetimes.test | 110 ++++++++++++ .../02_0012_function_datetimes_tz.test | 165 ++++++++++++++++++ 15 files changed, 1033 insertions(+), 142 deletions(-) diff --git a/src/query/ast/src/ast/expr.rs b/src/query/ast/src/ast/expr.rs index b6c0b40ed866..d19f00b2278b 100644 --- a/src/query/ast/src/ast/expr.rs +++ b/src/query/ast/src/ast/expr.rs @@ -229,6 +229,21 @@ pub enum Expr { unit: IntervalKind, date: Box, }, + LastDay { + span: Span, + unit: IntervalKind, + date: Box, + }, + PreviousDay { + span: Span, + unit: Weekday, + date: Box, + }, + NextDay { + span: Span, + unit: Weekday, + date: Box, + }, Hole { span: Span, name: String, @@ -269,6 +284,9 @@ impl Expr { | Expr::DateDiff { span, .. } | Expr::DateSub { span, .. } | Expr::DateTrunc { span, .. } + | Expr::LastDay { span, .. } + | Expr::PreviousDay { span, .. } + | Expr::NextDay { span, .. } | Expr::Hole { span, .. } => *span, } } @@ -411,6 +429,9 @@ impl Expr { .. } => merge_span(merge_span(*span, interval.whole_span()), date.whole_span()), Expr::DateTrunc { span, date, .. } => merge_span(*span, date.whole_span()), + Expr::LastDay { span, date, .. } => merge_span(*span, date.whole_span()), + Expr::PreviousDay { span, date, .. } => merge_span(*span, date.whole_span()), + Expr::NextDay { span, date, .. } => merge_span(*span, date.whole_span()), Expr::Hole { span, .. } => *span, } } @@ -734,6 +755,15 @@ impl Display for Expr { Expr::DateTrunc { unit, date, .. } => { write!(f, "DATE_TRUNC({unit}, {date})")?; } + Expr::LastDay { unit, date, .. } => { + write!(f, "LAST_DAY({date}, {unit})")?; + } + Expr::PreviousDay { unit, date, .. } => { + write!(f, "PREVIOUS_DAY({date}, {unit})")?; + } + Expr::NextDay { unit, date, .. } => { + write!(f, "NEXT_DAY({date}, {unit})")?; + } Expr::Hole { name, .. } => { write!(f, ":{name}")?; } @@ -750,6 +780,31 @@ impl Display for Expr { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Drive, DriveMut)] +pub enum Weekday { + Sunday, + Monday, + Tuesday, + Wednesday, + Thursday, + Friday, + Saturday, +} + +impl Display for Weekday { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(match self { + Weekday::Sunday => "SUNDAY", + Weekday::Monday => "MONDAY", + Weekday::Tuesday => "TUESDAY", + Weekday::Wednesday => "WEDNESDAY", + Weekday::Thursday => "THURSDAY", + Weekday::Friday => "FRIDAY", + Weekday::Saturday => "SATURDAY", + }) + } +} + #[derive(Debug, Copy, Clone, PartialEq, Eq, Drive, DriveMut)] pub enum IntervalKind { Year, diff --git a/src/query/ast/src/ast/format/syntax/expr.rs b/src/query/ast/src/ast/format/syntax/expr.rs index f0c6b97f20fa..cba0c8d2e786 100644 --- a/src/query/ast/src/ast/format/syntax/expr.rs +++ b/src/query/ast/src/ast/format/syntax/expr.rs @@ -440,6 +440,24 @@ pub(crate) fn pretty_expr(expr: Expr) -> RcDoc<'static> { .append(RcDoc::space()) .append(pretty_expr(*date)) .append(RcDoc::text(")")), + Expr::LastDay { unit, date, .. } => RcDoc::text("LAST_DAY(") + .append(pretty_expr(*date)) + .append(RcDoc::text(",")) + .append(RcDoc::space()) + .append(RcDoc::text(unit.to_string())) + .append(RcDoc::text(")")), + Expr::PreviousDay { unit, date, .. } => RcDoc::text("PREVIOUS_DAY(") + .append(pretty_expr(*date)) + .append(RcDoc::text(",")) + .append(RcDoc::space()) + .append(RcDoc::text(unit.to_string())) + .append(RcDoc::text(")")), + Expr::NextDay { unit, date, .. } => RcDoc::text("NEXT_DAY(") + .append(pretty_expr(*date)) + .append(RcDoc::text(",")) + .append(RcDoc::space()) + .append(RcDoc::text(unit.to_string())) + .append(RcDoc::text(")")), Expr::Hole { name, .. } => RcDoc::text(":").append(RcDoc::text(name.to_string())), } } diff --git a/src/query/ast/src/parser/expr.rs b/src/query/ast/src/parser/expr.rs index ad498d9b23eb..599748fcd497 100644 --- a/src/query/ast/src/parser/expr.rs +++ b/src/query/ast/src/parser/expr.rs @@ -310,6 +310,18 @@ pub enum ExprElement { unit: IntervalKind, date: Expr, }, + LastDay { + unit: IntervalKind, + date: Expr, + }, + PreviousDay { + unit: Weekday, + date: Expr, + }, + NextDay { + unit: Weekday, + date: Expr, + }, Hole { name: String, }, @@ -416,6 +428,9 @@ impl ExprElement { ExprElement::DateDiff { .. } => Affix::Nilfix, ExprElement::DateSub { .. } => Affix::Nilfix, ExprElement::DateTrunc { .. } => Affix::Nilfix, + ExprElement::LastDay { .. } => Affix::Nilfix, + ExprElement::PreviousDay { .. } => Affix::Nilfix, + ExprElement::NextDay { .. } => Affix::Nilfix, ExprElement::Hole { .. } => Affix::Nilfix, ExprElement::VariableAccess { .. } => Affix::Nilfix, } @@ -459,6 +474,9 @@ impl Expr { Expr::DateDiff { .. } => Affix::Nilfix, Expr::DateSub { .. } => Affix::Nilfix, Expr::DateTrunc { .. } => Affix::Nilfix, + Expr::LastDay { .. } => Affix::Nilfix, + Expr::PreviousDay { .. } => Affix::Nilfix, + Expr::NextDay { .. } => Affix::Nilfix, Expr::Hole { .. } => Affix::Nilfix, } } @@ -657,6 +675,21 @@ impl<'a, I: Iterator>> PrattParser for ExprP unit, date: Box::new(date), }, + ExprElement::LastDay { unit, date } => Expr::LastDay { + span: transform_span(elem.span.tokens), + unit, + date: Box::new(date), + }, + ExprElement::PreviousDay { unit, date } => Expr::PreviousDay { + span: transform_span(elem.span.tokens), + unit, + date: Box::new(date), + }, + ExprElement::NextDay { unit, date } => Expr::NextDay { + span: transform_span(elem.span.tokens), + unit, + date: Box::new(date), + }, ExprElement::Hole { name } => Expr::Hole { span: transform_span(elem.span.tokens), name, @@ -1222,6 +1255,27 @@ pub fn expr_element(i: Input) -> IResult> { |(_, _, unit, _, date, _)| ExprElement::DateTrunc { unit, date }, ); + let last_day = map( + rule! { + LAST_DAY ~ "(" ~ #subexpr(0) ~ "," ~ #interval_kind ~ ")" + }, + |(_, _, date, _, unit, _)| ExprElement::LastDay { unit, date }, + ); + + let previous_day = map( + rule! { + PREVIOUS_DAY ~ "(" ~ #subexpr(0) ~ "," ~ #weekday ~ ")" + }, + |(_, _, date, _, unit, _)| ExprElement::PreviousDay { unit, date }, + ); + + let next_day = map( + rule! { + NEXT_DAY ~ "(" ~ #subexpr(0) ~ "," ~ #weekday ~ ")" + }, + |(_, _, date, _, unit, _)| ExprElement::NextDay { unit, date }, + ); + let date_expr = map( rule! { DATE ~ #consumed(literal_string) @@ -1281,19 +1335,25 @@ pub fn expr_element(i: Input) -> IResult> { | #json_op : "" | #unary_op : "" | #cast : "`CAST(... AS ...)`" - | #date_add: "`DATE_ADD(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`" - | #date_diff: "`DATE_DIFF(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`" - | #date_sub: "`DATE_SUB(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`" - | #date_trunc: "`DATE_TRUNC((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND), ...)`" - | #date_expr: "`DATE `" - | #timestamp_expr: "`TIMESTAMP `" - | #interval: "`INTERVAL ... (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW)`" | #pg_cast : "`::`" - | #extract : "`EXTRACT((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | WEEK) FROM ...)`" - | #date_part : "`DATE_PART((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | WEEK), ...)`" | #position : "`POSITION(... IN ...)`" | #variable_access: "`$`" ), + rule! ( + #date_add : "`DATE_ADD(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`" + | #date_diff : "`DATE_DIFF(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`" + | #date_sub : "`DATE_SUB(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`" + | #date_trunc : "`DATE_TRUNC((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND), ...)`" + | #last_day : "`LAST_DAY(..., (YEAR | QUARTER | MONTH | WEEK)))`" + | #previous_day : "`PREVIOUS_DAY(..., (Sunday | Monday | Tuesday | Wednesday | Thursday | Friday | Saturday))`" + | #next_day : "`NEXT_DAY(..., (Sunday | Monday | Tuesday | Wednesday | Thursday | Friday | Saturday))`" + | #date_expr : "`DATE `" + | #timestamp_expr : "`TIMESTAMP `" + | #interval : "`INTERVAL ... (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW)`" + | #extract : "`EXTRACT((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | WEEK) FROM ...)`" + | #date_part : "`DATE_PART((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | WEEK), ...)`" + + ), rule!( #substring : "`SUBSTRING(... [FROM ...] [FOR ...])`" | #trim : "`TRIM(...)`" @@ -1692,6 +1752,46 @@ pub fn type_name(i: Input) -> IResult { )(i) } +pub fn weekday(i: Input) -> IResult { + alt(( + value(Weekday::Sunday, rule! { SUNDAY }), + value(Weekday::Monday, rule! { MONDAY }), + value(Weekday::Tuesday, rule! { TUESDAY }), + value(Weekday::Wednesday, rule! { WEDNESDAY }), + value(Weekday::Thursday, rule! { THURSDAY }), + value(Weekday::Friday, rule! { FRIDAY }), + value(Weekday::Saturday, rule! { SATURDAY }), + value( + Weekday::Sunday, + rule! { #literal_string_eq_ignore_case("SUNDAY") }, + ), + value( + Weekday::Monday, + rule! { #literal_string_eq_ignore_case("MONDAY") }, + ), + value( + Weekday::Tuesday, + rule! { #literal_string_eq_ignore_case("TUESDAY") }, + ), + value( + Weekday::Wednesday, + rule! { #literal_string_eq_ignore_case("WEDNESDAY") }, + ), + value( + Weekday::Thursday, + rule! { #literal_string_eq_ignore_case("THURSDAY") }, + ), + value( + Weekday::Friday, + rule! { #literal_string_eq_ignore_case("FRIDAY") }, + ), + value( + Weekday::Saturday, + rule! { #literal_string_eq_ignore_case("SATURDAY") }, + ), + ))(i) +} + pub fn interval_kind(i: Input) -> IResult { alt(( value(IntervalKind::Year, rule! { YEAR }), diff --git a/src/query/ast/src/parser/token.rs b/src/query/ast/src/parser/token.rs index 5905bdb1a4bb..657ca0ccf0c7 100644 --- a/src/query/ast/src/parser/token.rs +++ b/src/query/ast/src/parser/token.rs @@ -637,6 +637,8 @@ pub enum TokenKind { FORMATS, #[token("FRAGMENTS", ignore(ascii_case))] FRAGMENTS, + #[token("FRIDAY", ignore(ascii_case))] + FRIDAY, #[token("FROM", ignore(ascii_case))] FROM, #[token("FULL", ignore(ascii_case))] @@ -725,6 +727,8 @@ pub enum TokenKind { INTO, #[token("INVERTED", ignore(ascii_case))] INVERTED, + #[token("PREVIOUS_DAY", ignore(ascii_case))] + PREVIOUS_DAY, #[token("PROCEDURE", ignore(ascii_case))] PROCEDURE, #[token("PROCEDURES", ignore(ascii_case))] @@ -749,6 +753,8 @@ pub enum TokenKind { KEY, #[token("KILL", ignore(ascii_case))] KILL, + #[token("LAST_DAY", ignore(ascii_case))] + LAST_DAY, #[token("LATERAL", ignore(ascii_case))] LATERAL, #[token("LINEAR", ignore(ascii_case))] @@ -816,6 +822,8 @@ pub enum TokenKind { MATERIALIZED, #[token("MUST_CHANGE_PASSWORD", ignore(ascii_case))] MUST_CHANGE_PASSWORD, + #[token("NEXT_DAY", ignore(ascii_case))] + NEXT_DAY, #[token("NON_DISPLAY", ignore(ascii_case))] NON_DISPLAY, #[token("NATURAL", ignore(ascii_case))] @@ -1016,6 +1024,8 @@ pub enum TokenKind { OPTIMIZED, #[token("DECORRELATED", ignore(ascii_case))] DECORRELATED, + #[token("SATURDAY", ignore(ascii_case))] + SATURDAY, #[token("SCHEMA", ignore(ascii_case))] SCHEMA, #[token("SCHEMAS", ignore(ascii_case))] @@ -1062,6 +1072,8 @@ pub enum TokenKind { SIZE_LIMIT, #[token("MAX_FILES", ignore(ascii_case))] MAX_FILES, + #[token("MONDAY", ignore(ascii_case))] + MONDAY, #[token("SKIP_HEADER", ignore(ascii_case))] SKIP_HEADER, #[token("SMALLINT", ignore(ascii_case))] @@ -1138,6 +1150,8 @@ pub enum TokenKind { TENANT, #[token("THEN", ignore(ascii_case))] THEN, + #[token("THURSDAY", ignore(ascii_case))] + THURSDAY, #[token("TIMESTAMP", ignore(ascii_case))] TIMESTAMP, #[token("TIMEZONE_HOUR", ignore(ascii_case))] @@ -1166,6 +1180,8 @@ pub enum TokenKind { TRY_CAST, #[token("TSV", ignore(ascii_case))] TSV, + #[token("TUESDAY", ignore(ascii_case))] + TUESDAY, #[token("TUPLE", ignore(ascii_case))] TUPLE, #[token("TYPE", ignore(ascii_case))] @@ -1314,6 +1330,8 @@ pub enum TokenKind { ENABLED, #[token("WEBHOOK", ignore(ascii_case))] WEBHOOK, + #[token("WEDNESDAY", ignore(ascii_case))] + WEDNESDAY, #[token("ERROR_INTEGRATION", ignore(ascii_case))] ERROR_INTEGRATION, #[token("AUTO_INGEST", ignore(ascii_case))] @@ -1354,6 +1372,8 @@ pub enum TokenKind { SOURCE, #[token("SQL", ignore(ascii_case))] SQL, + #[token("SUNDAY", ignore(ascii_case))] + SUNDAY, } // Reference: https://www.postgresql.org/docs/current/sql-keywords-appendix.html @@ -1593,6 +1613,9 @@ impl TokenKind { | TokenKind::DATE_DIFF | TokenKind::DATE_SUB | TokenKind::DATE_TRUNC + | TokenKind::LAST_DAY + | TokenKind::PREVIOUS_DAY + | TokenKind::NEXT_DAY | TokenKind::IGNORE_RESULT ) } diff --git a/src/query/ast/tests/it/testdata/expr-error.txt b/src/query/ast/tests/it/testdata/expr-error.txt index 925d9338ce1f..01717c62edfc 100644 --- a/src/query/ast/tests/it/testdata/expr-error.txt +++ b/src/query/ast/tests/it/testdata/expr-error.txt @@ -52,7 +52,7 @@ error: --> SQL:1:10 | 1 | CAST(col1) - | ---- ^ unexpected `)`, expecting `AS`, `,`, `(`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, , , , , , `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, , , , , , `CAST`, `TRY_CAST`, `DATE_ADD`, `DATE_DIFF`, `DATE_SUB`, `DATE_TRUNC`, or 33 more ... + | ---- ^ unexpected `)`, expecting `AS`, `,`, `(`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, , , , , , `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, , , , , , `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, or 36 more ... | | | while parsing `CAST(... AS ...)` | while parsing expression @@ -81,7 +81,7 @@ error: 1 | $ abc + 3 | ^ | | - | unexpected `$`, expecting `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, , , , , , `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, , , , , , `CAST`, `TRY_CAST`, `DATE_ADD`, `DATE_DIFF`, `DATE_SUB`, `DATE_TRUNC`, `DATE`, `TIMESTAMP`, `INTERVAL`, or 31 more ... + | unexpected `$`, expecting `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, , , , , , `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, , , , , , `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, `DATE_DIFF`, `DATE_SUB`, `DATE_TRUNC`, or 34 more ... | while parsing expression diff --git a/src/query/ast/tests/it/testdata/stmt-error.txt b/src/query/ast/tests/it/testdata/stmt-error.txt index 5ee14c7618a8..18ec23cfb979 100644 --- a/src/query/ast/tests/it/testdata/stmt-error.txt +++ b/src/query/ast/tests/it/testdata/stmt-error.txt @@ -443,7 +443,7 @@ error: --> SQL:1:41 | 1 | SELECT * FROM t GROUP BY GROUPING SETS () - | ------ ^ unexpected `)`, expecting `(`, `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, , , , , , `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, , , , , , `CAST`, `TRY_CAST`, `DATE_ADD`, `DATE_DIFF`, `DATE_SUB`, `DATE_TRUNC`, `DATE`, `TIMESTAMP`, or 31 more ... + | ------ ^ unexpected `)`, expecting `(`, `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, , , , , , `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, , , , , , `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, `DATE_DIFF`, `DATE_SUB`, or 34 more ... | | | while parsing `SELECT ...` @@ -865,7 +865,7 @@ error: --> SQL:1:65 | 1 | CREATE FUNCTION IF NOT EXISTS isnotempty AS(p) -> not(is_null(p) - | ------ -- ---- ^ unexpected end of input, expecting `)`, `IGNORE`, `RESPECT`, `OVER`, `(`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, , , , , , `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, , , , , , `CAST`, `TRY_CAST`, `DATE_ADD`, `DATE_DIFF`, or 36 more ... + | ------ -- ---- ^ unexpected end of input, expecting `)`, `IGNORE`, `RESPECT`, `OVER`, `(`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, , , , , , `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, , , , , , `CAST`, `TRY_CAST`, `::`, `POSITION`, or 39 more ... | | | | | | | | | while parsing `( [, ...])` | | | while parsing expression diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index 175d63e185e9..dc8a5aa4f6ad 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -26,6 +26,7 @@ use chrono::Offset; use chrono::TimeZone; use chrono::Timelike; use chrono::Utc; +use chrono::Weekday; use chrono_tz::Tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -711,6 +712,24 @@ pub struct ToStartOfMonth; pub struct ToStartOfQuarter; pub struct ToStartOfYear; pub struct ToStartOfISOYear; +pub struct ToLastOfWeek; +pub struct ToLastOfMonth; +pub struct ToLastOfQuarter; +pub struct ToLastOfYear; +pub struct ToPreviousMonday; +pub struct ToPreviousTuesday; +pub struct ToPreviousWednesday; +pub struct ToPreviousThursday; +pub struct ToPreviousFriday; +pub struct ToPreviousSaturday; +pub struct ToPreviousSunday; +pub struct ToNextMonday; +pub struct ToNextTuesday; +pub struct ToNextWednesday; +pub struct ToNextThursday; +pub struct ToNextFriday; +pub struct ToNextSaturday; +pub struct ToNextSunday; impl ToNumber for ToLastMonday { fn to_number(dt: &DateTime) -> i32 { @@ -761,3 +780,130 @@ impl ToNumber for ToStartOfISOYear { datetime_to_date_inner_number(&iso_dt) } } + +impl ToNumber for ToLastOfWeek { + fn to_number(dt: &DateTime) -> i32 { + datetime_to_date_inner_number(dt) - dt.date_naive().weekday().num_days_from_monday() as i32 + + 6 + } +} + +impl ToNumber for ToLastOfMonth { + fn to_number(dt: &DateTime) -> i32 { + let day = last_day_of_year_month(dt.year(), dt.month()); + datetime_to_date_inner_number(&dt.with_day(day).unwrap()) + } +} + +impl ToNumber for ToLastOfQuarter { + fn to_number(dt: &DateTime) -> i32 { + let new_month = dt.month0() / 3 * 3 + 3; + let day = last_day_of_year_month(dt.year(), new_month); + datetime_to_date_inner_number(&dt.with_month(new_month).unwrap().with_day(day).unwrap()) + } +} + +impl ToNumber for ToLastOfYear { + fn to_number(dt: &DateTime) -> i32 { + let day = last_day_of_year_month(dt.year(), 12); + datetime_to_date_inner_number(&dt.with_month(12).unwrap().with_day(day).unwrap()) + } +} + +impl ToNumber for ToPreviousMonday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Mon, true) + } +} + +impl ToNumber for ToPreviousTuesday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Tue, true) + } +} + +impl ToNumber for ToPreviousWednesday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Wed, true) + } +} + +impl ToNumber for ToPreviousThursday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Thu, true) + } +} + +impl ToNumber for ToPreviousFriday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Fri, true) + } +} + +impl ToNumber for ToPreviousSaturday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Sat, true) + } +} + +impl ToNumber for ToPreviousSunday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Sun, true) + } +} + +impl ToNumber for ToNextMonday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Mon, false) + } +} + +impl ToNumber for ToNextTuesday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Tue, false) + } +} + +impl ToNumber for ToNextWednesday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Wed, false) + } +} + +impl ToNumber for ToNextThursday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Thu, false) + } +} + +impl ToNumber for ToNextFriday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Fri, false) + } +} + +impl ToNumber for ToNextSaturday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Sat, false) + } +} + +impl ToNumber for ToNextSunday { + fn to_number(dt: &DateTime) -> i32 { + previous_or_next_day(dt, Weekday::Sun, false) + } +} + +pub fn previous_or_next_day(dt: &DateTime, target: Weekday, is_previous: bool) -> i32 { + let dir = if is_previous { -1 } else { 1 }; + + let mut days_diff = (dir + * (target.num_days_from_monday() as i32 + - dt.date_naive().weekday().num_days_from_monday() as i32) + + 7) + % 7; + + days_diff = if days_diff == 0 { 7 } else { days_diff }; + + datetime_to_date_inner_number(dt) + dir * days_diff +} diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index ca3fe2d84d55..bf095ebdf225 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -1789,55 +1789,31 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { ); // date | timestamp -> date - registry.register_passthrough_nullable_1_arg::( - "to_monday", - |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match DateRounder::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } - }), - ); - registry.register_passthrough_nullable_1_arg::( - "to_monday", - |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) - }), - ); + rounder_functions_helper::(registry, "to_monday"); + rounder_functions_helper::(registry, "to_start_of_week"); + rounder_functions_helper::(registry, "to_start_of_month"); + rounder_functions_helper::(registry, "to_start_of_quarter"); + rounder_functions_helper::(registry, "to_start_of_year"); + rounder_functions_helper::(registry, "to_start_of_iso_year"); + rounder_functions_helper::(registry, "to_last_of_week"); + rounder_functions_helper::(registry, "to_last_of_month"); + rounder_functions_helper::(registry, "to_last_of_quarter"); + rounder_functions_helper::(registry, "to_last_of_year"); + rounder_functions_helper::(registry, "to_previous_monday"); + rounder_functions_helper::(registry, "to_previous_tuesday"); + rounder_functions_helper::(registry, "to_previous_wednesday"); + rounder_functions_helper::(registry, "to_previous_thursday"); + rounder_functions_helper::(registry, "to_previous_friday"); + rounder_functions_helper::(registry, "to_previous_saturday"); + rounder_functions_helper::(registry, "to_previous_sunday"); + rounder_functions_helper::(registry, "to_next_monday"); + rounder_functions_helper::(registry, "to_next_tuesday"); + rounder_functions_helper::(registry, "to_next_wednesday"); + rounder_functions_helper::(registry, "to_next_thursday"); + rounder_functions_helper::(registry, "to_next_friday"); + rounder_functions_helper::(registry, "to_next_saturday"); + rounder_functions_helper::(registry, "to_next_sunday"); - registry.register_passthrough_nullable_1_arg::( - "to_start_of_week", - |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match DateRounder::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } - }), - ); - registry.register_passthrough_nullable_1_arg::( - "to_start_of_week", - |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) - }), - ); registry.register_passthrough_nullable_2_arg::( "to_start_of_week", |_, _, _| FunctionDomain::Full, @@ -1880,87 +1856,15 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { } }), ); +} +fn rounder_functions_helper(registry: &mut FunctionRegistry, name: &str) +where T: ToNumber { registry.register_passthrough_nullable_1_arg::( - "to_start_of_month", - |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match DateRounder::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } - }), - ); - registry.register_passthrough_nullable_1_arg::( - "to_start_of_month", - |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) - }), - ); - - registry.register_passthrough_nullable_1_arg::( - "to_start_of_quarter", - |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match DateRounder::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } - }), - ); - registry.register_passthrough_nullable_1_arg::( - "to_start_of_quarter", - |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) - }), - ); - - registry.register_passthrough_nullable_1_arg::( - "to_start_of_year", - |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match DateRounder::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } - }), - ); - registry.register_passthrough_nullable_1_arg::( - "to_start_of_year", - |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) - }), - ); - - registry.register_passthrough_nullable_1_arg::( - "to_start_of_iso_year", + name, |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match DateRounder::eval_date::( + vectorize_with_builder_1_arg::(move |val, output, ctx| { + match DateRounder::eval_date::( val, ctx.func_ctx.tz, ctx.func_ctx.enable_dst_hour_fix, @@ -1974,10 +1878,10 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { }), ); registry.register_passthrough_nullable_1_arg::( - "to_start_of_iso_year", + name, |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) + vectorize_1_arg::(move |val, ctx| { + DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) }), ); } diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 783dc6b1f0ef..19c3f0870231 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -597,6 +597,33 @@ fn test_rounder_functions(file: &mut impl Write) { run_ast(file, "date_trunc(hour, to_timestamp(1630812366))", &[]); run_ast(file, "date_trunc(minute, to_timestamp(1630812366))", &[]); run_ast(file, "date_trunc(second, to_timestamp(1630812366))", &[]); + + run_ast(file, "last_day(to_timestamp(1630812366), year)", &[]); + run_ast(file, "last_day(to_timestamp(1630812366), quarter)", &[]); + run_ast(file, "last_day(to_timestamp(1630812366), month)", &[]); + run_ast(file, "last_day(to_timestamp(1630812366), week)", &[]); + + run_ast(file, "previous_day(to_timestamp(1630812366), monday)", &[]); + run_ast(file, "previous_day(to_timestamp(1630812366), tuesday)", &[]); + run_ast( + file, + "previous_day(to_timestamp(1630812366), wednesday)", + &[], + ); + run_ast(file, "previous_day(to_timestamp(1630812366), thursday)", &[ + ]); + run_ast(file, "previous_day(to_timestamp(1630812366), friday)", &[]); + run_ast(file, "previous_day(to_timestamp(1630812366), saturday)", &[ + ]); + run_ast(file, "previous_day(to_timestamp(1630812366), sunday)", &[]); + + run_ast(file, "next_day(to_timestamp(1630812366), monday)", &[]); + run_ast(file, "next_day(to_timestamp(1630812366), tuesday)", &[]); + run_ast(file, "next_day(to_timestamp(1630812366), wednesday)", &[]); + run_ast(file, "next_day(to_timestamp(1630812366), thursday)", &[]); + run_ast(file, "next_day(to_timestamp(1630812366), friday)", &[]); + run_ast(file, "next_day(to_timestamp(1630812366), saturday)", &[]); + run_ast(file, "next_day(to_timestamp(1630812366), sunday)", &[]); } fn test_date_date_diff(file: &mut impl Write) { diff --git a/src/query/functions/tests/it/scalars/parser.rs b/src/query/functions/tests/it/scalars/parser.rs index b1111bbdcc1a..238b4074942c 100644 --- a/src/query/functions/tests/it/scalars/parser.rs +++ b/src/query/functions/tests/it/scalars/parser.rs @@ -20,6 +20,7 @@ use databend_common_ast::ast::IntervalKind; use databend_common_ast::ast::Literal as ASTLiteral; use databend_common_ast::ast::MapAccessor; use databend_common_ast::ast::UnaryOperator; +use databend_common_ast::ast::Weekday; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; use databend_common_ast::parser::Dialect; @@ -56,6 +57,18 @@ macro_rules! with_interval_mapped_name { } } +macro_rules! with_weekday_mapped_name { + (| $t:tt | $($tail:tt)*) => { + match_template::match_template! { + $t = [ + Monday => "monday", Tuesday => "tuesday", Wednesday => "wednesday", Thursday => "thursday", Friday => "friday", + Saturday => "saturday", Sunday => "sunday", + ], + $($tail)* + } + } +} + macro_rules! transform_interval_add_sub { ($span: expr, $columns: expr, $op: expr, $unit: expr, $date: expr, $interval: expr) => { if $op == BinaryOperator::Plus { @@ -473,6 +486,39 @@ pub fn transform_expr(ast: AExpr, columns: &[(&str, DataType)]) -> RawExpr { } }) } + AExpr::LastDay { span, unit, date } => { + with_interval_mapped_name!(|INTERVAL| match unit { + IntervalKind::INTERVAL => RawExpr::FunctionCall { + span, + name: concat!("to_last_of_", INTERVAL).to_string(), + params: vec![], + args: vec![transform_expr(*date, columns),], + }, + kind => { + unimplemented!("{kind:?} is not supported") + } + }) + } + AExpr::PreviousDay { span, unit, date } => { + with_weekday_mapped_name!(|WEEKDAY| match unit { + Weekday::WEEKDAY => RawExpr::FunctionCall { + span, + name: concat!("to_previous_", WEEKDAY).to_string(), + params: vec![], + args: vec![transform_expr(*date, columns),], + }, + }) + } + AExpr::NextDay { span, unit, date } => { + with_weekday_mapped_name!(|WEEKDAY| match unit { + Weekday::WEEKDAY => RawExpr::FunctionCall { + span, + name: concat!("to_next_", WEEKDAY).to_string(), + params: vec![], + args: vec![transform_expr(*date, columns),], + }, + }) + } AExpr::InList { span, expr, diff --git a/src/query/functions/tests/it/scalars/testdata/datetime.txt b/src/query/functions/tests/it/scalars/testdata/datetime.txt index 673c15f75c74..90d727817217 100644 --- a/src/query/functions/tests/it/scalars/testdata/datetime.txt +++ b/src/query/functions/tests/it/scalars/testdata/datetime.txt @@ -3359,6 +3359,168 @@ output domain : {1630812366000000..=1630812366000000} output : '2021-09-05 03:26:06.000000' +ast : last_day(to_timestamp(1630812366), year) +raw expr : to_last_of_year(to_timestamp(1630812366)) +checked expr : to_last_of_year(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18992 +output type : Date +output domain : {18992..=18992} +output : '2021-12-31' + + +ast : last_day(to_timestamp(1630812366), quarter) +raw expr : to_last_of_quarter(to_timestamp(1630812366)) +checked expr : to_last_of_quarter(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18900 +output type : Date +output domain : {18900..=18900} +output : '2021-09-30' + + +ast : last_day(to_timestamp(1630812366), month) +raw expr : to_last_of_month(to_timestamp(1630812366)) +checked expr : to_last_of_month(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18900 +output type : Date +output domain : {18900..=18900} +output : '2021-09-30' + + +ast : last_day(to_timestamp(1630812366), week) +raw expr : to_last_of_week(to_timestamp(1630812366)) +checked expr : to_last_of_week(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18875 +output type : Date +output domain : {18875..=18875} +output : '2021-09-05' + + +ast : previous_day(to_timestamp(1630812366), monday) +raw expr : to_previous_monday(to_timestamp(1630812366)) +checked expr : to_previous_monday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18869 +output type : Date +output domain : {18869..=18869} +output : '2021-08-30' + + +ast : previous_day(to_timestamp(1630812366), tuesday) +raw expr : to_previous_tuesday(to_timestamp(1630812366)) +checked expr : to_previous_tuesday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18870 +output type : Date +output domain : {18870..=18870} +output : '2021-08-31' + + +ast : previous_day(to_timestamp(1630812366), wednesday) +raw expr : to_previous_wednesday(to_timestamp(1630812366)) +checked expr : to_previous_wednesday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18871 +output type : Date +output domain : {18871..=18871} +output : '2021-09-01' + + +ast : previous_day(to_timestamp(1630812366), thursday) +raw expr : to_previous_thursday(to_timestamp(1630812366)) +checked expr : to_previous_thursday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18872 +output type : Date +output domain : {18872..=18872} +output : '2021-09-02' + + +ast : previous_day(to_timestamp(1630812366), friday) +raw expr : to_previous_friday(to_timestamp(1630812366)) +checked expr : to_previous_friday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18873 +output type : Date +output domain : {18873..=18873} +output : '2021-09-03' + + +ast : previous_day(to_timestamp(1630812366), saturday) +raw expr : to_previous_saturday(to_timestamp(1630812366)) +checked expr : to_previous_saturday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18874 +output type : Date +output domain : {18874..=18874} +output : '2021-09-04' + + +ast : previous_day(to_timestamp(1630812366), sunday) +raw expr : to_previous_sunday(to_timestamp(1630812366)) +checked expr : to_previous_sunday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18868 +output type : Date +output domain : {18868..=18868} +output : '2021-08-29' + + +ast : next_day(to_timestamp(1630812366), monday) +raw expr : to_next_monday(to_timestamp(1630812366)) +checked expr : to_next_monday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18876 +output type : Date +output domain : {18876..=18876} +output : '2021-09-06' + + +ast : next_day(to_timestamp(1630812366), tuesday) +raw expr : to_next_tuesday(to_timestamp(1630812366)) +checked expr : to_next_tuesday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18877 +output type : Date +output domain : {18877..=18877} +output : '2021-09-07' + + +ast : next_day(to_timestamp(1630812366), wednesday) +raw expr : to_next_wednesday(to_timestamp(1630812366)) +checked expr : to_next_wednesday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18878 +output type : Date +output domain : {18878..=18878} +output : '2021-09-08' + + +ast : next_day(to_timestamp(1630812366), thursday) +raw expr : to_next_thursday(to_timestamp(1630812366)) +checked expr : to_next_thursday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18879 +output type : Date +output domain : {18879..=18879} +output : '2021-09-09' + + +ast : next_day(to_timestamp(1630812366), friday) +raw expr : to_next_friday(to_timestamp(1630812366)) +checked expr : to_next_friday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18880 +output type : Date +output domain : {18880..=18880} +output : '2021-09-10' + + +ast : next_day(to_timestamp(1630812366), saturday) +raw expr : to_next_saturday(to_timestamp(1630812366)) +checked expr : to_next_saturday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18881 +output type : Date +output domain : {18881..=18881} +output : '2021-09-11' + + +ast : next_day(to_timestamp(1630812366), sunday) +raw expr : to_next_sunday(to_timestamp(1630812366)) +checked expr : to_next_sunday(to_timestamp(to_int64(1630812366_u32))) +optimized expr : 18882 +output type : Date +output domain : {18882..=18882} +output : '2021-09-12' + + ast : date_diff(year, to_date(0), to_date(10000)) raw expr : diff_years(to_date(10000), to_date(0)) checked expr : diff_years(to_date(to_int64(10000_u16)), to_date(to_int64(0_u8))) diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index 89e237d43f5d..7517b17c9233 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -3985,6 +3985,22 @@ Functions overloads: 23 to_int8(Float64 NULL) :: Int8 NULL 24 to_int8(Boolean) :: Int8 25 to_int8(Boolean NULL) :: Int8 NULL +0 to_last_of_month(Date) :: Date +1 to_last_of_month(Date NULL) :: Date NULL +2 to_last_of_month(Timestamp) :: Date +3 to_last_of_month(Timestamp NULL) :: Date NULL +0 to_last_of_quarter(Date) :: Date +1 to_last_of_quarter(Date NULL) :: Date NULL +2 to_last_of_quarter(Timestamp) :: Date +3 to_last_of_quarter(Timestamp NULL) :: Date NULL +0 to_last_of_week(Date) :: Date +1 to_last_of_week(Date NULL) :: Date NULL +2 to_last_of_week(Timestamp) :: Date +3 to_last_of_week(Timestamp NULL) :: Date NULL +0 to_last_of_year(Date) :: Date +1 to_last_of_year(Date NULL) :: Date NULL +2 to_last_of_year(Timestamp) :: Date +3 to_last_of_year(Timestamp NULL) :: Date NULL 0 to_minute(Timestamp) :: UInt8 1 to_minute(Timestamp NULL) :: UInt8 NULL 0 to_monday(Date) :: Date @@ -3995,8 +4011,64 @@ Functions overloads: 1 to_month(Date NULL) :: UInt8 NULL 2 to_month(Timestamp) :: UInt8 3 to_month(Timestamp NULL) :: UInt8 NULL +0 to_next_friday(Date) :: Date +1 to_next_friday(Date NULL) :: Date NULL +2 to_next_friday(Timestamp) :: Date +3 to_next_friday(Timestamp NULL) :: Date NULL +0 to_next_monday(Date) :: Date +1 to_next_monday(Date NULL) :: Date NULL +2 to_next_monday(Timestamp) :: Date +3 to_next_monday(Timestamp NULL) :: Date NULL +0 to_next_saturday(Date) :: Date +1 to_next_saturday(Date NULL) :: Date NULL +2 to_next_saturday(Timestamp) :: Date +3 to_next_saturday(Timestamp NULL) :: Date NULL +0 to_next_sunday(Date) :: Date +1 to_next_sunday(Date NULL) :: Date NULL +2 to_next_sunday(Timestamp) :: Date +3 to_next_sunday(Timestamp NULL) :: Date NULL +0 to_next_thursday(Date) :: Date +1 to_next_thursday(Date NULL) :: Date NULL +2 to_next_thursday(Timestamp) :: Date +3 to_next_thursday(Timestamp NULL) :: Date NULL +0 to_next_tuesday(Date) :: Date +1 to_next_tuesday(Date NULL) :: Date NULL +2 to_next_tuesday(Timestamp) :: Date +3 to_next_tuesday(Timestamp NULL) :: Date NULL +0 to_next_wednesday(Date) :: Date +1 to_next_wednesday(Date NULL) :: Date NULL +2 to_next_wednesday(Timestamp) :: Date +3 to_next_wednesday(Timestamp NULL) :: Date NULL 0 to_nullable(NULL) :: NULL 1 to_nullable(T0 NULL) :: T0 NULL +0 to_previous_friday(Date) :: Date +1 to_previous_friday(Date NULL) :: Date NULL +2 to_previous_friday(Timestamp) :: Date +3 to_previous_friday(Timestamp NULL) :: Date NULL +0 to_previous_monday(Date) :: Date +1 to_previous_monday(Date NULL) :: Date NULL +2 to_previous_monday(Timestamp) :: Date +3 to_previous_monday(Timestamp NULL) :: Date NULL +0 to_previous_saturday(Date) :: Date +1 to_previous_saturday(Date NULL) :: Date NULL +2 to_previous_saturday(Timestamp) :: Date +3 to_previous_saturday(Timestamp NULL) :: Date NULL +0 to_previous_sunday(Date) :: Date +1 to_previous_sunday(Date NULL) :: Date NULL +2 to_previous_sunday(Timestamp) :: Date +3 to_previous_sunday(Timestamp NULL) :: Date NULL +0 to_previous_thursday(Date) :: Date +1 to_previous_thursday(Date NULL) :: Date NULL +2 to_previous_thursday(Timestamp) :: Date +3 to_previous_thursday(Timestamp NULL) :: Date NULL +0 to_previous_tuesday(Date) :: Date +1 to_previous_tuesday(Date NULL) :: Date NULL +2 to_previous_tuesday(Timestamp) :: Date +3 to_previous_tuesday(Timestamp NULL) :: Date NULL +0 to_previous_wednesday(Date) :: Date +1 to_previous_wednesday(Date NULL) :: Date NULL +2 to_previous_wednesday(Timestamp) :: Date +3 to_previous_wednesday(Timestamp NULL) :: Date NULL 0 to_quarter(Date) :: UInt8 1 to_quarter(Date NULL) :: UInt8 NULL 2 to_quarter(Timestamp) :: UInt8 diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index 479ed886c87d..fdb7355a241e 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -38,6 +38,7 @@ use databend_common_ast::ast::TrimWhere; use databend_common_ast::ast::TypeName; use databend_common_ast::ast::UnaryOperator; use databend_common_ast::ast::UriLocation; +use databend_common_ast::ast::Weekday as ASTWeekday; use databend_common_ast::ast::Window; use databend_common_ast::ast::WindowFrame; use databend_common_ast::ast::WindowFrameBound; @@ -1060,6 +1061,15 @@ impl<'a> TypeChecker<'a> { Expr::DateTrunc { span, unit, date, .. } => self.resolve_date_trunc(*span, date, unit)?, + Expr::LastDay { + span, unit, date, .. + } => self.resolve_last_day(*span, date, unit)?, + Expr::PreviousDay { + span, unit, date, .. + } => self.resolve_previous_or_next_day(*span, date, unit, true)?, + Expr::NextDay { + span, unit, date, .. + } => self.resolve_previous_or_next_day(*span, date, unit, false)?, Expr::Trim { span, expr, @@ -2981,6 +2991,59 @@ impl<'a> TypeChecker<'a> { } } + pub fn resolve_last_day( + &mut self, + span: Span, + date: &Expr, + kind: &ASTIntervalKind, + ) -> Result> { + match kind { + ASTIntervalKind::Year => { + self.resolve_function(span, "to_last_of_year", vec![], &[date]) + } + ASTIntervalKind::Quarter => { + self.resolve_function(span, "to_last_of_quarter", vec![], &[date]) + } + ASTIntervalKind::Month => { + self.resolve_function(span, "to_last_of_month", vec![], &[date]) + } + ASTIntervalKind::Week => { + self.resolve_function(span, "to_last_of_week", vec![], &[date]) + } + _ => Err(ErrorCode::SemanticError( + "Only these interval types are currently supported: [year, quarter, month, week]" + .to_string(), + ) + .set_span(span)), + } + } + + pub fn resolve_previous_or_next_day( + &mut self, + span: Span, + date: &Expr, + weekday: &ASTWeekday, + is_previous: bool, + ) -> Result> { + let prefix = if is_previous { + "to_previous_" + } else { + "to_next_" + }; + + let func_name = match weekday { + ASTWeekday::Monday => format!("{}monday", prefix), + ASTWeekday::Tuesday => format!("{}tuesday", prefix), + ASTWeekday::Wednesday => format!("{}wednesday", prefix), + ASTWeekday::Thursday => format!("{}thursday", prefix), + ASTWeekday::Friday => format!("{}friday", prefix), + ASTWeekday::Saturday => format!("{}saturday", prefix), + ASTWeekday::Sunday => format!("{}sunday", prefix), + }; + + self.resolve_function(span, &func_name, vec![], &[date]) + } + pub fn resolve_subquery( &mut self, typ: SubqueryType, diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index 74e047e658c2..ad9f1873ac4a 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -1170,6 +1170,116 @@ select to_datetime('2022-04-01 06:50:20') < '2022-04-02 04:50:20' ---- 1 +query T +select last_day(to_date('2024-10-22'), week); +---- +2024-10-27 + +query T +select last_day(to_date('2024-10-27'), week); +---- +2024-10-27 + +query T +select last_day(to_date('2024-02-01'), month); +---- +2024-02-29 + +query T +select last_day(to_date('2024-02-01'), quarter); +---- +2024-03-31 + +query T +select last_day(to_date('2024-02-01'), year); +---- +2024-12-31 + +query T +select last_day(to_timestamp('2024-10-24 09:38:18.165575'), week); +---- +2024-10-27 + +query T +select last_day(to_timestamp('2024-10-27 09:38:18.165575'), week); +---- +2024-10-27 + +query T +select last_day(to_timestamp('2024-02-11 09:38:18.165575'), month); +---- +2024-02-29 + +query T +select last_day(to_timestamp('2024-02-11 09:38:18.165575'), quarter); +---- +2024-03-31 + +query T +select last_day(to_timestamp('2024-02-11 09:38:18.165575'), year); +---- +2024-12-31 + +query T +select previous_day(to_date('2024-10-25'), monday); +---- +2024-10-21 + +query T +select previous_day(to_date('2024-10-25'), friday); +---- +2024-10-18 + +query T +select previous_day(to_date('2024-10-25'), saturday); +---- +2024-10-19 + +query T +select previous_day(to_timestamp('2024-10-25 09:38:18.165575'), monday); +---- +2024-10-21 + +query T +select previous_day(to_timestamp('2024-10-25 09:38:18.165575'), friday); +---- +2024-10-18 + +query T +select previous_day(to_timestamp('2024-10-25 09:38:18.165575'), sunday); +---- +2024-10-20 + +query T +select next_day(to_date('2024-10-25'), monday); +---- +2024-10-28 + +query T +select next_day(to_date('2024-10-25'), friday); +---- +2024-11-01 + +query T +select next_day(to_date('2024-10-25'), saturday); +---- +2024-10-26 + +query T +select next_day(to_timestamp('2024-10-25 09:38:18.165575'), monday); +---- +2024-10-28 + +query T +select next_day(to_timestamp('2024-10-25 09:38:18.165575'), friday); +---- +2024-11-01 + +query T +select next_day(to_timestamp('2024-10-25 09:38:18.165575'), sunday); +---- +2024-10-27 + statement ok drop table if exists ts diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test index 57b4984d6740..19c70da87074 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test @@ -21,6 +21,171 @@ select to_timestamp('2000-01-01 00:00:00') statement ok set timezone='Asia/Shanghai' +query T +select last_day(to_date('2024-10-22'), week); +---- +2024-10-27 + +query T +select last_day(to_date('2024-10-27'), week); +---- +2024-10-27 + +query T +select last_day(to_date('2024-02-01'), month); +---- +2024-02-29 + +query T +select last_day(to_date('2024-02-01'), quarter); +---- +2024-03-31 + +query T +select last_day(to_date('2024-02-01'), year); +---- +2024-12-31 + +query T +select last_day(to_timestamp('2024-10-24 01:00:00'), week); +---- +2024-10-27 + +query T +select last_day(to_timestamp('2024-10-24 23:00:00'), week); +---- +2024-10-27 + +query T +select last_day(to_timestamp('2024-10-27 01:00:00'), week); +---- +2024-10-27 + +query T +select last_day(to_timestamp('2024-10-27 23:00:00'), week); +---- +2024-10-27 + +query T +select last_day(to_timestamp('2024-02-11 01:00:00'), month); +---- +2024-02-29 + +query T +select last_day(to_timestamp('2024-02-11 23:00:00'), month); +---- +2024-02-29 + +query T +select last_day(to_timestamp('2024-02-11 01:00:00'), quarter); +---- +2024-03-31 + +query T +select last_day(to_timestamp('2024-02-11 23:00:00'), quarter); +---- +2024-03-31 + +query T +select last_day(to_timestamp('2024-02-11 01:00:00'), year); +---- +2024-12-31 + +query T +select last_day(to_timestamp('2024-02-11 23:00:00'), year); +---- +2024-12-31 + +query T +select previous_day(to_date('2024-10-25'), monday); +---- +2024-10-21 + +query T +select previous_day(to_date('2024-10-25'), friday); +---- +2024-10-18 + +query T +select previous_day(to_date('2024-10-25'), saturday); +---- +2024-10-19 + +query T +select previous_day(to_timestamp('2024-10-25 01:00:00'), monday); +---- +2024-10-21 + +query T +select previous_day(to_timestamp('2024-10-25 23:00:00'), monday); +---- +2024-10-21 + +query T +select previous_day(to_timestamp('2024-10-25 01:00:00'), friday); +---- +2024-10-18 + +query T +select previous_day(to_timestamp('2024-10-25 23:00:00'), friday); +---- +2024-10-18 + +query T +select previous_day(to_timestamp('2024-10-25 01:00:00'), sunday); +---- +2024-10-20 + +query T +select previous_day(to_timestamp('2024-10-25 23:00:00'), sunday); +---- +2024-10-20 + +query T +select next_day(to_date('2024-10-25'), monday); +---- +2024-10-28 + +query T +select next_day(to_date('2024-10-25'), friday); +---- +2024-11-01 + +query T +select next_day(to_date('2024-10-25'), saturday); +---- +2024-10-26 + +query T +select next_day(to_timestamp('2024-10-25 01:00:00'), monday); +---- +2024-10-28 + +query T +select next_day(to_timestamp('2024-10-25 23:00:00'), monday); +---- +2024-10-28 + +query T +select next_day(to_timestamp('2024-10-25 01:00:00'), friday); +---- +2024-11-01 + +query T +select next_day(to_timestamp('2024-10-25 23:00:00'), friday); +---- +2024-11-01 + +query T +select next_day(to_timestamp('2024-10-25 01:00:00'), sunday); +---- +2024-10-27 + +query T +select next_day(to_timestamp('2024-10-25 23:00:00'), sunday); +---- +2024-10-27 + query T select to_timestamp(1630320462000000) ---- From 9e56fe436ff57675224dfcfb424e12434428ee13 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 7 Nov 2024 23:18:09 +0800 Subject: [PATCH 07/92] chore: remove obsolete version (#16789) * chore: remove obsolete version * z --- docker/it-hive/hive-docker-compose.yml | 2 -- docker/it-iceberg-rest/docker-compose.yaml | 4 +--- tests/sqllogictests/scripts/docker-compose-iceberg-tpch.yml | 4 +--- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docker/it-hive/hive-docker-compose.yml b/docker/it-hive/hive-docker-compose.yml index acf4b325e9e9..5891d610c42f 100644 --- a/docker/it-hive/hive-docker-compose.yml +++ b/docker/it-hive/hive-docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: namenode: image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 diff --git a/docker/it-iceberg-rest/docker-compose.yaml b/docker/it-iceberg-rest/docker-compose.yaml index ee028d6001ab..dec8fa97ae31 100644 --- a/docker/it-iceberg-rest/docker-compose.yaml +++ b/docker/it-iceberg-rest/docker-compose.yaml @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -version: '3.8' - services: rest: image: tabulario/iceberg-rest:0.10.0 @@ -47,7 +45,7 @@ services: expose: - 9001 - 9000 - command: [ "server", "/data", "--console-address", ":9001" ] + command: ["server", "/data", "--console-address", ":9001"] ports: - "9000:9000" diff --git a/tests/sqllogictests/scripts/docker-compose-iceberg-tpch.yml b/tests/sqllogictests/scripts/docker-compose-iceberg-tpch.yml index 01d7fcb8119b..e3cdf7b741fd 100644 --- a/tests/sqllogictests/scripts/docker-compose-iceberg-tpch.yml +++ b/tests/sqllogictests/scripts/docker-compose-iceberg-tpch.yml @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -version: '3.8' - services: rest: image: tabulario/iceberg-rest:1.6.0 @@ -50,7 +48,7 @@ services: expose: - 9001 - 9000 - command: [ "server", "/data", "--console-address", ":9001" ] + command: ["server", "/data", "--console-address", ":9001"] ports: - "9000:9000" From a645b0d6969ea2568b7291f640b311ff7206f705 Mon Sep 17 00:00:00 2001 From: Yang Xiufeng Date: Fri, 8 Nov 2024 09:40:08 +0800 Subject: [PATCH 08/92] fix: token and cookie session id miss match. (#16786) * refactor: adjust response of login handler. * update tests. * fix: session id miss match * feat: logging of logout request. * fix --- Cargo.lock | 47 ++++++++++++++ Cargo.toml | 1 + src/query/service/src/auth.rs | 19 ++++++ .../src/servers/http/middleware/session.rs | 2 +- .../http/v1/session/client_session_manager.rs | 13 +--- .../servers/http/v1/session/login_handler.rs | 37 ++++++----- .../servers/http/v1/session/logout_handler.rs | 7 +++ .../http/v1/session/refresh_handler.rs | 25 +++++--- .../src/servers/http/v1/session/token.rs | 4 +- tests/sqllogictests/Cargo.toml | 4 +- .../src/client/global_cookie_store.rs | 62 +++++++++++++++++++ tests/sqllogictests/src/client/http_client.rs | 20 +++++- tests/sqllogictests/src/client/mod.rs | 1 + .../suites/temp_table/alter_temp_tables.sql | 4 +- .../09_http_handler/09_0007_token.py | 38 ++++++++---- .../09_http_handler/09_0007_token.result | 8 +-- 16 files changed, 234 insertions(+), 58 deletions(-) create mode 100644 tests/sqllogictests/src/client/global_cookie_store.rs diff --git a/Cargo.lock b/Cargo.lock index a40673d1de05..f8a8945c1322 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2423,6 +2423,23 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie_store" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4934e6b7e8419148b6ef56950d277af8561060b56afd59e2aadf98b59fce6baa" +dependencies = [ + "cookie", + "idna 0.5.0", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -5177,6 +5194,7 @@ version = "0.1.0" dependencies = [ "async-trait", "clap", + "cookie", "databend-common-base", "databend-common-exception", "env_logger 0.11.5", @@ -5193,6 +5211,7 @@ dependencies = [ "sqlparser 0.50.0", "thiserror", "tokio", + "url", "walkdir", ] @@ -8538,6 +8557,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.4.0" @@ -11736,6 +11765,12 @@ dependencies = [ "stacker", ] +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + [[package]] name = "psm" version = "0.1.21" @@ -11765,6 +11800,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "publicsuffix" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457" +dependencies = [ + "idna 0.3.0", + "psl-types", +] + [[package]] name = "pyo3" version = "0.21.2" @@ -12316,6 +12361,8 @@ checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "base64 0.22.1", "bytes", + "cookie", + "cookie_store", "futures-channel", "futures-core", "futures-util", diff --git a/Cargo.toml b/Cargo.toml index 2c2c0cf8ba42..4c93442f3d31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -260,6 +260,7 @@ cidr = { version = "0.2.2" } clap = { version = "4.4.2", features = ["derive"] } comfy-table = "7" convert_case = "0.6.0" +cookie = "0.18.1" crc32fast = "1.3.2" criterion = "0.5" cron = "0.12.0" diff --git a/src/query/service/src/auth.rs b/src/query/service/src/auth.rs index 0b1aead610cc..2d10848688cf 100644 --- a/src/query/service/src/auth.rs +++ b/src/query/service/src/auth.rs @@ -34,6 +34,14 @@ pub struct AuthMgr { jwt_auth: Option, } +#[derive(Debug)] +pub enum CredentialType { + DatabendToken, + Jwt, + Password, + NoNeed, +} + #[derive(Clone)] pub enum Credential { DatabendToken { @@ -51,6 +59,17 @@ pub enum Credential { NoNeed, } +impl Credential { + pub fn type_name(&self) -> CredentialType { + match self { + Credential::DatabendToken { .. } => CredentialType::DatabendToken, + Credential::Jwt { .. } => CredentialType::Jwt, + Credential::Password { .. } => CredentialType::Password, + Credential::NoNeed => CredentialType::NoNeed, + } + } +} + impl AuthMgr { pub fn init(cfg: &InnerConfig) -> Result<()> { GlobalInstance::set(AuthMgr::create(cfg)); diff --git a/src/query/service/src/servers/http/middleware/session.rs b/src/query/service/src/servers/http/middleware/session.rs index dca9ea39b75b..43399ea31de4 100644 --- a/src/query/service/src/servers/http/middleware/session.rs +++ b/src/query/service/src/servers/http/middleware/session.rs @@ -372,7 +372,7 @@ impl HTTPSessionEndpoint { (None, None) => { if cookie_enabled { let id = Uuid::new_v4().to_string(); - info!("new cookie session id: {}", id); + info!("new session id: {}", id); req.cookie() .add(Cookie::new_with_str(COOKIE_SESSION_ID, &id)); Some(id) diff --git a/src/query/service/src/servers/http/v1/session/client_session_manager.rs b/src/query/service/src/servers/http/v1/session/client_session_manager.rs index 62a109f40fd0..f9f4b9d39ec0 100644 --- a/src/query/service/src/servers/http/v1/session/client_session_manager.rs +++ b/src/query/service/src/servers/http/v1/session/client_session_manager.rs @@ -173,6 +173,7 @@ impl ClientSessionManager { pub async fn new_token_pair( &self, session: &Arc, + client_session_id: String, old_refresh_token: Option, old_session_token: Option, ) -> Result<(String, TokenPair)> { @@ -181,22 +182,12 @@ impl ClientSessionManager { let tenant_name = tenant.tenant_name().to_string(); let user = session.get_current_user()?.name; let auth_role = session.privilege_mgr().get_auth_role(); - let client_session_id = if let Some(old) = &old_refresh_token { - let (claim, _) = SessionClaim::decode(old)?; - assert_eq!(tenant_name, claim.tenant); - assert_eq!(user, claim.user); - assert_eq!(auth_role, claim.auth_role); - claim.session_id - } else { - uuid::Uuid::new_v4().to_string() - }; - let client_session_api = UserApiProvider::instance().client_session_api(&tenant); // new refresh token let now = unix_ts(); let mut claim = SessionClaim::new( - None, + client_session_id.clone(), &tenant_name, &user, &auth_role, diff --git a/src/query/service/src/servers/http/v1/session/login_handler.rs b/src/query/service/src/servers/http/v1/session/login_handler.rs index 9fde9496d051..6f388d8dbf9d 100644 --- a/src/query/service/src/servers/http/v1/session/login_handler.rs +++ b/src/query/service/src/servers/http/v1/session/login_handler.rs @@ -36,15 +36,19 @@ struct LoginRequest { pub settings: Option>, } +#[derive(Serialize, Clone, Debug)] +pub(crate) struct TokensInfo { + pub(crate) session_token_ttl_in_secs: u64, + pub(crate) session_token: String, + pub(crate) refresh_token: String, +} + #[derive(Serialize, Debug, Clone)] pub struct LoginResponse { version: String, session_id: String, - session_token_ttl_in_secs: u64, - - /// for now, only use session token when authed by user-password - session_token: Option, - refresh_token: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tokens: Option, } /// Although theses can be checked for each /v1/query for now, @@ -89,18 +93,20 @@ pub async fn login_handler( Json(req): Json, Query(query): Query, ) -> PoemResult { + let session_id = ctx + .client_session_id + .as_ref() + .expect("login_handler expect session id in ctx") + .clone(); let version = DATABEND_SEMVER.to_string(); check_login(ctx, &req) .await .map_err(HttpErrorCode::bad_request)?; let id_only = || { - let session_id = uuid::Uuid::new_v4().to_string(); Ok(Json(LoginResponse { version: version.clone(), - session_id, - session_token_ttl_in_secs: 0, - session_token: None, - refresh_token: None, + session_id: session_id.clone(), + tokens: None, })) }; @@ -111,16 +117,17 @@ pub async fn login_handler( id_only() } else { let (session_id, token_pair) = ClientSessionManager::instance() - .new_token_pair(&ctx.session, None, None) + .new_token_pair(&ctx.session, session_id, None, None) .await .map_err(HttpErrorCode::server_error)?; Ok(Json(LoginResponse { version, session_id, - - session_token_ttl_in_secs: SESSION_TOKEN_TTL.as_secs(), - session_token: Some(token_pair.session.clone()), - refresh_token: Some(token_pair.refresh.clone()), + tokens: Some(TokensInfo { + session_token_ttl_in_secs: SESSION_TOKEN_TTL.as_secs(), + session_token: token_pair.session.clone(), + refresh_token: token_pair.refresh.clone(), + }), })) } } diff --git a/src/query/service/src/servers/http/v1/session/logout_handler.rs b/src/query/service/src/servers/http/v1/session/logout_handler.rs index 489504cddf13..49221bed323e 100644 --- a/src/query/service/src/servers/http/v1/session/logout_handler.rs +++ b/src/query/service/src/servers/http/v1/session/logout_handler.rs @@ -13,6 +13,7 @@ // limitations under the License. use jwt_simple::prelude::Serialize; +use log::info; use poem::error::Result as PoemResult; use poem::web::Json; use poem::IntoResponse; @@ -32,6 +33,12 @@ pub struct LogoutResponse { #[async_backtrace::framed] pub async fn logout_handler(ctx: &HttpQueryContext) -> PoemResult { if let Some(id) = &ctx.client_session_id { + info!( + "logout with user={}, client session id={}, credential type={:?}", + ctx.user_name, + id, + ctx.credential.type_name() + ); ClientSessionManager::instance() .drop_client_session_state(&ctx.session.get_current_tenant(), &ctx.user_name, id) .await diff --git a/src/query/service/src/servers/http/v1/session/refresh_handler.rs b/src/query/service/src/servers/http/v1/session/refresh_handler.rs index 0231ebb6ecc0..d4a614777b77 100644 --- a/src/query/service/src/servers/http/v1/session/refresh_handler.rs +++ b/src/query/service/src/servers/http/v1/session/refresh_handler.rs @@ -22,6 +22,7 @@ use crate::auth::Credential; use crate::servers::http::error::HttpErrorCode; use crate::servers::http::v1::session::client_session_manager::ClientSessionManager; use crate::servers::http::v1::session::consts::SESSION_TOKEN_TTL; +use crate::servers::http::v1::session::login_handler::TokensInfo; use crate::servers::http::v1::HttpQueryContext; #[derive(Deserialize, Clone)] @@ -32,9 +33,7 @@ struct RefreshRequest { #[derive(Serialize, Debug, Clone)] pub struct RefreshResponse { - session_token: Option, - refresh_token: Option, - session_token_ttl_in_secs: u64, + tokens: TokensInfo, } #[poem::handler] @@ -43,17 +42,29 @@ pub async fn refresh_handler( ctx: &HttpQueryContext, Json(req): Json, ) -> PoemResult { + let client_session_id = ctx + .client_session_id + .as_ref() + .expect("login_handler expect session id in ctx") + .clone(); let mgr = ClientSessionManager::instance(); match &ctx.credential { Credential::DatabendToken { token, .. } => { let (_, token_pair) = mgr - .new_token_pair(&ctx.session, Some(token.clone()), req.session_token) + .new_token_pair( + &ctx.session, + client_session_id, + Some(token.clone()), + req.session_token, + ) .await .map_err(HttpErrorCode::server_error)?; Ok(Json(RefreshResponse { - session_token_ttl_in_secs: SESSION_TOKEN_TTL.as_secs(), - session_token: Some(token_pair.session.clone()), - refresh_token: Some(token_pair.refresh.clone()), + tokens: TokensInfo { + session_token_ttl_in_secs: SESSION_TOKEN_TTL.as_secs(), + session_token: token_pair.session.clone(), + refresh_token: token_pair.refresh.clone(), + }, })) } _ => { diff --git a/src/query/service/src/servers/http/v1/session/token.rs b/src/query/service/src/servers/http/v1/session/token.rs index 3355d3933ca4..5c6866bb31d3 100644 --- a/src/query/service/src/servers/http/v1/session/token.rs +++ b/src/query/service/src/servers/http/v1/session/token.rs @@ -50,7 +50,7 @@ pub fn unix_ts() -> Duration { impl SessionClaim { pub fn new( - session_id: Option, + session_id: String, tenant: &str, user: &str, auth_role: &Option, @@ -60,7 +60,7 @@ impl SessionClaim { tenant: tenant.to_string(), user: user.to_string(), auth_role: auth_role.clone(), - session_id: session_id.unwrap_or(uuid::Uuid::new_v4().to_string()), + session_id, nonce: generate_secure_nonce(), expire_at_in_secs: (unix_ts() + ttl).as_secs(), } diff --git a/tests/sqllogictests/Cargo.toml b/tests/sqllogictests/Cargo.toml index c2cae28cfad0..dbc6345fbb1b 100644 --- a/tests/sqllogictests/Cargo.toml +++ b/tests/sqllogictests/Cargo.toml @@ -16,6 +16,7 @@ name = "databend-sqllogictests" [dependencies] async-trait = { workspace = true } clap = { workspace = true } +cookie = { workspace = true } databend-common-base = { workspace = true } databend-common-exception = { workspace = true } env_logger = { workspace = true } @@ -25,13 +26,14 @@ mysql_async = { workspace = true } mysql_common = { workspace = true } rand = { workspace = true } regex = { workspace = true } -reqwest = { workspace = true } +reqwest = { workspace = true, features = ["cookies"] } serde = { workspace = true } serde_json = { workspace = true } sqllogictest = { workspace = true } sqlparser = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +url = { workspace = true } walkdir = { workspace = true } [lints] diff --git a/tests/sqllogictests/src/client/global_cookie_store.rs b/tests/sqllogictests/src/client/global_cookie_store.rs new file mode 100644 index 000000000000..498e2d82c9cd --- /dev/null +++ b/tests/sqllogictests/src/client/global_cookie_store.rs @@ -0,0 +1,62 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::RwLock; + +use cookie::Cookie; +use reqwest::cookie::CookieStore; +use reqwest::header::HeaderValue; +use url::Url; + +pub(crate) struct GlobalCookieStore { + cookies: RwLock>>, +} + +impl GlobalCookieStore { + pub fn new() -> Self { + GlobalCookieStore { + cookies: RwLock::new(HashMap::new()), + } + } +} + +impl CookieStore for GlobalCookieStore { + fn set_cookies(&self, cookie_headers: &mut dyn Iterator, _url: &Url) { + let iter = cookie_headers + .filter_map(|val| std::str::from_utf8(val.as_bytes()).ok()) + .filter_map(|kv| Cookie::parse(kv).map(|c| c.into_owned()).ok()); + + let mut guard = self.cookies.write().unwrap(); + for cookie in iter { + guard.insert(cookie.name().to_string(), cookie); + } + } + + fn cookies(&self, _url: &Url) -> Option { + let guard = self.cookies.read().unwrap(); + let s: String = guard + .values() + .map(|cookie| cookie.name_value()) + .map(|(name, value)| format!("{name}={value}")) + .collect::>() + .join("; "); + + if s.is_empty() { + return None; + } + + HeaderValue::from_str(&s).ok() + } +} diff --git a/tests/sqllogictests/src/client/http_client.rs b/tests/sqllogictests/src/client/http_client.rs index 4251933ca59e..4d2b56f2be47 100644 --- a/tests/sqllogictests/src/client/http_client.rs +++ b/tests/sqllogictests/src/client/http_client.rs @@ -13,9 +13,11 @@ // limitations under the License. use std::collections::HashMap; +use std::sync::Arc; use std::time::Duration; use std::time::Instant; +use reqwest::cookie::CookieStore; use reqwest::header::HeaderMap; use reqwest::header::HeaderValue; use reqwest::Client; @@ -23,7 +25,9 @@ use reqwest::ClientBuilder; use serde::Deserialize; use sqllogictest::DBOutput; use sqllogictest::DefaultColumnType; +use url::Url; +use crate::client::global_cookie_store::GlobalCookieStore; use crate::error::Result; use crate::util::parser_rows; use crate::util::HttpSessionConf; @@ -61,10 +65,15 @@ fn format_error(value: serde_json::Value) -> String { } #[derive(Deserialize)] -struct AuthResponse { +struct TokenInfo { session_token: String, } +#[derive(Deserialize)] +struct LoginResponse { + tokens: Option, +} + impl HttpClient { pub async fn create() -> Result { let mut header = HeaderMap::new(); @@ -73,7 +82,12 @@ impl HttpClient { HeaderValue::from_str("application/json").unwrap(), ); header.insert("Accept", HeaderValue::from_str("application/json").unwrap()); + let cookie_provider = GlobalCookieStore::new(); + let cookie = HeaderValue::from_str("cookie_enabled=true").unwrap(); + let mut initial_cookies = [&cookie].into_iter(); + cookie_provider.set_cookies(&mut initial_cookies, &Url::parse("https://a.com").unwrap()); let client = ClientBuilder::new() + .cookie_provider(Arc::new(cookie_provider)) .default_headers(header) // https://github.com/hyperium/hyper/issues/2136#issuecomment-589488526 .http2_keep_alive_timeout(Duration::from_secs(15)) @@ -91,11 +105,13 @@ impl HttpClient { .inspect_err(|e| { println!("fail to send to {}: {:?}", url, e); })? - .json::() + .json::() .await .inspect_err(|e| { println!("fail to decode json when call {}: {:?}", url, e); })? + .tokens + .unwrap() .session_token; Ok(Self { diff --git a/tests/sqllogictests/src/client/mod.rs b/tests/sqllogictests/src/client/mod.rs index 439bfb5e3ddc..d4f0098437c5 100644 --- a/tests/sqllogictests/src/client/mod.rs +++ b/tests/sqllogictests/src/client/mod.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod global_cookie_store; mod http_client; mod mysql_client; diff --git a/tests/sqllogictests/suites/temp_table/alter_temp_tables.sql b/tests/sqllogictests/suites/temp_table/alter_temp_tables.sql index 751646b8b6e2..c8b42a4c9d8b 100644 --- a/tests/sqllogictests/suites/temp_table/alter_temp_tables.sql +++ b/tests/sqllogictests/suites/temp_table/alter_temp_tables.sql @@ -193,8 +193,8 @@ alter table t modify c1 varchar comment 'c1-column', c2 int comment 'test'; query TTTTT select database,table,name,data_type,comment from system.columns where table='t' and database='default'; ---- -default t c1 VARCHAR 'c1-column' -default t c2 INT 'test' +default t c1 VARCHAR c1-column +default t c2 INT test statement ok alter table t comment='s1'; diff --git a/tests/suites/1_stateful/09_http_handler/09_0007_token.py b/tests/suites/1_stateful/09_http_handler/09_0007_token.py index 9f5b302381f8..25bb875d3cb4 100755 --- a/tests/suites/1_stateful/09_http_handler/09_0007_token.py +++ b/tests/suites/1_stateful/09_http_handler/09_0007_token.py @@ -7,6 +7,9 @@ import requests from pprint import pprint +from http.cookiejar import Cookie +from requests.cookies import RequestsCookieJar + # Define the URLs and credentials query_url = "http://localhost:8000/v1/query" query_url2 = "http://localhost:8002/v1/query" @@ -16,6 +19,19 @@ verify_url = "http://localhost:8000/v1/verify" auth = ("root", "") +class GlobalCookieJar(RequestsCookieJar): + def __init__(self): + super().__init__() + + def set_cookie(self, cookie: Cookie, *args, **kwargs): + cookie.domain = '' + cookie.path = '/' + super().set_cookie(cookie, *args, **kwargs) + +client = requests.session() +client.cookies = GlobalCookieJar() +client.cookies.set("cookie_enabled", "true") + def print_error(func): def wrapper(*args, **kwargs): @@ -34,7 +50,7 @@ def wrapper(*args, **kwargs): @print_error def do_login(): payload = {} - response = requests.post( + response = client.post( login_url, auth=auth, headers={"Content-Type": "application/json"}, @@ -45,7 +61,7 @@ def do_login(): @print_error def do_logout(_case_id, session_token): - response = requests.post( + response = client.post( logout_url, headers={"Authorization": f"Bearer {session_token}"}, ) @@ -55,7 +71,7 @@ def do_logout(_case_id, session_token): def do_verify(session_token): for token in [session_token, "xxx"]: print("---- verify token ", token) - response = requests.get( + response = client.get( verify_url, headers={"Authorization": f"Bearer {token}"}, ) @@ -64,7 +80,7 @@ def do_verify(session_token): for a in [auth, ("u", "p")]: print("---- verify password: ", a) - response = requests.post( + response = client.post( verify_url, auth=a, ) @@ -72,7 +88,7 @@ def do_verify(session_token): print(response.text) print("---- verify no auth header ", token) - response = requests.get( + response = client.get( verify_url, ) print(response.status_code) @@ -82,7 +98,7 @@ def do_verify(session_token): @print_error def do_refresh(_case_id, refresh_token, session_token): payload = {"session_token": session_token} - response = requests.post( + response = client.post( renew_url, headers={ "Content-Type": "application/json", @@ -96,7 +112,7 @@ def do_refresh(_case_id, refresh_token, session_token): @print_error def do_query(query, session_token, url=query_url): query_payload = {"sql": query, "pagination": {"wait_time_secs": 11}} - response = requests.post( + response = client.post( url, headers={ "Content-Type": "application/json", @@ -127,8 +143,8 @@ def fake_expired_token(ty): def main(): login_resp = do_login() pprint(sorted(login_resp.keys())) - session_token = login_resp.get("session_token") - refresh_token = login_resp.get("refresh_token") + session_token = login_resp.get("tokens").get("session_token") + refresh_token = login_resp.get("tokens").get("refresh_token") # print(session_token) # ok @@ -163,8 +179,8 @@ def main(): renew_resp = do_refresh(1, refresh_token, session_token) pprint(sorted(renew_resp.keys())) - new_session_token = renew_resp.get("session_token") - new_refresh_token = renew_resp.get("refresh_token") + new_session_token = renew_resp.get("tokens").get("session_token") + new_refresh_token = renew_resp.get("tokens").get("refresh_token") # old session_token still valid query_resp = do_query("select 6", session_token) diff --git a/tests/suites/1_stateful/09_http_handler/09_0007_token.result b/tests/suites/1_stateful/09_http_handler/09_0007_token.result index 6af46b402752..f6ad94092791 100644 --- a/tests/suites/1_stateful/09_http_handler/09_0007_token.result +++ b/tests/suites/1_stateful/09_http_handler/09_0007_token.result @@ -1,10 +1,6 @@ ---- do_login() 200 -['refresh_token', - 'session_id', - 'session_token', - 'session_token_ttl_in_secs', - 'version'] +['session_id', 'tokens', 'version'] ---- do_query('select 1',) 200 [['1']] @@ -39,7 +35,7 @@ False {'code': 5100, 'message': 'wrong data token type'} ---- do_refresh(1,) 200 -['refresh_token', 'session_token', 'session_token_ttl_in_secs'] +['tokens'] ---- do_query('select 6',) 200 [['6']] From f4e599fa888ed02de4b4c3b234f7e721995d4ca2 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 8 Nov 2024 10:37:04 +0800 Subject: [PATCH 09/92] feat: implement StringColumn using StringViewArray (#16610) * feat: implement StringColumn using StringViewArray * fix * convert binaryview between arrow1 and arrow2 * fix * fix * fix * fix * fix * fix some issue * fix view slice bug * fix view slice bug * fix * support native read write * fix * fix * fix tests * add with_data_type * add with_data_type * fix gen_random_uuid commit row * move record batch to block * remove unused dep * fix lint * fix commit row * fix commit row * fix size * fix size * add NewBinaryColumnBuilder and NewStringColumnBulder * fix incorrect serialize_size * fix incorrect serialize_size * lint * lint * fix tests * use binary state * use binary state * update tests * update tests * update tests * fix native view encoding * fix * [ci skip] updata kernel concat for view types * [ci skip] improve kernels for view types * [ci skip] only string type use string view type * [ci skip] only string type use string view type * fix tests * [ci skip] fix tests * [ci skip] fix * fix * use NewStringColumnBuilder * rename NewString -> String * fmt * [ci skip] update tests * optimize take * add bench * fix tests * update * improve compare * implement compare using string view prefix * fix * fix * fix * fix-length * disable spill * [ci skip] add put_and_commit * [ci skip] update * update test * lint * [ci skip] add maybe gc * fix endiness * fix endiness * fix * update string compare * update --------- Co-authored-by: sundy-li <543950155@qq.com> --- Cargo.lock | 35 +- Cargo.toml | 6 +- .../arrow/src/arrow/array/binview/ffi.rs | 1 - .../arrow/src/arrow/array/binview/from.rs | 76 ++ .../arrow/src/arrow/array/binview/mod.rs | 93 +- .../arrow/src/arrow/array/binview/mutable.rs | 31 +- .../arrow/src/arrow/array/binview/view.rs | 96 ++ src/common/arrow/src/arrow/array/mod.rs | 6 +- src/common/arrow/src/arrow/datatypes/mod.rs | 7 +- .../arrow/src/native/compression/basic.rs | 40 + .../src/native/compression/binary/mod.rs | 31 +- src/common/arrow/src/native/read/array/mod.rs | 4 + .../arrow/src/native/read/array/view.rs | 274 ++++++ .../arrow/src/native/read/batch_read.rs | 13 + .../arrow/src/native/read/deserialize.rs | 12 + src/common/arrow/src/native/read/reader.rs | 2 + src/common/arrow/src/native/stat.rs | 3 +- src/common/arrow/src/native/write/mod.rs | 1 + .../arrow/src/native/write/serialize.rs | 28 + src/common/arrow/src/native/write/view.rs | 56 ++ .../arrow/tests/it/arrow/array/binview/mod.rs | 22 + src/common/arrow/tests/it/native/io.rs | 25 + src/meta/api/src/txn_backoff.rs | 2 +- src/query/catalog/src/plan/internal_column.rs | 22 +- src/query/catalog/src/table.rs | 4 + src/query/ee/Cargo.toml | 1 - .../background_service_handler.rs | 4 +- .../src/background_service/compaction_job.rs | 135 ++- .../ee_features/background_service/Cargo.toml | 2 +- .../src/background_service.rs | 6 +- src/query/expression/Cargo.toml | 6 + src/query/expression/benches/bench.rs | 157 ++++ .../expression/src/aggregate/payload_flush.rs | 25 +- .../expression/src/aggregate/payload_row.rs | 110 ++- src/query/expression/src/block.rs | 9 + src/query/expression/src/converts/arrow/to.rs | 1 + .../expression/src/converts/arrow2/from.rs | 295 +++--- .../expression/src/converts/arrow2/to.rs | 33 +- .../expression/src/converts/meta/bincode.rs | 46 +- src/query/expression/src/evaluator.rs | 8 +- .../expression/src/filter/filter_executor.rs | 5 +- .../select_value/select_column_scalar.rs | 96 +- src/query/expression/src/kernels/concat.rs | 167 +--- src/query/expression/src/kernels/filter.rs | 164 +--- src/query/expression/src/kernels/group_by.rs | 2 +- .../src/kernels/group_by_hash/method.rs | 4 +- .../group_by_hash/method_dict_serializer.rs | 21 +- .../group_by_hash/method_serializer.rs | 12 +- .../group_by_hash/method_single_string.rs | 28 +- .../src/kernels/group_by_hash/utils.rs | 23 +- src/query/expression/src/kernels/mod.rs | 1 + src/query/expression/src/kernels/scatter.rs | 34 +- src/query/expression/src/kernels/sort.rs | 2 +- .../expression/src/kernels/sort_compare.rs | 13 +- src/query/expression/src/kernels/take.rs | 150 ++-- .../expression/src/kernels/take_chunks.rs | 125 +-- .../expression/src/kernels/take_compact.rs | 111 +-- .../expression/src/kernels/take_ranges.rs | 71 +- src/query/expression/src/kernels/utils.rs | 190 ---- src/query/expression/src/row/row_converter.rs | 5 +- src/query/expression/src/types/array.rs | 1 + src/query/expression/src/types/bitmap.rs | 2 +- src/query/expression/src/types/geography.rs | 32 +- src/query/expression/src/types/geometry.rs | 2 +- src/query/expression/src/types/string.rs | 483 ++++------ src/query/expression/src/types/variant.rs | 2 +- src/query/expression/src/utils/display.rs | 6 +- src/query/expression/src/values.rs | 30 +- src/query/expression/tests/it/common.rs | 2 +- src/query/expression/tests/it/kernel.rs | 56 +- src/query/expression/tests/it/row.rs | 38 +- .../tests/it/testdata/kernel-pass.txt | 36 +- .../formats/src/field_decoder/fast_values.rs | 2 +- src/query/formats/src/field_decoder/nested.rs | 2 +- .../src/field_decoder/separated_text.rs | 3 +- src/query/functions/Cargo.toml | 2 +- .../aggregates/aggregate_distinct_state.rs | 5 +- .../src/aggregates/aggregate_histogram.rs | 5 +- .../src/aggregates/aggregate_min_max_any.rs | 124 ++- .../src/aggregates/aggregate_scalar_state.rs | 33 + .../src/aggregates/aggregate_string_agg.rs | 5 +- src/query/functions/src/scalars/arithmetic.rs | 29 +- src/query/functions/src/scalars/binary.rs | 72 +- src/query/functions/src/scalars/bitmap.rs | 5 +- src/query/functions/src/scalars/comparison.rs | 93 +- src/query/functions/src/scalars/datetime.rs | 13 +- .../functions/src/scalars/decimal/cast.rs | 2 +- src/query/functions/src/scalars/geo_h3.rs | 5 +- src/query/functions/src/scalars/hash.rs | 83 +- src/query/functions/src/scalars/other.rs | 24 +- src/query/functions/src/scalars/string.rs | 333 ++----- .../src/scalars/string_multi_args.rs | 13 +- src/query/functions/src/scalars/variant.rs | 5 +- src/query/functions/src/scalars/vector.rs | 11 +- src/query/functions/src/srfs/variant.rs | 13 +- .../tests/it/aggregates/testdata/agg.txt | 114 +-- .../it/aggregates/testdata/agg_group_by.txt | 66 +- .../functions/tests/it/scalars/comparison.rs | 1 + .../tests/it/scalars/testdata/arithmetic.txt | 96 +- .../tests/it/scalars/testdata/array.txt | 36 +- .../tests/it/scalars/testdata/binary.txt | 72 +- .../tests/it/scalars/testdata/cast.txt | 276 +++--- .../tests/it/scalars/testdata/comparison.txt | 305 ++++--- .../tests/it/scalars/testdata/geo_h3.txt | 24 +- .../tests/it/scalars/testdata/geometry.txt | 76 +- .../tests/it/scalars/testdata/hash.txt | 100 +-- .../tests/it/scalars/testdata/map.txt | 322 +++---- .../tests/it/scalars/testdata/regexp.txt | 446 ++++----- .../tests/it/scalars/testdata/string.txt | 848 +++++++++--------- .../tests/it/scalars/testdata/tuple.txt | 36 +- .../tests/it/scalars/testdata/variant.txt | 810 ++++++++--------- .../processors/transforms/sort/rows/common.rs | 6 +- .../interpreter_table_modify_column.rs | 6 +- .../src/pipelines/builders/builder_join.rs | 5 - .../group_by/aggregator_groups_builder.rs | 73 +- .../hash_join/hash_join_build_state.rs | 36 +- .../hash_join/hash_join_probe_state.rs | 4 - .../merge_into_hash_join_optimization.rs | 5 +- .../hash_join/probe_join/inner_join.rs | 7 +- .../hash_join/probe_join/left_anti_join.rs | 14 +- .../hash_join/probe_join/left_join.rs | 13 +- .../hash_join/probe_join/left_mark_join.rs | 7 +- .../hash_join/probe_join/left_semi_join.rs | 9 +- .../hash_join/probe_join/right_join.rs | 7 +- .../hash_join/probe_join/right_mark_join.rs | 7 +- .../probe_join/right_semi_anti_join.rs | 7 +- .../transforms/hash_join/probe_state.rs | 12 +- .../processors/transforms/hash_join/row.rs | 2 - .../hash_join/transform_hash_join_probe.rs | 2 - .../suggested_background_compaction_tasks.rs | 148 ++- .../others/suggested_background_tasks.rs | 12 +- .../service/src/table_functions/others/udf.rs | 3 +- src/query/service/src/test_kits/fixture.rs | 12 +- src/query/settings/src/settings_default.rs | 14 +- .../common/index/benches/build_from_block.rs | 2 +- .../storages/common/index/src/bloom_index.rs | 2 +- src/query/storages/fuse/src/fuse_table.rs | 4 + .../fuse/src/io/read/block/block_reader.rs | 1 + .../block/block_reader_native_deserialize.rs | 1 + .../operations/read/runtime_filter_prunner.rs | 4 +- .../fuse/src/statistics/cluster_statistics.rs | 2 +- .../table_functions/clustering_statistics.rs | 5 +- .../fuse/src/table_functions/fuse_block.rs | 5 +- .../fuse/src/table_functions/fuse_column.rs | 15 +- .../fuse/src/table_functions/fuse_encoding.rs | 11 +- src/query/storages/parquet/Cargo.toml | 1 - .../row_based/formats/csv/block_builder.rs | 16 +- .../system/src/malloc_stats_totals_table.rs | 2 +- tests/sqllogictests/src/main.rs | 5 +- .../base/05_ddl/05_0003_ddl_alter_table.test | 4 +- .../09_0027_func_fuse_encoding.test | 8 +- .../20_0013_query_result_cache.test | 4 +- ...elete.sql.test => distributed_delete.test} | 12 +- .../mode/cluster/memo/aggregate_property.test | 4 +- .../mode/cluster/memo/join_property.test | 10 +- .../mode/cluster/memo/mix_property.test | 2 +- .../suites/mode/cluster/shuffle_join.test | 40 +- .../mode/standalone/explain/explain.test | 66 +- .../suites/mode/standalone/limit.test | 9 +- .../suites/mode/standalone/pr15804.test | 11 +- .../functions/02_0005_function_compare.test | 10 + .../suites/query/join/runtime_filter.test | 4 +- .../formats/parquet/options/null_if.test | 2 +- .../20+_others/20_0014_sort_spill.result | 20 +- .../20+_others/20_0014_sort_spill.sql | 21 +- .../00_stage/00_0001_copy_into_stage.result | 2 +- .../05_05_01_parquet_load_unload.result | 4 +- .../07_0000_insert_with_stage.result | 2 +- .../07_0001_replace_with_stage.result | 2 +- ..._0002_insert_with_stage_deduplicate.result | 2 +- ..._0003_insert_with_stage_file_format.result | 6 +- .../07_0003_insert_with_stage_file_format.sh | 2 +- .../08_00_parquet/08_00_00_basic.result | 4 +- 173 files changed, 4535 insertions(+), 4355 deletions(-) create mode 100644 src/common/arrow/src/native/read/array/view.rs create mode 100644 src/common/arrow/src/native/write/view.rs create mode 100644 src/query/expression/benches/bench.rs rename tests/sqllogictests/suites/mode/cluster/{distributed_delete.sql.test => distributed_delete.test} (86%) mode change 100755 => 100644 tests/suites/1_stateful/00_stage/00_0001_copy_into_stage.result mode change 100755 => 100644 tests/suites/1_stateful/05_formats/05_05_parquet/05_05_01_parquet_load_unload.result mode change 100755 => 100644 tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_00_basic.result diff --git a/Cargo.lock b/Cargo.lock index f8a8945c1322..f4c02090d96f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -575,8 +575,7 @@ dependencies = [ [[package]] name = "arrow-udf-js" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6151bb7f26cde846e14adb17e08282153f7a9250dd78bbab3fa462b66d7b623" +source = "git+https://github.com/arrow-udf/arrow-udf?rev=80b09d6#80b09d67ee0c7b796bf7a492a71842ac64622406" dependencies = [ "anyhow", "arrow-array", @@ -589,8 +588,7 @@ dependencies = [ [[package]] name = "arrow-udf-python" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b80da061a53aac237e711fddb01709002ba2e006f9fd4c72a430d4938dd921" +source = "git+https://github.com/arrow-udf/arrow-udf?rev=80b09d6#80b09d67ee0c7b796bf7a492a71842ac64622406" dependencies = [ "anyhow", "arrow-array", @@ -604,8 +602,7 @@ dependencies = [ [[package]] name = "arrow-udf-wasm" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe31144804e093dd60b4e7a749b64b9454040c05a34ccbeb641fc60fcf5ee92d" +source = "git+https://github.com/arrow-udf/arrow-udf?rev=80b09d6#80b09d67ee0c7b796bf7a492a71842ac64622406" dependencies = [ "anyhow", "arrow-array", @@ -2973,9 +2970,9 @@ dependencies = [ [[package]] name = "databend-client" -version = "0.22.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8770a1c49fa21e62a768a0de442cc3f77998a357303d56ddd3485cb7c58d3a" +checksum = "0819048a792e2eac58b455bbbcf6077c81e2780b3cc4f565a6e72e92dde56bd1" dependencies = [ "async-trait", "log", @@ -3317,6 +3314,7 @@ dependencies = [ "chrono", "chrono-tz 0.8.6", "comfy-table", + "criterion", "dashmap 6.1.0", "databend-common-arrow", "databend-common-ast", @@ -3327,6 +3325,7 @@ dependencies = [ "databend-common-hashtable", "databend-common-io", "educe 0.4.23", + "either", "enum-as-inner 0.5.1", "ethnum", "futures", @@ -4422,7 +4421,6 @@ dependencies = [ "databend-common-metrics", "databend-common-pipeline-core", "databend-common-settings", - "databend-common-sql", "databend-common-storage", "databend-storages-common-cache", "databend-storages-common-pruner", @@ -4673,9 +4671,9 @@ dependencies = [ [[package]] name = "databend-driver" -version = "0.22.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "393daaf83f32c5685887103b04bd9762d43298f7820b9f888144877fcf1d89e8" +checksum = "cbea10312fa203003b572b701b6121cecd8eaf260e7ec7687796b552044541c6" dependencies = [ "arrow", "async-compression 0.4.12", @@ -4698,9 +4696,9 @@ dependencies = [ [[package]] name = "databend-driver-core" -version = "0.22.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd1abd8ab1c4200648510204cdfcb538af1eca2c9433e12276dee26a1806144" +checksum = "e1b8f3bf7bb87d0f2fce8a1992eea4a0b442c01165a0d68aab2727a386c945ab" dependencies = [ "arrow", "chrono", @@ -4721,9 +4719,9 @@ dependencies = [ [[package]] name = "databend-driver-macros" -version = "0.22.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5038d47b403ce2351b109fb82627959a27a16a53ce16397bf1ca39307e052b5c" +checksum = "40bed5c66f36e79baea7c95d2d0dc8433671606ca7152012562c7145e3b909a1" dependencies = [ "quote", "syn 2.0.58", @@ -4757,11 +4755,11 @@ dependencies = [ name = "databend-enterprise-background-service" version = "0.1.0" dependencies = [ - "arrow-array", "async-backtrace", "async-trait", "databend-common-base", "databend-common-exception", + "databend-common-expression", "databend-common-meta-app", "serde", ] @@ -4811,7 +4809,6 @@ dependencies = [ name = "databend-enterprise-query" version = "0.1.0" dependencies = [ - "arrow-array", "async-backtrace", "async-trait", "aws-config", @@ -8419,9 +8416,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", diff --git a/Cargo.toml b/Cargo.toml index 4c93442f3d31..b6ce2ec48e71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -219,9 +219,9 @@ arrow-ipc = { version = "53" } arrow-ord = { version = "53" } arrow-schema = { version = "53", features = ["serde"] } arrow-select = { version = "53" } -arrow-udf-js = "0.5.0" -arrow-udf-python = "0.4.0" -arrow-udf-wasm = "0.4.0" +arrow-udf-js = { git = "https://github.com/arrow-udf/arrow-udf", rev = "80b09d6" } +arrow-udf-python = { git = "https://github.com/arrow-udf/arrow-udf", rev = "80b09d6" } +arrow-udf-wasm = { git = "https://github.com/arrow-udf/arrow-udf", rev = "80b09d6" } async-backtrace = "0.2" async-channel = "1.7.1" async-compression = { git = "https://github.com/datafuse-extras/async-compression", rev = "dc81082", features = [ diff --git a/src/common/arrow/src/arrow/array/binview/ffi.rs b/src/common/arrow/src/arrow/array/binview/ffi.rs index b28b349c5244..8d98daa62a95 100644 --- a/src/common/arrow/src/arrow/array/binview/ffi.rs +++ b/src/common/arrow/src/arrow/array/binview/ffi.rs @@ -64,7 +64,6 @@ unsafe impl ToFfi for BinaryViewArrayGeneric { validity, views: self.views.clone(), buffers: self.buffers.clone(), - raw_buffers: self.raw_buffers.clone(), phantom: Default::default(), total_bytes_len: AtomicU64::new(self.total_bytes_len.load(Ordering::Relaxed)), total_buffer_len: self.total_buffer_len, diff --git a/src/common/arrow/src/arrow/array/binview/from.rs b/src/common/arrow/src/arrow/array/binview/from.rs index 7559b19d8f54..40304c7c0279 100644 --- a/src/common/arrow/src/arrow/array/binview/from.rs +++ b/src/common/arrow/src/arrow/array/binview/from.rs @@ -12,9 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +use arrow_data::ArrayData; +use arrow_data::ArrayDataBuilder; +use arrow_schema::DataType; + +use crate::arrow::array::Arrow2Arrow; +use crate::arrow::array::BinaryViewArray; use crate::arrow::array::BinaryViewArrayGeneric; use crate::arrow::array::MutableBinaryViewArray; +use crate::arrow::array::Utf8ViewArray; use crate::arrow::array::ViewType; +use crate::arrow::bitmap::Bitmap; impl> FromIterator> for BinaryViewArrayGeneric { #[inline] @@ -22,3 +30,71 @@ impl> FromIterator> for BinaryViewAr MutableBinaryViewArray::::from_iter(iter).into() } } + +impl Arrow2Arrow for BinaryViewArray { + fn to_data(&self) -> ArrayData { + let builder = ArrayDataBuilder::new(DataType::BinaryView) + .len(self.len()) + .add_buffer(self.views.clone().into()) + .add_buffers( + self.buffers + .iter() + .map(|x| x.clone().into()) + .collect::>(), + ) + .nulls(self.validity.clone().map(Into::into)); + unsafe { builder.build_unchecked() } + } + + fn from_data(data: &ArrayData) -> Self { + let views = crate::arrow::buffer::Buffer::from(data.buffers()[0].clone()); + let buffers = data.buffers()[1..] + .iter() + .map(|x| crate::arrow::buffer::Buffer::from(x.clone())) + .collect(); + let validity = data.nulls().map(|x| Bitmap::from_null_buffer(x.clone())); + unsafe { + Self::new_unchecked_unknown_md( + crate::arrow::datatypes::DataType::BinaryView, + views, + buffers, + validity, + None, + ) + } + } +} + +impl Arrow2Arrow for Utf8ViewArray { + fn to_data(&self) -> ArrayData { + let builder = ArrayDataBuilder::new(DataType::Utf8View) + .len(self.len()) + .add_buffer(self.views.clone().into()) + .add_buffers( + self.buffers + .iter() + .map(|x| x.clone().into()) + .collect::>(), + ) + .nulls(self.validity.clone().map(Into::into)); + unsafe { builder.build_unchecked() } + } + + fn from_data(data: &ArrayData) -> Self { + let views = crate::arrow::buffer::Buffer::from(data.buffers()[0].clone()); + let buffers = data.buffers()[1..] + .iter() + .map(|x| crate::arrow::buffer::Buffer::from(x.clone())) + .collect(); + let validity = data.nulls().map(|x| Bitmap::from_null_buffer(x.clone())); + unsafe { + Self::new_unchecked_unknown_md( + crate::arrow::datatypes::DataType::Utf8View, + views, + buffers, + validity, + None, + ) + } + } +} diff --git a/src/common/arrow/src/arrow/array/binview/mod.rs b/src/common/arrow/src/arrow/array/binview/mod.rs index 4d4637a95a8c..35850b10e0ab 100644 --- a/src/common/arrow/src/arrow/array/binview/mod.rs +++ b/src/common/arrow/src/arrow/array/binview/mod.rs @@ -41,9 +41,7 @@ pub use mutable::MutableBinaryViewArray; use private::Sealed; pub use view::View; -use crate::arrow::array::binview::view::validate_binary_view; use crate::arrow::array::binview::view::validate_utf8_only; -use crate::arrow::array::binview::view::validate_utf8_view; use crate::arrow::array::iterator::NonNullValuesIter; use crate::arrow::array::Array; use crate::arrow::bitmap::utils::BitmapIter; @@ -128,8 +126,6 @@ pub struct BinaryViewArrayGeneric { data_type: DataType, views: Buffer, buffers: Arc<[Buffer]>, - // Raw buffer access. (pointer, len). - raw_buffers: Arc<[(*const u8, usize)]>, validity: Option, phantom: PhantomData, /// Total bytes length if we would concat them all @@ -150,7 +146,6 @@ impl Clone for BinaryViewArrayGeneric { data_type: self.data_type.clone(), views: self.views.clone(), buffers: self.buffers.clone(), - raw_buffers: self.raw_buffers.clone(), validity: self.validity.clone(), phantom: Default::default(), total_bytes_len: AtomicU64::new(self.total_bytes_len.load(Ordering::Relaxed)), @@ -163,13 +158,6 @@ unsafe impl Send for BinaryViewArrayGeneric {} unsafe impl Sync for BinaryViewArrayGeneric {} -fn buffers_into_raw(buffers: &[Buffer]) -> Arc<[(*const T, usize)]> { - buffers - .iter() - .map(|buf| (buf.data_ptr(), buf.len())) - .collect() -} - impl BinaryViewArrayGeneric { pub fn new_unchecked( data_type: DataType, @@ -179,7 +167,6 @@ impl BinaryViewArrayGeneric { total_bytes_len: usize, total_buffer_len: usize, ) -> Self { - let raw_buffers = buffers_into_raw(&buffers); // # Safety // The caller must ensure // - the data is valid utf8 (if required) @@ -188,7 +175,6 @@ impl BinaryViewArrayGeneric { data_type, views, buffers, - raw_buffers, validity, phantom: Default::default(), total_bytes_len: AtomicU64::new(total_bytes_len as u64), @@ -242,10 +228,20 @@ impl BinaryViewArrayGeneric { "BinaryViewArray can only be initialized with DataType::BinaryView or DataType::Utf8View", )); } - if T::IS_UTF8 { - validate_utf8_view(views.as_ref(), buffers.as_ref())?; - } else { - validate_binary_view(views.as_ref(), buffers.as_ref())?; + + #[cfg(debug_assertions)] + { + if T::IS_UTF8 { + crate::arrow::array::binview::view::validate_utf8_view( + views.as_ref(), + buffers.as_ref(), + )?; + } else { + crate::arrow::array::binview::view::validate_binary_view( + views.as_ref(), + buffers.as_ref(), + )?; + } } if let Some(validity) = &validity { @@ -303,29 +299,8 @@ impl BinaryViewArrayGeneric { /// Assumes that the `i < self.len`. #[inline] pub unsafe fn value_unchecked(&self, i: usize) -> &T { - let v = *self.views.get_unchecked(i); - let len = v.length; - - // view layout: - // length: 4 bytes - // prefix: 4 bytes - // buffer_index: 4 bytes - // offset: 4 bytes - - // inlined layout: - // length: 4 bytes - // data: 12 bytes - - let bytes = if len <= 12 { - let ptr = self.views.data_ptr() as *const u8; - std::slice::from_raw_parts(ptr.add(i * 16 + 4), len as usize) - } else { - let (data_ptr, data_len) = *self.raw_buffers.get_unchecked(v.buffer_idx as usize); - let data = std::slice::from_raw_parts(data_ptr, data_len); - let offset = v.offset as usize; - data.get_unchecked(offset..offset + len as usize) - }; - T::from_bytes_unchecked(bytes) + let v = self.views.get_unchecked(i); + T::from_bytes_unchecked(v.get_slice_unchecked(&self.buffers)) } /// Returns an iterator of `Option<&T>` over every element of this array. @@ -381,6 +356,21 @@ impl BinaryViewArrayGeneric { } } + fn total_unshared_buffer_len(&self) -> usize { + // Given this function is only called in `maybe_gc()`, + // it may not be worthy to add an extra field for this. + self.buffers + .iter() + .map(|buf| { + if buf.shared_count_strong() > 1 { + 0 + } else { + buf.len() + } + }) + .sum() + } + /// Get the length of bytes that are stored in the variadic buffers. pub fn total_buffer_len(&self) -> usize { self.total_buffer_len @@ -402,10 +392,10 @@ impl BinaryViewArrayGeneric { return self; } let mut mutable = MutableBinaryViewArray::with_capacity(self.len()); - let buffers = self.raw_buffers.as_ref(); + let buffers = self.buffers.as_ref(); for view in self.views.as_ref() { - unsafe { mutable.push_view(*view, buffers) } + unsafe { mutable.push_view_unchecked(*view, buffers) } } mutable.freeze().with_validity(self.validity) } @@ -421,6 +411,13 @@ impl BinaryViewArrayGeneric { return self; } + // if Arc::strong_count(&self.buffers) != 1 { + // // There are multiple holders of this `buffers`. + // // If we allow gc in this case, + // // it may end up copying the same content multiple times. + // return self; + // } + // Subtract the maximum amount of inlined strings to get a lower bound // on the number of buffer bytes needed (assuming no dedup). let total_bytes_len = self.total_bytes_len(); @@ -531,11 +528,18 @@ impl BinaryViewArrayGeneric { pub fn default_data_type() -> &'static DataType { T::data_type() } + + pub fn with_data_type(mut self, data_type: DataType) -> Self { + self.data_type = data_type; + self + } } pub type BinaryViewArray = BinaryViewArrayGeneric<[u8]>; pub type Utf8ViewArray = BinaryViewArrayGeneric; +pub type MutableUtf8ViewArray = MutableBinaryViewArray; + impl BinaryViewArray { /// Validate the underlying bytes on UTF-8. pub fn validate_utf8(&self) -> Result<()> { @@ -593,7 +597,7 @@ impl Array for BinaryViewArrayGeneric { } fn data_type(&self) -> &DataType { - T::data_type() + &self.data_type } fn validity(&self) -> Option<&Bitmap> { @@ -616,6 +620,7 @@ impl Array for BinaryViewArrayGeneric { .map(|bitmap| bitmap.sliced_unchecked(offset, length)) .filter(|bitmap| bitmap.unset_bits() > 0); self.views.slice_unchecked(offset, length); + self.total_bytes_len.store(UNKNOWN_LEN, Ordering::Relaxed) } diff --git a/src/common/arrow/src/arrow/array/binview/mutable.rs b/src/common/arrow/src/arrow/array/binview/mutable.rs index 64ef7880bcf8..abf2530b6a38 100644 --- a/src/common/arrow/src/arrow/array/binview/mutable.rs +++ b/src/common/arrow/src/arrow/array/binview/mutable.rs @@ -41,9 +41,9 @@ pub struct MutableBinaryViewArray { pub(super) validity: Option, pub(super) phantom: std::marker::PhantomData, /// Total bytes length if we would concatenate them all. - pub(super) total_bytes_len: usize, + pub total_bytes_len: usize, /// Total bytes in the buffer (excluding remaining capacity) - pub(super) total_buffer_len: usize, + pub total_buffer_len: usize, } impl Clone for MutableBinaryViewArray { @@ -153,16 +153,15 @@ impl MutableBinaryViewArray { /// - caller must allocate enough capacity /// - caller must ensure the view and buffers match. #[inline] - pub unsafe fn push_view(&mut self, v: View, buffers: &[(*const u8, usize)]) { + pub(crate) unsafe fn push_view_unchecked(&mut self, v: View, buffers: &[Buffer]) { let len = v.length; self.total_bytes_len += len as usize; if len <= 12 { debug_assert!(self.views.capacity() > self.views.len()); - self.views.push(v); + self.views.push(v) } else { self.total_buffer_len += len as usize; - let (data_ptr, data_len) = *buffers.get_unchecked(v.buffer_idx as usize); - let data = std::slice::from_raw_parts(data_ptr, data_len); + let data = buffers.get_unchecked(v.buffer_idx as usize); let offset = v.offset as usize; let bytes = data.get_unchecked(offset..offset + len as usize); let t = T::from_bytes_unchecked(bytes); @@ -191,7 +190,11 @@ impl MutableBinaryViewArray { // buffer index + offset -> real binary data self.total_buffer_len += bytes.len(); let required_cap = self.in_progress_buffer.len() + bytes.len(); - if self.in_progress_buffer.capacity() < required_cap { + + let does_not_fit_in_buffer = self.in_progress_buffer.capacity() < required_cap; + let offset_will_not_fit = self.in_progress_buffer.len() > u32::MAX as usize; + + if does_not_fit_in_buffer || offset_will_not_fit { let new_capacity = (self.in_progress_buffer.capacity() * 2) .clamp(DEFAULT_BLOCK_SIZE, 16 * 1024 * 1024) .max(bytes.len()); @@ -406,6 +409,20 @@ impl MutableBinaryViewArray<[u8]> { } } +impl MutableBinaryViewArray { + pub fn pop(&mut self) -> Option { + if self.is_empty() { + return None; + } + + let value = unsafe { self.value_unchecked(self.len() - 1).to_string() }; + + self.views.pop(); + + Some(value) + } +} + impl> Extend> for MutableBinaryViewArray { #[inline] fn extend>>(&mut self, iter: I) { diff --git a/src/common/arrow/src/arrow/array/binview/view.rs b/src/common/arrow/src/arrow/array/binview/view.rs index b49993fb77ff..1707b6a58a9c 100644 --- a/src/common/arrow/src/arrow/array/binview/view.rs +++ b/src/common/arrow/src/arrow/array/binview/view.rs @@ -36,13 +36,109 @@ pub struct View { pub buffer_idx: u32, /// The offset into the buffer. pub offset: u32, + pub _align: [u128; 0], } impl View { + pub const MAX_INLINE_SIZE: u32 = 12; + #[inline(always)] pub fn as_u128(self) -> u128 { unsafe { std::mem::transmute(self) } } + + /// Create a new inline view without verifying the length + /// + /// # Safety + /// + /// It needs to hold that `bytes.len() <= View::MAX_INLINE_SIZE`. + #[inline] + pub unsafe fn new_inline_unchecked(bytes: &[u8]) -> Self { + debug_assert!(bytes.len() <= u32::MAX as usize); + debug_assert!(bytes.len() as u32 <= Self::MAX_INLINE_SIZE); + + let mut view = Self { + length: bytes.len() as u32, + ..Default::default() + }; + + let view_ptr = &mut view as *mut _ as *mut u8; + + // SAFETY: + // - bytes length <= 12, + // - size_of:: == 16 + // - View is laid out as [length, prefix, buffer_idx, offset] (using repr(C)) + // - By grabbing the view_ptr and adding 4, we have provenance over prefix, buffer_idx and + // offset. (i.e. the same could not be achieved with &mut self.prefix as *mut _ as *mut u8) + unsafe { + let inline_data_ptr = view_ptr.add(4); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), inline_data_ptr, bytes.len()); + } + view + } + + /// Create a new inline view + /// + /// # Panics + /// + /// Panics if the `bytes.len() > View::MAX_INLINE_SIZE`. + #[inline] + pub fn new_inline(bytes: &[u8]) -> Self { + assert!(bytes.len() as u32 <= Self::MAX_INLINE_SIZE); + unsafe { Self::new_inline_unchecked(bytes) } + } + + /// Create a new inline view + /// + /// # Safety + /// + /// It needs to hold that `bytes.len() > View::MAX_INLINE_SIZE`. + #[inline] + pub unsafe fn new_noninline_unchecked(bytes: &[u8], buffer_idx: u32, offset: u32) -> Self { + debug_assert!(bytes.len() <= u32::MAX as usize); + debug_assert!(bytes.len() as u32 > View::MAX_INLINE_SIZE); + + // SAFETY: The invariant of this function guarantees that this is safe. + let prefix = unsafe { u32::from_le_bytes(bytes[0..4].try_into().unwrap_unchecked()) }; + Self { + length: bytes.len() as u32, + prefix, + buffer_idx, + offset, + ..Default::default() + } + } + + #[inline] + pub fn new_from_bytes(bytes: &[u8], buffer_idx: u32, offset: u32) -> Self { + debug_assert!(bytes.len() <= u32::MAX as usize); + + // SAFETY: We verify the invariant with the outer if statement + unsafe { + if bytes.len() as u32 <= Self::MAX_INLINE_SIZE { + Self::new_inline_unchecked(bytes) + } else { + Self::new_noninline_unchecked(bytes, buffer_idx, offset) + } + } + } + + /// Constructs a byteslice from this view. + /// + /// # Safety + /// Assumes that this view is valid for the given buffers. + pub unsafe fn get_slice_unchecked<'a>(&'a self, buffers: &'a [Buffer]) -> &'a [u8] { + unsafe { + if self.length <= Self::MAX_INLINE_SIZE { + let ptr = self as *const View as *const u8; + std::slice::from_raw_parts(ptr.add(4), self.length as usize) + } else { + let data = buffers.get_unchecked(self.buffer_idx as usize); + let offset = self.offset as usize; + data.get_unchecked(offset..offset + self.length as usize) + } + } + } } impl Display for View { diff --git a/src/common/arrow/src/arrow/array/mod.rs b/src/common/arrow/src/arrow/array/mod.rs index 3f893ebaf15f..349a3345f6db 100644 --- a/src/common/arrow/src/arrow/array/mod.rs +++ b/src/common/arrow/src/arrow/array/mod.rs @@ -496,7 +496,8 @@ pub fn to_data(array: &dyn Array) -> arrow_data::ArrayData { }) } Map => to_data_dyn!(array, MapArray), - BinaryView | Utf8View => unimplemented!(), + BinaryView => to_data_dyn!(array, BinaryViewArray), + Utf8View => to_data_dyn!(array, Utf8ViewArray), } } @@ -527,7 +528,8 @@ pub fn from_data(data: &arrow_data::ArrayData) -> Box { }) } Map => Box::new(MapArray::from_data(data)), - BinaryView | Utf8View => unimplemented!(), + BinaryView => Box::new(BinaryViewArray::from_data(data)), + Utf8View => Box::new(Utf8ViewArray::from_data(data)), } } diff --git a/src/common/arrow/src/arrow/datatypes/mod.rs b/src/common/arrow/src/arrow/datatypes/mod.rs index c49ca12d9104..2a0075a68573 100644 --- a/src/common/arrow/src/arrow/datatypes/mod.rs +++ b/src/common/arrow/src/arrow/datatypes/mod.rs @@ -237,9 +237,8 @@ impl From for arrow_schema::DataType { DataType::Decimal(precision, scale) => Self::Decimal128(precision as _, scale as _), DataType::Decimal256(precision, scale) => Self::Decimal256(precision as _, scale as _), DataType::Extension(_, d, _) => (*d).into(), - DataType::BinaryView | DataType::Utf8View => { - panic!("view datatypes are not supported by arrow-rs") - } + DataType::BinaryView => Self::BinaryView, + DataType::Utf8View => Self::Utf8View, } } } @@ -302,6 +301,8 @@ impl From for DataType { } DataType::Decimal128(precision, scale) => Self::Decimal(precision as _, scale as _), DataType::Decimal256(precision, scale) => Self::Decimal256(precision as _, scale as _), + DataType::BinaryView => Self::BinaryView, + DataType::Utf8View => Self::Utf8View, v => panic!("{:?} encoding not supported by arrow2", v), } } diff --git a/src/common/arrow/src/native/compression/basic.rs b/src/common/arrow/src/native/compression/basic.rs index 77e6af20cdc4..56e76da03c22 100644 --- a/src/common/arrow/src/native/compression/basic.rs +++ b/src/common/arrow/src/native/compression/basic.rs @@ -148,3 +148,43 @@ pub fn compress_snappy(input_buf: &[u8], output_buf: &mut Vec) -> Result( + &self, + reader: &mut R, + uncompressed_size: usize, + compressed_size: usize, + values: &mut Vec, + scratch: &mut Vec, + ) -> Result<()> { + // values + values.reserve(uncompressed_size); + let mut use_inner = false; + reader.fill_buf()?; + let input = if reader.buffer_bytes().len() >= compressed_size { + use_inner = true; + reader.buffer_bytes() + } else { + scratch.resize(compressed_size, 0); + reader.read_exact(scratch.as_mut_slice())?; + scratch.as_slice() + }; + + let out_slice = unsafe { + core::slice::from_raw_parts_mut( + values.as_mut_ptr().add(values.len()), + uncompressed_size, + ) + }; + self.decompress(&input[..compressed_size], out_slice)?; + unsafe { values.set_len(values.len() + uncompressed_size) }; + + if use_inner { + reader.consume(compressed_size); + } + Ok(()) + } +} diff --git a/src/common/arrow/src/native/compression/binary/mod.rs b/src/common/arrow/src/native/compression/binary/mod.rs index 42b7d427639c..0cf9875288ff 100644 --- a/src/common/arrow/src/native/compression/binary/mod.rs +++ b/src/common/arrow/src/native/compression/binary/mod.rs @@ -159,30 +159,13 @@ pub fn decompress_binary( // values let (_, compressed_size, uncompressed_size) = read_compress_header(reader, scratch)?; - use_inner = false; - reader.fill_buf()?; - let input = if reader.buffer_bytes().len() >= compressed_size { - use_inner = true; - reader.buffer_bytes() - } else { - scratch.resize(compressed_size, 0); - reader.read_exact(scratch.as_mut_slice())?; - scratch.as_slice() - }; - - values.reserve(uncompressed_size); - let out_slice = unsafe { - core::slice::from_raw_parts_mut( - values.as_mut_ptr().add(values.len()), - uncompressed_size, - ) - }; - c.decompress(&input[..compressed_size], out_slice)?; - unsafe { values.set_len(values.len() + uncompressed_size) }; - - if use_inner { - reader.consume(compressed_size); - } + c.decompress_common_binary( + reader, + uncompressed_size, + compressed_size, + values, + scratch, + )?; } BinaryCompressor::Extend(c) => { c.decompress(input, length, offsets, values)?; diff --git a/src/common/arrow/src/native/read/array/mod.rs b/src/common/arrow/src/native/read/array/mod.rs index 887f6426edee..a1443034ea1a 100644 --- a/src/common/arrow/src/native/read/array/mod.rs +++ b/src/common/arrow/src/native/read/array/mod.rs @@ -21,6 +21,10 @@ mod boolean; pub use boolean::*; mod binary; pub use binary::*; + +mod view; +pub use view::*; + mod null; pub use null::*; mod struct_; diff --git a/src/common/arrow/src/native/read/array/view.rs b/src/common/arrow/src/native/read/array/view.rs new file mode 100644 index 000000000000..2d577ebdcf3b --- /dev/null +++ b/src/common/arrow/src/native/read/array/view.rs @@ -0,0 +1,274 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Cursor; + +use byteorder::LittleEndian; +use byteorder::ReadBytesExt; +use parquet2::metadata::ColumnDescriptor; + +use crate::arrow::array::Array; +use crate::arrow::array::BinaryViewArray; +use crate::arrow::array::View; +use crate::arrow::bitmap::Bitmap; +use crate::arrow::bitmap::MutableBitmap; +use crate::arrow::buffer::Buffer; +use crate::arrow::compute::concatenate::concatenate; +use crate::arrow::datatypes::DataType; +use crate::arrow::error::Result; +use crate::arrow::io::parquet::read::InitNested; +use crate::arrow::io::parquet::read::NestedState; +use crate::native::read::read_basic::*; +use crate::native::read::BufReader; +use crate::native::read::NativeReadBuf; +use crate::native::read::PageIterator; +use crate::native::CommonCompression; +use crate::native::PageMeta; + +#[derive(Debug)] +pub struct ViewArrayIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + iter: I, + is_nullable: bool, + data_type: DataType, + scratch: Vec, +} + +impl ViewArrayIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + pub fn new(iter: I, is_nullable: bool, data_type: DataType) -> Self { + Self { + iter, + is_nullable, + data_type, + scratch: vec![], + } + } +} + +impl ViewArrayIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result> { + let length = num_values as usize; + let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); + let validity = if self.is_nullable { + let mut validity_builder = MutableBitmap::with_capacity(length); + read_validity(&mut reader, length, &mut validity_builder)?; + Some(std::mem::take(&mut validity_builder).into()) + } else { + None + }; + + read_view_array(&mut reader, length, self.data_type.clone(), validity) + } +} + +impl Iterator for ViewArrayIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + type Item = Result>; + + fn nth(&mut self, n: usize) -> Option { + match self.iter.nth(n) { + Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), + Some(Err(err)) => Some(Result::Err(err)), + None => None, + } + } + + fn next(&mut self) -> Option { + match self.iter.next() { + Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), + Some(Err(err)) => Some(Result::Err(err)), + None => None, + } + } +} + +#[derive(Debug)] +pub struct ViewArrayNestedIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + iter: I, + data_type: DataType, + leaf: ColumnDescriptor, + init: Vec, + scratch: Vec, +} + +impl ViewArrayNestedIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + pub fn new( + iter: I, + data_type: DataType, + leaf: ColumnDescriptor, + init: Vec, + ) -> Self { + Self { + iter, + data_type, + leaf, + init, + scratch: vec![], + } + } +} + +impl ViewArrayNestedIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + fn deserialize( + &mut self, + num_values: u64, + buffer: Vec, + ) -> Result<(NestedState, Box)> { + let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); + let (mut nested, validity) = read_validity_nested( + &mut reader, + num_values as usize, + &self.leaf, + self.init.clone(), + )?; + let length = nested.nested.pop().unwrap().len(); + let array = read_view_array(&mut reader, length, self.data_type.clone(), validity)?; + Ok((nested, array)) + } +} + +impl Iterator for ViewArrayNestedIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + type Item = Result<(NestedState, Box)>; + + fn nth(&mut self, n: usize) -> Option { + match self.iter.nth(n) { + Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), + Some(Err(err)) => Some(Result::Err(err)), + None => None, + } + } + + fn next(&mut self) -> Option { + match self.iter.next() { + Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), + Some(Err(err)) => Some(Result::Err(err)), + None => None, + } + } +} + +pub fn read_view( + reader: &mut R, + is_nullable: bool, + data_type: DataType, + page_metas: Vec, +) -> Result> { + let num_values = page_metas.iter().map(|p| p.num_values as usize).sum(); + let mut validity_builder = if is_nullable { + Some(MutableBitmap::with_capacity(num_values)) + } else { + None + }; + let mut arrays = vec![]; + for page_meta in page_metas { + let length = page_meta.num_values as usize; + if let Some(ref mut validity_builder) = validity_builder { + read_validity(reader, length, validity_builder)?; + } + let array = read_view_array(reader, length, data_type.clone(), None)?; + arrays.push(array); + } + + let validity = + validity_builder.map(|mut validity_builder| std::mem::take(&mut validity_builder).into()); + let arrays = arrays.iter().map(|x| x.as_ref()).collect::>(); + let array = concatenate(&arrays)?; + let array = array.with_validity(validity); + Ok(array) +} + +pub fn read_nested_view_array( + reader: &mut R, + data_type: DataType, + leaf: ColumnDescriptor, + init: Vec, + page_metas: Vec, +) -> Result)>> { + let mut results = Vec::with_capacity(page_metas.len()); + + for page_meta in page_metas { + let num_values = page_meta.num_values as usize; + let (mut nested, validity) = read_validity_nested(reader, num_values, &leaf, init.clone())?; + let length = nested.nested.pop().unwrap().len(); + + let array = read_view_array(reader, length, data_type.clone(), validity)?; + results.push((nested, array)); + } + Ok(results) +} + +fn read_view_array( + reader: &mut R, + length: usize, + data_type: DataType, + validity: Option, +) -> Result> { + let mut scratch = vec![0; 9]; + let (_c, _compressed_size, _uncompressed_size) = read_compress_header(reader, &mut scratch)?; + + let mut buffer = vec![View::default(); length]; + let temp_data = + unsafe { std::slice::from_raw_parts_mut(buffer.as_mut_ptr() as *mut u8, length * 16) }; + reader.read_exact(temp_data)?; + let views = Buffer::from(buffer); + + let buffer_len = reader.read_u32::()?; + let mut buffers = Vec::with_capacity(buffer_len as _); + + for _ in 0..buffer_len { + scratch.clear(); + let (compression, compressed_size, uncompressed_size) = + read_compress_header(reader, &mut scratch)?; + let c = CommonCompression::try_from(&compression)?; + let mut buffer = vec![]; + c.decompress_common_binary( + reader, + uncompressed_size, + compressed_size, + &mut buffer, + &mut scratch, + )?; + buffers.push(Buffer::from(buffer)); + } + + let array = unsafe { + BinaryViewArray::new_unchecked_unknown_md( + data_type.clone(), + views, + buffers.into(), + validity, + None, + ) + }; + + if matches!(data_type, DataType::Utf8View) { + Ok(Box::new(array.to_utf8view()?)) + } else { + Ok(Box::new(array)) + } +} diff --git a/src/common/arrow/src/native/read/batch_read.rs b/src/common/arrow/src/native/read/batch_read.rs index 97cceaea614f..7c9ed0c1d7a2 100644 --- a/src/common/arrow/src/native/read/batch_read.rs +++ b/src/common/arrow/src/native/read/batch_read.rs @@ -60,6 +60,7 @@ pub fn read_simple( ) }), Binary | Utf8 => read_binary::(reader, is_nullable, data_type, page_metas), + BinaryView | Utf8View => read_view::<_>(reader, is_nullable, data_type, page_metas), LargeBinary | LargeUtf8 => { read_binary::(reader, is_nullable, data_type, page_metas) } @@ -121,6 +122,18 @@ pub fn read_nested( page_metas.pop().unwrap(), )? } + + BinaryView | Utf8View => { + init.push(InitNested::Primitive(field.is_nullable)); + read_nested_view_array::<_>( + &mut readers.pop().unwrap(), + field.data_type().clone(), + leaves.pop().unwrap(), + init, + page_metas.pop().unwrap(), + )? + } + LargeBinary | LargeUtf8 => { init.push(InitNested::Primitive(field.is_nullable)); read_nested_binary::( diff --git a/src/common/arrow/src/native/read/deserialize.rs b/src/common/arrow/src/native/read/deserialize.rs index 72e6e06104cc..8ed459e8d201 100644 --- a/src/common/arrow/src/native/read/deserialize.rs +++ b/src/common/arrow/src/native/read/deserialize.rs @@ -126,6 +126,9 @@ where } ), Binary | Utf8 => DynIter::new(BinaryIter::<_, i32>::new(reader, is_nullable, data_type)), + BinaryView | Utf8View => { + DynIter::new(ViewArrayIter::<_>::new(reader, is_nullable, data_type)) + } LargeBinary | LargeUtf8 => { DynIter::new(BinaryIter::<_, i64>::new(reader, is_nullable, data_type)) } @@ -185,6 +188,15 @@ where init, )) } + BinaryView | Utf8View => { + init.push(InitNested::Primitive(field.is_nullable)); + DynIter::new(ViewArrayNestedIter::<_>::new( + readers.pop().unwrap(), + field.data_type().clone(), + leaves.pop().unwrap(), + init, + )) + } LargeBinary | LargeUtf8 => { init.push(InitNested::Primitive(field.is_nullable)); DynIter::new(BinaryNestedIter::<_, i64>::new( diff --git a/src/common/arrow/src/native/read/reader.rs b/src/common/arrow/src/native/read/reader.rs index c9652c36b323..a9912b274e22 100644 --- a/src/common/arrow/src/native/read/reader.rs +++ b/src/common/arrow/src/native/read/reader.rs @@ -42,6 +42,8 @@ pub fn is_primitive(data_type: &DataType) -> bool { | PhysicalType::Utf8 | PhysicalType::LargeUtf8 | PhysicalType::Binary + | PhysicalType::Utf8View + | PhysicalType::BinaryView | PhysicalType::LargeBinary | PhysicalType::FixedSizeBinary | PhysicalType::Dictionary(_) diff --git a/src/common/arrow/src/native/stat.rs b/src/common/arrow/src/native/stat.rs index b11baed77c74..62e1aee7c6fe 100644 --- a/src/common/arrow/src/native/stat.rs +++ b/src/common/arrow/src/native/stat.rs @@ -152,7 +152,7 @@ fn size_of_primitive(p: PrimitiveType) -> usize { PrimitiveType::Int16 => 2, PrimitiveType::Int32 => 4, PrimitiveType::Int64 => 8, - PrimitiveType::Int128 => 16, + PrimitiveType::Int128 | PrimitiveType::UInt128 => 16, PrimitiveType::Int256 => 32, PrimitiveType::UInt8 => 1, PrimitiveType::UInt16 => 2, @@ -163,7 +163,6 @@ fn size_of_primitive(p: PrimitiveType) -> usize { PrimitiveType::Float64 => 8, PrimitiveType::DaysMs => unimplemented!(), PrimitiveType::MonthDayNano => unimplemented!(), - PrimitiveType::UInt128 => unimplemented!(), } } diff --git a/src/common/arrow/src/native/write/mod.rs b/src/common/arrow/src/native/write/mod.rs index 157d463c40d9..63a95c2dce22 100644 --- a/src/common/arrow/src/native/write/mod.rs +++ b/src/common/arrow/src/native/write/mod.rs @@ -17,6 +17,7 @@ pub(crate) mod boolean; pub(crate) mod common; pub(crate) mod primitive; mod serialize; +pub(crate) mod view; pub(crate) mod writer; pub use common::WriteOptions; diff --git a/src/common/arrow/src/native/write/serialize.rs b/src/common/arrow/src/native/write/serialize.rs index 303fcf1ecd91..c285065b9dc2 100644 --- a/src/common/arrow/src/native/write/serialize.rs +++ b/src/common/arrow/src/native/write/serialize.rs @@ -31,6 +31,7 @@ use crate::arrow::io::parquet::write::write_rep_and_def; use crate::arrow::io::parquet::write::Nested; use crate::arrow::io::parquet::write::Version; use crate::native::write::binary::write_binary; +use crate::native::write::view::write_view; use crate::with_match_primitive_type; /// Writes an [`Array`] to the file @@ -120,6 +121,24 @@ pub fn write_simple( ); write_binary::(w, &binary_array, write_options, scratch)?; } + BinaryView => { + let array: &BinaryViewArray = array.as_any().downcast_ref().unwrap(); + if is_optional { + write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; + } + + write_view::(w, array, write_options, scratch)?; + } + Utf8View => { + let array: &Utf8ViewArray = array.as_any().downcast_ref().unwrap(); + let array = array.clone().to_binview(); + + if is_optional { + write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; + } + + write_view::(w, &array, write_options, scratch)?; + } Struct => unreachable!(), List => unreachable!(), FixedSizeList => unreachable!(), @@ -186,6 +205,15 @@ pub fn write_nested( write_binary::(w, &binary_array, write_options, scratch)?; } + BinaryView => { + let array: &BinaryViewArray = array.as_any().downcast_ref().unwrap(); + write_view::(w, array, write_options, scratch)?; + } + Utf8View => { + let array: &Utf8ViewArray = array.as_any().downcast_ref().unwrap(); + let array = array.clone().to_binview(); + write_view::(w, &array, write_options, scratch)?; + } Struct => unreachable!(), List => unreachable!(), FixedSizeList => unreachable!(), diff --git a/src/common/arrow/src/native/write/view.rs b/src/common/arrow/src/native/write/view.rs new file mode 100644 index 000000000000..fc5b416b04c1 --- /dev/null +++ b/src/common/arrow/src/native/write/view.rs @@ -0,0 +1,56 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Write; + +use super::WriteOptions; +use crate::arrow::array::BinaryViewArray; +use crate::arrow::array::View; +use crate::arrow::error::Result; + +pub(crate) fn write_view( + w: &mut W, + array: &BinaryViewArray, + write_options: WriteOptions, + buf: &mut Vec, +) -> Result<()> { + // TODO: adaptive gc and dict by stats + let array = array.clone().gc(); + let c = write_options.default_compression; + let codec = c.to_compression(); + + let total_size = array.len() * std::mem::size_of::() + + array.data_buffers().iter().map(|x| x.len()).sum::(); + w.write_all(&[codec as u8])?; + w.write_all(&(total_size as u32).to_le_bytes())?; + w.write_all(&(total_size as u32).to_le_bytes())?; + + let input_buf: &[u8] = bytemuck::cast_slice(array.views().as_slice()); + w.write_all(input_buf)?; + w.write_all(&(array.data_buffers().len() as u32).to_le_bytes())?; + + for buffer in array.data_buffers().iter() { + buf.clear(); + let pos = buf.len(); + w.write_all(&[codec as u8])?; + buf.extend_from_slice(&[0u8; 8]); + + let compressed_size = c.compress(buffer.as_slice(), buf)?; + buf[pos..pos + 4].copy_from_slice(&(compressed_size as u32).to_le_bytes()); + buf[pos + 4..pos + 8].copy_from_slice(&(buffer.len() as u32).to_le_bytes()); + w.write_all(buf.as_slice())?; + buf.clear(); + } + Ok(()) +} diff --git a/src/common/arrow/tests/it/arrow/array/binview/mod.rs b/src/common/arrow/tests/it/arrow/array/binview/mod.rs index 32f10baecf39..ff1183375a02 100644 --- a/src/common/arrow/tests/it/arrow/array/binview/mod.rs +++ b/src/common/arrow/tests/it/arrow/array/binview/mod.rs @@ -187,3 +187,25 @@ fn iter_nth() { assert_eq!(array.iter().nth(1), Some(Some(" ".as_bytes()))); assert_eq!(array.iter().nth(10), None); } + +#[test] +fn test_slice() { + let data = vec![ + Some("hello"), + Some("world"), + Some("databend"), + None, + Some("y"), + Some("z"), + Some("abc"), + ]; + + let array: Utf8ViewArray = data.into_iter().collect(); + + let a3 = array.sliced(2, 3); + assert_eq!(a3.into_iter().collect::>(), vec![ + Some("databend"), + None, + Some("y"), + ]); +} diff --git a/src/common/arrow/tests/it/native/io.rs b/src/common/arrow/tests/it/native/io.rs index 3b978312effd..2dd0816d73ff 100644 --- a/src/common/arrow/tests/it/native/io.rs +++ b/src/common/arrow/tests/it/native/io.rs @@ -17,6 +17,7 @@ use std::io::BufReader; use databend_common_arrow::arrow::array::Array; use databend_common_arrow::arrow::array::BinaryArray; +use databend_common_arrow::arrow::array::BinaryViewArray; use databend_common_arrow::arrow::array::BooleanArray; use databend_common_arrow::arrow::array::Float32Array; use databend_common_arrow::arrow::array::Float64Array; @@ -33,6 +34,7 @@ use databend_common_arrow::arrow::array::UInt32Array; use databend_common_arrow::arrow::array::UInt64Array; use databend_common_arrow::arrow::array::UInt8Array; use databend_common_arrow::arrow::array::Utf8Array; +use databend_common_arrow::arrow::array::Utf8ViewArray; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_arrow::arrow::chunk::Chunk; @@ -81,6 +83,12 @@ pub fn new_test_chunk() -> Chunk> { Box::new(BinaryArray::::from_iter_values( ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), )) as _, + Box::new(Utf8ViewArray::from_slice_values( + ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), + )) as _, + Box::new(BinaryViewArray::from_slice_values( + ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), + )) as _, ]) } @@ -97,6 +105,7 @@ fn test_random_nonull() { Box::new(create_random_index(size, 0.0, size)) as _, Box::new(create_random_double(size, 0.0, size)) as _, Box::new(create_random_string(size, 0.0, size)) as _, + Box::new(create_random_view(size, 0.0, size)) as _, ]); test_write_read(chunk); } @@ -112,6 +121,7 @@ fn test_random() { Box::new(create_random_index(size, 0.4, size)) as _, Box::new(create_random_double(size, 0.5, size)) as _, Box::new(create_random_string(size, 0.4, size)) as _, + Box::new(create_random_view(size, 0.4, size)) as _, ]); test_write_read(chunk); } @@ -127,6 +137,7 @@ fn test_dict() { Box::new(create_random_index(size, 0.4, 8)) as _, Box::new(create_random_double(size, 0.5, 8)) as _, Box::new(create_random_string(size, 0.4, 8)) as _, + Box::new(create_random_view(size, 0.4, size)) as _, ]); test_write_read(chunk); } @@ -411,6 +422,20 @@ fn create_random_string(size: usize, null_density: f32, uniq: usize) -> BinaryAr .collect::>() } +fn create_random_view(size: usize, null_density: f32, uniq: usize) -> Utf8ViewArray { + let mut rng = StdRng::seed_from_u64(42); + (0..size) + .map(|_| { + if rng.gen::() > null_density { + let value = rng.gen_range::(0i32..uniq as i32); + Some(format!("{value}")) + } else { + None + } + }) + .collect::() +} + fn create_random_offsets(size: usize, null_density: f32) -> (Vec, Option) { let mut offsets = Vec::with_capacity(size + 1); offsets.push(0i32); diff --git a/src/meta/api/src/txn_backoff.rs b/src/meta/api/src/txn_backoff.rs index f33ebb54f7b1..e741d3a9a519 100644 --- a/src/meta/api/src/txn_backoff.rs +++ b/src/meta/api/src/txn_backoff.rs @@ -154,7 +154,7 @@ mod tests { let elapsed = now.elapsed().as_secs_f64(); println!("elapsed: {elapsed}"); assert!( - (0.041..0.090).contains(&elapsed), + (0.041..0.120).contains(&elapsed), "{} is expected to be 2 + 5 + 10 + 14 + 20 milliseconds", elapsed ); diff --git a/src/query/catalog/src/plan/internal_column.rs b/src/query/catalog/src/plan/internal_column.rs index 2872e0dc4df9..10438eb7f40c 100644 --- a/src/query/catalog/src/plan/internal_column.rs +++ b/src/query/catalog/src/plan/internal_column.rs @@ -226,34 +226,24 @@ impl InternalColumn { ) } InternalColumnType::BlockName => { - let mut builder = StringColumnBuilder::with_capacity(1, meta.block_location.len()); - builder.put_str(&meta.block_location); - builder.commit_row(); + let mut builder = StringColumnBuilder::with_capacity(1); + builder.put_and_commit(&meta.block_location); BlockEntry::new( DataType::String, Value::Scalar(Scalar::String(builder.build_scalar())), ) } InternalColumnType::SegmentName => { - let mut builder = - StringColumnBuilder::with_capacity(1, meta.segment_location.len()); - builder.put_str(&meta.segment_location); - builder.commit_row(); + let mut builder = StringColumnBuilder::with_capacity(1); + builder.put_and_commit(&meta.segment_location); BlockEntry::new( DataType::String, Value::Scalar(Scalar::String(builder.build_scalar())), ) } InternalColumnType::SnapshotName => { - let mut builder = StringColumnBuilder::with_capacity( - 1, - meta.snapshot_location - .clone() - .unwrap_or("".to_string()) - .len(), - ); - builder.put_str(&meta.snapshot_location.clone().unwrap_or("".to_string())); - builder.commit_row(); + let mut builder = StringColumnBuilder::with_capacity(1); + builder.put_and_commit(meta.snapshot_location.clone().unwrap_or("".to_string())); BlockEntry::new( DataType::String, Value::Scalar(Scalar::String(builder.build_scalar())), diff --git a/src/query/catalog/src/table.rs b/src/query/catalog/src/table.rs index 1f8dd4fe8045..916d170f1563 100644 --- a/src/query/catalog/src/table.rs +++ b/src/query/catalog/src/table.rs @@ -151,6 +151,10 @@ pub trait Table: Sync + Send { false } + fn storage_format_as_parquet(&self) -> bool { + false + } + #[async_backtrace::framed] async fn alter_table_cluster_keys( &self, diff --git a/src/query/ee/Cargo.toml b/src/query/ee/Cargo.toml index 01d7c480c790..29824d20420f 100644 --- a/src/query/ee/Cargo.toml +++ b/src/query/ee/Cargo.toml @@ -12,7 +12,6 @@ doctest = false test = true [dependencies] -arrow-array = { workspace = true } async-backtrace = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } diff --git a/src/query/ee/src/background_service/background_service_handler.rs b/src/query/ee/src/background_service/background_service_handler.rs index 06d9220c1e31..daaec48e0a1e 100644 --- a/src/query/ee/src/background_service/background_service_handler.rs +++ b/src/query/ee/src/background_service/background_service_handler.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use arrow_array::RecordBatch; use databend_common_base::base::tokio::sync::mpsc::Sender; use databend_common_base::base::tokio::sync::Mutex; use databend_common_base::base::uuid::Uuid; @@ -22,6 +21,7 @@ use databend_common_base::base::GlobalInstance; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::DataBlock; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_api::BackgroundApi; @@ -61,7 +61,7 @@ pub struct RealBackgroundService { #[async_trait::async_trait] impl BackgroundServiceHandler for RealBackgroundService { #[async_backtrace::framed] - async fn execute_sql(&self, sql: String) -> Result> { + async fn execute_sql(&self, sql: String) -> Result> { let session = create_session(&self.conf).await?; let ctx = session.create_query_context().await?; SuggestedBackgroundTasksSource::do_execute_sql(ctx, sql).await diff --git a/src/query/ee/src/background_service/compaction_job.rs b/src/query/ee/src/background_service/compaction_job.rs index 4940d38f9c2a..3fc35b1c5e99 100644 --- a/src/query/ee/src/background_service/compaction_job.rs +++ b/src/query/ee/src/background_service/compaction_job.rs @@ -16,10 +16,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use arrow_array::BooleanArray; -use arrow_array::LargeStringArray; -use arrow_array::RecordBatch; -use arrow_array::UInt64Array; use chrono::Utc; use databend_common_base::base::tokio::sync::mpsc::Sender; use databend_common_base::base::tokio::sync::Mutex; @@ -27,6 +23,10 @@ use databend_common_base::base::tokio::time::Instant; use databend_common_base::base::uuid::Uuid; use databend_common_config::InnerConfig; use databend_common_exception::Result; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::ValueType; +use databend_common_expression::DataBlock; use databend_common_meta_api::BackgroundApi; use databend_common_meta_app::background::BackgroundJobIdent; use databend_common_meta_app::background::BackgroundJobInfo; @@ -172,31 +172,28 @@ impl CompactionJob { for records in Self::do_get_target_tables_from_config(&self.conf, ctx.clone()).await? { debug!(records :? =(&records); "target_tables"); - let db_names = records - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let db_ids = records - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let tb_names = records - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let tb_ids = records - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); + let records = records.consume_convert_to_full(); + + let db_names = + StringType::try_downcast_column(records.columns()[0].value.as_column().unwrap()) + .unwrap(); + let db_ids = + UInt64Type::try_downcast_column(records.columns()[1].value.as_column().unwrap()) + .unwrap(); + let tb_names = + StringType::try_downcast_column(records.columns()[2].value.as_column().unwrap()) + .unwrap(); + + let tb_ids = + UInt64Type::try_downcast_column(records.columns()[3].value.as_column().unwrap()) + .unwrap(); + for i in 0..records.num_rows() { - let db_name = db_names.value(i).to_owned(); - let db_id = db_ids.value(i); - let tb_name = tb_names.value(i).to_owned(); - let tb_id = tb_ids.value(i); + let db_name: String = db_names.index(i).unwrap().to_string(); + let db_id = db_ids[i]; + let tb_name = tb_names.index(i).unwrap().to_string(); + let tb_id = tb_ids[i]; + match self .compact_table( session.clone(), @@ -460,7 +457,7 @@ impl CompactionJob { pub async fn do_get_target_tables_from_config( config: &InnerConfig, ctx: Arc, - ) -> Result> { + ) -> Result> { if !config.background.compaction.has_target_tables() { let res = SuggestedBackgroundTasksSource::do_get_all_suggested_compaction_tables(ctx).await; @@ -529,7 +526,7 @@ impl CompactionJob { pub async fn do_get_target_tables( configs: &InnerConfig, ctx: Arc, - ) -> Result> { + ) -> Result> { let all_target_tables = configs.background.compaction.target_tables.as_ref(); let all_target_tables = Self::parse_all_target_tables(all_target_tables); let future_res = all_target_tables @@ -574,57 +571,55 @@ impl CompactionJob { return Ok((false, false, TableStatistics::default())); } let res = res.unwrap(); - let need_segment_compact = res - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(0); - let need_block_compact = res - .column(1) - .as_any() - .downcast_ref::() - .unwrap() - .value(0); + let res = res.consume_convert_to_full(); + + let need_segment_compact = *res.value_at(0, 0).unwrap().as_boolean().unwrap(); + let need_block_compact = *res.value_at(1, 0).unwrap().as_boolean().unwrap(); let table_statistics = TableStatistics { - number_of_rows: res - .column(2) - .as_any() - .downcast_ref::() + number_of_rows: *res + .value_at(2, 0) + .unwrap() + .as_number() .unwrap() - .value(0), - data_bytes: res - .column(3) - .as_any() - .downcast_ref::() + .as_u_int64() + .unwrap(), + data_bytes: *res + .value_at(3, 0) .unwrap() - .value(0), - compressed_data_bytes: res - .column(4) - .as_any() - .downcast_ref::() + .as_number() .unwrap() - .value(0), - index_data_bytes: res - .column(5) - .as_any() - .downcast_ref::() + .as_u_int64() + .unwrap(), + compressed_data_bytes: *res + .value_at(4, 0) .unwrap() - .value(0), + .as_number() + .unwrap() + .as_u_int64() + .unwrap(), + index_data_bytes: *res + .value_at(5, 0) + .unwrap() + .as_number() + .unwrap() + .as_u_int64() + .unwrap(), number_of_segments: Some( - res.column(6) - .as_any() - .downcast_ref::() + *res.value_at(6, 0) .unwrap() - .value(0), + .as_number() + .unwrap() + .as_u_int64() + .unwrap(), ), number_of_blocks: Some( - res.column(7) - .as_any() - .downcast_ref::() + *res.value_at(7, 0) + .unwrap() + .as_number() .unwrap() - .value(0), + .as_u_int64() + .unwrap(), ), }; diff --git a/src/query/ee_features/background_service/Cargo.toml b/src/query/ee_features/background_service/Cargo.toml index 4d5dff29ab21..027bca904e52 100644 --- a/src/query/ee_features/background_service/Cargo.toml +++ b/src/query/ee_features/background_service/Cargo.toml @@ -12,11 +12,11 @@ doctest = false test = true [dependencies] -arrow-array = { workspace = true } async-backtrace = { workspace = true } async-trait = { workspace = true } databend-common-base = { workspace = true } databend-common-exception = { workspace = true } +databend-common-expression = { workspace = true } databend-common-meta-app = { workspace = true } serde = { workspace = true } diff --git a/src/query/ee_features/background_service/src/background_service.rs b/src/query/ee_features/background_service/src/background_service.rs index a91c4dab8bb4..e55e9ac687ac 100644 --- a/src/query/ee_features/background_service/src/background_service.rs +++ b/src/query/ee_features/background_service/src/background_service.rs @@ -14,15 +14,15 @@ use std::sync::Arc; -use arrow_array::RecordBatch; use databend_common_base::base::GlobalInstance; use databend_common_exception::Result; +use databend_common_expression::DataBlock; use databend_common_meta_app::principal::UserIdentity; use databend_common_meta_app::tenant::Tenant; #[async_trait::async_trait] pub trait BackgroundServiceHandler: Sync + Send { - async fn execute_sql(&self, sql: String) -> Result>; + async fn execute_sql(&self, sql: String) -> Result>; async fn execute_scheduled_job( &self, @@ -43,7 +43,7 @@ impl BackgroundServiceHandlerWrapper { } #[async_backtrace::framed] - pub async fn execute_sql(&self, sql: String) -> Result> { + pub async fn execute_sql(&self, sql: String) -> Result> { self.handler.execute_sql(sql).await } diff --git a/src/query/expression/Cargo.toml b/src/query/expression/Cargo.toml index 80e347176c2b..e44cabed9b12 100644 --- a/src/query/expression/Cargo.toml +++ b/src/query/expression/Cargo.toml @@ -32,6 +32,7 @@ databend-common-grpc = { workspace = true } databend-common-hashtable = { workspace = true } databend-common-io = { workspace = true } educe = { workspace = true } +either = { workspace = true } enum-as-inner = { workspace = true } ethnum = { workspace = true, features = ["serde", "macros", "borsh"] } futures = { workspace = true } @@ -64,11 +65,16 @@ unicode-segmentation = { workspace = true } [dev-dependencies] arrow-ord = { workspace = true } +criterion = { workspace = true } goldenfile = { workspace = true } pretty_assertions = { workspace = true } rand = { workspace = true } rmp-serde = { workspace = true } +[[bench]] +name = "bench" +harness = false + [lints] workspace = true diff --git a/src/query/expression/benches/bench.rs b/src/query/expression/benches/bench.rs new file mode 100644 index 000000000000..b0e18083fc80 --- /dev/null +++ b/src/query/expression/benches/bench.rs @@ -0,0 +1,157 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[macro_use] +extern crate criterion; + +use criterion::Criterion; +use databend_common_expression::arrow::deserialize_column; +use databend_common_expression::arrow::serialize_column; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::StringType; +use databend_common_expression::Column; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; +use rand::rngs::StdRng; +use rand::Rng; +use rand::SeedableRng; + +fn bench(c: &mut Criterion) { + let mut group = c.benchmark_group("bench_kernels"); + + let mut rng = StdRng::seed_from_u64(0); + // concats + { + for length in [12, 20, 500] { + let (s, b) = generate_random_string_data(&mut rng, length); + let bin_col = (0..5).map(|_| BinaryType::from_data(b.clone())); + let str_col = (0..5).map(|_| StringType::from_data(s.clone())); + + group.bench_function(format!("concat_string_offset/{length}"), |b| { + b.iter(|| Column::concat_columns(bin_col.clone()).unwrap()) + }); + + group.bench_function(format!("concat_string_view/{length}"), |b| { + b.iter(|| Column::concat_columns(str_col.clone()).unwrap()) + }); + } + } + + // take compact + { + for length in [12, 20, 500] { + let (s, b) = generate_random_string_data(&mut rng, length); + let block_bin = DataBlock::new_from_columns(vec![BinaryType::from_data(b.clone())]); + let block_view = DataBlock::new_from_columns(vec![StringType::from_data(s.clone())]); + + let indices: Vec<(u32, u32)> = (0..s.len()) + .filter(|x| x % 10 == 0) + .map(|x| (x as u32, 1000)) + .collect(); + let num_rows = indices.len() * 1000; + + group.bench_function(format!("take_compact_string_offset/{length}"), |b| { + b.iter(|| { + block_bin + .take_compacted_indices(&indices, num_rows) + .unwrap() + }) + }); + + group.bench_function(format!("take_compact_string_view/{length}"), |b| { + b.iter(|| { + block_view + .take_compacted_indices(&indices, num_rows) + .unwrap() + }) + }); + } + } + + // IPC + // bench_kernels/serialize_string_offset/12 + // time: [183.25 µs 183.49 µs 183.93 µs] + // Found 7 outliers among 100 measurements (7.00%) + // 3 (3.00%) high mild + // 4 (4.00%) high severe + // bench_kernels/serialize_string_view/12 + // time: [415.25 µs 415.36 µs 415.47 µs] + // Found 6 outliers among 100 measurements (6.00%) + // 3 (3.00%) high mild + // 3 (3.00%) high severe + // bench_kernels/serialize_string_offset/20 + // time: [195.09 µs 195.15 µs 195.23 µs] + // Found 6 outliers among 100 measurements (6.00%) + // 6 (6.00%) high mild + // bench_kernels/serialize_string_view/20 + // time: [464.96 µs 465.08 µs 465.21 µs] + // Found 4 outliers among 100 measurements (4.00%) + // 4 (4.00%) high mild + // bench_kernels/serialize_string_offset/500 + // time: [3.3092 ms 3.3139 ms 3.3194 ms] + // Found 2 outliers among 100 measurements (2.00%) + // 1 (1.00%) high mild + // 1 (1.00%) high severe + // bench_kernels/serialize_string_view/500 + // time: [3.9254 ms 3.9303 ms 3.9366 ms] + // Found 9 outliers among 100 measurements (9.00%) + // 4 (4.00%) high mild + // 5 (5.00%) high severe + + { + for length in [12, 20, 500] { + let (s, b) = generate_random_string_data(&mut rng, length); + let b_c = BinaryType::from_data(b.clone()); + let s_c = StringType::from_data(s.clone()); + + group.bench_function(format!("serialize_string_offset/{length}"), |b| { + b.iter(|| { + let bs = serialize_column(&b_c); + deserialize_column(&bs).unwrap(); + }) + }); + + group.bench_function(format!("serialize_string_view/{length}"), |b| { + b.iter(|| { + let bs = serialize_column(&s_c); + deserialize_column(&bs).unwrap(); + }) + }); + } + } +} + +criterion_group!(benches, bench); +criterion_main!(benches); + +fn generate_random_string_data(rng: &mut StdRng, length: usize) -> (Vec, Vec>) { + let iter_str: Vec<_> = (0..10000) + .map(|_| { + let random_string: String = (0..length) + .map(|_| { + // Generate a random character (ASCII printable characters) + rng.gen_range(32..=126) as u8 as char + }) + .collect(); + random_string + }) + .collect(); + + let iter_binary: Vec<_> = iter_str + .iter() + .map(|x| x.clone().as_bytes().to_vec()) + .collect(); + + (iter_str, iter_binary) +} diff --git a/src/query/expression/src/aggregate/payload_flush.rs b/src/query/expression/src/aggregate/payload_flush.rs index 1b09b3e43937..632ec78f6999 100644 --- a/src/query/expression/src/aggregate/payload_flush.rs +++ b/src/query/expression/src/aggregate/payload_flush.rs @@ -26,6 +26,7 @@ use crate::types::decimal::Decimal; use crate::types::decimal::DecimalType; use crate::types::nullable::NullableColumn; use crate::types::string::StringColumn; +use crate::types::string::StringColumnBuilder; use crate::types::ArgType; use crate::types::BooleanType; use crate::types::DataType; @@ -36,7 +37,6 @@ use crate::types::NumberType; use crate::types::TimestampType; use crate::types::ValueType; use crate::with_number_mapped_type; -use crate::AggregateFunctionRef; use crate::Column; use crate::ColumnBuilder; use crate::DataBlock; @@ -133,7 +133,7 @@ impl Payload { let mut state_builders: Vec = self .aggrs .iter() - .map(|agg| state_serializer(agg, row_count)) + .map(|_| BinaryColumnBuilder::with_capacity(row_count, row_count * 4)) .collect(); for place in state.state_places.as_slice()[0..row_count].iter() { @@ -322,7 +322,21 @@ impl Payload { col_offset: usize, state: &mut PayloadFlushState, ) -> StringColumn { - unsafe { StringColumn::from_binary_unchecked(self.flush_binary_column(col_offset, state)) } + let len = state.probe_state.row_count; + let mut binary_builder = StringColumnBuilder::with_capacity(len); + + unsafe { + for idx in 0..len { + let str_len = read::(state.addresses[idx].add(col_offset) as _) as usize; + let data_address = read::(state.addresses[idx].add(col_offset + 4) as _) + as usize as *const u8; + + let scalar = std::slice::from_raw_parts(data_address, str_len); + + binary_builder.put_and_commit(std::str::from_utf8(scalar).unwrap()); + } + } + binary_builder.build() } fn flush_generic_column( @@ -349,8 +363,3 @@ impl Payload { builder.build() } } - -fn state_serializer(func: &AggregateFunctionRef, row: usize) -> BinaryColumnBuilder { - let size = func.serialize_size_per_row().unwrap_or(4); - BinaryColumnBuilder::with_capacity(row, row * size) -} diff --git a/src/query/expression/src/aggregate/payload_row.rs b/src/query/expression/src/aggregate/payload_row.rs index 2de484893858..3a171d2d8b1f 100644 --- a/src/query/expression/src/aggregate/payload_row.rs +++ b/src/query/expression/src/aggregate/payload_row.rs @@ -31,6 +31,8 @@ use crate::types::DataType; use crate::types::DateType; use crate::types::NumberColumn; use crate::types::NumberType; +use crate::types::StringColumn; +use crate::types::StringType; use crate::types::TimestampType; use crate::types::ValueType; use crate::with_decimal_mapped_type; @@ -302,6 +304,18 @@ pub unsafe fn row_match_column( no_match, no_match_count, ), + Column::String(v) => row_match_string_column( + v, + validity, + address, + select_vector, + temp_vector, + count, + validity_offset, + col_offset, + no_match, + no_match_count, + ), Column::Bitmap(v) | Column::Binary(v) | Column::Variant(v) | Column::Geometry(v) => { row_match_binary_column( v, @@ -316,21 +330,6 @@ pub unsafe fn row_match_column( no_match_count, ) } - Column::String(v) => { - let v = &BinaryColumn::from(v.clone()); - row_match_binary_column( - v, - validity, - address, - select_vector, - temp_vector, - count, - validity_offset, - col_offset, - no_match, - no_match_count, - ) - } Column::Nullable(_) => unreachable!("nullable is unwrapped"), other => row_match_generic_column( other, @@ -426,6 +425,87 @@ unsafe fn row_match_binary_column( *count = match_count; } +unsafe fn row_match_string_column( + col: &StringColumn, + validity: Option<&Bitmap>, + address: &[*const u8], + select_vector: &mut SelectVector, + temp_vector: &mut SelectVector, + count: &mut usize, + validity_offset: usize, + col_offset: usize, + no_match: &mut SelectVector, + no_match_count: &mut usize, +) { + let mut match_count = 0; + let mut equal: bool; + + if let Some(validity) = validity { + let is_all_set = validity.unset_bits() == 0; + for idx in select_vector[..*count].iter() { + let idx = *idx; + let validity_address = address[idx].add(validity_offset); + let is_set2 = read::(validity_address as _) != 0; + let is_set = is_all_set || validity.get_bit_unchecked(idx); + + if is_set && is_set2 { + let len_address = address[idx].add(col_offset); + let address = address[idx].add(col_offset + 4); + let len = read::(len_address as _) as usize; + + let value = StringType::index_column_unchecked(col, idx); + if len != value.len() { + equal = false; + } else { + let data_address = read::(address as _) as usize as *const u8; + let scalar = std::slice::from_raw_parts(data_address, len); + equal = databend_common_hashtable::fast_memcmp(scalar, value.as_bytes()); + } + } else { + equal = is_set == is_set2; + } + + if equal { + temp_vector[match_count] = idx; + match_count += 1; + } else { + no_match[*no_match_count] = idx; + *no_match_count += 1; + } + } + } else { + for idx in select_vector[..*count].iter() { + let idx = *idx; + let len_address = address[idx].add(col_offset); + let address = address[idx].add(col_offset + 4); + + let len = read::(len_address as _) as usize; + + let value = StringType::index_column_unchecked(col, idx); + if len != value.len() { + equal = false; + } else { + let data_address = read::(address as _) as usize as *const u8; + let scalar = std::slice::from_raw_parts(data_address, len); + + equal = databend_common_hashtable::fast_memcmp(scalar, value.as_bytes()); + } + + if equal { + temp_vector[match_count] = idx; + match_count += 1; + } else { + no_match[*no_match_count] = idx; + *no_match_count += 1; + } + } + } + + select_vector.clone_from_slice(temp_vector); + + *count = match_count; +} + unsafe fn row_match_column_type( col: &Column, validity: Option<&Bitmap>, diff --git a/src/query/expression/src/block.rs b/src/query/expression/src/block.rs index 64b497636421..47715b137db7 100644 --- a/src/query/expression/src/block.rs +++ b/src/query/expression/src/block.rs @@ -32,6 +32,7 @@ use crate::DataField; use crate::DataSchemaRef; use crate::Domain; use crate::Scalar; +use crate::ScalarRef; use crate::TableSchemaRef; use crate::Value; @@ -608,6 +609,14 @@ impl DataBlock { .collect(); DataSchema::new(fields) } + + // This is inefficient, don't use it in hot path + pub fn value_at(&self, col: usize, row: usize) -> Option> { + if col >= self.columns.len() { + return None; + } + self.columns[col].value.index(row) + } } impl TryFrom for ArrowChunk { diff --git a/src/query/expression/src/converts/arrow/to.rs b/src/query/expression/src/converts/arrow/to.rs index 3860fc64d503..34f929edbd2a 100644 --- a/src/query/expression/src/converts/arrow/to.rs +++ b/src/query/expression/src/converts/arrow/to.rs @@ -113,6 +113,7 @@ impl DataBlock { .zip(arrow_schema.fields()) { let column = entry.value.to_owned().into_column().unwrap(); + let column = column.maybe_gc(); let array = column.into_arrow_rs(); // Adjust struct array names diff --git a/src/query/expression/src/converts/arrow2/from.rs b/src/query/expression/src/converts/arrow2/from.rs index d354cb9c13af..87822a5045b2 100644 --- a/src/query/expression/src/converts/arrow2/from.rs +++ b/src/query/expression/src/converts/arrow2/from.rs @@ -12,12 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_arrow::arrow::array::BinaryArray; +use databend_common_arrow::arrow::array::FixedSizeBinaryArray; +use databend_common_arrow::arrow::array::Utf8Array; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::buffer::Buffer; use databend_common_arrow::arrow::datatypes::DataType as ArrowDataType; use databend_common_arrow::arrow::datatypes::Field as ArrowField; use databend_common_arrow::arrow::datatypes::Schema as ArrowSchema; use databend_common_arrow::arrow::datatypes::TimeUnit; +use databend_common_arrow::arrow::types::Offset; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -28,10 +32,12 @@ use super::ARROW_EXT_TYPE_GEOMETRY; use super::ARROW_EXT_TYPE_VARIANT; use crate::types::array::ArrayColumn; use crate::types::binary::BinaryColumn; +use crate::types::binary::BinaryColumnBuilder; use crate::types::decimal::DecimalColumn; use crate::types::geography::GeographyColumn; use crate::types::nullable::NullableColumn; use crate::types::string::StringColumn; +use crate::types::string::StringColumnBuilder; use crate::types::DataType; use crate::types::DecimalDataType; use crate::types::DecimalSize; @@ -119,7 +125,8 @@ fn arrow_type_to_table_type(ty: &ArrowDataType, is_nullable: bool) -> Result TableDataType::Binary, - ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => TableDataType::String, + ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 | ArrowDataType::Utf8View => + TableDataType::String, ArrowDataType::Timestamp(_, _) => TableDataType::Timestamp, ArrowDataType::Date32 | ArrowDataType::Date64 => TableDataType::Date, @@ -342,14 +349,13 @@ impl Column { ); let offsets = arrow_col .offsets() - .buffer() .iter() .map(|x| *x as u64) .collect::>(); - Column::Binary(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) + Column::Binary(BinaryColumn { + data: arrow_col.values().clone(), + offsets: offsets.into(), + }) } (DataType::Binary, ArrowDataType::LargeBinary) => { let arrow_col = arrow_col @@ -361,22 +367,19 @@ impl Column { let offsets = arrow_col.offsets().clone().into_inner(); let offsets = unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Binary(BinaryColumn::new(arrow_col.values().clone(), offsets)) + Column::Binary(BinaryColumn { + data: arrow_col.values().clone(), + offsets, + }) } - (DataType::Binary, ArrowDataType::FixedSizeBinary(size)) => { + (DataType::Binary, ArrowDataType::FixedSizeBinary(_)) => { let arrow_col = arrow_col .as_any() .downcast_ref::() .expect( "fail to read `Binary` from arrow: array should be `FixedSizeBinaryArray`", ); - let offsets = (0..arrow_col.len() as u64 + 1) - .map(|x| x * (*size) as u64) - .collect::>(); - Column::Binary(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) + Column::Binary(fixed_size_binary_array_to_binary_column(arrow_col)) } (DataType::Binary, ArrowDataType::Utf8) => { let arrow_col = arrow_col @@ -387,14 +390,13 @@ impl Column { ); let offsets = arrow_col .offsets() - .buffer() .iter() .map(|x| *x as u64) .collect::>(); - Column::Binary(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) + Column::Binary(BinaryColumn { + data: arrow_col.values().clone(), + offsets: offsets.into(), + }) } (DataType::Binary, ArrowDataType::LargeUtf8) => { let arrow_col = arrow_col @@ -403,10 +405,16 @@ impl Column { .expect( "fail to read `Binary` from arrow: array should be `Utf8Array`", ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Binary(BinaryColumn::new(arrow_col.values().clone(), offsets)) + + let offsets = unsafe { + std::mem::transmute::, Buffer>( + arrow_col.offsets().clone().into_inner(), + ) + }; + Column::Binary(BinaryColumn { + data: arrow_col.values().clone(), + offsets, + }) } (DataType::String, ArrowDataType::Binary) => { let arrow_col = arrow_col @@ -415,14 +423,8 @@ impl Column { .expect( "fail to read `String` from arrow: array should be `BinaryArray`", ); - let offsets = arrow_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - let column = StringColumn::new(arrow_col.values().clone(), offsets.into()); - Column::String(column) + let col = binary_array_to_string_column(arrow_col); + Column::String(col) } (DataType::String, ArrowDataType::LargeBinary) => { let arrow_col = arrow_col @@ -431,24 +433,18 @@ impl Column { .expect( "fail to read `String` from arrow: array should be `BinaryArray`", ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - let column = StringColumn::new(arrow_col.values().clone(), offsets); - Column::String(column) + let col = binary_array_to_string_column(arrow_col); + Column::String(col) } - (DataType::String, ArrowDataType::FixedSizeBinary(size)) => { + (DataType::String, ArrowDataType::FixedSizeBinary(_)) => { let arrow_col = arrow_col .as_any() .downcast_ref::() .expect( "fail to read `String` from arrow: array should be `FixedSizeBinaryArray`", ); - let offsets = (0..arrow_col.len() as u64 + 1) - .map(|x| x * (*size) as u64) - .collect::>(); - let column = StringColumn::new(arrow_col.values().clone(), offsets.into()); - Column::String(column) + let col = fixed_size_binary_array_to_string_column(arrow_col); + Column::String(col) } (DataType::String, ArrowDataType::Utf8) => { let arrow_col = arrow_col @@ -457,18 +453,8 @@ impl Column { .expect( "fail to read `String` from arrow: array should be `Utf8Array`", ); - let offsets = arrow_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - unsafe { - Column::String(StringColumn::new_unchecked( - arrow_col.values().clone(), - offsets.into(), - )) - } + let col = utf8_array_to_string_column(arrow_col); + Column::String(col) } (DataType::String, ArrowDataType::LargeUtf8) => { let arrow_col = arrow_col @@ -477,15 +463,17 @@ impl Column { .expect( "fail to read `String` from arrow: array should be `Utf8Array`", ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - unsafe { - Column::String(StringColumn::new_unchecked( - arrow_col.values().clone(), - offsets, - )) - } + let col = utf8_array_to_string_column(arrow_col); + Column::String(col) + } + (DataType::String, ArrowDataType::Utf8View) => { + let arrow_col = arrow_col + .as_any() + .downcast_ref::() + .expect( + "fail to read `String` from arrow: array should be `Utf8ViewArray`", + ); + Column::String(StringColumn::new(arrow_col.clone())) } (DataType::Timestamp, ArrowDataType::Timestamp(uint, _)) => { let values = arrow_col @@ -642,27 +630,6 @@ impl Column { .collect::>>()?; Column::Tuple(field_cols) } - ( - DataType::Bitmap, - ArrowDataType::Extension(name, box ArrowDataType::Binary, None), - ) if name == ARROW_EXT_TYPE_BITMAP => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Bitmap` from arrow: array should be `BinaryArray`", - ); - let offsets = arrow_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Bitmap(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) - } ( DataType::Bitmap, ArrowDataType::Extension(name, box ArrowDataType::LargeBinary, None), @@ -673,28 +640,15 @@ impl Column { .expect( "fail to read `Bitmap` from arrow: array should be `BinaryArray`", ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Bitmap(BinaryColumn::new(arrow_col.values().clone(), offsets)) - } - (DataType::Bitmap, ArrowDataType::Binary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Bitmap` from arrow: array should be `BinaryArray`", - ); - let offsets = arrow_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Bitmap(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) + let offsets = unsafe { + std::mem::transmute::, Buffer>( + arrow_col.offsets().clone().into_inner(), + ) + }; + Column::Bitmap(BinaryColumn { + data: arrow_col.values().clone(), + offsets, + }) } (DataType::Bitmap, ArrowDataType::LargeBinary) => { let arrow_col = arrow_col @@ -703,31 +657,15 @@ impl Column { .expect( "fail to read `Bitmap` from arrow: array should be `BinaryArray`", ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Bitmap(BinaryColumn::new(arrow_col.values().clone(), offsets)) - } - ( - DataType::Geometry, - ArrowDataType::Extension(name, box ArrowDataType::Binary, None), - ) if name == ARROW_EXT_TYPE_GEOMETRY => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Geometry` from arrow: array should be `BinaryArray`", - ); - let offsets = arrow_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Geometry(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) + let offsets = unsafe { + std::mem::transmute::, Buffer>( + arrow_col.offsets().clone().into_inner(), + ) + }; + Column::Bitmap(BinaryColumn { + data: arrow_col.values().clone(), + offsets, + }) } ( DataType::Geometry, @@ -739,28 +677,15 @@ impl Column { .expect( "fail to read `Geometry` from arrow: array should be `BinaryArray`", ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Geometry(BinaryColumn::new(arrow_col.values().clone(), offsets)) - } - (DataType::Geometry, ArrowDataType::Binary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Geometry` from arrow: array should be `BinaryArray`", - ); - let offsets = arrow_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Geometry(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) + let offsets = unsafe { + std::mem::transmute::, Buffer>( + arrow_col.offsets().clone().into_inner(), + ) + }; + Column::Geometry(BinaryColumn { + data: arrow_col.values().clone(), + offsets, + }) } (DataType::Geometry, ArrowDataType::LargeBinary) => { let arrow_col = arrow_col @@ -769,10 +694,16 @@ impl Column { .expect( "fail to read `Geometry` from arrow: array should be `BinaryArray`", ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Geometry(BinaryColumn::new(arrow_col.values().clone(), offsets)) + + let offsets = unsafe { + std::mem::transmute::, Buffer>( + arrow_col.offsets().clone().into_inner(), + ) + }; + Column::Geometry(BinaryColumn { + data: arrow_col.values().clone(), + offsets, + }) } (DataType::Geography, ArrowDataType::LargeBinary) => { let arrow_col = arrow_col @@ -781,13 +712,16 @@ impl Column { .expect( "fail to read `Geography` from arrow: array should be `BinaryArray`", ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Geography(GeographyColumn(BinaryColumn::new( - arrow_col.values().clone(), + + let offsets = unsafe { + std::mem::transmute::, Buffer>( + arrow_col.offsets().clone().into_inner(), + ) + }; + Column::Geography(GeographyColumn(BinaryColumn { + data: arrow_col.values().clone(), offsets, - ))) + })) } (data_type, ArrowDataType::Extension(_, arrow_type, _)) => { from_arrow_with_arrow_type(arrow_col, arrow_type, data_type)? @@ -812,3 +746,36 @@ impl Column { from_arrow_with_arrow_type(arrow_col, arrow_col.data_type(), data_type) } } + +fn binary_array_to_string_column(array: &BinaryArray) -> StringColumn { + let mut builder = StringColumnBuilder::with_capacity(array.len()); + for value in array.values_iter() { + builder.put_and_commit(std::str::from_utf8(value).unwrap()); + } + builder.build() +} + +fn utf8_array_to_string_column(array: &Utf8Array) -> StringColumn { + let mut builder = StringColumnBuilder::with_capacity(array.len()); + for value in array.values_iter() { + builder.put_and_commit(value); + } + builder.build() +} + +fn fixed_size_binary_array_to_string_column(array: &FixedSizeBinaryArray) -> StringColumn { + let mut builder = StringColumnBuilder::with_capacity(array.len()); + for value in array.values_iter() { + builder.put_and_commit(std::str::from_utf8(value).unwrap()); + } + builder.build() +} + +fn fixed_size_binary_array_to_binary_column(array: &FixedSizeBinaryArray) -> BinaryColumn { + let mut builder = BinaryColumnBuilder::with_capacity(array.len(), array.len() * array.size()); + for value in array.values_iter() { + builder.put_slice(value); + builder.commit_row(); + } + builder.build() +} diff --git a/src/query/expression/src/converts/arrow2/to.rs b/src/query/expression/src/converts/arrow2/to.rs index f248a1a38c0d..89d51b70126a 100644 --- a/src/query/expression/src/converts/arrow2/to.rs +++ b/src/query/expression/src/converts/arrow2/to.rs @@ -93,7 +93,7 @@ fn table_type_to_arrow_type(ty: &TableDataType) -> ArrowDataType { ), TableDataType::Boolean => ArrowDataType::Boolean, TableDataType::Binary => ArrowDataType::LargeBinary, - TableDataType::String => ArrowDataType::LargeUtf8, + TableDataType::String => ArrowDataType::Utf8View, TableDataType::Number(ty) => with_number_type!(|TYPE| match ty { NumberDataType::TYPE => ArrowDataType::TYPE, }), @@ -315,32 +315,7 @@ impl Column { ) .unwrap(), ), - Column::Binary(col) => { - let offsets: Buffer = - col.offsets().iter().map(|offset| *offset as i64).collect(); - Box::new( - databend_common_arrow::arrow::array::BinaryArray::::try_new( - arrow_type, - unsafe { OffsetsBuffer::new_unchecked(offsets) }, - col.data().clone(), - None, - ) - .unwrap(), - ) - } - Column::String(col) => { - let offsets: Buffer = - col.offsets().iter().map(|offset| *offset as i64).collect(); - Box::new( - databend_common_arrow::arrow::array::Utf8Array::::try_new( - arrow_type, - unsafe { OffsetsBuffer::new_unchecked(offsets) }, - col.data().clone(), - None, - ) - .unwrap(), - ) - } + Column::String(col) => Box::new(col.clone().into_inner()), Column::Timestamp(col) => Box::new( databend_common_arrow::arrow::array::PrimitiveArray::::try_new( arrow_type, @@ -409,7 +384,9 @@ impl Column { ) .unwrap(), ), - Column::Bitmap(col) + + Column::Binary(col) + | Column::Bitmap(col) | Column::Variant(col) | Column::Geometry(col) | Column::Geography(GeographyColumn(col)) => { diff --git a/src/query/expression/src/converts/meta/bincode.rs b/src/query/expression/src/converts/meta/bincode.rs index aa8828bacaae..e839a10d99b0 100644 --- a/src/query/expression/src/converts/meta/bincode.rs +++ b/src/query/expression/src/converts/meta/bincode.rs @@ -60,15 +60,21 @@ pub enum LegacyColumn { Number(NumberColumn), Decimal(DecimalColumn), Boolean(Bitmap), - String(BinaryColumn), + String(LegacyBinaryColumn), Timestamp(Buffer), Date(Buffer), Array(Box), Map(Box), - Bitmap(BinaryColumn), + Bitmap(LegacyBinaryColumn), Nullable(Box), Tuple(Vec), - Variant(BinaryColumn), + Variant(LegacyBinaryColumn), +} + +#[derive(Clone)] +pub struct LegacyBinaryColumn { + pub(crate) data: Buffer, + pub(crate) offsets: Buffer, } #[derive(Clone)] @@ -104,6 +110,24 @@ impl From for Scalar { } } +impl From for BinaryColumn { + fn from(value: LegacyBinaryColumn) -> Self { + BinaryColumn { + data: value.data, + offsets: value.offsets, + } + } +} + +impl From for LegacyBinaryColumn { + fn from(value: BinaryColumn) -> Self { + LegacyBinaryColumn { + data: value.data, + offsets: value.offsets, + } + } +} + impl From for Column { fn from(value: LegacyColumn) -> Self { match value { @@ -113,7 +137,9 @@ impl From for Column { LegacyColumn::Number(num_col) => Column::Number(num_col), LegacyColumn::Decimal(dec_col) => Column::Decimal(dec_col), LegacyColumn::Boolean(bmp) => Column::Boolean(bmp), - LegacyColumn::String(str_col) => Column::String(str_col.try_into().unwrap()), + LegacyColumn::String(str_col) => { + Column::String(StringColumn::try_from_binary(BinaryColumn::from(str_col)).unwrap()) + } LegacyColumn::Timestamp(buf) => Column::Timestamp(buf), LegacyColumn::Date(buf) => Column::Date(buf), LegacyColumn::Array(arr_col) => Column::Array(Box::new(ArrayColumn:: { @@ -124,7 +150,7 @@ impl From for Column { values: map_col.values.into(), offsets: map_col.offsets, })), - LegacyColumn::Bitmap(str_col) => Column::Bitmap(str_col), + LegacyColumn::Bitmap(str_col) => Column::Bitmap(BinaryColumn::from(str_col)), LegacyColumn::Nullable(nullable_col) => { Column::Nullable(Box::new(NullableColumn:: { column: nullable_col.column.into(), @@ -134,7 +160,7 @@ impl From for Column { LegacyColumn::Tuple(tuple) => { Column::Tuple(tuple.into_iter().map(|c| c.into()).collect()) } - LegacyColumn::Variant(variant) => Column::Variant(variant), + LegacyColumn::Variant(variant) => Column::Variant(BinaryColumn::from(variant)), } } } @@ -171,7 +197,9 @@ impl From for LegacyColumn { Column::Decimal(dec_col) => LegacyColumn::Decimal(dec_col), Column::Boolean(bmp) => LegacyColumn::Boolean(bmp), Column::Binary(_) | Column::Geometry(_) | Column::Geography(_) => unreachable!(), - Column::String(str_col) => LegacyColumn::String(str_col.into()), + Column::String(str_col) => { + LegacyColumn::String(LegacyBinaryColumn::from(BinaryColumn::from(str_col))) + } Column::Timestamp(buf) => LegacyColumn::Timestamp(buf), Column::Date(buf) => LegacyColumn::Date(buf), Column::Array(arr_col) => LegacyColumn::Array(Box::new(LegacyArrayColumn { @@ -182,7 +210,7 @@ impl From for LegacyColumn { values: map_col.values.into(), offsets: map_col.offsets, })), - Column::Bitmap(str_col) => LegacyColumn::Bitmap(str_col), + Column::Bitmap(str_col) => LegacyColumn::Bitmap(LegacyBinaryColumn::from(str_col)), Column::Nullable(nullable_col) => { LegacyColumn::Nullable(Box::new(LegacyNullableColumn { column: nullable_col.column.into(), @@ -192,7 +220,7 @@ impl From for LegacyColumn { Column::Tuple(tuple) => { LegacyColumn::Tuple(tuple.into_iter().map(|c| c.into()).collect()) } - Column::Variant(variant) => LegacyColumn::Variant(variant), + Column::Variant(variant) => LegacyColumn::Variant(LegacyBinaryColumn::from(variant)), } } } diff --git a/src/query/expression/src/evaluator.rs b/src/query/expression/src/evaluator.rs index 59bd71379e57..85ec550fc8f5 100644 --- a/src/query/expression/src/evaluator.rs +++ b/src/query/expression/src/evaluator.rs @@ -613,10 +613,9 @@ impl<'a> Evaluator<'a> { }; let validity = None; - let mut key_builder = StringColumnBuilder::with_capacity(obj.len(), 0); + let mut key_builder = StringColumnBuilder::with_capacity(obj.len()); for k in obj.keys() { - key_builder.put_str(k.as_str()); - key_builder.commit_row(); + key_builder.put_and_commit(k.as_str()); } let key_column = Column::String(key_builder.build()); @@ -659,8 +658,7 @@ impl<'a> Evaluator<'a> { }; for (k, v) in obj.iter() { - key_builder.put_str(k.as_str()); - key_builder.commit_row(); + key_builder.put_and_commit(k.as_str()); v.write_to_vec(&mut value_builder.builder.data); value_builder.builder.commit_row(); } diff --git a/src/query/expression/src/filter/filter_executor.rs b/src/query/expression/src/filter/filter_executor.rs index 007f6d319e9c..321d9c68731a 100644 --- a/src/query/expression/src/filter/filter_executor.rs +++ b/src/query/expression/src/filter/filter_executor.rs @@ -26,6 +26,7 @@ use crate::Expr; use crate::FunctionContext; use crate::FunctionRegistry; use crate::SelectExprBuilder; +use crate::SELECTIVITY_THRESHOLD; // FilterExecutor is used to filter `DataBlock` by `SelectExpr`. pub struct FilterExecutor { @@ -114,7 +115,7 @@ impl FilterExecutor { // (3) Otherwise, use `take` to generate a new `DataBlock`. if result_count == origin_count { Ok(data_block) - } else if result_count as f64 > data_block.num_rows() as f64 * 0.8 + } else if result_count as f64 > data_block.num_rows() as f64 * SELECTIVITY_THRESHOLD && data_block.num_columns() > 1 { let range_count = self.build_selection_range(result_count); @@ -125,7 +126,7 @@ impl FilterExecutor { if self.keep_order && self.has_or { self.true_selection[0..result_count].sort(); } - data_block.take(&self.true_selection[0..result_count], &mut None) + data_block.take_with_optimize_size(&self.true_selection[0..result_count]) } } diff --git a/src/query/expression/src/filter/select_value/select_column_scalar.rs b/src/query/expression/src/filter/select_value/select_column_scalar.rs index 3d36884cde64..8c2bad57e108 100644 --- a/src/query/expression/src/filter/select_value/select_column_scalar.rs +++ b/src/query/expression/src/filter/select_value/select_column_scalar.rs @@ -239,48 +239,19 @@ impl<'a> Selector<'a> { Some(validity) => { // search the whole string buffer if let LikePattern::SurroundByPercent(searcher) = like_pattern { - let needle = searcher.needle(); - let needle_byte_len = needle.len(); - let data = column.data().as_slice(); - let offsets = column.offsets().as_slice(); - let mut idx = 0; - let mut pos = (*offsets.first().unwrap()) as usize; - let end = (*offsets.last().unwrap()) as usize; - - while pos < end && idx < count { - if let Some(p) = searcher.search(&data[pos..end]) { - while offsets[idx + 1] as usize <= pos + p { - let ret = NOT && validity.get_bit_unchecked(idx); - update_index( - ret, - idx as u32, - true_selection, - false_selection, - ); - idx += 1; - } - - // check if the substring is in bound - let ret = - pos + p + needle_byte_len <= offsets[idx + 1] as usize; - - let ret = if NOT { - validity.get_bit_unchecked(idx) && !ret - } else { - validity.get_bit_unchecked(idx) && ret - }; - update_index(ret, idx as u32, true_selection, false_selection); - - pos = offsets[idx + 1] as usize; - idx += 1; + for idx in 0u32..count as u32 { + let ret = if NOT { + validity.get_bit_unchecked(idx as usize) + && searcher + .search(column.index_unchecked_bytes(idx as usize)) + .is_none() } else { - break; - } - } - while idx < count { - let ret = NOT && validity.get_bit_unchecked(idx); - update_index(ret, idx as u32, true_selection, false_selection); - idx += 1; + validity.get_bit_unchecked(idx as usize) + && searcher + .search(column.index_unchecked_bytes(idx as usize)) + .is_some() + }; + update_index(ret, idx, true_selection, false_selection); } } else { for idx in 0u32..count as u32 { @@ -300,40 +271,17 @@ impl<'a> Selector<'a> { None => { // search the whole string buffer if let LikePattern::SurroundByPercent(searcher) = like_pattern { - let needle = searcher.needle(); - let needle_byte_len = needle.len(); - let data = column.data().as_slice(); - let offsets = column.offsets().as_slice(); - let mut idx = 0; - let mut pos = (*offsets.first().unwrap()) as usize; - let end = (*offsets.last().unwrap()) as usize; - - while pos < end && idx < count { - if let Some(p) = searcher.search(&data[pos..end]) { - while offsets[idx + 1] as usize <= pos + p { - update_index( - NOT, - idx as u32, - true_selection, - false_selection, - ); - idx += 1; - } - // check if the substring is in bound - let ret = - pos + p + needle_byte_len <= offsets[idx + 1] as usize; - let ret = if NOT { !ret } else { ret }; - update_index(ret, idx as u32, true_selection, false_selection); - - pos = offsets[idx + 1] as usize; - idx += 1; + for idx in 0u32..count as u32 { + let ret = if NOT { + searcher + .search(column.index_unchecked_bytes(idx as usize)) + .is_none() } else { - break; - } - } - while idx < count { - update_index(NOT, idx as u32, true_selection, false_selection); - idx += 1; + searcher + .search(column.index_unchecked_bytes(idx as usize)) + .is_some() + }; + update_index(ret, idx, true_selection, false_selection); } } else { for idx in 0u32..count as u32 { diff --git a/src/query/expression/src/kernels/concat.rs b/src/query/expression/src/kernels/concat.rs index d327fff0eb2c..782031de7c10 100644 --- a/src/query/expression/src/kernels/concat.rs +++ b/src/query/expression/src/kernels/concat.rs @@ -13,8 +13,10 @@ // limitations under the License. use std::iter::TrustedLen; -use std::sync::Arc; +use databend_common_arrow::arrow::array::growable::make_growable; +use databend_common_arrow::arrow::array::Array; +use databend_common_arrow::arrow::array::BooleanArray; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::ErrorCode; @@ -22,35 +24,22 @@ use databend_common_exception::Result; use ethnum::i256; use itertools::Itertools; -use crate::copy_continuous_bits; -use crate::kernels::take::BIT_MASK; -use crate::kernels::utils::copy_advance_aligned; -use crate::kernels::utils::set_vec_len_by_ptr; -use crate::store_advance_aligned; use crate::types::array::ArrayColumnBuilder; -use crate::types::binary::BinaryColumn; use crate::types::decimal::Decimal; use crate::types::decimal::DecimalColumn; -use crate::types::geography::GeographyColumn; -use crate::types::geometry::GeometryType; use crate::types::map::KvColumnBuilder; use crate::types::nullable::NullableColumn; use crate::types::number::NumberColumn; -use crate::types::string::StringColumn; use crate::types::AnyType; use crate::types::ArrayType; -use crate::types::BinaryType; -use crate::types::BitmapType; use crate::types::BooleanType; +use crate::types::DataType; use crate::types::DateType; use crate::types::DecimalType; -use crate::types::GeographyType; use crate::types::MapType; use crate::types::NumberType; -use crate::types::StringType; use crate::types::TimestampType; use crate::types::ValueType; -use crate::types::VariantType; use crate::with_decimal_mapped_type; use crate::with_number_mapped_type; use crate::BlockEntry; @@ -164,14 +153,6 @@ impl Column { columns.map(|col| col.into_boolean().unwrap()), capacity, )), - Column::Binary(_) => BinaryType::upcast_column(Self::concat_binary_types( - columns.map(|col| col.into_binary().unwrap()), - capacity, - )), - Column::String(_) => StringType::upcast_column(Self::concat_string_types( - columns.map(|col| col.into_string().unwrap()), - capacity, - )), Column::Timestamp(_) => { let buffer = Self::concat_primitive_types( columns.map(|col| TimestampType::try_downcast_column(&col).unwrap()), @@ -210,10 +191,6 @@ impl Column { let builder = ArrayColumnBuilder { builder, offsets }; Self::concat_value_types::>(builder, columns) } - Column::Bitmap(_) => BitmapType::upcast_column(Self::concat_binary_types( - columns.map(|col| col.into_bitmap().unwrap()), - capacity, - )), Column::Nullable(_) => { let column: Vec = columns .clone() @@ -239,19 +216,13 @@ impl Column { .collect::>()?; Column::Tuple(fields) } - Column::Variant(_) => VariantType::upcast_column(Self::concat_binary_types( - columns.map(|col| col.into_variant().unwrap()), - capacity, - )), - Column::Geometry(_) => GeometryType::upcast_column(Self::concat_binary_types( - columns.map(|col| col.into_geometry().unwrap()), - capacity, - )), - Column::Geography(_) => { - GeographyType::upcast_column(GeographyColumn(Self::concat_binary_types( - columns.map(|col| col.into_geography().unwrap().0), - capacity, - ))) + Column::Variant(_) + | Column::Geometry(_) + | Column::Geography(_) + | Column::Binary(_) + | Column::String(_) + | Column::Bitmap(_) => { + Self::concat_use_grows(columns, first_column.data_type(), capacity) } }; Ok(column) @@ -271,108 +242,38 @@ impl Column { builder.into() } - pub fn concat_binary_types( - cols: impl Iterator + Clone, + pub fn concat_use_grows( + cols: impl Iterator, + data_type: DataType, num_rows: usize, - ) -> BinaryColumn { - // [`BinaryColumn`] consists of [`data`] and [`offset`], we build [`data`] and [`offset`] respectively, - // and then call `BinaryColumn::new(data.into(), offsets.into())` to create [`BinaryColumn`]. - let mut offsets: Vec = Vec::with_capacity(num_rows + 1); - let mut data_size = 0; - - // Build [`offset`] and calculate `data_size` required by [`data`]. - offsets.push(0); - for col in cols.clone() { - let mut start = col.offsets()[0]; - for end in col.offsets()[1..].iter() { - data_size += end - start; - start = *end; - offsets.push(data_size); - } - } - - // Build [`data`]. - let mut data: Vec = Vec::with_capacity(data_size as usize); - let mut data_ptr = data.as_mut_ptr(); + ) -> Column { + let arrays: Vec> = + cols.map(|c| c.as_arrow()).collect(); - unsafe { - for col in cols { - let offsets = col.offsets(); - let col_data = &(col.data().as_slice()) - [offsets[0] as usize..offsets[offsets.len() - 1] as usize]; - copy_advance_aligned(col_data.as_ptr(), &mut data_ptr, col_data.len()); - } - set_vec_len_by_ptr(&mut data, data_ptr); - } + let arrays = arrays.iter().map(|c| c.as_ref()).collect::>(); + let mut grow = make_growable(&arrays, false, num_rows); - BinaryColumn::new(data.into(), offsets.into()) - } - - pub fn concat_string_types( - cols: impl Iterator + Clone, - num_rows: usize, - ) -> StringColumn { - unsafe { - StringColumn::from_binary_unchecked(Self::concat_binary_types( - cols.map(Into::into), - num_rows, - )) + for (idx, array) in arrays.iter().enumerate() { + grow.extend(idx, 0, array.len()); } + let array = grow.as_box(); + Column::from_arrow(array.as_ref(), &data_type).unwrap() } pub fn concat_boolean_types(bitmaps: impl Iterator, num_rows: usize) -> Bitmap { - let capacity = num_rows.saturating_add(7) / 8; - let mut builder: Vec = Vec::with_capacity(capacity); - let mut builder_ptr = builder.as_mut_ptr(); - let mut builder_idx = 0; - let mut unset_bits = 0; - let mut buf = 0; - - unsafe { - for bitmap in bitmaps { - let (bitmap_slice, bitmap_offset, _) = bitmap.as_slice(); - let mut idx = 0; - let len = bitmap.len(); - if builder_idx % 8 != 0 { - while idx < len { - if bitmap.get_bit_unchecked(idx) { - buf |= BIT_MASK[builder_idx % 8]; - } else { - unset_bits += 1; - } - builder_idx += 1; - idx += 1; - if builder_idx % 8 == 0 { - store_advance_aligned(buf, &mut builder_ptr); - buf = 0; - break; - } - } - } - let remaining = len - idx; - if remaining > 0 { - let (cur_buf, cur_unset_bits) = copy_continuous_bits( - &mut builder_ptr, - bitmap_slice, - builder_idx, - idx + bitmap_offset, - remaining, - ); - builder_idx += remaining; - unset_bits += cur_unset_bits; - buf = cur_buf; - } - } - - if builder_idx % 8 != 0 { - store_advance_aligned(buf, &mut builder_ptr); - } + use databend_common_arrow::arrow::datatypes::DataType as ArrowType; + let arrays: Vec = bitmaps + .map(|c| BooleanArray::new(ArrowType::Boolean, c, None)) + .collect(); + let arrays = arrays.iter().map(|c| c as &dyn Array).collect::>(); + let mut grow = make_growable(&arrays, false, num_rows); - set_vec_len_by_ptr(&mut builder, builder_ptr); - Bitmap::from_inner(Arc::new(builder.into()), 0, num_rows, unset_bits) - .ok() - .unwrap() + for (idx, array) in arrays.iter().enumerate() { + grow.extend(idx, 0, array.len()); } + let array = grow.as_box(); + let array = array.as_any().downcast_ref::().unwrap(); + array.values().clone() } fn concat_value_types( diff --git a/src/query/expression/src/kernels/filter.rs b/src/query/expression/src/kernels/filter.rs index 894ceb412cfb..11bfffe08812 100644 --- a/src/query/expression/src/kernels/filter.rs +++ b/src/query/expression/src/kernels/filter.rs @@ -12,18 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::utils::BitChunkIterExact; -use databend_common_arrow::arrow::bitmap::utils::BitChunksExact; +use binary::BinaryColumnBuilder; +use databend_common_arrow::arrow::array::Array; +use databend_common_arrow::arrow::array::Utf8ViewArray; use databend_common_arrow::arrow::bitmap::utils::SlicesIterator; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_arrow::arrow::bitmap::TrueIdxIter; use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::Result; +use string::StringColumnBuilder; -use crate::kernels::utils::copy_advance_aligned; -use crate::kernels::utils::set_vec_len_by_ptr; -use crate::kernels::utils::store_advance_aligned; use crate::types::binary::BinaryColumn; use crate::types::nullable::NullableColumn; use crate::types::string::StringColumn; @@ -118,7 +117,7 @@ pub enum IterationStrategy { } /// based on -const SELECTIVITY_THRESHOLD: f64 = 0.8; +pub const SELECTIVITY_THRESHOLD: f64 = 0.8; impl IterationStrategy { fn default_strategy(length: usize, true_count: usize) -> Self { @@ -172,7 +171,7 @@ impl<'a> ValueVisitor for FilterVisitor<'a> { match value { Value::Scalar(c) => self.visit_scalar(c), Value::Column(c) => { - assert!(c.len() == self.original_rows); + assert_eq!(c.len(), self.original_rows); match self.strategy { IterationStrategy::None => self.result = Some(Value::Column(c.slice(0..0))), IterationStrategy::All => self.result = Some(Value::Column(c)), @@ -226,6 +225,7 @@ impl<'a> ValueVisitor for FilterVisitor<'a> { }); } } + self.result = Some(Value::Column(T::upcast_column(T::build_column(builder)))); Ok(()) } @@ -304,10 +304,9 @@ impl<'a> ValueVisitor for FilterVisitor<'a> { } fn visit_string(&mut self, column: StringColumn) -> Result<()> { - let column: BinaryColumn = column.into(); - self.result = Some(Value::Column(StringType::upcast_column(unsafe { - StringColumn::from_binary_unchecked(self.filter_binary_types(&column)) - }))); + self.result = Some(Value::Column(StringType::upcast_column( + self.filter_string_types(&column), + ))); Ok(()) } @@ -337,126 +336,45 @@ impl<'a> FilterVisitor<'a> { } } - // TODO: optimize this after BinaryView is introduced by @andy - fn filter_binary_types(&mut self, values: &BinaryColumn) -> BinaryColumn { - // Each element of `items` is (string pointer(u64), string length). - let mut items: Vec<(u64, usize)> = Vec::with_capacity(self.filter_rows); - // [`BinaryColumn`] consists of [`data`] and [`offset`], we build [`data`] and [`offset`] respectively, - // and then call `BinaryColumn::new(data.into(), offsets.into())` to create [`BinaryColumn`]. - let values_offset = values.offsets().as_slice(); - let values_data_ptr = values.data().as_slice().as_ptr(); - let mut offsets: Vec = Vec::with_capacity(self.filter_rows + 1); - let mut offsets_ptr = offsets.as_mut_ptr(); - let mut items_ptr = items.as_mut_ptr(); - let mut data_size = 0; - - // Build [`offset`] and calculate `data_size` required by [`data`]. - unsafe { - store_advance_aligned::(0, &mut offsets_ptr); - let mut idx = 0; - let (mut slice, offset, mut length) = self.filter.as_slice(); - if offset > 0 { - let mut mask = slice[0]; - while mask != 0 { - let n = mask.trailing_zeros() as usize; - // If `offset` > 0, the valid bits of this byte start at `offset`, we also - // need to ensure that we cannot iterate more than `length` bits. - if n >= offset && n < offset + length { - let start = *values_offset.get_unchecked(n - offset) as usize; - let len = *values_offset.get_unchecked(n - offset + 1) as usize - start; - data_size += len as u64; - store_advance_aligned(data_size, &mut offsets_ptr); - store_advance_aligned( - (values_data_ptr.add(start) as u64, len), - &mut items_ptr, - ); - } - mask = mask & (mask - 1); - } - let bits_to_align = 8 - offset; - length = if length >= bits_to_align { - length - bits_to_align - } else { - 0 - }; - slice = &slice[1..]; - idx += bits_to_align; - } + fn filter_string_types(&mut self, values: &StringColumn) -> StringColumn { + match self.strategy { + IterationStrategy::IndexIterator => { + let mut builder = StringColumnBuilder::with_capacity(self.filter_rows); - const CHUNK_SIZE: usize = 64; - let mut mask_chunks = BitChunksExact::::new(slice, length); - let mut continuous_selected = 0; - for mut mask in mask_chunks.by_ref() { - if mask == u64::MAX { - continuous_selected += CHUNK_SIZE; - } else { - if continuous_selected > 0 { - let start = *values_offset.get_unchecked(idx) as usize; - let len = *values_offset.get_unchecked(idx + continuous_selected) as usize - - start; - store_advance_aligned( - (values_data_ptr.add(start) as u64, len), - &mut items_ptr, - ); - for i in 0..continuous_selected { - data_size += *values_offset.get_unchecked(idx + i + 1) - - *values_offset.get_unchecked(idx + i); - store_advance_aligned(data_size, &mut offsets_ptr); - } - idx += continuous_selected; - continuous_selected = 0; - } - while mask != 0 { - let n = mask.trailing_zeros() as usize; - let start = *values_offset.get_unchecked(idx + n) as usize; - let len = *values_offset.get_unchecked(idx + n + 1) as usize - start; - data_size += len as u64; - store_advance_aligned( - (values_data_ptr.add(start) as u64, len), - &mut items_ptr, - ); - store_advance_aligned(data_size, &mut offsets_ptr); - mask = mask & (mask - 1); + let iter = TrueIdxIter::new(self.original_rows, Some(self.filter)); + for i in iter { + unsafe { + builder.put_and_commit(values.index_unchecked(i)); } - idx += CHUNK_SIZE; - } - } - if continuous_selected > 0 { - let start = *values_offset.get_unchecked(idx) as usize; - let len = *values_offset.get_unchecked(idx + continuous_selected) as usize - start; - store_advance_aligned((values_data_ptr.add(start) as u64, len), &mut items_ptr); - for i in 0..continuous_selected { - data_size += *values_offset.get_unchecked(idx + i + 1) - - *values_offset.get_unchecked(idx + i); - store_advance_aligned(data_size, &mut offsets_ptr); } - idx += continuous_selected; + builder.build() } - - for (i, is_selected) in mask_chunks.remainder_iter().enumerate() { - if is_selected { - let start = *values_offset.get_unchecked(idx + i) as usize; - let len = *values_offset.get_unchecked(idx + i + 1) as usize - start; - data_size += len as u64; - store_advance_aligned((values_data_ptr.add(start) as u64, len), &mut items_ptr); - store_advance_aligned(data_size, &mut offsets_ptr); - } + _ => { + // reuse the buffers + let new_views = self.filter_primitive_types(values.data.views().clone()); + let new_col = unsafe { + Utf8ViewArray::new_unchecked_unknown_md( + values.data.data_type().clone(), + new_views, + values.data.data_buffers().clone(), + None, + Some(values.data.total_buffer_len()), + ) + }; + StringColumn::new(new_col) } - set_vec_len_by_ptr(&mut items, items_ptr); - set_vec_len_by_ptr(&mut offsets, offsets_ptr); } + } - // Build [`data`]. - let mut data: Vec = Vec::with_capacity(data_size as usize); - let mut data_ptr = data.as_mut_ptr(); - - unsafe { - for (str_ptr, len) in items.iter() { - copy_advance_aligned(*str_ptr as *const u8, &mut data_ptr, *len); + fn filter_binary_types(&mut self, values: &BinaryColumn) -> BinaryColumn { + let mut builder = BinaryColumnBuilder::with_capacity(self.filter_rows, 0); + let iter = TrueIdxIter::new(self.original_rows, Some(self.filter)); + for i in iter { + unsafe { + builder.put_slice(values.index_unchecked(i)); + builder.commit_row(); } - set_vec_len_by_ptr(&mut data, data_ptr); } - - BinaryColumn::new(data.into(), offsets.into()) + builder.build() } } diff --git a/src/query/expression/src/kernels/group_by.rs b/src/query/expression/src/kernels/group_by.rs index ebebf4dd86e0..ebef511ae4f4 100644 --- a/src/query/expression/src/kernels/group_by.rs +++ b/src/query/expression/src/kernels/group_by.rs @@ -52,7 +52,7 @@ impl DataBlock { if hash_key_types.len() == 1 && matches!( hash_key_types[0], - DataType::Binary | DataType::String | DataType::Variant | DataType::Bitmap + DataType::Binary | DataType::Variant | DataType::Bitmap ) { return Ok(HashMethodKind::SingleBinary( diff --git a/src/query/expression/src/kernels/group_by_hash/method.rs b/src/query/expression/src/kernels/group_by_hash/method.rs index 0485c07ab17a..b8215a0d213f 100644 --- a/src/query/expression/src/kernels/group_by_hash/method.rs +++ b/src/query/expression/src/kernels/group_by_hash/method.rs @@ -20,6 +20,7 @@ use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::Result; use databend_common_hashtable::DictionaryKeys; use databend_common_hashtable::FastHash; +use either::Either; use ethnum::i256; use ethnum::u256; @@ -28,6 +29,7 @@ use crate::types::decimal::Decimal; use crate::types::DataType; use crate::types::DecimalDataType; use crate::types::NumberDataType; +use crate::types::StringColumn; use crate::Column; use crate::HashMethodDictionarySerializer; use crate::HashMethodKeysU128; @@ -46,7 +48,7 @@ pub enum KeysState { U128(Buffer), U256(Buffer), Dictionary { - columns: Vec, + columns: Vec>, keys_point: Vec>, dictionaries: Vec, }, diff --git a/src/query/expression/src/kernels/group_by_hash/method_dict_serializer.rs b/src/query/expression/src/kernels/group_by_hash/method_dict_serializer.rs index a2ce1d4c57e9..cf0cdd91967b 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_dict_serializer.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_dict_serializer.rs @@ -17,6 +17,7 @@ use std::ptr::NonNull; use databend_common_exception::Result; use databend_common_hashtable::DictionaryKeys; use databend_common_hashtable::FastHash; +use either::Either; use super::utils::serialize_group_columns; use crate::Column; @@ -46,11 +47,11 @@ impl HashMethod for HashMethodDictionarySerializer { match group_column { Column::Binary(v) | Column::Variant(v) | Column::Bitmap(v) => { debug_assert_eq!(v.len(), num_rows); - dictionary_columns.push(v.clone()); + dictionary_columns.push(Either::Right(v.clone())); } Column::String(v) => { debug_assert_eq!(v.len(), num_rows); - dictionary_columns.push(v.clone().into()); + dictionary_columns.push(Either::Left(v.clone())); } _ => serialize_columns.push(group_column.clone()), } @@ -62,11 +63,9 @@ impl HashMethod for HashMethodDictionarySerializer { for column in serialize_columns.iter() { serialize_size += column.serialize_size(); } - dictionary_columns.push(serialize_group_columns( - (&serialize_columns).into(), - num_rows, - serialize_size, - )); + let state = + serialize_group_columns((&serialize_columns).into(), num_rows, serialize_size); + dictionary_columns.push(Either::Right(state)); } let mut keys = Vec::with_capacity(num_rows * dictionary_columns.len()); @@ -76,9 +75,11 @@ impl HashMethod for HashMethodDictionarySerializer { let start = points.len(); for dictionary_column in &dictionary_columns { - points.push(NonNull::from(unsafe { - dictionary_column.index_unchecked(row) - })); + let data = match dictionary_column { + Either::Left(l) => unsafe { l.index_unchecked(row).as_bytes() }, + Either::Right(r) => unsafe { r.index_unchecked(row) }, + }; + points.push(NonNull::from(data)); } keys.push(DictionaryKeys::create(&points[start..])) diff --git a/src/query/expression/src/kernels/group_by_hash/method_serializer.rs b/src/query/expression/src/kernels/group_by_hash/method_serializer.rs index 4cb277bf416c..b11f028e07c6 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_serializer.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_serializer.rs @@ -17,7 +17,6 @@ use databend_common_hashtable::hash_join_fast_string_hash; use super::utils::serialize_group_columns; use crate::types::binary::BinaryIterator; -use crate::BinaryKeyAccessor; use crate::Column; use crate::HashMethod; use crate::InputColumns; @@ -51,7 +50,7 @@ impl HashMethod for HashMethodSerializer { fn build_keys_iter<'a>(&self, key_state: &'a KeysState) -> Result> { match key_state { - KeysState::Column(Column::Binary(col)) => Ok(col.iter()), + KeysState::Column(Column::Binary(state)) => Ok(state.iter()), _ => unreachable!(), } } @@ -61,18 +60,15 @@ impl HashMethod for HashMethodSerializer { keys_state: KeysState, ) -> Result>> { match keys_state { - KeysState::Column(Column::Binary(col)) => { - let (data, offsets) = col.into_buffer(); - Ok(Box::new(BinaryKeyAccessor::new(data, offsets))) - } + KeysState::Column(Column::Binary(state)) => Ok(Box::new(state)), _ => unreachable!(), } } fn build_keys_hashes(&self, keys_state: &KeysState, hashes: &mut Vec) { match keys_state { - KeysState::Column(Column::Binary(col)) => { - hashes.extend(col.iter().map(hash_join_fast_string_hash)); + KeysState::Column(Column::Binary(state)) => { + hashes.extend(state.iter().map(hash_join_fast_string_hash)); } _ => unreachable!(), } diff --git a/src/query/expression/src/kernels/group_by_hash/method_single_string.rs b/src/query/expression/src/kernels/group_by_hash/method_single_string.rs index 3c77a7bd58af..f2791534e55a 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_single_string.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_single_string.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::Result; use databend_common_hashtable::hash_join_fast_string_hash; use crate::types::binary::BinaryIterator; +use crate::types::BinaryColumn; use crate::Column; use crate::HashMethod; use crate::InputColumns; @@ -44,7 +44,6 @@ impl HashMethod for HashMethodSingleBinary { KeysState::Column(Column::Binary(col)) | KeysState::Column(Column::Variant(col)) | KeysState::Column(Column::Bitmap(col)) => Ok(col.iter()), - KeysState::Column(Column::String(col)) => Ok(col.iter_binary()), _ => unreachable!(), } } @@ -56,14 +55,7 @@ impl HashMethod for HashMethodSingleBinary { match keys_state { KeysState::Column(Column::Binary(col)) | KeysState::Column(Column::Variant(col)) - | KeysState::Column(Column::Bitmap(col)) => { - let (data, offsets) = col.into_buffer(); - Ok(Box::new(BinaryKeyAccessor::new(data, offsets))) - } - KeysState::Column(Column::String(col)) => { - let (data, offsets) = col.into_buffer(); - Ok(Box::new(BinaryKeyAccessor::new(data, offsets))) - } + | KeysState::Column(Column::Bitmap(col)) => Ok(Box::new(col)), _ => unreachable!(), } } @@ -75,26 +67,12 @@ impl HashMethod for HashMethodSingleBinary { | KeysState::Column(Column::Bitmap(col)) => { hashes.extend(col.iter().map(hash_join_fast_string_hash)); } - KeysState::Column(Column::String(col)) => { - hashes.extend(col.iter_binary().map(hash_join_fast_string_hash)); - } _ => unreachable!(), } } } -pub struct BinaryKeyAccessor { - data: Buffer, - offsets: Buffer, -} - -impl BinaryKeyAccessor { - pub fn new(data: Buffer, offsets: Buffer) -> Self { - Self { data, offsets } - } -} - -impl KeyAccessor for BinaryKeyAccessor { +impl KeyAccessor for BinaryColumn { type Key = [u8]; /// # Safety diff --git a/src/query/expression/src/kernels/group_by_hash/utils.rs b/src/query/expression/src/kernels/group_by_hash/utils.rs index badd8056cf17..8b844452a798 100644 --- a/src/query/expression/src/kernels/group_by_hash/utils.rs +++ b/src/query/expression/src/kernels/group_by_hash/utils.rs @@ -15,8 +15,9 @@ use databend_common_base::vec_ext::VecExt; use databend_common_base::vec_ext::VecU8Ext; -use crate::types::binary::BinaryColumn; +use crate::types::binary::BinaryColumnBuilder; use crate::types::decimal::DecimalColumn; +use crate::types::BinaryColumn; use crate::types::NumberColumn; use crate::with_decimal_mapped_type; use crate::with_number_mapped_type; @@ -29,21 +30,17 @@ pub fn serialize_group_columns( num_rows: usize, serialize_size: usize, ) -> BinaryColumn { - // [`BinaryColumn`] consists of [`data`] and [`offset`], we build [`data`] and [`offset`] respectively, - // and then call `BinaryColumn::new(data.into(), offsets.into())` to create [`BinaryColumn`]. - let mut data: Vec = Vec::with_capacity(serialize_size); - let mut offsets: Vec = Vec::with_capacity(num_rows + 1); - unsafe { - offsets.push_unchecked(0); - for i in 0..num_rows { - for col in columns.iter() { - serialize_column_binary(col, i, &mut data); + let mut builder = BinaryColumnBuilder::with_capacity(num_rows, serialize_size); + + for i in 0..num_rows { + for col in columns.iter() { + unsafe { + serialize_column_binary(col, i, &mut builder.data); } - offsets.push_unchecked(data.len() as u64); } + builder.commit_row(); } - - BinaryColumn::new(data.into(), offsets.into()) + builder.build() } /// This function must be consistent with the `push_binary` function of `src/query/expression/src/values.rs`. diff --git a/src/query/expression/src/kernels/mod.rs b/src/query/expression/src/kernels/mod.rs index 738598406d23..01b1f2f02171 100644 --- a/src/query/expression/src/kernels/mod.rs +++ b/src/query/expression/src/kernels/mod.rs @@ -28,6 +28,7 @@ mod utils; pub use filter::FilterVisitor; pub use filter::IterationStrategy; +pub use filter::SELECTIVITY_THRESHOLD; pub use group_by_hash::*; pub use sort::*; pub use sort_compare::*; diff --git a/src/query/expression/src/kernels/scatter.rs b/src/query/expression/src/kernels/scatter.rs index 11d4ffbea53f..1d690ac929a8 100644 --- a/src/query/expression/src/kernels/scatter.rs +++ b/src/query/expression/src/kernels/scatter.rs @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_base::vec_ext::VecExt; use databend_common_exception::Result; -use itertools::Itertools; -use crate::kernels::utils::set_vec_len_by_ptr; -use crate::kernels::utils::store_advance_aligned; use crate::DataBlock; impl DataBlock { @@ -33,24 +31,9 @@ impl DataBlock { let scatter_indices = Self::divide_indices_by_scatter_size(indices, scatter_size); - let has_string_column = self - .columns() - .iter() - .any(|col| col.data_type.is_string_column()); - let mut string_items_buf = if has_string_column { - let max_num_rows = scatter_indices - .iter() - .map(|indices| indices.len()) - .max() - .unwrap(); - Some(vec![(0, 0); max_num_rows]) - } else { - None - }; - let mut results = Vec::with_capacity(scatter_size); for indices in scatter_indices.iter().take(scatter_size) { - let block = self.take(indices, &mut string_items_buf)?; + let block = self.take_with_optimize_size(indices)?; results.push(block); } @@ -68,18 +51,9 @@ impl DataBlock { for num_rows in scatter_num_rows.iter().take(scatter_size) { scatter_indices.push(Vec::with_capacity(*num_rows)); } - let mut scatter_indices_ptrs = scatter_indices - .iter_mut() - .map(|indices| indices.as_mut_ptr()) - .collect_vec(); + for (i, index) in indices.iter().enumerate() { - store_advance_aligned( - i as u32, - scatter_indices_ptrs.get_unchecked_mut(index.to_usize()), - ); - } - for i in 0..scatter_size { - set_vec_len_by_ptr(&mut scatter_indices[i], scatter_indices_ptrs[i]); + scatter_indices[index.to_usize()].push_unchecked(i as u32); } } scatter_indices diff --git a/src/query/expression/src/kernels/sort.rs b/src/query/expression/src/kernels/sort.rs index 4a9699c0e859..91bc7ebff299 100644 --- a/src/query/expression/src/kernels/sort.rs +++ b/src/query/expression/src/kernels/sort.rs @@ -105,7 +105,7 @@ impl DataBlock { } let permutations = sort_compare.take_permutation(); - DataBlock::take(block, &permutations, &mut None) + DataBlock::take(block, &permutations) } } diff --git a/src/query/expression/src/kernels/sort_compare.rs b/src/query/expression/src/kernels/sort_compare.rs index 45d91f0098d0..9f5078c621c3 100644 --- a/src/query/expression/src/kernels/sort_compare.rs +++ b/src/query/expression/src/kernels/sort_compare.rs @@ -23,6 +23,7 @@ use memchr::memchr; use crate::types::AnyType; use crate::types::NullableColumn; use crate::types::Number; +use crate::types::StringColumn; use crate::types::ValueType; use crate::visitor::ValueVisitor; use crate::LimitType; @@ -276,11 +277,21 @@ impl ValueVisitor for SortCompare { self.visit_number(buffer) } + fn visit_string(&mut self, column: StringColumn) -> Result<()> { + assert!(column.len() == self.rows); + self.generic_sort( + &column, + |col, idx| (col, idx as usize), + |(col1, idx1), (col2, idx2)| StringColumn::compare(col1, idx1, col2, idx2), + ); + Ok(()) + } + fn visit_typed_column(&mut self, col: T::Column) -> Result<()> { assert!(T::column_len(&col) == self.rows); self.generic_sort( &col, - |c, idx| -> T::ScalarRef<'_> { unsafe { T::index_column_unchecked(c, idx as _) } }, + |c, idx| unsafe { T::index_column_unchecked(c, idx as _) }, |a, b| T::compare(a, b), ); Ok(()) diff --git a/src/query/expression/src/kernels/take.rs b/src/query/expression/src/kernels/take.rs index 84d120b0390c..f4957a1c19f3 100644 --- a/src/query/expression/src/kernels/take.rs +++ b/src/query/expression/src/kernels/take.rs @@ -14,39 +14,52 @@ use std::sync::Arc; +use binary::BinaryColumnBuilder; +use databend_common_arrow::arrow::array::Array; +use databend_common_arrow::arrow::array::Utf8ViewArray; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_base::slice_ext::GetSaferUnchecked; use databend_common_exception::Result; +use string::StringColumnBuilder; -use crate::kernels::utils::copy_advance_aligned; -use crate::kernels::utils::set_vec_len_by_ptr; use crate::types::binary::BinaryColumn; use crate::types::nullable::NullableColumn; use crate::types::string::StringColumn; use crate::types::*; use crate::visitor::ValueVisitor; use crate::BlockEntry; +use crate::Column; use crate::ColumnBuilder; use crate::DataBlock; use crate::Value; +use crate::SELECTIVITY_THRESHOLD; pub const BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; impl DataBlock { - pub fn take( - &self, - indices: &[I], - string_items_buf: &mut Option>, - ) -> Result - where - I: databend_common_arrow::arrow::types::Index, - { + pub fn take(&self, indices: &[I]) -> Result + where I: databend_common_arrow::arrow::types::Index { if indices.is_empty() { return Ok(self.slice(0..0)); } - let mut taker = TakeVisitor::new(indices, string_items_buf); + let taker = TakeVisitor::new(indices); + self.take_inner(taker) + } + + pub fn take_with_optimize_size(&self, indices: &[I]) -> Result + where I: databend_common_arrow::arrow::types::Index { + if indices.is_empty() { + return Ok(self.slice(0..0)); + } + + let taker = TakeVisitor::new(indices).with_optimize_size_enable(true); + self.take_inner(taker) + } + fn take_inner(&self, mut taker: TakeVisitor) -> Result + where I: databend_common_arrow::arrow::types::Index { let after_columns = self .columns() .iter() @@ -62,7 +75,7 @@ impl DataBlock { Ok(DataBlock::new_with_meta( after_columns, - indices.len(), + taker.indices.len(), self.get_meta().cloned(), )) } @@ -72,20 +85,30 @@ struct TakeVisitor<'a, I> where I: databend_common_arrow::arrow::types::Index { indices: &'a [I], - string_items_buf: &'a mut Option>, result: Option>, + optimize_size_enable: bool, } impl<'a, I> TakeVisitor<'a, I> where I: databend_common_arrow::arrow::types::Index { - fn new(indices: &'a [I], string_items_buf: &'a mut Option>) -> Self { + fn new(indices: &'a [I]) -> Self { Self { indices, - string_items_buf, result: None, + optimize_size_enable: false, } } + + fn with_optimize_size_enable(mut self, optimize_size_enable: bool) -> Self { + self.optimize_size_enable = optimize_size_enable; + self + } + + fn should_optimize_size(&self, num_rows: usize) -> bool { + self.optimize_size_enable + || num_rows as f64 * SELECTIVITY_THRESHOLD > self.indices.len() as f64 + } } impl<'a, I> ValueVisitor for TakeVisitor<'a, I> @@ -168,9 +191,9 @@ where I: databend_common_arrow::arrow::types::Index // If this [`Bitmap`] is all true or all false and `num_rows <= bitmap.len()``, // we can just slice it. if num_rows <= col.len() && (col.unset_bits() == 0 || col.unset_bits() == col.len()) { - let mut bitmap = col.clone(); - bitmap.slice(0, num_rows); - self.result = Some(Value::Column(BooleanType::upcast_column(bitmap))); + self.result = Some(Value::Column(BooleanType::upcast_column( + col.sliced(0, num_rows), + ))); return Ok(()); } @@ -213,10 +236,9 @@ where I: databend_common_arrow::arrow::types::Index } fn visit_string(&mut self, column: StringColumn) -> Result<()> { - let column: BinaryColumn = column.into(); - self.result = Some(Value::Column(StringType::upcast_column(unsafe { - StringColumn::from_binary_unchecked(self.take_binary_types(&column)) - }))); + self.result = Some(Value::Column(StringType::upcast_column( + self.take_string_types(&column), + ))); Ok(()) } @@ -236,57 +258,61 @@ where I: databend_common_arrow::arrow::types::Index let result: Vec = self .indices .iter() - .map(|index| unsafe { *col.get_unchecked(index.to_usize()) }) + .map(|index| unsafe { *col.get_unchecked_release(index.to_usize()) }) .collect(); result.into() } fn take_binary_types(&mut self, col: &BinaryColumn) -> BinaryColumn { let num_rows = self.indices.len(); - - // Each element of `items` is (string pointer(u64), string length), if `string_items_buf` - // can be reused, we will not re-allocate memory. - let mut items: Option> = match &self.string_items_buf { - Some(string_items_buf) if string_items_buf.capacity() >= num_rows => None, - _ => Some(Vec::with_capacity(num_rows)), - }; - let items = match items.is_some() { - true => items.as_mut().unwrap(), - false => self.string_items_buf.as_mut().unwrap(), - }; - - // [`BinaryColumn`] consists of [`data`] and [`offset`], we build [`data`] and [`offset`] respectively, - // and then call `BinaryColumn::new(data.into(), offsets.into())` to create [`BinaryColumn`]. - let col_offset = col.offsets().as_slice(); - let col_data_ptr = col.data().as_slice().as_ptr(); - let mut offsets: Vec = Vec::with_capacity(num_rows + 1); - let mut data_size = 0; - - // Build [`offset`] and calculate `data_size` required by [`data`]. - unsafe { - items.set_len(num_rows); - offsets.set_len(num_rows + 1); - *offsets.get_unchecked_mut(0) = 0; - for (i, index) in self.indices.iter().enumerate() { - let start = *col_offset.get_unchecked(index.to_usize()) as usize; - let len = *col_offset.get_unchecked(index.to_usize() + 1) as usize - start; - data_size += len as u64; - *items.get_unchecked_mut(i) = (col_data_ptr.add(start) as u64, len); - *offsets.get_unchecked_mut(i + 1) = data_size; + let mut builder = BinaryColumnBuilder::with_capacity(num_rows, 0); + for index in self.indices.iter() { + unsafe { + builder.put_slice(col.index_unchecked(index.to_usize())); + builder.commit_row(); } } + builder.build() + } - // Build [`data`]. - let mut data: Vec = Vec::with_capacity(data_size as usize); - let mut data_ptr = data.as_mut_ptr(); - - unsafe { - for (str_ptr, len) in items.iter() { - copy_advance_aligned(*str_ptr as *const u8, &mut data_ptr, *len); + fn take_string_types(&mut self, col: &StringColumn) -> StringColumn { + if self.should_optimize_size(col.len()) { + let mut builder = StringColumnBuilder::with_capacity(self.indices.len()); + for index in self.indices.iter() { + unsafe { + builder.put_and_commit(col.index_unchecked(index.to_usize())); + } } - set_vec_len_by_ptr(&mut data, data_ptr); + builder.build() + } else { + let new_views = self.take_primitive_types(col.data.views().clone()); + let new_col = unsafe { + Utf8ViewArray::new_unchecked_unknown_md( + col.data.data_type().clone(), + new_views, + col.data.data_buffers().clone(), + None, + Some(col.data.total_buffer_len()), + ) + }; + StringColumn::new(new_col) } + } +} - BinaryColumn::new(data.into(), offsets.into()) +impl Column { + pub fn maybe_gc(self) -> Self { + match self { + Column::String(c) => { + let data = c.data.maybe_gc(); + let c = StringColumn::new(data); + Column::String(c) + } + Column::Nullable(n) => { + let c = n.column.maybe_gc(); + NullableColumn::new_column(c, n.validity) + } + other => other, + } } } diff --git a/src/query/expression/src/kernels/take_chunks.rs b/src/query/expression/src/kernels/take_chunks.rs index 34b0d2598fb9..d532a67fabb4 100644 --- a/src/query/expression/src/kernels/take_chunks.rs +++ b/src/query/expression/src/kernels/take_chunks.rs @@ -14,15 +14,15 @@ use std::sync::Arc; +use binary::BinaryColumnBuilder; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::buffer::Buffer; use databend_common_arrow::arrow::compute::merge_sort::MergeSlice; use databend_common_hashtable::RowPtr; use itertools::Itertools; +use string::StringColumnBuilder; use crate::kernels::take::BIT_MASK; -use crate::kernels::utils::copy_advance_aligned; -use crate::kernels::utils::set_vec_len_by_ptr; use crate::types::array::ArrayColumnBuilder; use crate::types::binary::BinaryColumn; use crate::types::bitmap::BitmapType; @@ -111,7 +111,6 @@ impl DataBlock { build_columns_data_type: &[DataType], indices: &[RowPtr], result_size: usize, - binary_items_buf: &mut Option>, ) -> Self { let num_columns = build_columns.len(); let result_columns = (0..num_columns) @@ -122,7 +121,6 @@ impl DataBlock { data_type.clone(), indices, result_size, - binary_items_buf, ); BlockEntry::new(data_type.clone(), Value::Column(column)) }) @@ -631,7 +629,6 @@ impl Column { data_type: DataType, indices: &[RowPtr], result_size: usize, - binary_items_buf: &mut Option>, ) -> Column { match &columns { ColumnVec::Null { .. } => Column::Null { len: result_size }, @@ -655,12 +652,12 @@ impl Column { ColumnVec::Boolean(columns) => { Column::Boolean(Self::take_block_vec_boolean_types(columns, indices)) } - ColumnVec::Binary(columns) => BinaryType::upcast_column( - Self::take_block_vec_binary_types(columns, indices, binary_items_buf.as_mut()), - ), - ColumnVec::String(columns) => StringType::upcast_column( - Self::take_block_vec_string_types(columns, indices, binary_items_buf.as_mut()), - ), + ColumnVec::Binary(columns) => { + BinaryType::upcast_column(Self::take_block_vec_binary_types(columns, indices)) + } + ColumnVec::String(columns) => { + StringType::upcast_column(Self::take_block_vec_string_types(columns, indices)) + } ColumnVec::Timestamp(columns) => { let builder = Self::take_block_vec_primitive_types(columns, indices); let ts = >::upcast_column(>::column_from_vec( @@ -713,9 +710,9 @@ impl Column { columns, builder, indices, ) } - ColumnVec::Bitmap(columns) => BitmapType::upcast_column( - Self::take_block_vec_binary_types(columns, indices, binary_items_buf.as_mut()), - ), + ColumnVec::Bitmap(columns) => { + BitmapType::upcast_column(Self::take_block_vec_binary_types(columns, indices)) + } ColumnVec::Nullable(columns) => { let inner_data_type = data_type.as_nullable().unwrap(); let inner_column = Self::take_column_vec_indices( @@ -723,7 +720,6 @@ impl Column { *inner_data_type.clone(), indices, result_size, - binary_items_buf, ); let inner_bitmap = Self::take_column_vec_indices( @@ -731,7 +727,6 @@ impl Column { DataType::Boolean, indices, result_size, - binary_items_buf, ); NullableColumn::new_column( @@ -750,25 +745,22 @@ impl Column { ty.clone(), indices, result_size, - binary_items_buf, ) }) .collect(); Column::Tuple(fields) } - ColumnVec::Variant(columns) => VariantType::upcast_column( - Self::take_block_vec_binary_types(columns, indices, binary_items_buf.as_mut()), - ), - ColumnVec::Geometry(columns) => GeometryType::upcast_column( - Self::take_block_vec_binary_types(columns, indices, binary_items_buf.as_mut()), - ), + ColumnVec::Variant(columns) => { + VariantType::upcast_column(Self::take_block_vec_binary_types(columns, indices)) + } + ColumnVec::Geometry(columns) => { + GeometryType::upcast_column(Self::take_block_vec_binary_types(columns, indices)) + } ColumnVec::Geography(columns) => { let columns = columns.iter().map(|x| x.0.clone()).collect::>(); GeographyType::upcast_column(GeographyColumn(Self::take_block_vec_binary_types( - &columns, - indices, - binary_items_buf.as_mut(), + &columns, indices, ))) } } @@ -784,73 +776,30 @@ impl Column { builder } - pub fn take_block_vec_binary_types( - col: &[BinaryColumn], - indices: &[RowPtr], - binary_items_buf: Option<&mut Vec<(u64, usize)>>, - ) -> BinaryColumn { - let num_rows = indices.len(); - - // Each element of `items` is (string pointer(u64), string length), if `binary_items_buf` - // can be reused, we will not re-allocate memory. - let mut items: Option> = match &binary_items_buf { - Some(binary_items_buf) if binary_items_buf.capacity() >= num_rows => None, - _ => Some(Vec::with_capacity(num_rows)), - }; - let items = match items.is_some() { - true => items.as_mut().unwrap(), - false => binary_items_buf.unwrap(), - }; - - // [`BinaryColumn`] consists of [`data`] and [`offset`], we build [`data`] and [`offset`] respectively, - // and then call `BinaryColumn::new(data.into(), offsets.into())` to create [`BinaryColumn`]. - let mut offsets: Vec = Vec::with_capacity(num_rows + 1); - let mut data_size = 0; - - // Build [`offset`] and calculate `data_size` required by [`data`]. - unsafe { - items.set_len(num_rows); - offsets.set_len(num_rows + 1); - *offsets.get_unchecked_mut(0) = 0; - for (i, row_ptr) in indices.iter().enumerate() { - let item = - col[row_ptr.chunk_index as usize].index_unchecked(row_ptr.row_index as usize); - data_size += item.len() as u64; - *items.get_unchecked_mut(i) = (item.as_ptr() as u64, item.len()); - *offsets.get_unchecked_mut(i + 1) = data_size; - } - } - - // Build [`data`]. - let mut data: Vec = Vec::with_capacity(data_size as usize); - let mut data_ptr = data.as_mut_ptr(); - - unsafe { - for (str_ptr, len) in items.iter() { - copy_advance_aligned(*str_ptr as *const u8, &mut data_ptr, *len); + // TODO: reuse the buffer by `SELECTIVITY_THRESHOLD` + pub fn take_block_vec_binary_types(col: &[BinaryColumn], indices: &[RowPtr]) -> BinaryColumn { + let mut builder = BinaryColumnBuilder::with_capacity(indices.len(), 0); + for row_ptr in indices { + unsafe { + builder.put_slice( + col[row_ptr.chunk_index as usize].index_unchecked(row_ptr.row_index as usize), + ); + builder.commit_row(); } - set_vec_len_by_ptr(&mut data, data_ptr); } - - BinaryColumn::new(data.into(), offsets.into()) + builder.build() } - pub fn take_block_vec_string_types( - cols: &[StringColumn], - indices: &[RowPtr], - binary_items_buf: Option<&mut Vec<(u64, usize)>>, - ) -> StringColumn { - let binary_cols = cols - .iter() - .map(|col| col.clone().into()) - .collect::>(); - unsafe { - StringColumn::from_binary_unchecked(Self::take_block_vec_binary_types( - &binary_cols, - indices, - binary_items_buf, - )) + pub fn take_block_vec_string_types(col: &[StringColumn], indices: &[RowPtr]) -> StringColumn { + let mut builder = StringColumnBuilder::with_capacity(indices.len()); + for row_ptr in indices { + unsafe { + builder.put_and_commit( + col[row_ptr.chunk_index as usize].index_unchecked(row_ptr.row_index as usize), + ); + } } + builder.build() } pub fn take_block_vec_boolean_types(col: &[Bitmap], indices: &[RowPtr]) -> Bitmap { diff --git a/src/query/expression/src/kernels/take_compact.rs b/src/query/expression/src/kernels/take_compact.rs index a2f97b894956..2cf400264b6d 100644 --- a/src/query/expression/src/kernels/take_compact.rs +++ b/src/query/expression/src/kernels/take_compact.rs @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use binary::BinaryColumnBuilder; +use databend_common_arrow::arrow::array::Array; +use databend_common_arrow::arrow::array::Utf8ViewArray; use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_base::vec_ext::VecExt; use databend_common_exception::Result; -use crate::kernels::utils::copy_advance_aligned; -use crate::kernels::utils::set_vec_len_by_ptr; -use crate::kernels::utils::store_advance_aligned; use crate::types::binary::BinaryColumn; use crate::types::nullable::NullableColumn; use crate::types::string::StringColumn; @@ -161,11 +162,10 @@ impl<'a> ValueVisitor for TakeCompactVisitor<'a> { Ok(()) } - fn visit_string(&mut self, column: StringColumn) -> Result<()> { - let column: BinaryColumn = column.into(); - self.result = Some(Value::Column(StringType::upcast_column(unsafe { - StringColumn::from_binary_unchecked(self.take_binary_types(&column)) - }))); + fn visit_string(&mut self, col: StringColumn) -> Result<()> { + self.result = Some(Value::Column(StringType::upcast_column( + self.take_string_types(&col), + ))); Ok(()) } @@ -179,15 +179,14 @@ impl<'a> ValueVisitor for TakeCompactVisitor<'a> { impl<'a> TakeCompactVisitor<'a> { fn take_primitive_types(&mut self, buffer: Buffer) -> Buffer { - let col_ptr = buffer.as_slice().as_ptr(); + let buffer = buffer.as_slice(); let mut builder: Vec = Vec::with_capacity(self.num_rows); - let mut ptr = builder.as_mut_ptr(); let mut remain; unsafe { for (index, cnt) in self.indices.iter() { if *cnt == 1 { - copy_advance_aligned(col_ptr.add(*index as usize), &mut ptr, 1); + builder.push_unchecked(buffer[*index as usize]); continue; } @@ -195,11 +194,12 @@ impl<'a> TakeCompactVisitor<'a> { // [___________] => [x__________] => [xx_________] => [xxxx_______] => [xxxxxxxx___] // Since cnt > 0, then 31 - cnt.leading_zeros() >= 0. let max_segment = 1 << (31 - cnt.leading_zeros()); - let base_ptr = ptr; - copy_advance_aligned(col_ptr.add(*index as usize), &mut ptr, 1); + let base_pos = builder.len(); + builder.push_unchecked(buffer[*index as usize]); + let mut cur_segment = 1; while cur_segment < max_segment { - copy_advance_aligned(base_ptr, &mut ptr, cur_segment); + builder.extend_from_within(base_pos..base_pos + cur_segment); cur_segment <<= 1; } @@ -208,78 +208,39 @@ impl<'a> TakeCompactVisitor<'a> { // ^^^^ ---> ^^^^ remain = *cnt as usize - max_segment; if remain > 0 { - copy_advance_aligned(base_ptr, &mut ptr, remain); + builder.extend_from_within(base_pos..base_pos + remain) } } - set_vec_len_by_ptr(&mut builder, ptr); } builder.into() } fn take_binary_types(&mut self, col: &BinaryColumn) -> BinaryColumn { - // Each element of `items` is (string(&[u8]), repeat times). - let mut items = Vec::with_capacity(self.indices.len()); - let mut items_ptr = items.as_mut_ptr(); - - // [`BinaryColumn`] consists of [`data`] and [`offset`], we build [`data`] and [`offset`] respectively, - // and then call `BinaryColumn::new(data.into(), offsets.into())` to create [`BinaryColumn`]. - let mut offsets = Vec::with_capacity(self.num_rows + 1); - let mut offsets_ptr = offsets.as_mut_ptr(); - let mut data_size = 0; - - // Build [`offset`] and calculate `data_size` required by [`data`]. - unsafe { - store_advance_aligned::(0, &mut offsets_ptr); - for (index, cnt) in self.indices.iter() { - let item = col.index_unchecked(*index as usize); - store_advance_aligned((item, *cnt), &mut items_ptr); - for _ in 0..*cnt { - data_size += item.len() as u64; - store_advance_aligned(data_size, &mut offsets_ptr); - } - } - set_vec_len_by_ptr(&mut offsets, offsets_ptr); - set_vec_len_by_ptr(&mut items, items_ptr); - } - - // Build [`data`]. - let mut data: Vec = Vec::with_capacity(data_size as usize); - let mut data_ptr = data.as_mut_ptr(); - let mut remain; - - unsafe { - for (item, cnt) in items { - let len = item.len(); - if cnt == 1 { - copy_advance_aligned(item.as_ptr(), &mut data_ptr, len); - continue; - } - - // Using the doubling method to copy the max segment memory. - // [___________] => [x__________] => [xx_________] => [xxxx_______] => [xxxxxxxx___] - // Since cnt > 0, then 31 - cnt.leading_zeros() >= 0. - let max_bit_num = 1 << (31 - cnt.leading_zeros()); - let max_segment = max_bit_num * len; - let base_data_ptr = data_ptr; - copy_advance_aligned(item.as_ptr(), &mut data_ptr, len); - let mut cur_segment = len; - while cur_segment < max_segment { - copy_advance_aligned(base_data_ptr, &mut data_ptr, cur_segment); - cur_segment <<= 1; - } - - // Copy the remaining memory directly. - // [xxxxxxxxxx____] => [xxxxxxxxxxxxxx] - // ^^^^ ---> ^^^^ - remain = cnt as usize - max_bit_num; - if remain > 0 { - copy_advance_aligned(base_data_ptr, &mut data_ptr, remain * len); + let num_rows = self.num_rows; + let mut builder = BinaryColumnBuilder::with_capacity(num_rows, 0); + for (index, cnt) in self.indices.iter() { + for _ in 0..*cnt { + unsafe { + builder.put_slice(col.index_unchecked(*index as usize)); + builder.commit_row(); } } - set_vec_len_by_ptr(&mut data, data_ptr); } + builder.build() + } - BinaryColumn::new(data.into(), offsets.into()) + fn take_string_types(&mut self, col: &StringColumn) -> StringColumn { + let new_views = self.take_primitive_types(col.data.views().clone()); + let new_col = unsafe { + Utf8ViewArray::new_unchecked_unknown_md( + col.data.data_type().clone(), + new_views, + col.data.data_buffers().clone(), + None, + Some(col.data.total_buffer_len()), + ) + }; + StringColumn::new(new_col) } } diff --git a/src/query/expression/src/kernels/take_ranges.rs b/src/query/expression/src/kernels/take_ranges.rs index 3c9159abb44b..872f3f5829ef 100644 --- a/src/query/expression/src/kernels/take_ranges.rs +++ b/src/query/expression/src/kernels/take_ranges.rs @@ -14,9 +14,13 @@ use core::ops::Range; +use binary::BinaryColumnBuilder; +use databend_common_arrow::arrow::array::Array; +use databend_common_arrow::arrow::array::Utf8ViewArray; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_base::vec_ext::VecExt; use databend_common_exception::Result; use crate::types::binary::BinaryColumn; @@ -31,6 +35,7 @@ use crate::Value; impl DataBlock { // Generate a new `DataBlock` by the specified indices ranges. + // ranges already cover most data pub fn take_ranges(self, ranges: &[Range], num_rows: usize) -> Result { debug_assert_eq!( ranges @@ -154,8 +159,19 @@ impl<'a> ValueVisitor for TakeRangeVisitor<'a> { } fn visit_boolean(&mut self, bitmap: Bitmap) -> Result<()> { - let mut builder = MutableBitmap::with_capacity(self.num_rows); + // Fast path: avoid iterating column to generate a new bitmap. + // If this [`Bitmap`] is all true or all false and `num_rows <= bitmap.len()``, + // we can just slice it. + if self.num_rows <= bitmap.len() + && (bitmap.unset_bits() == 0 || bitmap.unset_bits() == bitmap.len()) + { + self.result = Some(Value::Column(BooleanType::upcast_column( + bitmap.sliced(0, self.num_rows), + ))); + return Ok(()); + } + let mut builder = MutableBitmap::with_capacity(self.num_rows); let src = bitmap.values(); let offset = bitmap.offset(); self.ranges.iter().for_each(|range| { @@ -176,10 +192,9 @@ impl<'a> ValueVisitor for TakeRangeVisitor<'a> { } fn visit_string(&mut self, column: StringColumn) -> Result<()> { - let column: BinaryColumn = column.into(); - self.result = Some(Value::Column(StringType::upcast_column(unsafe { - StringColumn::from_binary_unchecked(self.take_binary_types(&column)) - }))); + self.result = Some(Value::Column(StringType::upcast_column( + self.take_string_types(&column), + ))); Ok(()) } @@ -196,37 +211,37 @@ impl<'a> TakeRangeVisitor<'a> { let mut builder: Vec = Vec::with_capacity(self.num_rows); let values = buffer.as_slice(); for range in self.ranges { - builder.extend(&values[range.start as usize..range.end as usize]); + unsafe { + builder + .extend_from_slice_unchecked(&values[range.start as usize..range.end as usize]) + }; } builder.into() } fn take_binary_types(&mut self, values: &BinaryColumn) -> BinaryColumn { - let mut offsets: Vec = Vec::with_capacity(self.num_rows + 1); - let mut data_size = 0; - - let value_data = values.data().as_slice(); - let values_offset = values.offsets().as_slice(); - // Build [`offset`] and calculate `data_size` required by [`data`]. - offsets.push(0); + let mut builder = BinaryColumnBuilder::with_capacity(self.num_rows, 0); for range in self.ranges { - let mut offset_start = values_offset[range.start as usize]; - for offset_end in values_offset[range.start as usize + 1..range.end as usize + 1].iter() - { - data_size += offset_end - offset_start; - offset_start = *offset_end; - offsets.push(data_size); + for index in range.start as usize..range.end as usize { + let value = unsafe { values.index_unchecked(index) }; + builder.put_slice(value); + builder.commit_row(); } } + builder.build() + } - // Build [`data`]. - let mut data: Vec = Vec::with_capacity(data_size as usize); - for range in self.ranges { - let col_data = &value_data[values_offset[range.start as usize] as usize - ..values_offset[range.end as usize] as usize]; - data.extend_from_slice(col_data); - } - - BinaryColumn::new(data.into(), offsets.into()) + fn take_string_types(&mut self, col: &StringColumn) -> StringColumn { + let new_views = self.take_primitive_types(col.data.views().clone()); + let new_col = unsafe { + Utf8ViewArray::new_unchecked_unknown_md( + col.data.data_type().clone(), + new_views, + col.data.data_buffers().clone(), + None, + Some(col.data.total_buffer_len()), + ) + }; + StringColumn::new(new_col) } } diff --git a/src/query/expression/src/kernels/utils.rs b/src/query/expression/src/kernels/utils.rs index 0e37bcb4e470..b5f8b0d98b34 100644 --- a/src/query/expression/src/kernels/utils.rs +++ b/src/query/expression/src/kernels/utils.rs @@ -12,100 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::take::BIT_MASK; - -/// # Safety -/// -/// * `ptr` must be [valid] for writes of `size_of::()` bytes. -/// * The region of memory beginning at `val` with a size of `size_of::()` -/// bytes must *not* overlap with the region of memory beginning at `ptr` -/// with the same size. -#[inline] -pub unsafe fn store_advance(val: &T, ptr: &mut *mut u8) { - unsafe { - std::ptr::copy_nonoverlapping(val as *const T as *const u8, *ptr, std::mem::size_of::()); - *ptr = ptr.add(std::mem::size_of::()) - } -} - -/// # Safety -/// -/// * `ptr` must be [valid] for writes. -/// * `ptr` must be properly aligned. -#[inline] -pub unsafe fn store_advance_aligned(val: T, ptr: &mut *mut T) { - unsafe { - std::ptr::write(*ptr, val); - *ptr = ptr.add(1) - } -} - -/// # Safety -/// -/// * `src` must be [valid] for reads of `count * size_of::()` bytes. -/// * `ptr` must be [valid] for writes of `count * size_of::()` bytes. -/// * Both `src` and `dst` must be properly aligned. -/// * The region of memory beginning at `val` with a size of `count * size_of::()` -/// bytes must *not* overlap with the region of memory beginning at `ptr` with the -/// same size. -#[inline] -pub unsafe fn copy_advance_aligned(src: *const T, ptr: &mut *mut T, count: usize) { - unsafe { - std::ptr::copy_nonoverlapping(src, *ptr, count); - *ptr = ptr.add(count); - } -} - -/// # Safety -/// * `src` + `src_idx`(in bits) must be [valid] for reads of `len` bits. -/// * `ptr` must be [valid] for writes of `len` bits. -pub unsafe fn copy_continuous_bits( - ptr: &mut *mut u8, - src: &[u8], - mut dst_idx: usize, - mut src_idx: usize, - len: usize, -) -> (u8, usize) { - let mut unset_bits = 0; - let chunks = BitChunks::new(src, src_idx, len); - chunks.iter().for_each(|chunk| { - unset_bits += chunk.count_zeros(); - copy_advance_aligned(&chunk as *const _ as *const u8, ptr, 8); - }); - - let mut remainder = chunks.remainder_len(); - dst_idx += len - remainder; - src_idx += len - remainder; - - let mut buf = 0; - while remainder > 0 { - if (*src.as_ptr().add(src_idx >> 3) & BIT_MASK[src_idx & 7]) != 0 { - buf |= BIT_MASK[dst_idx % 8]; - } else { - unset_bits += 1; - } - src_idx += 1; - dst_idx += 1; - remainder -= 1; - if dst_idx % 8 == 0 { - store_advance_aligned(buf, ptr); - buf = 0; - } - } - (buf, unset_bits as usize) -} - -/// # Safety -/// -/// * `(ptr as usize - vec.as_ptr() as usize) / std::mem::size_of::()` must be -/// less than or equal to the capacity of Vec. -#[inline] -pub unsafe fn set_vec_len_by_ptr(vec: &mut Vec, ptr: *const T) { - unsafe { - vec.set_len(ptr.offset_from(vec.as_ptr()) as usize); - } -} - /// # Safety /// # As: core::ptr::copy_nonoverlapping #[inline] @@ -119,99 +25,3 @@ pub unsafe fn store(val: &T, ptr: *mut u8) { pub unsafe fn read(ptr: *const u8) -> T { core::ptr::read_unaligned::(ptr as _) } - -/// Iterates over an arbitrarily aligned byte buffer -/// -/// Yields an iterator of u64, and a remainder. The first byte in the buffer -/// will be the least significant byte in output u64 -#[derive(Debug)] -pub struct BitChunks<'a> { - buffer: &'a [u8], - /// offset inside a byte, guaranteed to be between 0 and 7 (inclusive) - bit_offset: usize, - /// number of complete u64 chunks - chunk_len: usize, - /// number of remaining bits, guaranteed to be between 0 and 63 (inclusive) - remainder_len: usize, -} - -impl<'a> BitChunks<'a> { - pub fn new(buffer: &'a [u8], offset: usize, len: usize) -> Self { - assert!((offset + len + 7) / 8 <= buffer.len() * 8); - - let byte_offset = offset / 8; - let bit_offset = offset % 8; - - // number of complete u64 chunks - let chunk_len = len / 64; - // number of remaining bits - let remainder_len = len % 64; - - BitChunks::<'a> { - buffer: &buffer[byte_offset..], - bit_offset, - chunk_len, - remainder_len, - } - } -} - -#[derive(Debug)] -pub struct BitChunkIterator { - buffer: *const u64, - bit_offset: usize, - chunk_len: usize, - index: usize, -} - -impl<'a> BitChunks<'a> { - /// Returns the number of remaining bits, guaranteed to be between 0 and 63 (inclusive) - #[inline] - pub const fn remainder_len(&self) -> usize { - self.remainder_len - } - - /// Returns an iterator over chunks of 64 bits represented as an u64 - #[inline] - pub const fn iter(&self) -> BitChunkIterator { - BitChunkIterator { - buffer: self.buffer.as_ptr() as *const u64, - bit_offset: self.bit_offset, - chunk_len: self.chunk_len, - index: 0, - } - } -} - -impl Iterator for BitChunkIterator { - type Item = u64; - - #[inline] - fn next(&mut self) -> Option { - let index = self.index; - if index >= self.chunk_len { - return None; - } - - // bit-packed buffers are stored starting with the least-significant byte first - // so when reading as u64 on a big-endian machine, the bytes need to be swapped - let current = unsafe { std::ptr::read_unaligned(self.buffer.add(index)).to_le() }; - - let bit_offset = self.bit_offset; - - let combined = if bit_offset == 0 { - current - } else { - // the constructor ensures that bit_offset is in 0..8 - // that means we need to read at most one additional byte to fill in the high bits - let next = - unsafe { std::ptr::read_unaligned(self.buffer.add(index + 1) as *const u8) as u64 }; - - (current >> bit_offset) | (next << (64 - bit_offset)) - }; - - self.index = index + 1; - - Some(combined) - } -} diff --git a/src/query/expression/src/row/row_converter.rs b/src/query/expression/src/row/row_converter.rs index 3c4d967ae5c0..f259df073192 100644 --- a/src/query/expression/src/row/row_converter.rs +++ b/src/query/expression/src/row/row_converter.rs @@ -84,10 +84,7 @@ impl RowConverter { encode_column(&mut builder, column, field.asc, field.nulls_first); } - let rows = builder.build(); - debug_assert_eq!(*rows.offsets().last().unwrap(), rows.data().len() as u64); - debug_assert!(rows.offsets().windows(2).all(|w| w[0] <= w[1])); - rows + builder.build() } fn new_empty_rows(&self, cols: &[Column], num_rows: usize) -> BinaryColumnBuilder { diff --git a/src/query/expression/src/types/array.rs b/src/query/expression/src/types/array.rs index 83d25cdb4adf..058954d323b0 100755 --- a/src/query/expression/src/types/array.rs +++ b/src/query/expression/src/types/array.rs @@ -232,6 +232,7 @@ impl ArrayColumn { } pub fn slice(&self, range: Range) -> Self { + // We need keep the last offsets in slice let offsets = self .offsets .clone() diff --git a/src/query/expression/src/types/bitmap.rs b/src/query/expression/src/types/bitmap.rs index 1823941ba2b7..ab411346980a 100644 --- a/src/query/expression/src/types/bitmap.rs +++ b/src/query/expression/src/types/bitmap.rs @@ -161,7 +161,7 @@ impl ValueType for BitmapType { } fn column_memory_size(col: &Self::Column) -> usize { - col.data().len() + col.offsets().len() * 8 + col.memory_size() } #[inline(always)] diff --git a/src/query/expression/src/types/geography.rs b/src/query/expression/src/types/geography.rs index 8c0d95e92c4e..b41aad37d9df 100644 --- a/src/query/expression/src/types/geography.rs +++ b/src/query/expression/src/types/geography.rs @@ -19,6 +19,7 @@ use std::ops::Range; use borsh::BorshDeserialize; use borsh::BorshSerialize; +use databend_common_arrow::arrow::trusted_len::TrustedLen; use databend_common_exception::Result; use databend_common_io::geography::*; use databend_common_io::wkb::make_point; @@ -29,8 +30,7 @@ use geozero::ToWkt; use serde::Deserialize; use serde::Serialize; -use super::binary::BinaryLike; -use super::binary::BinaryLikeIterator; +use super::binary::BinaryIterator; use crate::property::Domain; use crate::types::binary::BinaryColumn; use crate::types::binary::BinaryColumnBuilder; @@ -83,12 +83,6 @@ impl<'a> GeographyRef<'a> { } } -impl<'a> BinaryLike<'a> for GeographyRef<'a> { - fn from(value: &'a [u8]) -> Self { - GeographyRef(value) - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct GeographyType; @@ -281,10 +275,8 @@ impl GeographyColumn { } pub fn iter(&self) -> GeographyIterator<'_> { - BinaryLikeIterator { - data: &self.0.data, - offsets: self.0.offsets.windows(2), - _t: std::marker::PhantomData, + GeographyIterator { + inner: self.0.iter(), } } @@ -293,4 +285,18 @@ impl GeographyColumn { } } -pub type GeographyIterator<'a> = BinaryLikeIterator<'a, GeographyRef<'a>>; +pub struct GeographyIterator<'a> { + inner: BinaryIterator<'a>, +} + +impl<'a> Iterator for GeographyIterator<'a> { + type Item = GeographyRef<'a>; + + fn next(&mut self) -> Option { + self.inner.next().map(GeographyRef) + } +} + +unsafe impl<'a> TrustedLen for GeographyIterator<'a> {} + +unsafe impl<'a> std::iter::TrustedLen for GeographyIterator<'a> {} diff --git a/src/query/expression/src/types/geometry.rs b/src/query/expression/src/types/geometry.rs index 67f1afc95ec9..d1dfe01911cc 100644 --- a/src/query/expression/src/types/geometry.rs +++ b/src/query/expression/src/types/geometry.rs @@ -165,7 +165,7 @@ impl ValueType for GeometryType { } fn column_memory_size(col: &Self::Column) -> usize { - col.data().len() + col.offsets().len() * 8 + col.memory_size() } #[inline(always)] diff --git a/src/query/expression/src/types/string.rs b/src/query/expression/src/types/string.rs index c4fe32534c96..bc06218b358f 100644 --- a/src/query/expression/src/types/string.rs +++ b/src/query/expression/src/types/string.rs @@ -13,26 +13,23 @@ // limitations under the License. use std::cmp::Ordering; -use std::iter::once; use std::ops::Range; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_arrow::arrow::array::MutableBinaryViewArray; +use databend_common_arrow::arrow::array::Utf8ViewArray; use databend_common_arrow::arrow::trusted_len::TrustedLen; +use databend_common_base::slice_ext::GetSaferUnchecked; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use serde::Deserialize; -use serde::Serialize; use super::binary::BinaryColumn; use super::binary::BinaryColumnBuilder; -use super::binary::BinaryIterator; use crate::property::Domain; use crate::types::ArgType; use crate::types::DataType; use crate::types::DecimalSize; use crate::types::GenericMap; use crate::types::ValueType; -use crate::utils::arrow::buffer_into_mut; use crate::values::Column; use crate::values::Scalar; use crate::ColumnBuilder; @@ -137,8 +134,7 @@ impl ValueType for StringType { } fn push_item(builder: &mut Self::ColumnBuilder, item: Self::ScalarRef<'_>) { - builder.put_str(item); - builder.commit_row(); + builder.put_and_commit(item); } fn push_item_repeat(builder: &mut Self::ColumnBuilder, item: Self::ScalarRef<'_>, n: usize) { @@ -146,7 +142,7 @@ impl ValueType for StringType { } fn push_default(builder: &mut Self::ColumnBuilder) { - builder.commit_row(); + builder.put_and_commit(""); } fn append_column(builder: &mut Self::ColumnBuilder, other_builder: &Self::Column) { @@ -166,7 +162,7 @@ impl ValueType for StringType { } fn column_memory_size(col: &Self::Column) -> usize { - col.data().len() + col.offsets().len() * 8 + col.memory_size() } #[inline(always)] @@ -218,85 +214,38 @@ impl ArgType for StringType { } fn create_builder(capacity: usize, _: &GenericMap) -> Self::ColumnBuilder { - StringColumnBuilder::with_capacity(capacity, 0) + StringColumnBuilder::with_capacity(capacity) } } -#[derive(Clone, PartialEq)] +#[derive(Clone)] pub struct StringColumn { - data: Buffer, - offsets: Buffer, + pub(crate) data: Utf8ViewArray, } impl StringColumn { - pub fn new(data: Buffer, offsets: Buffer) -> Self { - let col = BinaryColumn::new(data, offsets); - - col.check_utf8().unwrap(); - - unsafe { Self::from_binary_unchecked(col) } - } - - /// # Safety - /// This function is unsound iff: - /// * the offsets are not monotonically increasing - /// * The `data` between two consecutive `offsets` are not valid utf8 - pub unsafe fn new_unchecked(data: Buffer, offsets: Buffer) -> Self { - let col = BinaryColumn::new(data, offsets); - - #[cfg(debug_assertions)] - col.check_utf8().unwrap(); - - unsafe { Self::from_binary_unchecked(col) } - } - - /// # Safety - /// This function is unsound iff: - /// * the offsets are not monotonically increasing - /// * The `data` between two consecutive `offsets` are not valid utf8 - pub unsafe fn from_binary_unchecked(col: BinaryColumn) -> Self { - #[cfg(debug_assertions)] - col.check_utf8().unwrap(); - - StringColumn { - data: col.data, - offsets: col.offsets, - } + pub fn new(data: Utf8ViewArray) -> Self { + Self { data } } pub fn len(&self) -> usize { - self.offsets.len() - 1 + self.data.len() } pub fn current_buffer_len(&self) -> usize { - (*self.offsets().last().unwrap() - *self.offsets().first().unwrap()) as _ - } - - pub fn data(&self) -> &Buffer { - &self.data - } - - pub fn offsets(&self) -> &Buffer { - &self.offsets + self.data.total_bytes_len() } pub fn memory_size(&self) -> usize { - let offsets = self.offsets.as_slice(); - let len = offsets.len(); - len * 8 + (offsets[len - 1] - offsets[0]) as usize + self.data.total_buffer_len() + self.len() * 12 } pub fn index(&self, index: usize) -> Option<&str> { - if index + 1 >= self.offsets.len() { + if index >= self.len() { return None; } - let bytes = &self.data[(self.offsets[index] as usize)..(self.offsets[index + 1] as usize)]; - - #[cfg(debug_assertions)] - bytes.check_utf8().unwrap(); - - unsafe { Some(std::str::from_utf8_unchecked(bytes)) } + Some(unsafe { self.index_unchecked(index) }) } /// # Safety @@ -304,16 +253,9 @@ impl StringColumn { /// Calling this method with an out-of-bounds index is *[undefined behavior]* #[inline] pub unsafe fn index_unchecked(&self, index: usize) -> &str { - debug_assert!(index + 1 < self.offsets.len()); - - let start = *self.offsets.get_unchecked(index) as usize; - let end = *self.offsets.get_unchecked(index + 1) as usize; - let bytes = &self.data.get_unchecked(start..end); - - #[cfg(debug_assertions)] - bytes.check_utf8().unwrap(); + debug_assert!(index < self.data.len()); - std::str::from_utf8_unchecked(bytes) + self.data.value_unchecked(index) } /// # Safety @@ -321,71 +263,113 @@ impl StringColumn { /// Calling this method with an out-of-bounds index is *[undefined behavior]* #[inline] pub unsafe fn index_unchecked_bytes(&self, index: usize) -> &[u8] { - debug_assert!(index + 1 < self.offsets.len()); + debug_assert!(index < self.data.len()); - let start = *self.offsets.get_unchecked(index) as usize; - let end = *self.offsets.get_unchecked(index + 1) as usize; - self.data.get_unchecked(start..end) + self.data.value_unchecked(index).as_bytes() } pub fn slice(&self, range: Range) -> Self { - let offsets = self - .offsets + let data = self + .data .clone() - .sliced(range.start, range.end - range.start + 1); - StringColumn { - data: self.data.clone(), - offsets, - } + .sliced(range.start, range.end - range.start); + Self { data } } pub fn iter(&self) -> StringIterator { StringIterator { - data: &self.data, - offsets: self.offsets.windows(2), + col: self, + index: 0, } } - pub fn iter_binary(&self) -> BinaryIterator { - BinaryIterator { - data: &self.data, - offsets: self.offsets.windows(2), - _t: std::marker::PhantomData, - } + pub fn into_inner(self) -> Utf8ViewArray { + self.data } - pub fn into_buffer(self) -> (Buffer, Buffer) { - (self.data, self.offsets) + pub fn try_from_binary(col: BinaryColumn) -> Result { + let builder = StringColumnBuilder::try_from_bin_column(col)?; + Ok(builder.build()) } - pub fn check_valid(&self) -> Result<()> { - let offsets = self.offsets.as_slice(); - let len = offsets.len(); - if len < 1 { - return Err(ErrorCode::Internal(format!( - "StringColumn offsets length must be equal or greater than 1, but got {}", - len - ))); - } + pub fn compare(col_i: &Self, i: usize, col_j: &Self, j: usize) -> Ordering { + let view_i = unsafe { col_i.data.views().as_slice().get_unchecked_release(i) }; + let view_j = unsafe { col_j.data.views().as_slice().get_unchecked_release(j) }; - for i in 1..len { - if offsets[i] < offsets[i - 1] { - return Err(ErrorCode::Internal(format!( - "StringColumn offsets value must be equal or greater than previous value, but got {}", - offsets[i] - ))); + if view_i.prefix == view_j.prefix { + unsafe { + let value_i = col_i.data.value_unchecked(i); + let value_j = col_j.data.value_unchecked(j); + value_i.cmp(value_j) } + } else { + view_i + .prefix + .to_le_bytes() + .cmp(&view_j.prefix.to_le_bytes()) + } + } + + pub fn compare_str(col: &Self, i: usize, value: &str) -> Ordering { + let view = unsafe { col.data.views().as_slice().get_unchecked_release(i) }; + let prefix = load_prefix(value.as_bytes()); + + if view.prefix == prefix { + let value_i = unsafe { col.data.value_unchecked(i) }; + value_i.cmp(value) + } else { + view.prefix.to_le_bytes().as_slice().cmp(value.as_bytes()) } - Ok(()) } } -impl From for BinaryColumn { - fn from(col: StringColumn) -> BinaryColumn { - BinaryColumn { - data: col.data, - offsets: col.offsets, +// Loads (up to) the first 4 bytes of s as little-endian, padded with zeros. +#[inline] +fn load_prefix(s: &[u8]) -> u32 { + let start = &s[..s.len().min(4)]; + let mut tmp = [0u8; 4]; + tmp[..start.len()].copy_from_slice(start); + u32::from_le_bytes(tmp) +} + +impl PartialEq for StringColumn { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == Ordering::Equal + } +} + +impl Eq for StringColumn {} + +impl PartialOrd for StringColumn { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for StringColumn { + fn cmp(&self, other: &Self) -> Ordering { + for i in 0..self.len().max(other.len()) { + match (self.data.views().get(i), other.data.views().get(i)) { + (Some(left), Some(right)) => { + match left.prefix.to_le_bytes().cmp(&right.prefix.to_le_bytes()) { + Ordering::Equal => unsafe { + let left = self.data.value_unchecked(i); + let right = other.data.value_unchecked(i); + match left.cmp(right) { + Ordering::Equal => continue, + non_eq => return non_eq, + } + }, + non_eq => return non_eq, + } + } + (Some(_), None) => return Ordering::Greater, + (None, Some(_)) => return Ordering::Less, + (None, None) => return Ordering::Equal, + } } + + Ordering::Equal } } @@ -393,36 +377,36 @@ impl TryFrom for StringColumn { type Error = ErrorCode; fn try_from(col: BinaryColumn) -> Result { - col.check_utf8()?; - Ok(StringColumn { - data: col.data, - offsets: col.offsets, - }) + StringColumn::try_from_binary(col) + } +} + +impl From for BinaryColumn { + fn from(col: StringColumn) -> BinaryColumn { + BinaryColumnBuilder::from_iter(col.iter().map(|x| x.as_bytes())).build() } } pub struct StringIterator<'a> { - data: &'a [u8], - offsets: std::slice::Windows<'a, u64>, + col: &'a StringColumn, + index: usize, } impl<'a> Iterator for StringIterator<'a> { type Item = &'a str; fn next(&mut self) -> Option { - let bytes = self - .offsets - .next() - .map(|range| &self.data[(range[0] as usize)..(range[1] as usize)])?; - - #[cfg(debug_assertions)] - bytes.check_utf8().unwrap(); - - unsafe { Some(std::str::from_utf8_unchecked(bytes)) } + if self.index >= self.col.len() { + return None; + } + let value = self.col.index(self.index)?; + self.index += 1; + Some(value) } fn size_hint(&self) -> (usize, Option) { - self.offsets.size_hint() + let remaining = self.col.len() - self.index; + (remaining, Some(remaining)) } } @@ -430,252 +414,158 @@ unsafe impl<'a> TrustedLen for StringIterator<'a> {} unsafe impl<'a> std::iter::TrustedLen for StringIterator<'a> {} -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +type MutableUtf8ViewArray = MutableBinaryViewArray; + +#[derive(Debug, Clone)] pub struct StringColumnBuilder { - // if the StringColumnBuilder is created with `data_capacity`, need_estimated is false - pub need_estimated: bool, - pub data: Vec, - pub offsets: Vec, + pub data: MutableUtf8ViewArray, + pub row_buffer: Vec, } impl StringColumnBuilder { - pub fn with_capacity(len: usize, data_capacity: usize) -> Self { - let mut offsets = Vec::with_capacity(len + 1); - offsets.push(0); + pub fn with_capacity(len: usize) -> Self { + let data = MutableUtf8ViewArray::with_capacity(len); StringColumnBuilder { - need_estimated: data_capacity == 0 && len > 0, - data: Vec::with_capacity(data_capacity), - offsets, + data, + row_buffer: Vec::new(), } } pub fn from_column(col: StringColumn) -> Self { + let data = col.data.make_mut(); StringColumnBuilder { - need_estimated: col.data.is_empty(), - data: buffer_into_mut(col.data), - offsets: col.offsets.to_vec(), + data, + row_buffer: Vec::new(), } } - pub fn from_data(data: Vec, offsets: Vec) -> Self { - let builder = BinaryColumnBuilder::from_data(data, offsets); - builder.check_utf8().unwrap(); - unsafe { StringColumnBuilder::from_binary_unchecked(builder) } - } - - /// # Safety - /// This function is unsound iff: - /// * the offsets are not monotonically increasing - /// * The `data` between two consecutive `offsets` are not valid utf8 - pub unsafe fn from_binary_unchecked(col: BinaryColumnBuilder) -> Self { - #[cfg(debug_assertions)] - col.check_utf8().unwrap(); - - StringColumnBuilder { - need_estimated: col.need_estimated, - data: col.data, - offsets: col.offsets, + pub fn try_from_bin_column(col: BinaryColumn) -> Result { + let mut data = MutableUtf8ViewArray::with_capacity(col.len()); + col.data.as_slice().check_utf8()?; + for v in col.iter() { + data.push_value(unsafe { std::str::from_utf8_unchecked(v) }); } + + Ok(StringColumnBuilder { + data, + row_buffer: Vec::new(), + }) } pub fn repeat(scalar: &str, n: usize) -> Self { - let len = scalar.len(); - let data = scalar.as_bytes().repeat(n); - let offsets = once(0) - .chain((0..n).map(|i| (len * (i + 1)) as u64)) - .collect(); + let mut data = MutableUtf8ViewArray::with_capacity(n); + data.extend_constant(n, Some(scalar)); StringColumnBuilder { data, - offsets, - need_estimated: false, + row_buffer: Vec::new(), } } pub fn repeat_default(n: usize) -> Self { + let mut data = MutableUtf8ViewArray::with_capacity(n); + data.extend_constant(n, Some("")); StringColumnBuilder { - data: vec![], - offsets: vec![0; n + 1], - need_estimated: false, + data, + row_buffer: Vec::new(), } } pub fn len(&self) -> usize { - self.offsets.len() - 1 + self.data.len() } pub fn memory_size(&self) -> usize { - self.offsets.len() * 8 + self.data.len() + self.data.total_buffer_len } pub fn put_char(&mut self, item: char) { - self.data - .extend_from_slice(item.encode_utf8(&mut [0; 4]).as_bytes()); + match item.len_utf8() { + 1 => self.row_buffer.push(item as u8), + _ => self + .row_buffer + .extend_from_slice(item.encode_utf8(&mut [0; 4]).as_bytes()), + } } #[inline] - #[deprecated] - pub fn put_slice(&mut self, item: &[u8]) { - #[cfg(debug_assertions)] - item.check_utf8().unwrap(); + pub fn put_str(&mut self, item: &str) { + self.row_buffer.extend_from_slice(item.as_bytes()); + } - self.data.extend_from_slice(item); + #[inline] + pub fn put_and_commit>(&mut self, item: V) { + self.data.push_value_ignore_validity(item); } #[inline] - pub fn put_str(&mut self, item: &str) { - self.data.extend_from_slice(item.as_bytes()); + pub fn put_slice(&mut self, item: &[u8]) { + self.row_buffer.extend_from_slice(item); } pub fn put_char_iter(&mut self, iter: impl Iterator) { for c in iter { - let mut buf = [0; 4]; - let result = c.encode_utf8(&mut buf); - self.data.extend_from_slice(result.as_bytes()); + self.put_char(c); } } #[inline] pub fn commit_row(&mut self) { - self.offsets.push(self.data.len() as u64); - - if self.need_estimated - && self.offsets.len() - 1 == 64 - && self.offsets.len() < self.offsets.capacity() - { - let bytes_per_row = self.data.len() / 64 + 1; - let bytes_estimate = bytes_per_row * self.offsets.capacity(); - - const MAX_HINT_SIZE: usize = 1_000_000_000; - // if we are more than 10% over the capacity, we reserve more - if bytes_estimate < MAX_HINT_SIZE - && bytes_estimate as f64 > self.data.capacity() as f64 * 1.10f64 - { - self.data.reserve(bytes_estimate - self.data.capacity()); - } - } + debug_assert!(std::str::from_utf8(&self.row_buffer).is_ok()); + let str = unsafe { std::str::from_utf8_unchecked(&self.row_buffer) }; + self.data.push_value(str); + self.row_buffer.clear(); } pub fn append_column(&mut self, other: &StringColumn) { - // the first offset of other column may not be zero - let other_start = *other.offsets.first().unwrap(); - let other_last = *other.offsets.last().unwrap(); - let start = self.offsets.last().cloned().unwrap(); - self.data - .extend_from_slice(&other.data[(other_start as usize)..(other_last as usize)]); - self.offsets.extend( - other - .offsets - .iter() - .skip(1) - .map(|offset| start + offset - other_start), - ); + self.data.extend_values(other.iter()); } pub fn build(self) -> StringColumn { - unsafe { StringColumn::new_unchecked(self.data.into(), self.offsets.into()) } + StringColumn { + data: self.data.into(), + } } pub fn build_scalar(self) -> String { - assert_eq!(self.offsets.len(), 2); - - let bytes = self.data[(self.offsets[0] as usize)..(self.offsets[1] as usize)].to_vec(); + assert_eq!(self.len(), 1); - #[cfg(debug_assertions)] - bytes.check_utf8().unwrap(); - - unsafe { String::from_utf8_unchecked(bytes) } - } - - #[inline] - pub fn may_resize(&self, add_size: usize) -> bool { - self.data.len() + add_size > self.data.capacity() + self.data.values()[0].to_string() } /// # Safety /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* pub unsafe fn index_unchecked(&self, row: usize) -> &str { - debug_assert!(row + 1 < self.offsets.len()); - - let start = *self.offsets.get_unchecked(row) as usize; - let end = *self.offsets.get_unchecked(row + 1) as usize; - let bytes = self.data.get_unchecked(start..end); - - #[cfg(debug_assertions)] - bytes.check_utf8().unwrap(); - - std::str::from_utf8_unchecked(bytes) + self.data.value_unchecked(row) } pub fn push_repeat(&mut self, item: &str, n: usize) { - self.data.reserve(item.len() * n); - if self.need_estimated && self.offsets.len() - 1 < 64 { - for _ in 0..n { - self.data.extend_from_slice(item.as_bytes()); - self.commit_row(); - } - } else { - let start = self.data.len(); - let len = item.len(); - for _ in 0..n { - self.data.extend_from_slice(item.as_bytes()); - } - self.offsets - .extend((1..=n).map(|i| (start + len * i) as u64)); - } + self.data.extend_constant(n, Some(item)); } pub fn pop(&mut self) -> Option { - if self.len() > 0 { - let index = self.len() - 1; - let start = unsafe { *self.offsets.get_unchecked(index) as usize }; - self.offsets.pop(); - let val = self.data.split_off(start); - - #[cfg(debug_assertions)] - val.check_utf8().unwrap(); - - Some(unsafe { String::from_utf8_unchecked(val) }) - } else { - None - } + self.data.pop() } } impl<'a> FromIterator<&'a str> for StringColumnBuilder { fn from_iter>(iter: T) -> Self { let iter = iter.into_iter(); - let mut builder = StringColumnBuilder::with_capacity(iter.size_hint().0, 0); + let mut builder = StringColumnBuilder::with_capacity(iter.size_hint().0); for item in iter { - builder.put_str(item); - builder.commit_row(); + builder.put_and_commit(item); } builder } } -impl From for BinaryColumnBuilder { - fn from(builder: StringColumnBuilder) -> BinaryColumnBuilder { - BinaryColumnBuilder { - need_estimated: builder.need_estimated, - data: builder.data, - offsets: builder.offsets, - } +impl PartialEq for StringColumnBuilder { + fn eq(&self, other: &Self) -> bool { + self.data.values_iter().eq(other.data.values_iter()) } } -impl TryFrom for StringColumnBuilder { - type Error = ErrorCode; - - fn try_from(builder: BinaryColumnBuilder) -> Result { - builder.check_utf8()?; - Ok(StringColumnBuilder { - need_estimated: builder.need_estimated, - data: builder.data, - offsets: builder.offsets, - }) - } -} +impl Eq for StringColumnBuilder {} #[derive(Debug, Clone, PartialEq, Eq)] pub struct StringDomain { @@ -712,7 +602,10 @@ impl CheckUTF8 for Vec { impl CheckUTF8 for BinaryColumn { fn check_utf8(&self) -> Result<()> { - check_utf8_column(&self.offsets, &self.data) + for bytes in self.iter() { + bytes.check_utf8()?; + } + Ok(()) } } diff --git a/src/query/expression/src/types/variant.rs b/src/query/expression/src/types/variant.rs index 6d7ab89a3c8d..262d2f36aa72 100644 --- a/src/query/expression/src/types/variant.rs +++ b/src/query/expression/src/types/variant.rs @@ -176,7 +176,7 @@ impl ValueType for VariantType { } fn column_memory_size(col: &Self::Column) -> usize { - col.data().len() + col.offsets().len() * 8 + col.memory_size() } #[inline(always)] diff --git a/src/query/expression/src/utils/display.rs b/src/query/expression/src/utils/display.rs index c0023fc7b8a9..32287b84c8e0 100755 --- a/src/query/expression/src/utils/display.rs +++ b/src/query/expression/src/utils/display.rs @@ -429,11 +429,7 @@ impl Debug for BinaryColumn { impl Debug for StringColumn { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { f.debug_struct("StringColumn") - .field( - "data", - &format_args!("0x{}", &hex::encode(self.data().as_slice())), - ) - .field("offsets", &self.offsets()) + .field("data", &format_args!("{:?}", self.data)) .finish() } } diff --git a/src/query/expression/src/values.rs b/src/query/expression/src/values.rs index 0196bab4a955..6c63fd6e5392 100755 --- a/src/query/expression/src/values.rs +++ b/src/query/expression/src/values.rs @@ -20,6 +20,7 @@ use std::ops::Range; use base64::engine::general_purpose; use base64::prelude::*; +use binary::BinaryColumnBuilder; use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_arrow::arrow::bitmap::Bitmap; @@ -43,12 +44,12 @@ use serde::Deserialize; use serde::Deserializer; use serde::Serialize; use serde::Serializer; +use string::StringColumnBuilder; use crate::property::Domain; use crate::types::array::ArrayColumn; use crate::types::array::ArrayColumnBuilder; use crate::types::binary::BinaryColumn; -use crate::types::binary::BinaryColumnBuilder; use crate::types::bitmap::BitmapType; use crate::types::boolean::BooleanDomain; use crate::types::date::DATE_MAX; @@ -76,7 +77,6 @@ use crate::types::number::SimpleDomain; use crate::types::number::F32; use crate::types::number::F64; use crate::types::string::StringColumn; -use crate::types::string::StringColumnBuilder; use crate::types::string::StringDomain; use crate::types::timestamp::clamp_timestamp; use crate::types::timestamp::TIMESTAMP_MAX; @@ -1172,7 +1172,6 @@ impl Column { pub fn check_valid(&self) -> Result<()> { match self { Column::Binary(x) => x.check_valid(), - Column::String(x) => x.check_valid(), Column::Variant(x) => x.check_valid(), Column::Geometry(x) => x.check_valid(), Column::Geography(x) => x.check_valid(), @@ -1442,11 +1441,12 @@ impl Column { Column::Decimal(DecimalColumn::Decimal256(col, _)) => col.len() * 32, Column::Geography(col) => GeographyType::column_memory_size(col), Column::Boolean(c) => c.len(), + // 8 * len + size of bytes Column::Binary(col) | Column::Bitmap(col) | Column::Variant(col) | Column::Geometry(col) => col.memory_size(), - Column::String(col) => col.memory_size(), + Column::String(col) => col.len() * 8 + col.current_buffer_len(), Column::Array(col) | Column::Map(col) => col.values.serialize_size() + col.len() * 8, Column::Nullable(c) => c.column.serialize_size() + c.len(), Column::Tuple(fields) => fields.iter().map(|f| f.serialize_size()).sum(), @@ -1666,7 +1666,7 @@ impl ColumnBuilder { } ColumnBuilder::Boolean(c) => c.as_slice().len(), ColumnBuilder::Binary(col) => col.data.len() + col.offsets.len() * 8, - ColumnBuilder::String(col) => col.data.len() + col.offsets.len() * 8, + ColumnBuilder::String(col) => col.memory_size(), ColumnBuilder::Timestamp(col) => col.len() * 8, ColumnBuilder::Date(col) => col.len() * 4, ColumnBuilder::Array(col) => col.builder.memory_size() + col.offsets.len() * 8, @@ -1742,10 +1742,7 @@ impl ColumnBuilder { let data_capacity = if enable_datasize_hint { 0 } else { capacity }; ColumnBuilder::Binary(BinaryColumnBuilder::with_capacity(capacity, data_capacity)) } - DataType::String => { - let data_capacity = if enable_datasize_hint { 0 } else { capacity }; - ColumnBuilder::String(StringColumnBuilder::with_capacity(capacity, data_capacity)) - } + DataType::String => ColumnBuilder::String(StringColumnBuilder::with_capacity(capacity)), DataType::Timestamp => ColumnBuilder::Timestamp(Vec::with_capacity(capacity)), DataType::Date => ColumnBuilder::Date(Vec::with_capacity(capacity)), DataType::Nullable(ty) => ColumnBuilder::Nullable(Box::new(NullableColumnBuilder { @@ -1829,8 +1826,8 @@ impl ColumnBuilder { // binary based DataType::Binary => ColumnBuilder::Binary(BinaryColumnBuilder::repeat_default(len)), - DataType::Bitmap => ColumnBuilder::Bitmap(BinaryColumnBuilder::repeat_default(len)), DataType::String => ColumnBuilder::String(StringColumnBuilder::repeat_default(len)), + DataType::Bitmap => ColumnBuilder::Bitmap(BinaryColumnBuilder::repeat_default(len)), DataType::Variant => ColumnBuilder::Variant(BinaryColumnBuilder::repeat_default(len)), DataType::Geometry => ColumnBuilder::Geometry(BinaryColumnBuilder::repeat_default(len)), DataType::Geography => { @@ -2046,13 +2043,13 @@ impl ColumnBuilder { } ColumnBuilder::String(builder) => { let offset = reader.read_scalar::()? as usize; - builder.data.resize(offset + builder.data.len(), 0); - let last = *builder.offsets.last().unwrap() as usize; - reader.read_exact(&mut builder.data[last..last + offset])?; - builder.commit_row(); + builder.row_buffer.resize(offset, 0); + reader.read_exact(&mut builder.row_buffer)?; #[cfg(debug_assertions)] - string::CheckUTF8::check_utf8(&(&builder.data[last..last + offset])).unwrap(); + string::CheckUTF8::check_utf8(&builder.row_buffer).unwrap(); + + builder.commit_row(); } ColumnBuilder::Timestamp(builder) => { let mut value: i64 = reader.read_scalar()?; @@ -2150,8 +2147,7 @@ impl ColumnBuilder { string::CheckUTF8::check_utf8(&bytes).unwrap(); let s = unsafe { std::str::from_utf8_unchecked(bytes) }; - builder.put_str(s); - builder.commit_row(); + builder.put_and_commit(s); } } ColumnBuilder::Timestamp(builder) => { diff --git a/src/query/expression/tests/it/common.rs b/src/query/expression/tests/it/common.rs index c2dd01b0f952..38a3bdaf16ad 100644 --- a/src/query/expression/tests/it/common.rs +++ b/src/query/expression/tests/it/common.rs @@ -127,7 +127,7 @@ pub fn run_scatter(file: &mut impl Write, block: &DataBlock, indices: &[u32], sc } pub fn run_take(file: &mut impl Write, indices: &[u32], block: &DataBlock) { - let result = DataBlock::take(block, indices, &mut None); + let result = DataBlock::take(block, indices); match result { Ok(result_block) => { diff --git a/src/query/expression/tests/it/kernel.rs b/src/query/expression/tests/it/kernel.rs index 4d0d8a2af572..d374f553fb6e 100644 --- a/src/query/expression/tests/it/kernel.rs +++ b/src/query/expression/tests/it/kernel.rs @@ -14,14 +14,18 @@ use core::ops::Range; +use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_expression::block_debug::assert_block_value_eq; use databend_common_expression::types::number::*; +use databend_common_expression::types::AnyType; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; +use databend_common_expression::types::ValueType; use databend_common_expression::visitor::ValueVisitor; use databend_common_expression::BlockEntry; use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::FilterVisitor; use databend_common_expression::FromData; @@ -281,10 +285,14 @@ pub fn test_take_and_filter_and_concat() -> databend_common_exception::Result<() FilterVisitor::new(&filter).with_strategy(IterationStrategy::SlicesIterator); let mut f2 = FilterVisitor::new(&filter).with_strategy(IterationStrategy::IndexIterator); + for col in random_block.columns() { f1.visit_value(col.value.clone())?; f2.visit_value(col.value.clone())?; - assert_eq!(f1.take_result(), f2.take_result()); + + let l = f1.take_result(); + let r = f2.take_result(); + assert_eq!(l, r); } } @@ -326,15 +334,9 @@ pub fn test_take_and_filter_and_concat() -> databend_common_exception::Result<() .collect_vec(); let concated_blocks = DataBlock::concat(&blocks)?; - let block_1 = concated_blocks.take(&take_indices, &mut None)?; + let block_1 = concated_blocks.take(&take_indices)?; let block_2 = concated_blocks.take_compacted_indices(&take_compact_indices, count)?; - let block_3 = DataBlock::take_column_vec( - &column_vec, - &data_types, - &take_chunks_indices, - count, - &mut None, - ); + let block_3 = DataBlock::take_column_vec(&column_vec, &data_types, &take_chunks_indices, count); let block_4 = DataBlock::concat(&filtered_blocks)?; let block_5 = concated_blocks.take_ranges( &build_range_selection(&take_indices, take_indices.len()), @@ -433,7 +435,7 @@ pub fn test_take_compact() -> databend_common_exception::Result<()> { take_indices.extend(std::iter::repeat(batch_index as u32).take(batch_size)); take_compact_indices.push((batch_index as u32, batch_size as u32)); } - let block_1 = block.take(&take_indices, &mut None)?; + let block_1 = block.take(&take_indices)?; let block_2 = block.take_compacted_indices(&take_compact_indices, count)?; assert_eq!(block_1.num_columns(), block_2.num_columns()); @@ -501,8 +503,8 @@ pub fn test_filters() -> databend_common_exception::Result<()> { .map(|(i, _)| i as u32) .collect::>(); - let t_b = bb.take(&indices, &mut None)?; - let t_c = cc.take(&indices, &mut None)?; + let t_b = bb.take(&indices)?; + let t_c = cc.take(&indices)?; let f_b = bb.filter_with_bitmap(&f)?; let f_c = cc.filter_with_bitmap(&f)?; @@ -590,7 +592,7 @@ pub fn test_scatter() -> databend_common_exception::Result<()> { } } - let block_1 = random_block.take(&take_indices, &mut None)?; + let block_1 = random_block.take(&take_indices)?; let block_2 = DataBlock::concat(&scattered_blocks)?; assert_eq!(block_1.num_columns(), block_2.num_columns()); @@ -606,3 +608,31 @@ pub fn test_scatter() -> databend_common_exception::Result<()> { Ok(()) } + +#[test] +fn test_builder() { + let ty = DataType::String; + let len = 30; + let col = Column::random(&ty, len, None); + + let bitmap = Bitmap::from_iter((0..len).map(|x| x % 4 != 0)); + + let mut builder1 = ColumnBuilder::with_capacity(&col.data_type(), col.len()); + let mut builder2 = ColumnBuilder::with_capacity(&col.data_type(), col.len()); + + for i in 0..len { + if bitmap.get_bit(i) { + builder1.push(col.index(i).unwrap()); + } + } + + for (start, len) in databend_common_arrow::arrow::bitmap::utils::SlicesIterator::new(&bitmap) { + let sub_col = col.slice(start..start + len); + AnyType::append_column(&mut builder2, &sub_col); + } + + let r1 = builder1.build(); + let r2 = builder2.build(); + + assert_eq!(r1, r2) +} diff --git a/src/query/expression/tests/it/row.rs b/src/query/expression/tests/it/row.rs index f7ba9b75dd9c..07b9c2e196c9 100644 --- a/src/query/expression/tests/it/row.rs +++ b/src/query/expression/tests/it/row.rs @@ -18,7 +18,6 @@ use arrow_ord::sort::LexicographicalComparator; use arrow_ord::sort::SortColumn; use arrow_schema::SortOptions; use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::offset::OffsetsBuffer; use databend_common_base::base::OrderedFloat; use databend_common_expression::converts::arrow2::set_validities; use databend_common_expression::types::binary::BinaryColumnBuilder; @@ -70,31 +69,6 @@ fn test_fixed_width() { let rows = converter.convert_columns(&cols, cols[0].len()); - assert_eq!( - rows.offsets().clone(), - vec![0, 8, 16, 24, 32, 40, 48, 56].into() - ); - assert_eq!( - rows.data().clone(), - vec![ - 1, 128, 1, // - 1, 191, 166, 102, 102, // - 1, 128, 2, // - 1, 192, 32, 0, 0, // - 0, 0, 0, // - 0, 0, 0, 0, 0, // - 1, 127, 251, // - 1, 192, 128, 0, 0, // - 1, 128, 2, // - 1, 189, 204, 204, 205, // - 1, 128, 2, // - 1, 63, 127, 255, 255, // - 1, 128, 0, // - 1, 127, 255, 255, 255 // - ] - .into() - ); - unsafe { assert!(rows.index_unchecked(3) < rows.index_unchecked(6)); assert!(rows.index_unchecked(0) < rows.index_unchecked(1)); @@ -549,17 +523,7 @@ fn fuzz_test() { // arrow_ord does not support LargeBinary converted from Databend String Column::Nullable(c) => match &c.column { Column::String(sc) => { - let offsets = - sc.offsets().iter().map(|offset| *offset as i64).collect(); - let array = Box::new( - databend_common_arrow::arrow::array::Utf8Array::::try_new( - databend_common_arrow::arrow::datatypes::DataType::LargeUtf8, - unsafe { OffsetsBuffer::new_unchecked(offsets) }, - sc.data().clone(), - None, - ) - .unwrap(), - ); + let array = Box::new(sc.clone().into_inner()); set_validities(array, &c.validity) } _ => col.as_arrow(), diff --git a/src/query/expression/tests/it/testdata/kernel-pass.txt b/src/query/expression/tests/it/testdata/kernel-pass.txt index 53da6e912d6b..cfe2e20d5bda 100644 --- a/src/query/expression/tests/it/testdata/kernel-pass.txt +++ b/src/query/expression/tests/it/testdata/kernel-pass.txt @@ -19,25 +19,25 @@ Result: Concat-Column 0: -+-----------+----------------+-----------------------------------------------------------------------------------------------------------------------------+ -| Column ID | Type | Column Data | -+-----------+----------------+-----------------------------------------------------------------------------------------------------------------------------+ -| 0 | Int32 | Column(Int32([0, 1, 2, 3, -4])) | -| 1 | UInt8 NULL | Column(NullableColumn { column: UInt8([10, 11, 12, 13, 14]), validity: [0b___00010] }) | -| 2 | NULL | Column(Null { len: 5 }) | -| 3 | Array(Nothing) | Column(EmptyArray { len: 5 }) | -| 4 | String NULL | Column(NullableColumn { column: StringColumn { data: 0x78797a6162, offsets: [0, 1, 2, 3, 4, 5] }, validity: [0b___00110] }) | -+-----------+----------------+-----------------------------------------------------------------------------------------------------------------------------+ ++-----------+----------------+----------------------------------------------------------------------------------------------------------------+ +| Column ID | Type | Column Data | ++-----------+----------------+----------------------------------------------------------------------------------------------------------------+ +| 0 | Int32 | Column(Int32([0, 1, 2, 3, -4])) | +| 1 | UInt8 NULL | Column(NullableColumn { column: UInt8([10, 11, 12, 13, 14]), validity: [0b___00010] }) | +| 2 | NULL | Column(Null { len: 5 }) | +| 3 | Array(Nothing) | Column(EmptyArray { len: 5 }) | +| 4 | String NULL | Column(NullableColumn { column: StringColumn { data: Utf8ViewArray[x, y, z, a, b] }, validity: [0b___00110] }) | ++-----------+----------------+----------------------------------------------------------------------------------------------------------------+ Concat-Column 1: -+-----------+----------------+--------------------------------------------------------------------------------------------------------------+ -| Column ID | Type | Column Data | -+-----------+----------------+--------------------------------------------------------------------------------------------------------------+ -| 0 | Int32 | Column(Int32([5, 6])) | -| 1 | UInt8 NULL | Column(NullableColumn { column: UInt8([15, 16]), validity: [0b______10] }) | -| 2 | NULL | Column(Null { len: 2 }) | -| 3 | Array(Nothing) | Column(EmptyArray { len: 2 }) | -| 4 | String NULL | Column(NullableColumn { column: StringColumn { data: 0x7879, offsets: [0, 1, 2] }, validity: [0b______10] }) | -+-----------+----------------+--------------------------------------------------------------------------------------------------------------+ ++-----------+----------------+-------------------------------------------------------------------------------------------------------+ +| Column ID | Type | Column Data | ++-----------+----------------+-------------------------------------------------------------------------------------------------------+ +| 0 | Int32 | Column(Int32([5, 6])) | +| 1 | UInt8 NULL | Column(NullableColumn { column: UInt8([15, 16]), validity: [0b______10] }) | +| 2 | NULL | Column(Null { len: 2 }) | +| 3 | Array(Nothing) | Column(EmptyArray { len: 2 }) | +| 4 | String NULL | Column(NullableColumn { column: StringColumn { data: Utf8ViewArray[x, y] }, validity: [0b______10] }) | ++-----------+----------------+-------------------------------------------------------------------------------------------------------+ Result: +----------+----------+----------+----------+----------+ | Column 0 | Column 1 | Column 2 | Column 3 | Column 4 | diff --git a/src/query/formats/src/field_decoder/fast_values.rs b/src/query/formats/src/field_decoder/fast_values.rs index 56ce1f46c3fa..c3a5c16ebd55 100644 --- a/src/query/formats/src/field_decoder/fast_values.rs +++ b/src/query/formats/src/field_decoder/fast_values.rs @@ -270,7 +270,7 @@ impl FastFieldDecoderValues { reader: &mut Cursor, positions: &mut VecDeque, ) -> Result<()> { - self.read_string_inner(reader, &mut column.data, positions)?; + self.read_string_inner(reader, &mut column.row_buffer, positions)?; column.commit_row(); Ok(()) } diff --git a/src/query/formats/src/field_decoder/nested.rs b/src/query/formats/src/field_decoder/nested.rs index cf690d719f7c..1b7b5ba958f7 100644 --- a/src/query/formats/src/field_decoder/nested.rs +++ b/src/query/formats/src/field_decoder/nested.rs @@ -196,7 +196,7 @@ impl NestedValues { column: &mut StringColumnBuilder, reader: &mut Cursor, ) -> Result<()> { - reader.read_quoted_text(&mut column.data, b'\'')?; + reader.read_quoted_text(&mut column.row_buffer, b'\'')?; column.commit_row(); Ok(()) } diff --git a/src/query/formats/src/field_decoder/separated_text.rs b/src/query/formats/src/field_decoder/separated_text.rs index 0ca734b07d3a..31f0226f032b 100644 --- a/src/query/formats/src/field_decoder/separated_text.rs +++ b/src/query/formats/src/field_decoder/separated_text.rs @@ -124,8 +124,7 @@ impl SeparatedTextDecoder { Ok(()) } ColumnBuilder::String(c) => { - c.put_str(std::str::from_utf8(data)?); - c.commit_row(); + c.put_and_commit(std::str::from_utf8(data)?); Ok(()) } ColumnBuilder::Boolean(c) => self.read_bool(c, data), diff --git a/src/query/functions/Cargo.toml b/src/query/functions/Cargo.toml index 34a0a89f3a63..a0a3d1872c8c 100644 --- a/src/query/functions/Cargo.toml +++ b/src/query/functions/Cargo.toml @@ -18,7 +18,6 @@ bumpalo = { workspace = true } chrono = { workspace = true } chrono-tz = { workspace = true } crc32fast = { workspace = true } -criterion = { workspace = true } ctor = { workspace = true } databend-common-arrow = { workspace = true } databend-common-base = { workspace = true } @@ -67,6 +66,7 @@ twox-hash = { workspace = true } [dev-dependencies] comfy-table = { workspace = true } +criterion = { workspace = true } databend-common-ast = { workspace = true } goldenfile = { workspace = true } diff --git a/src/query/functions/src/aggregates/aggregate_distinct_state.rs b/src/query/functions/src/aggregates/aggregate_distinct_state.rs index 424043c2903b..4db1b13b5b2f 100644 --- a/src/query/functions/src/aggregates/aggregate_distinct_state.rs +++ b/src/query/functions/src/aggregates/aggregate_distinct_state.rs @@ -232,10 +232,9 @@ impl DistinctStateFunc for AggregateDistinctStringState { } fn build_columns(&mut self, _types: &[DataType]) -> Result> { - let mut builder = StringColumnBuilder::with_capacity(self.set.len(), self.set.len() * 2); + let mut builder = StringColumnBuilder::with_capacity(self.set.len()); for key in self.set.iter() { - builder.put_str(unsafe { std::str::from_utf8_unchecked(key.key()) }); - builder.commit_row(); + builder.put_and_commit(unsafe { std::str::from_utf8_unchecked(key.key()) }); } Ok(vec![Column::String(builder.build())]) } diff --git a/src/query/functions/src/aggregates/aggregate_histogram.rs b/src/query/functions/src/aggregates/aggregate_histogram.rs index d3f40ec178d9..d083d8461039 100644 --- a/src/query/functions/src/aggregates/aggregate_histogram.rs +++ b/src/query/functions/src/aggregates/aggregate_histogram.rs @@ -24,7 +24,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::decimal::*; use databend_common_expression::types::number::*; -use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::*; use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggregateFunctionRef; @@ -32,6 +31,7 @@ use databend_common_expression::Scalar; use ethnum::i256; use serde::Deserialize; use serde::Serialize; +use string::StringColumnBuilder; use super::FunctionData; use crate::aggregates::aggregate_function_factory::AggregateFunctionDescription; @@ -157,8 +157,7 @@ where }) .collect::>>(), )?; - builder.put_str(&json_str); - builder.commit_row(); + builder.put_and_commit(json_str); Ok(()) } diff --git a/src/query/functions/src/aggregates/aggregate_min_max_any.rs b/src/query/functions/src/aggregates/aggregate_min_max_any.rs index 6ab35d792d80..bbe7325b93d2 100644 --- a/src/query/functions/src/aggregates/aggregate_min_max_any.rs +++ b/src/query/functions/src/aggregates/aggregate_min_max_any.rs @@ -42,7 +42,114 @@ use super::UnaryState; use crate::aggregates::assert_unary_arguments; use crate::aggregates::AggregateFunction; use crate::with_compare_mapped_type; -use crate::with_simple_no_number_mapped_type; +use crate::with_simple_no_number_no_string_mapped_type; + +#[derive(BorshSerialize, BorshDeserialize)] +pub struct MinMaxStringState +where C: ChangeIf +{ + pub value: Option, + #[borsh(skip)] + _c: PhantomData, +} + +impl Default for MinMaxStringState +where C: ChangeIf + Default +{ + fn default() -> Self { + Self { + value: None, + _c: PhantomData, + } + } +} + +impl UnaryState for MinMaxStringState +where C: ChangeIf + Default +{ + fn add( + &mut self, + other: ::ScalarRef<'_>, + _function_data: Option<&dyn FunctionData>, + ) -> Result<()> { + match &self.value { + Some(v) => { + if C::change_if(&StringType::to_scalar_ref(v), &other) { + self.value = Some(StringType::to_owned_scalar(other)); + } + } + None => { + self.value = Some(StringType::to_owned_scalar(other)); + } + } + Ok(()) + } + + fn add_batch( + &mut self, + other: StringColumn, + validity: Option<&Bitmap>, + function_data: Option<&dyn FunctionData>, + ) -> Result<()> { + let column_len = StringType::column_len(&other); + if column_len == 0 { + return Ok(()); + } + + let column_iter = 0..other.len(); + if let Some(validity) = validity { + if validity.unset_bits() == column_len { + return Ok(()); + } + let v = column_iter + .zip(validity) + .filter(|(_, valid)| *valid) + .map(|(idx, _)| idx) + .reduce(|l, r| { + if !C::change_if_ordering(StringColumn::compare(&other, l, &other, r)) { + l + } else { + r + } + }); + if let Some(v) = v { + let _ = self.add(other.index(v).unwrap(), function_data); + } + } else { + let v = column_iter.reduce(|l, r| { + if !C::change_if_ordering(StringColumn::compare(&other, l, &other, r)) { + l + } else { + r + } + }); + if let Some(v) = v { + let _ = self.add(other.index(v).unwrap(), function_data); + } + } + Ok(()) + } + + fn merge(&mut self, rhs: &Self) -> Result<()> { + if let Some(v) = &rhs.value { + self.add(v.as_str(), None)?; + } + Ok(()) + } + + fn merge_result( + &mut self, + builder: &mut ::ColumnBuilder, + _function_data: Option<&dyn FunctionData>, + ) -> Result<()> { + if let Some(v) = &self.value { + StringType::push_item(builder, v.as_str()); + } else { + StringType::push_default(builder); + } + Ok(()) + } +} #[derive(BorshSerialize, BorshDeserialize)] pub struct MinMaxAnyState @@ -161,7 +268,7 @@ pub fn try_create_aggregate_min_max_any_function( with_compare_mapped_type!(|CMP| match CMP_TYPE { CMP => { - with_simple_no_number_mapped_type!(|T| match data_type { + with_simple_no_number_no_string_mapped_type!(|T| match data_type { DataType::T => { let return_type = data_type.clone(); let func = AggregateUnaryFunction::, T, T>::try_create( @@ -174,6 +281,19 @@ pub fn try_create_aggregate_min_max_any_function( Ok(Arc::new(func)) } + DataType::String => { + let return_type = data_type.clone(); + let func = AggregateUnaryFunction::< + MinMaxStringState, + StringType, + StringType, + >::try_create( + display_name, return_type, params, data_type + ) + .with_need_drop(need_drop); + + Ok(Arc::new(func)) + } DataType::Number(num_type) => { with_number_mapped_type!(|NUM| match num_type { NumberDataType::NUM => { diff --git a/src/query/functions/src/aggregates/aggregate_scalar_state.rs b/src/query/functions/src/aggregates/aggregate_scalar_state.rs index 3a5c450761c8..26ccc51d1f91 100644 --- a/src/query/functions/src/aggregates/aggregate_scalar_state.rs +++ b/src/query/functions/src/aggregates/aggregate_scalar_state.rs @@ -44,6 +44,23 @@ macro_rules! with_simple_no_number_mapped_type { } } +#[macro_export] +macro_rules! with_simple_no_number_no_string_mapped_type { + (| $t:tt | $($tail:tt)*) => { + match_template::match_template! { + $t = [ + Boolean => BooleanType, + Timestamp => TimestampType, + Null => NullType, + EmptyArray => EmptyArrayType, + EmptyMap => EmptyMapType, + Date => DateType, + ], + $($tail)* + } + } +} + pub const TYPE_ANY: u8 = 0; pub const TYPE_MIN: u8 = 1; pub const TYPE_MAX: u8 = 2; @@ -64,6 +81,7 @@ macro_rules! with_compare_mapped_type { pub trait ChangeIf: Send + Sync + 'static { fn change_if(l: &T::ScalarRef<'_>, r: &T::ScalarRef<'_>) -> bool; + fn change_if_ordering(ordering: Ordering) -> bool; } #[derive(Default)] @@ -78,6 +96,11 @@ where fn change_if<'a>(l: &T::ScalarRef<'_>, r: &T::ScalarRef<'_>) -> bool { matches!(l.partial_cmp(r), Some(Ordering::Greater)) } + + #[inline] + fn change_if_ordering(ordering: Ordering) -> bool { + ordering == Ordering::Greater + } } #[derive(Default)] @@ -92,6 +115,11 @@ where fn change_if<'a>(l: &T::ScalarRef<'_>, r: &T::ScalarRef<'_>) -> bool { matches!(l.partial_cmp(r), Some(Ordering::Less)) } + + #[inline] + fn change_if_ordering(ordering: Ordering) -> bool { + ordering == Ordering::Less + } } #[derive(Default)] @@ -102,6 +130,11 @@ impl ChangeIf for CmpAny { fn change_if(_: &T::ScalarRef<'_>, _: &T::ScalarRef<'_>) -> bool { false } + + #[inline] + fn change_if_ordering(_: Ordering) -> bool { + false + } } pub trait ScalarStateFunc: diff --git a/src/query/functions/src/aggregates/aggregate_string_agg.rs b/src/query/functions/src/aggregates/aggregate_string_agg.rs index 60ff6fd4cf3f..3bb9259d11a9 100644 --- a/src/query/functions/src/aggregates/aggregate_string_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_string_agg.rs @@ -147,9 +147,10 @@ impl AggregateFunction for AggregateStringAggFunction { let builder = StringType::try_downcast_builder(builder).unwrap(); if !state.values.is_empty() { let len = state.values.len() - self.delimiter.len(); - builder.put_str(&state.values[..len]); + builder.put_and_commit(&state.values[..len]); + } else { + builder.put_and_commit(""); } - builder.commit_row(); Ok(()) } diff --git a/src/query/functions/src/scalars/arithmetic.rs b/src/query/functions/src/scalars/arithmetic.rs index 7cf90d5efd12..081cba3653d4 100644 --- a/src/query/functions/src/scalars/arithmetic.rs +++ b/src/query/functions/src/scalars/arithmetic.rs @@ -22,6 +22,7 @@ use std::sync::Arc; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_expression::serialize::read_decimal_with_size; +use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::decimal::DecimalDomain; use databend_common_expression::types::decimal::DecimalType; use databend_common_expression::types::nullable::NullableColumn; @@ -37,6 +38,7 @@ use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberClass; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::SimpleDomain; +use databend_common_expression::types::StringColumn; use databend_common_expression::types::StringType; use databend_common_expression::types::ALL_FLOAT_TYPES; use databend_common_expression::types::ALL_INTEGER_TYPES; @@ -972,28 +974,25 @@ pub fn register_number_to_string(registry: &mut FunctionRegistry) { let options = NUM_TYPE::lexical_options(); const FORMAT: u128 = lexical_core::format::STANDARD; - let mut builder = - StringColumnBuilder::with_capacity(from.len(), from.len() + 1); - let values = &mut builder.data; - type Native = ::Native; - let mut offset: usize = 0; + + let mut builder = StringColumnBuilder::with_capacity(from.len()); + unsafe { for x in from.iter() { - values.reserve(offset + Native::FORMATTED_SIZE_DECIMAL); - values.set_len(offset + Native::FORMATTED_SIZE_DECIMAL); - let bytes = &mut values[offset..]; - + builder.row_buffer.resize( + ::Native::FORMATTED_SIZE_DECIMAL, + 0, + ); let len = lexical_core::write_with_options::<_, FORMAT>( Native::from(*x), - bytes, + &mut builder.row_buffer, &options, ) .len(); - offset += len; - builder.offsets.push(offset as u64); + builder.row_buffer.truncate(len); + builder.commit_row(); } - values.set_len(offset); } Value::Column(builder.build()) } @@ -1008,7 +1007,7 @@ pub fn register_number_to_string(registry: &mut FunctionRegistry) { let options = NUM_TYPE::lexical_options(); const FORMAT: u128 = lexical_core::format::STANDARD; let mut builder = - StringColumnBuilder::with_capacity(from.len(), from.len() + 1); + BinaryColumnBuilder::with_capacity(from.len(), from.len() + 1); let values = &mut builder.data; type Native = ::Native; @@ -1029,7 +1028,7 @@ pub fn register_number_to_string(registry: &mut FunctionRegistry) { } values.set_len(offset); } - let result = builder.build(); + let result = StringColumn::try_from(builder.build()).unwrap(); Value::Column(NullableColumn::new( result, Bitmap::new_constant(true, from.len()), diff --git a/src/query/functions/src/scalars/binary.rs b/src/query/functions/src/scalars/binary.rs index 6c76598768f6..1e865888b98b 100644 --- a/src/query/functions/src/scalars/binary.rs +++ b/src/query/functions/src/scalars/binary.rs @@ -30,7 +30,7 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::types::UInt8Type; -use databend_common_expression::types::ValueType; +use databend_common_expression::vectorize_1_arg; use databend_common_expression::Column; use databend_common_expression::EvalContext; use databend_common_expression::Function; @@ -49,19 +49,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::, _, _>( "length", |_, _| FunctionDomain::Full, - |val, _| match val { - ValueRef::Scalar(s) => Value::Scalar(s.len() as u64), - ValueRef::Column(c) => { - let diffs = c - .offsets() - .iter() - .zip(c.offsets().iter().skip(1)) - .map(|(a, b)| b - a) - .collect::>(); - - Value::Column(diffs.into()) - } - }, + vectorize_1_arg::>(|val, _| val.len() as u64), ); registry.register_passthrough_nullable_1_arg::( @@ -101,12 +89,11 @@ pub fn register(registry: &mut FunctionRegistry) { "to_hex", |_, _| FunctionDomain::Full, vectorize_binary_to_string( - |col| col.data().len() * 2, + |col| col.current_buffer_len() * 2, |val, output, _| { - let old_len = output.data.len(); let extra_len = val.len() * 2; - output.data.resize(old_len + extra_len, 0); - hex::encode_to_slice(val, &mut output.data[old_len..]).unwrap(); + output.row_buffer.resize(extra_len, 0); + hex::encode_to_slice(val, &mut output.row_buffer).unwrap(); output.commit_row(); }, ), @@ -128,10 +115,10 @@ pub fn register(registry: &mut FunctionRegistry) { "to_base64", |_, _| FunctionDomain::Full, vectorize_binary_to_string( - |col| col.data().len() * 4 / 3 + col.len() * 4, + |col| col.current_buffer_len() * 4 / 3 + col.len() * 4, |val, output, _| { base64::write::EncoderWriter::new( - &mut output.data, + &mut output.row_buffer, &base64::engine::general_purpose::STANDARD, ) .write_all(val) @@ -203,7 +190,7 @@ pub fn register(registry: &mut FunctionRegistry) { fn eval_binary_to_string(val: ValueRef, ctx: &mut EvalContext) -> Value { vectorize_binary_to_string( - |col| col.data().len(), + |col| col.current_buffer_len(), |val, output, ctx| { if let Ok(val) = simdutf8::basic::from_utf8(val) { output.put_str(val); @@ -217,7 +204,7 @@ fn eval_binary_to_string(val: ValueRef, ctx: &mut EvalContext) -> Va fn eval_unhex(val: ValueRef, ctx: &mut EvalContext) -> Value { vectorize_string_to_binary( - |col| col.data().len() / 2, + |col| col.current_buffer_len() / 2, |val, output, ctx| { let old_len = output.data.len(); let extra_len = val.len() / 2; @@ -232,7 +219,7 @@ fn eval_unhex(val: ValueRef, ctx: &mut EvalContext) -> Value, ctx: &mut EvalContext) -> Value { vectorize_string_to_binary( - |col| col.data().len() * 4 / 3 + col.len() * 4, + |col| col.current_buffer_len() * 4 / 3 + col.len() * 4, |val, output, ctx| { if let Err(err) = base64::Engine::decode_vec( &base64::engine::general_purpose::STANDARD, @@ -248,18 +235,17 @@ fn eval_from_base64(val: ValueRef, ctx: &mut EvalContext) -> Value usize + Copy, + _estimate_bytes: impl Fn(&BinaryColumn) -> usize + Copy, func: impl Fn(&[u8], &mut StringColumnBuilder, &mut EvalContext) + Copy, ) -> impl Fn(ValueRef, &mut EvalContext) -> Value + Copy { move |arg1, ctx| match arg1 { ValueRef::Scalar(val) => { - let mut builder = StringColumnBuilder::with_capacity(1, 0); + let mut builder = StringColumnBuilder::with_capacity(1); func(val, &mut builder, ctx); Value::Scalar(builder.build_scalar()) } ValueRef::Column(col) => { - let data_capacity = estimate_bytes(&col); - let mut builder = StringColumnBuilder::with_capacity(col.len(), data_capacity); + let mut builder = StringColumnBuilder::with_capacity(col.len()); for val in col.iter() { func(val, &mut builder, ctx); } @@ -304,34 +290,18 @@ fn char_fn(args: &[ValueRef], _: &mut EvalContext) -> Value { }); let input_rows = len.unwrap_or(1); - let mut values: Vec = vec![0; input_rows * args.len()]; - let values_ptr = values.as_mut_ptr(); + let mut builder = BinaryColumnBuilder::with_capacity(input_rows, 0); - for (i, arg) in args.iter().enumerate() { - match arg { - ValueRef::Scalar(v) => { - for j in 0..input_rows { - unsafe { - *values_ptr.add(args.len() * j + i) = *v; - } - } - } - ValueRef::Column(c) => { - for (j, ch) in UInt8Type::iter_column(c).enumerate() { - unsafe { - *values_ptr.add(args.len() * j + i) = ch; - } - } - } + for _ in 0..input_rows { + for arg in &args { + let val = arg.index(0).unwrap(); + builder.put_u8(val); } + builder.commit_row(); } - let offsets = (0..(input_rows + 1) as u64 * args.len() as u64) - .step_by(args.len()) - .collect::>(); - let result = BinaryColumn::new(values.into(), offsets.into()); match len { - Some(_) => Value::Column(Column::Binary(result)), - _ => Value::Scalar(Scalar::Binary(result.index(0).unwrap().to_vec())), + Some(_) => Value::Column(Column::Binary(builder.build())), + _ => Value::Scalar(Scalar::Binary(builder.build_scalar())), } } diff --git a/src/query/functions/src/scalars/bitmap.rs b/src/query/functions/src/scalars/bitmap.rs index 59b7fbfbf8f3..0cb936957c80 100644 --- a/src/query/functions/src/scalars/bitmap.rs +++ b/src/query/functions/src/scalars/bitmap.rs @@ -182,14 +182,13 @@ pub fn register(registry: &mut FunctionRegistry) { Ok(rb) => { let raw = rb.into_iter().collect::>(); let s = join(raw.iter(), ","); - builder.put_str(&s); + builder.put_and_commit(s); } Err(e) => { ctx.set_error(builder.len(), e.to_string()); + builder.commit_row(); } } - - builder.commit_row(); }), ); diff --git a/src/query/functions/src/scalars/comparison.rs b/src/query/functions/src/scalars/comparison.rs index 71f667de8bd5..8346066d128e 100644 --- a/src/query/functions/src/scalars/comparison.rs +++ b/src/query/functions/src/scalars/comparison.rs @@ -30,6 +30,7 @@ use databend_common_expression::types::EmptyArrayType; use databend_common_expression::types::GenericType; use databend_common_expression::types::NumberClass; use databend_common_expression::types::NumberType; +use databend_common_expression::types::StringColumn; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::ValueType; @@ -158,7 +159,66 @@ macro_rules! register_simple_domain_type_cmp { } fn register_string_cmp(registry: &mut FunctionRegistry) { - register_simple_domain_type_cmp!(registry, StringType); + registry.register_passthrough_nullable_2_arg::( + "eq", + |_, d1, d2| d1.domain_eq(d2), + vectorize_string_cmp(|cmp| cmp == Ordering::Equal), + ); + registry.register_passthrough_nullable_2_arg::( + "noteq", + |_, d1, d2| d1.domain_noteq(d2), + vectorize_string_cmp(|cmp| cmp != Ordering::Equal), + ); + registry.register_passthrough_nullable_2_arg::( + "gt", + |_, d1, d2| d1.domain_gt(d2), + vectorize_string_cmp(|cmp| cmp == Ordering::Greater), + ); + registry.register_passthrough_nullable_2_arg::( + "gte", + |_, d1, d2| d1.domain_gte(d2), + vectorize_string_cmp(|cmp| cmp != Ordering::Less), + ); + registry.register_passthrough_nullable_2_arg::( + "lt", + |_, d1, d2| d1.domain_lt(d2), + vectorize_string_cmp(|cmp| cmp == Ordering::Less), + ); + registry.register_passthrough_nullable_2_arg::( + "lte", + |_, d1, d2| d1.domain_lte(d2), + vectorize_string_cmp(|cmp| cmp != Ordering::Greater), + ); +} + +fn vectorize_string_cmp( + func: impl Fn(Ordering) -> bool + Copy, +) -> impl Fn(ValueRef, ValueRef, &mut EvalContext) -> Value + Copy +{ + move |arg1, arg2, _ctx| match (arg1, arg2) { + (ValueRef::Scalar(arg1), ValueRef::Scalar(arg2)) => Value::Scalar(func(arg1.cmp(arg2))), + (ValueRef::Column(arg1), ValueRef::Scalar(arg2)) => { + let mut builder = MutableBitmap::with_capacity(arg1.len()); + for i in 0..arg1.len() { + builder.push(func(StringColumn::compare_str(&arg1, i, arg2))); + } + Value::Column(builder.into()) + } + (ValueRef::Scalar(arg1), ValueRef::Column(arg2)) => { + let mut builder = MutableBitmap::with_capacity(arg1.len()); + for i in 0..arg2.len() { + builder.push(func(StringColumn::compare_str(&arg2, i, arg1).reverse())); + } + Value::Column(builder.into()) + } + (ValueRef::Column(arg1), ValueRef::Column(arg2)) => { + let mut builder = MutableBitmap::with_capacity(arg1.len()); + for i in 0..arg1.len() { + builder.push(func(StringColumn::compare(&arg1, i, &arg2, i))); + } + Value::Column(builder.into()) + } + } } fn register_date_cmp(registry: &mut FunctionRegistry) { @@ -531,35 +591,8 @@ fn vectorize_like( let mut builder = MutableBitmap::with_capacity(arg1.len()); let pattern_type = generate_like_pattern(arg2.as_bytes(), arg1.current_buffer_len()); if let LikePattern::SurroundByPercent(searcher) = pattern_type { - let needle_byte_len = searcher.needle().len(); - let data = arg1.data().as_slice(); - let offsets = arg1.offsets().as_slice(); - let mut idx = 0; - let mut pos = (*offsets.first().unwrap()) as usize; - let end = (*offsets.last().unwrap()) as usize; - - while pos < end { - if let Some(p) = searcher.search(&data[pos..end]) { - // data: {3x}googlex|{3x}googlex|{3x}googlex - // needle_size: 6 - // offsets: 0, 10, 20, 30 - // (pos, p): (0, 3) , (10, 3), (20, 3), () - while offsets[idx + 1] as usize <= pos + p { - builder.push(false); - idx += 1; - } - // check if the substring is in bound - builder.push(pos + p + needle_byte_len <= offsets[idx + 1] as usize); - pos = offsets[idx + 1] as usize; - idx += 1; - } else { - break; - } - } - - while idx < arg1.len() { - builder.push(false); - idx += 1; + for arg1 in arg1_iter { + builder.push(searcher.search(arg1.as_bytes()).is_some()); } } else { for arg1 in arg1_iter { diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index bf095ebdf225..b4ed0508f772 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -706,7 +706,12 @@ fn register_to_string(registry: &mut FunctionRegistry) { "to_string", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - write!(output.data, "{}", date_to_string(val, ctx.func_ctx.tz.tz)).unwrap(); + write!( + output.row_buffer, + "{}", + date_to_string(val, ctx.func_ctx.tz.tz) + ) + .unwrap(); output.commit_row(); }), ); @@ -716,7 +721,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { write!( - output.data, + output.row_buffer, "{}", timestamp_to_string(val, ctx.func_ctx.tz.tz) ) @@ -738,7 +743,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { }, vectorize_with_builder_1_arg::>(|val, output, ctx| { write!( - output.builder.data, + output.builder.row_buffer, "{}", date_to_string(val, ctx.func_ctx.tz.tz) ) @@ -762,7 +767,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { vectorize_with_builder_1_arg::>( |val, output, ctx| { write!( - output.builder.data, + output.builder.row_buffer, "{}", timestamp_to_string(val, ctx.func_ctx.tz.tz) ) diff --git a/src/query/functions/src/scalars/decimal/cast.rs b/src/query/functions/src/scalars/decimal/cast.rs index 3547e2611d55..540bbad5f84b 100644 --- a/src/query/functions/src/scalars/decimal/cast.rs +++ b/src/query/functions/src/scalars/decimal/cast.rs @@ -313,7 +313,7 @@ fn decimal_to_string( match arg { ValueRef::Column(col) => { - let mut builder = StringColumnBuilder::with_capacity(col.len(), col.len() * 10); + let mut builder = StringColumnBuilder::with_capacity(col.len()); for x in DecimalType::::iter_column(&col) { builder.put_str(&DECIMAL_TYPE::display(x, from_size.scale)); builder.commit_row(); diff --git a/src/query/functions/src/scalars/geo_h3.rs b/src/query/functions/src/scalars/geo_h3.rs index 045327555552..17fbf78ebd60 100644 --- a/src/query/functions/src/scalars/geo_h3.rs +++ b/src/query/functions/src/scalars/geo_h3.rs @@ -272,13 +272,12 @@ pub fn register(registry: &mut FunctionRegistry) { |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|h3, builder, ctx| { match CellIndex::try_from(h3) { - Ok(index) => builder.put_str(&index.to_string()), + Ok(index) => builder.put_and_commit(index.to_string()), Err(err) => { ctx.set_error(builder.len(), err.to_string()); - builder.put_str(""); + builder.put_and_commit(""); } } - builder.commit_row(); }), ); diff --git a/src/query/functions/src/scalars/hash.rs b/src/query/functions/src/scalars/hash.rs index 4e00a7f4aeab..252e3b4eb5cb 100644 --- a/src/query/functions/src/scalars/hash.rs +++ b/src/query/functions/src/scalars/hash.rs @@ -49,8 +49,6 @@ use num_traits::AsPrimitive; use twox_hash::XxHash32; use twox_hash::XxHash64; -use crate::scalars::string::vectorize_string_to_string; - pub fn register(registry: &mut FunctionRegistry) { registry.register_aliases("siphash64", &["siphash"]); registry.register_aliases("sha", &["sha1"]); @@ -79,62 +77,48 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "md5", |_, _| FunctionDomain::MayThrow, - vectorize_string_to_string( - |col| col.data().len() * 32, - |val, output, ctx| { - // TODO md5 lib doesn't allow encode into buffer... - let old_len = output.data.len(); - output.data.resize(old_len + 32, 0); - if let Err(err) = hex::encode_to_slice( - Md5Hasher::digest(val).as_slice(), - &mut output.data[old_len..], - ) { - ctx.set_error(output.len(), err.to_string()); - } - output.commit_row(); - }, - ), + vectorize_with_builder_1_arg::(|val, output, ctx| { + // TODO md5 lib doesn't allow encode into buffer... + output.row_buffer.resize(32, 0); + if let Err(err) = + hex::encode_to_slice(Md5Hasher::digest(val).as_slice(), &mut output.row_buffer) + { + ctx.set_error(output.len(), err.to_string()); + } + output.commit_row(); + }), ); registry.register_passthrough_nullable_1_arg::( "sha", |_, _| FunctionDomain::MayThrow, - vectorize_string_to_string( - |col| col.data().len() * 40, - |val, output, ctx| { - let old_len = output.data.len(); - output.data.resize(old_len + 40, 0); - // TODO sha1 lib doesn't allow encode into buffer... - let mut m = ::sha1::Sha1::new(); - sha1::digest::Update::update(&mut m, val.as_bytes()); - - if let Err(err) = - hex::encode_to_slice(m.finalize().as_slice(), &mut output.data[old_len..]) - { - ctx.set_error(output.len(), err.to_string()); - } - output.commit_row(); - }, - ), + vectorize_with_builder_1_arg::(|val, output, ctx| { + output.row_buffer.resize(40, 0); + // TODO sha1 lib doesn't allow encode into buffer... + let mut m = ::sha1::Sha1::new(); + sha1::digest::Update::update(&mut m, val.as_bytes()); + + if let Err(err) = hex::encode_to_slice(m.finalize().as_slice(), &mut output.row_buffer) + { + ctx.set_error(output.len(), err.to_string()); + } + output.commit_row(); + }), ); registry.register_passthrough_nullable_1_arg::( "blake3", |_, _| FunctionDomain::MayThrow, - vectorize_string_to_string( - |col| col.data().len() * 64, - |val, output, ctx| { - let old_len = output.data.len(); - output.data.resize(old_len + 64, 0); - if let Err(err) = hex::encode_to_slice( - blake3::hash(val.as_bytes()).as_bytes(), - &mut output.data[old_len..], - ) { - ctx.set_error(output.len(), err.to_string()); - } - output.commit_row(); - }, - ), + vectorize_with_builder_1_arg::(|val, output, ctx| { + output.row_buffer.resize(64, 0); + if let Err(err) = hex::encode_to_slice( + blake3::hash(val.as_bytes()).as_bytes(), + &mut output.row_buffer, + ) { + ctx.set_error(output.len(), err.to_string()); + } + output.commit_row(); + }), ); registry.register_passthrough_nullable_2_arg::, StringType, _, _>( @@ -175,8 +159,7 @@ pub fn register(registry: &mut FunctionRegistry) { String::new() }, }; - output.put_str(&res); - output.commit_row(); + output.put_and_commit(res); }, ), ); diff --git a/src/query/functions/src/scalars/other.rs b/src/query/functions/src/scalars/other.rs index 2ed8f5eeb456..0dd6f34ec2e2 100644 --- a/src/query/functions/src/scalars/other.rs +++ b/src/query/functions/src/scalars/other.rs @@ -22,6 +22,7 @@ use databend_common_base::base::convert_number_size; use databend_common_base::base::uuid::Uuid; use databend_common_base::base::OrderedFloat; use databend_common_expression::error_to_null; +use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::boolean::BooleanDomain; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::number::Float32Type; @@ -93,8 +94,7 @@ pub fn register(registry: &mut FunctionRegistry) { |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(move |val, output, _| { let new_val = convert_byte_size(val.into()); - output.put_str(&new_val); - output.commit_row(); + output.put_and_commit(new_val); }), ); @@ -103,8 +103,7 @@ pub fn register(registry: &mut FunctionRegistry) { |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(move |val, output, _| { let new_val = convert_number_size(val.into()); - output.put_str(&new_val); - output.commit_row(); + output.put_and_commit(new_val); }), ); @@ -232,16 +231,15 @@ pub fn register(registry: &mut FunctionRegistry) { "gen_random_uuid", |_| FunctionDomain::Full, |ctx| { - let mut values: Vec = Vec::with_capacity(ctx.num_rows * 36); - let mut offsets: Vec = Vec::with_capacity(ctx.num_rows); - offsets.push(0); + let mut builder = BinaryColumnBuilder::with_capacity(ctx.num_rows, 0); for _ in 0..ctx.num_rows { let value = Uuid::new_v4(); - offsets.push(offsets.last().unwrap() + 36u64); - write!(&mut values, "{:x}", value).unwrap(); + write!(&mut builder.data, "{}", value).unwrap(); + builder.commit_row(); } - let col = StringColumn::new(values.into(), offsets.into()); + + let col = StringColumn::try_from(builder.build()).unwrap(); Value::Column(col) }, ); @@ -294,8 +292,7 @@ fn register_inet_ntoa(registry: &mut FunctionRegistry) { match num_traits::cast::cast::(val) { Some(val) => { let addr_str = Ipv4Addr::from(val.to_be_bytes()).to_string(); - output.put_str(&addr_str); - output.commit_row(); + output.put_and_commit(addr_str); } None => { ctx.set_error( @@ -407,8 +404,7 @@ fn register_num_to_char(registry: &mut FunctionRegistry) { .and_then(|entry| entry.process_i64(value)) { Ok(s) => { - builder.put_str(&s); - builder.commit_row() + builder.put_and_commit(s); } Err(e) => { ctx.set_error(builder.len(), e.to_string()); diff --git a/src/query/functions/src/scalars/string.rs b/src/query/functions/src/scalars/string.rs index 25f10b82ccd2..273a1d9dc9ed 100644 --- a/src/query/functions/src/scalars/string.rs +++ b/src/query/functions/src/scalars/string.rs @@ -19,22 +19,19 @@ use databend_common_base::base::uuid::Uuid; use databend_common_expression::types::decimal::Decimal128Type; use databend_common_expression::types::number::SimpleDomain; use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::string::StringColumn; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::string::StringDomain; use databend_common_expression::types::ArrayType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::unify_string; +use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::vectorize_with_builder_4_arg; -use databend_common_expression::EvalContext; use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionRegistry; -use databend_common_expression::Value; -use databend_common_expression::ValueRef; use stringslice::StringSlice; pub const ALL_STRING_FUNC_NAMES: &[&str] = &[ @@ -102,41 +99,35 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "upper", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| col.data().len(), - |val, output, _| { - for ch in val.chars() { - if ch.is_ascii() { - output.put_char(ch.to_ascii_uppercase()); - } else { - for x in ch.to_uppercase() { - output.put_char(x); - } + vectorize_with_builder_1_arg::(|val, output, _| { + for ch in val.chars() { + if ch.is_ascii() { + output.put_char(ch.to_ascii_uppercase()); + } else { + for x in ch.to_uppercase() { + output.put_char(x); } } - output.commit_row(); - }, - ), + } + output.commit_row(); + }), ); registry.register_passthrough_nullable_1_arg::( "lower", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| col.data().len(), - |val, output, _| { - for ch in val.chars() { - if ch.is_ascii() { - output.put_char(ch.to_ascii_lowercase()); - } else { - for x in ch.to_lowercase() { - output.put_char(x); - } + vectorize_with_builder_1_arg::(|val, output, _| { + for ch in val.chars() { + if ch.is_ascii() { + output.put_char(ch.to_ascii_lowercase()); + } else { + for x in ch.to_lowercase() { + output.put_char(x); } } - output.commit_row(); - }, - ), + } + output.commit_row(); + }), ); registry.register_1_arg::, _, _>( @@ -148,19 +139,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::, _, _>( "octet_length", |_, _| FunctionDomain::Full, - |val, _| match val { - ValueRef::Scalar(s) => Value::Scalar(s.len() as u64), - ValueRef::Column(c) => { - let diffs = c - .offsets() - .iter() - .zip(c.offsets().iter().skip(1)) - .map(|(a, b)| b - a) - .collect::>(); - - Value::Column(diffs.into()) - } - }, + vectorize_1_arg::>(|val, _| val.len() as u64), ); registry.register_1_arg::, _, _>( @@ -382,39 +361,33 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "quote", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| col.data().len() * 2, - |val, output, _| { - for ch in val.chars() { - match ch { - '\0' => output.put_str("\\0"), - '\'' => output.put_str("\\\'"), - '\"' => output.put_str("\\\""), - '\u{8}' => output.put_str("\\b"), - '\n' => output.put_str("\\n"), - '\r' => output.put_str("\\r"), - '\t' => output.put_str("\\t"), - '\\' => output.put_str("\\\\"), - c => output.put_char(c), - } + vectorize_with_builder_1_arg::(|val, output, _| { + for ch in val.chars() { + match ch { + '\0' => output.put_str("\\0"), + '\'' => output.put_str("\\\'"), + '\"' => output.put_str("\\\""), + '\u{8}' => output.put_str("\\b"), + '\n' => output.put_str("\\n"), + '\r' => output.put_str("\\r"), + '\t' => output.put_str("\\t"), + '\\' => output.put_str("\\\\"), + c => output.put_char(c), } - output.commit_row(); - }, - ), + } + output.commit_row(); + }), ); registry.register_passthrough_nullable_1_arg::( "reverse", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| col.data().len(), - |val, output, _| { - for char in val.chars().rev() { - output.put_char(char); - } - output.commit_row(); - }, - ), + vectorize_with_builder_1_arg::(|val, output, _| { + for char in val.chars().rev() { + output.put_char(char); + } + output.commit_row(); + }), ); registry.register_1_arg::, _, _>( @@ -436,53 +409,38 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "ltrim", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| col.data().len(), - |val, output, _| { - output.put_str(val.trim_start()); - output.commit_row(); - }, - ), + vectorize_with_builder_1_arg::(|val, output, _| { + output.put_and_commit(val.trim_start()); + }), ); registry.register_passthrough_nullable_1_arg::( "rtrim", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| col.data().len(), - |val, output, _| { - output.put_str(val.trim_end()); - output.commit_row(); - }, - ), + vectorize_with_builder_1_arg::(|val, output, _| { + output.put_and_commit(val.trim_end()); + }), ); registry.register_passthrough_nullable_1_arg::( "trim", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| col.data().len(), - |val, output, _| { - output.put_str(val.trim()); - output.commit_row(); - }, - ), + vectorize_with_builder_1_arg::(|val, output, _| { + output.put_and_commit(val.trim()); + }), ); registry.register_passthrough_nullable_2_arg::( "trim_leading", |_, _, _| FunctionDomain::Full, - vectorize_string_to_string_2_arg( - |col, _| col.data().len(), - |val, trim_str, _, output| { + vectorize_with_builder_2_arg::( + |val, trim_str, output, _| { if trim_str.is_empty() { - output.put_str(val); - output.commit_row(); + output.put_and_commit(val); return; } - output.put_str(val.trim_start_matches(trim_str)); - output.commit_row(); + output.put_and_commit(val.trim_start_matches(trim_str)); }, ), ); @@ -490,17 +448,14 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_2_arg::( "trim_trailing", |_, _, _| FunctionDomain::Full, - vectorize_string_to_string_2_arg( - |col, _| col.data().len(), - |val, trim_str, _, output| { + vectorize_with_builder_2_arg::( + |val, trim_str, output, _| { if trim_str.is_empty() { - output.put_str(val); - output.commit_row(); + output.put_and_commit(val); return; } - output.put_str(val.trim_end_matches(trim_str)); - output.commit_row(); + output.put_and_commit(val.trim_end_matches(trim_str)); }, ), ); @@ -508,12 +463,10 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_2_arg::( "trim_both", |_, _, _| FunctionDomain::Full, - vectorize_string_to_string_2_arg( - |col, _| col.data().len(), - |val, trim_str, _, output| { + vectorize_with_builder_2_arg::( + |val, trim_str, output, _| { if trim_str.is_empty() { - output.put_str(val); - output.commit_row(); + output.put_and_commit(val); return; } @@ -526,8 +479,7 @@ pub fn register(registry: &mut FunctionRegistry) { res = &res[..res.len() - trim_str.len()]; } - output.put_str(res); - output.commit_row(); + output.put_and_commit(res); }, ), ); @@ -535,16 +487,12 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "to_hex", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| col.data().len() * 2, - |val, output, _| { - let old_len = output.data.len(); - let extra_len = val.len() * 2; - output.data.resize(old_len + extra_len, 0); - hex::encode_to_slice(val, &mut output.data[old_len..]).unwrap(); - output.commit_row(); - }, - ), + vectorize_with_builder_1_arg::(|val, output, _| { + let len = val.len() * 2; + output.row_buffer.resize(len, 0); + hex::encode_to_slice(val, &mut output.row_buffer).unwrap(); + output.commit_row(); + }), ); // TODO: generalize them to be alias of [CONV](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_conv) @@ -553,7 +501,7 @@ pub fn register(registry: &mut FunctionRegistry) { "bin", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::, StringType>(|val, output, _| { - write!(output.data, "{val:b}").unwrap(); + write!(output.row_buffer, "{val:b}").unwrap(); output.commit_row(); }), ); @@ -561,7 +509,7 @@ pub fn register(registry: &mut FunctionRegistry) { "oct", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::, StringType>(|val, output, _| { - write!(output.data, "{val:o}").unwrap(); + write!(output.row_buffer, "{val:o}").unwrap(); output.commit_row(); }), ); @@ -569,7 +517,7 @@ pub fn register(registry: &mut FunctionRegistry) { "to_hex", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::, StringType>(|val, output, _| { - write!(output.data, "{val:x}").unwrap(); + write!(output.row_buffer, "{val:x}").unwrap(); output.commit_row(); }), ); @@ -624,56 +572,26 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "soundex", |_, _| FunctionDomain::Full, - vectorize_string_to_string( - |col| usize::max(col.data().len(), 4 * col.len()), - soundex::soundex, - ), + vectorize_with_builder_1_arg::(soundex::soundex), ); const MAX_SPACE_LENGTH: u64 = 1000000; registry.register_passthrough_nullable_1_arg::, StringType, _, _>( "space", |_, _| FunctionDomain::MayThrow, - |times, ctx| match times { - ValueRef::Scalar(times) => { - if times > MAX_SPACE_LENGTH { - ctx.set_error( - 0, - format!("space length is too big, max is: {}", MAX_SPACE_LENGTH), - ); - Value::Scalar("".to_string()) - } else { - Value::Scalar(" ".repeat(times as usize)) - } - } - ValueRef::Column(col) => { - let mut total_space: u64 = 0; - let mut offsets: Vec = Vec::with_capacity(col.len() + 1); - offsets.push(0); - for (i, times) in col.iter().enumerate() { - if times > &MAX_SPACE_LENGTH { - ctx.set_error( - i, - format!("space length is too big, max is: {}", MAX_SPACE_LENGTH), - ); - break; - } - total_space += times; - offsets.push(total_space); - } - if ctx.errors.is_some() { - offsets.truncate(1); - total_space = 0; - } - let col = StringColumnBuilder { - data: " ".repeat(total_space as usize).into_bytes(), - offsets, - need_estimated: false, + vectorize_with_builder_1_arg::, StringType>(|times, output, ctx| { + if times > MAX_SPACE_LENGTH { + ctx.set_error( + output.len(), + format!("space length is too big, max is: {}", MAX_SPACE_LENGTH), + ); + } else { + for _ in 0..times { + output.put_char(' '); } - .build(); - Value::Column(col) } - }, + output.commit_row(); + }), ); registry.register_passthrough_nullable_2_arg::, StringType, _, _>( @@ -702,11 +620,10 @@ pub fn register(registry: &mut FunctionRegistry) { let n = n as usize; let s_len = s.chars().count(); if n < s_len { - output.put_str(s.slice(0..n)); + output.put_and_commit(s.slice(0..n)); } else { - output.put_str(s); + output.put_and_commit(s); } - output.commit_row(); }, ), ); @@ -719,11 +636,10 @@ pub fn register(registry: &mut FunctionRegistry) { let n = n as usize; let s_len = s.chars().count(); if n < s_len { - output.put_str(s.slice(s_len - n..)); + output.put_and_commit(s.slice(s_len - n..)); } else { - output.put_str(s); + output.put_and_commit(s); } - output.commit_row(); }, ), ); @@ -783,12 +699,10 @@ pub fn register(registry: &mut FunctionRegistry) { if s == sep { output.builder.commit_row(); } else if sep.is_empty() { - output.builder.put_str(s); - output.builder.commit_row(); + output.builder.put_and_commit(s); } else { for v in s.split(sep) { - output.builder.put_str(v); - output.builder.commit_row(); + output.builder.put_and_commit(v); } } output.commit_row(); @@ -916,68 +830,3 @@ fn substr(builder: &mut StringColumnBuilder, str: &str, pos: i64, len: u64) { builder.put_char_iter(str.chars().skip(start).take(len as usize)); builder.commit_row(); } - -/// String to String scalar function with estimated output column capacity. -pub fn vectorize_string_to_string( - estimate_bytes: impl Fn(&StringColumn) -> usize + Copy, - func: impl Fn(&str, &mut StringColumnBuilder, &mut EvalContext) + Copy, -) -> impl Fn(ValueRef, &mut EvalContext) -> Value + Copy { - move |arg1, ctx| match arg1 { - ValueRef::Scalar(val) => { - let mut builder = StringColumnBuilder::with_capacity(1, 0); - func(val, &mut builder, ctx); - Value::Scalar(builder.build_scalar()) - } - ValueRef::Column(col) => { - let data_capacity = estimate_bytes(&col); - let mut builder = StringColumnBuilder::with_capacity(col.len(), data_capacity); - for val in col.iter() { - func(val, &mut builder, ctx); - } - - Value::Column(builder.build()) - } - } -} - -/// (String, String) to String scalar function with estimated output column capacity. -fn vectorize_string_to_string_2_arg( - estimate_bytes: impl Fn(&StringColumn, &StringColumn) -> usize + Copy, - func: impl Fn(&str, &str, &mut EvalContext, &mut StringColumnBuilder) + Copy, -) -> impl Fn(ValueRef, ValueRef, &mut EvalContext) -> Value + Copy -{ - move |arg1, arg2, ctx| match (arg1, arg2) { - (ValueRef::Scalar(arg1), ValueRef::Scalar(arg2)) => { - let mut builder = StringColumnBuilder::with_capacity(1, 0); - func(arg1, arg2, ctx, &mut builder); - Value::Scalar(builder.build_scalar()) - } - (ValueRef::Scalar(arg1), ValueRef::Column(arg2)) => { - let data_capacity = - estimate_bytes(&StringColumnBuilder::repeat(arg1, 1).build(), &arg2); - let mut builder = StringColumnBuilder::with_capacity(arg2.len(), data_capacity); - for val in arg2.iter() { - func(arg1, val, ctx, &mut builder); - } - Value::Column(builder.build()) - } - (ValueRef::Column(arg1), ValueRef::Scalar(arg2)) => { - let data_capacity = - estimate_bytes(&arg1, &StringColumnBuilder::repeat(arg2, 1).build()); - let mut builder = StringColumnBuilder::with_capacity(arg1.len(), data_capacity); - for val in arg1.iter() { - func(val, arg2, ctx, &mut builder); - } - Value::Column(builder.build()) - } - (ValueRef::Column(arg1), ValueRef::Column(arg2)) => { - let data_capacity = estimate_bytes(&arg1, &arg2); - let mut builder = StringColumnBuilder::with_capacity(arg1.len(), data_capacity); - let iter = arg1.iter().zip(arg2.iter()); - for (val1, val2) in iter { - func(val1, val2, ctx, &mut builder); - } - Value::Column(builder.build()) - } - } -} diff --git a/src/query/functions/src/scalars/string_multi_args.rs b/src/query/functions/src/scalars/string_multi_args.rs index ebbae81fbee2..cb5547b9ae7f 100644 --- a/src/query/functions/src/scalars/string_multi_args.rs +++ b/src/query/functions/src/scalars/string_multi_args.rs @@ -19,7 +19,6 @@ use databend_common_expression::passthrough_nullable; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::number::Int64Type; use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::string::StringDomain; use databend_common_expression::types::NumberColumn; use databend_common_expression::types::*; @@ -34,6 +33,7 @@ use databend_common_expression::FunctionSignature; use databend_common_expression::Scalar; use databend_common_expression::Value; use databend_common_expression::ValueRef; +use string::StringColumnBuilder; pub fn register(registry: &mut FunctionRegistry) { registry.register_function_factory("concat", |_, args_type| { @@ -114,7 +114,7 @@ pub fn register(registry: &mut FunctionRegistry) { .collect::>(); let size = len.unwrap_or(1); - let mut builder = StringColumnBuilder::with_capacity(size, 0); + let mut builder = StringColumnBuilder::with_capacity(size); match &args[0] { ValueRef::Scalar(sep) => { @@ -430,7 +430,7 @@ fn concat_fn(args: &[ValueRef], _: &mut EvalContext) -> Value .collect::>(); let size = len.unwrap_or(1); - let mut builder = StringColumnBuilder::with_capacity(size, 0); + let mut builder = StringColumnBuilder::with_capacity(size); for idx in 0..size { for arg in &args { builder.put_str(unsafe { arg.index_unchecked(idx) }); @@ -641,7 +641,7 @@ fn regexp_replace_fn(args: &[ValueRef], ctx: &mut EvalContext) -> Value }; let size = len.unwrap_or(1); - let mut builder = StringColumnBuilder::with_capacity(size, 0); + let mut builder = StringColumnBuilder::with_capacity(size); let cached_reg = match (&pat_arg, &mt_arg) { (ValueRef::Scalar(pat), Some(ValueRef::Scalar(mt))) => { @@ -691,8 +691,7 @@ fn regexp_replace_fn(args: &[ValueRef], ctx: &mut EvalContext) -> Value } if source.is_empty() || pat.is_empty() { - builder.put_str(source); - builder.commit_row(); + builder.put_and_commit(source); continue; } @@ -766,7 +765,7 @@ fn regexp_substr_fn(args: &[ValueRef], ctx: &mut EvalContext) -> Value< }; let size = len.unwrap_or(1); - let mut builder = StringColumnBuilder::with_capacity(size, 0); + let mut builder = StringColumnBuilder::with_capacity(size); let mut validity = MutableBitmap::with_capacity(size); for idx in 0..size { let source = unsafe { source_arg.index_unchecked(idx) }; diff --git a/src/query/functions/src/scalars/variant.rs b/src/query/functions/src/scalars/variant.rs index f118929c8014..79e156ed8cb3 100644 --- a/src/query/functions/src/scalars/variant.rs +++ b/src/query/functions/src/scalars/variant.rs @@ -1009,8 +1009,7 @@ pub fn register(registry: &mut FunctionRegistry) { } } let json_str = cast_to_string(val); - output.put_str(&json_str); - output.commit_row(); + output.put_and_commit(json_str); }), ); @@ -1953,7 +1952,7 @@ fn get_by_keypath_fn( let len = len_opt.unwrap_or(1); let mut builder = if string_res { - ColumnBuilder::String(StringColumnBuilder::with_capacity(len, len * 50)) + ColumnBuilder::String(StringColumnBuilder::with_capacity(len)) } else { ColumnBuilder::Variant(BinaryColumnBuilder::with_capacity(len, len * 50)) }; diff --git a/src/query/functions/src/scalars/vector.rs b/src/query/functions/src/scalars/vector.rs index b551f2539bf3..f64032e5ca9a 100644 --- a/src/query/functions/src/scalars/vector.rs +++ b/src/query/functions/src/scalars/vector.rs @@ -188,16 +188,14 @@ pub fn register(registry: &mut FunctionRegistry) { vectorize_with_builder_1_arg::(|data, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { - output.put_str(""); - output.commit_row(); + output.put_and_commit(""); return; } } if ctx.func_ctx.openai_api_key.is_empty() { ctx.set_error(output.len(), "openai_api_key is empty".to_string()); - output.put_str(""); - output.commit_row(); + output.put_and_commit(""); return; } let api_base = ctx.func_ctx.openai_api_chat_base_url.clone(); @@ -216,17 +214,16 @@ pub fn register(registry: &mut FunctionRegistry) { let result = openai.completion_text_request(data.to_string()); match result { Ok((resp, _)) => { - output.put_str(&resp); + output.put_and_commit(resp); } Err(e) => { ctx.set_error( output.len(), format!("openai completion request error:{:?}", e), ); - output.put_str(""); + output.put_and_commit(""); } } - output.commit_row(); }), ); } diff --git a/src/query/functions/src/srfs/variant.rs b/src/query/functions/src/srfs/variant.rs index eb4c8db7e3a0..f6d73b140577 100644 --- a/src/query/functions/src/srfs/variant.rs +++ b/src/query/functions/src/srfs/variant.rs @@ -678,13 +678,12 @@ fn unnest_variant_obj( Some(vals) if !vals.is_empty() => { let len = vals.len(); let mut val_builder = BinaryColumnBuilder::with_capacity(0, 0); - let mut key_builder = StringColumnBuilder::with_capacity(0, 0); + let mut key_builder = StringColumnBuilder::with_capacity(0); max_nums_per_row[row] = std::cmp::max(max_nums_per_row[row], len); for (key, val) in vals { - key_builder.put_str(&String::from_utf8_lossy(&key)); - key_builder.commit_row(); + key_builder.put_and_commit(String::from_utf8_lossy(&key)); val_builder.put_slice(&val); val_builder.commit_row(); } @@ -807,8 +806,7 @@ impl FlattenGenerator { key_builder.push_null(); } if let Some(path_builder) = path_builder { - path_builder.put_str(&inner_path); - path_builder.commit_row(); + path_builder.put_and_commit(&inner_path); } if let Some(index_builder) = index_builder { index_builder.push(i.try_into().unwrap()); @@ -867,8 +865,7 @@ impl FlattenGenerator { key_builder.push(name.as_ref()); } if let Some(path_builder) = path_builder { - path_builder.put_str(&inner_path); - path_builder.commit_row(); + path_builder.put_and_commit(&inner_path); } if let Some(index_builder) = index_builder { index_builder.push_null(); @@ -908,7 +905,7 @@ impl FlattenGenerator { None }; let mut path_builder = if params.is_empty() || params.contains(&3) { - Some(StringColumnBuilder::with_capacity(0, 0)) + Some(StringColumnBuilder::with_capacity(0)) } else { None }; diff --git a/src/query/functions/tests/it/aggregates/testdata/agg.txt b/src/query/functions/tests/it/aggregates/testdata/agg.txt index 85300eac60a7..777b5b540fa0 100644 --- a/src/query/functions/tests/it/aggregates/testdata/agg.txt +++ b/src/query/functions/tests/it/aggregates/testdata/agg.txt @@ -941,12 +941,12 @@ evaluation (internal): ast: array_agg('a') evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------+ -| a | Int64([4, 3, 2, 1]) | -| Output | ArrayColumn { values: StringColumn { data: 0x61616161, offsets: [0, 1, 2, 3, 4] }, offsets: [0, 4] } | -+--------+------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------+ +| a | Int64([4, 3, 2, 1]) | +| Output | ArrayColumn { values: StringColumn { data: Utf8ViewArray[a, a, a, a] }, offsets: [0, 4] } | ++--------+-------------------------------------------------------------------------------------------+ ast: array_agg(NULL) @@ -1031,42 +1031,42 @@ evaluation (internal): ast: string_agg(s) evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 3, 6, 9, 12] } | -| Output | NullableColumn { column: StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 12] }, validity: [0b_______1] } | -+--------+------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abcdefopqxyz] }, validity: [0b_______1] } | ++--------+-------------------------------------------------------------------------------------------------------+ ast: string_agg(s_null) evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------+ -| s_null | NullableColumn { column: StringColumn { data: 0x616364, offsets: [0, 1, 1, 2, 3] }, validity: [0b____1101] } | -| Output | NullableColumn { column: StringColumn { data: 0x616364, offsets: [0, 3] }, validity: [0b_______1] } | -+--------+--------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------+ +| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[acd] }, validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------------------------------+ ast: string_agg(s, '|') evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 3, 6, 9, 12] } | -| Output | NullableColumn { column: StringColumn { data: 0x6162637c6465667c6f70717c78797a, offsets: [0, 15] }, validity: [0b_______1] } | -+--------+------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc|def|opq|xyz] }, validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------------------------------------+ ast: string_agg(s_null, '-') evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------+ -| s_null | NullableColumn { column: StringColumn { data: 0x616364, offsets: [0, 1, 1, 2, 3] }, validity: [0b____1101] } | -| Output | NullableColumn { column: StringColumn { data: 0x612d632d64, offsets: [0, 5] }, validity: [0b_______1] } | -+--------+--------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------+ +| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a-c-d] }, validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------------------------------+ ast: bitmap_and_count(bm) @@ -1317,42 +1317,42 @@ evaluation (internal): ast: histogram(all_null) evaluation (internal): -+----------+-----------------------------------------------------------------------------------------------+ -| Column | Data | -+----------+-----------------------------------------------------------------------------------------------+ -| all_null | NullableColumn { column: UInt64([1, 2, 3, 4]), validity: [0b____0000] } | -| Output | NullableColumn { column: StringColumn { data: 0x, offsets: [0, 0] }, validity: [0b_______0] } | -+----------+-----------------------------------------------------------------------------------------------+ ++----------+-------------------------------------------------------------------------------------------+ +| Column | Data | ++----------+-------------------------------------------------------------------------------------------+ +| all_null | NullableColumn { column: UInt64([1, 2, 3, 4]), validity: [0b____0000] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[] }, validity: [0b_______0] } | ++----------+-------------------------------------------------------------------------------------------+ ast: histogram(x_null) evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| x_null | NullableColumn { column: UInt64([1, 2, 3, 4]), validity: [0b____0011] } | -| Output | NullableColumn { column: StringColumn { data: 0x5b7b226c6f776572223a2231222c227570706572223a2231222c226e6476223a312c22636f756e74223a312c227072655f73756d223a307d2c7b226c6f776572223a2232222c227570706572223a2232222c226e6476223a312c22636f756e74223a312c227072655f73756d223a317d5d, offsets: [0, 113] }, validity: [0b_______1] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| x_null | NullableColumn { column: UInt64([1, 2, 3, 4]), validity: [0b____0011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1}]] }, validity: [0b_______1] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast: histogram(a) evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | Int64([4, 3, 2, 1]) | -| Output | NullableColumn { column: StringColumn { data: 0x5b7b226c6f776572223a2231222c227570706572223a2231222c226e6476223a312c22636f756e74223a312c227072655f73756d223a307d2c7b226c6f776572223a2232222c227570706572223a2232222c226e6476223a312c22636f756e74223a312c227072655f73756d223a317d2c7b226c6f776572223a2233222c227570706572223a2233222c226e6476223a312c22636f756e74223a312c227072655f73756d223a327d2c7b226c6f776572223a2234222c227570706572223a2234222c226e6476223a312c22636f756e74223a312c227072655f73756d223a337d5d, offsets: [0, 225] }, validity: [0b_______1] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | Int64([4, 3, 2, 1]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1},{"lower":"3","upper":"3","ndv":1,"count":1,"pre_sum":2},{"lower":"4","upper":"4","ndv":1,"count":1,"pre_sum":3}]] }, validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast: histogram(a, 1) evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | Int64([4, 3, 2, 1]) | -| Output | NullableColumn { column: StringColumn { data: 0x5b7b226c6f776572223a2231222c227570706572223a2234222c226e6476223a342c22636f756e74223a342c227072655f73756d223a307d5d, offsets: [0, 57] }, validity: [0b_______1] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | Int64([4, 3, 2, 1]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[[{"lower":"1","upper":"4","ndv":4,"count":4,"pre_sum":0}]] }, validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------+ ast: json_array_agg(1) @@ -1463,7 +1463,7 @@ evaluation (internal): | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ | a | Int64([4, 3, 2, 1]) | -| s | StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 3, 6, 9, 12] } | +| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | | Output | BinaryColumn { data: 0x4000000410000003100000031000000310000003200000022000000220000002200000026162636465666f707178797a4004400340024001, offsets: [0, 56] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1474,7 +1474,7 @@ evaluation (internal): | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------+ | b | UInt64([1, 2, 3, 4]) | -| s_null | NullableColumn { column: StringColumn { data: 0x616364, offsets: [0, 1, 1, 2, 3] }, validity: [0b____1101] } | +| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | | Output | BinaryColumn { data: 0x40000003100000011000000110000001200000022000000220000002616364500150035004, offsets: [0, 37] } | +--------+-----------------------------------------------------------------------------------------------------------------------+ @@ -1486,7 +1486,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 3, 6, 9, 12] } | +| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | | dec | NullableColumn { column: Decimal128([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | | Output | BinaryColumn { data: 0x4000000310000003100000031000000320000009200000092000000961626364656678797a603ff199999999999a60400199999999999a60400a666666666666, offsets: [0, 64] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt b/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt index 2a4854afe6c9..1b69c2840d76 100644 --- a/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt +++ b/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt @@ -919,12 +919,12 @@ evaluation (internal): ast: array_agg('a') evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------+ -| a | Int64([4, 3, 2, 1]) | -| Output | ArrayColumn { values: StringColumn { data: 0x61616161, offsets: [0, 1, 2, 3, 4] }, offsets: [0, 2, 4] } | -+--------+---------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------+ +| a | Int64([4, 3, 2, 1]) | +| Output | ArrayColumn { values: StringColumn { data: Utf8ViewArray[a, a, a, a] }, offsets: [0, 2, 4] } | ++--------+----------------------------------------------------------------------------------------------+ ast: array_agg(NULL) @@ -1009,42 +1009,42 @@ evaluation (internal): ast: string_agg(s) evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 3, 6, 9, 12] } | -| Output | NullableColumn { column: StringColumn { data: 0x6162636f707164656678797a, offsets: [0, 6, 12] }, validity: [0b______11] } | -+--------+---------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abcopq, defxyz] }, validity: [0b______11] } | ++--------+---------------------------------------------------------------------------------------------------------+ ast: string_agg(s_null) evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------+ -| s_null | NullableColumn { column: StringColumn { data: 0x616364, offsets: [0, 1, 1, 2, 3] }, validity: [0b____1101] } | -| Output | NullableColumn { column: StringColumn { data: 0x616364, offsets: [0, 2, 3] }, validity: [0b______11] } | -+--------+--------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------+ +| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[ac, d] }, validity: [0b______11] } | ++--------+----------------------------------------------------------------------------------------------------+ ast: string_agg(s, '|') evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 3, 6, 9, 12] } | -| Output | NullableColumn { column: StringColumn { data: 0x6162637c6f70716465667c78797a, offsets: [0, 7, 14] }, validity: [0b______11] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc|opq, def|xyz] }, validity: [0b______11] } | ++--------+-----------------------------------------------------------------------------------------------------------+ ast: string_agg(s_null, '-') evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------+ -| s_null | NullableColumn { column: StringColumn { data: 0x616364, offsets: [0, 1, 1, 2, 3] }, validity: [0b____1101] } | -| Output | NullableColumn { column: StringColumn { data: 0x612d6364, offsets: [0, 3, 4] }, validity: [0b______11] } | -+--------+--------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------+ +| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a-c, d] }, validity: [0b______11] } | ++--------+----------------------------------------------------------------------------------------------------+ ast: bitmap_and_count(bm) @@ -1401,7 +1401,7 @@ evaluation (internal): | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ | a | Int64([4, 3, 2, 1]) | -| s | StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 3, 6, 9, 12] } | +| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | | Output | BinaryColumn { data: 0x4000000410000003100000031000000310000003200000022000000220000002200000026162636465666f707178797a4004400340024001, offsets: [0, 56] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1412,7 +1412,7 @@ evaluation (internal): | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------+ | b | UInt64([1, 2, 3, 4]) | -| s_null | NullableColumn { column: StringColumn { data: 0x616364, offsets: [0, 1, 1, 2, 3] }, validity: [0b____1101] } | +| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | | Output | BinaryColumn { data: 0x40000003100000011000000110000001200000022000000220000002616364500150035004, offsets: [0, 37] } | +--------+-----------------------------------------------------------------------------------------------------------------------+ @@ -1424,7 +1424,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6162636465666f707178797a, offsets: [0, 3, 6, 9, 12] } | +| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | | dec | NullableColumn { column: Decimal128([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | | Output | BinaryColumn { data: 0x4000000310000003100000031000000320000009200000092000000961626364656678797a603ff199999999999a60400199999999999a60400a666666666666, offsets: [0, 64] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/comparison.rs b/src/query/functions/tests/it/scalars/comparison.rs index bb0f85c3ff17..318673980cb0 100644 --- a/src/query/functions/tests/it/scalars/comparison.rs +++ b/src/query/functions/tests/it/scalars/comparison.rs @@ -365,6 +365,7 @@ fn test_like(file: &mut impl Write) { )]; run_ast(file, "lhs like 'a%'", &columns); run_ast(file, "lhs like 'b%'", &columns); + run_ast(file, "lhs like 'ab%'", &columns); run_ast(file, "lhs like 'c'", &columns); let columns = [ diff --git a/src/query/functions/tests/it/scalars/testdata/arithmetic.txt b/src/query/functions/tests/it/scalars/testdata/arithmetic.txt index dedd9bdb5499..281ddc8e843d 100644 --- a/src/query/functions/tests/it/scalars/testdata/arithmetic.txt +++ b/src/query/functions/tests/it/scalars/testdata/arithmetic.txt @@ -1617,12 +1617,12 @@ evaluation: | Row 2 | 3 | '3' | +--------+---------+--------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| a | Int8([1, 2, 3]) | -| Output | StringColumn { data: 0x313233, offsets: [0, 1, 2, 3] } | -+--------+--------------------------------------------------------+ ++--------+-----------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------+ +| a | Int8([1, 2, 3]) | +| Output | StringColumn { data: Utf8ViewArray[1, 2, 3] } | ++--------+-----------------------------------------------+ ast : to_string(a2) @@ -1639,12 +1639,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+------------------+-----------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: 0x313233, offsets: [0, 1, 2, 3] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 2, 3] }, validity: [0b_____011] } | ++--------+--------------------------------------------------------------------------------------------------+ ast : to_string(b) @@ -1661,12 +1661,12 @@ evaluation: | Row 2 | 6 | '6' | +--------+---------+--------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| b | Int16([2, 4, 6]) | -| Output | StringColumn { data: 0x323436, offsets: [0, 1, 2, 3] } | -+--------+--------------------------------------------------------+ ++--------+-----------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------+ +| b | Int16([2, 4, 6]) | +| Output | StringColumn { data: Utf8ViewArray[2, 4, 6] } | ++--------+-----------------------------------------------+ ast : to_string(c) @@ -1683,12 +1683,12 @@ evaluation: | Row 2 | 30 | '30' | +--------+-----------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| c | UInt32([10, 20, 30]) | -| Output | StringColumn { data: 0x313032303330, offsets: [0, 2, 4, 6] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------+ +| c | UInt32([10, 20, 30]) | +| Output | StringColumn { data: Utf8ViewArray[10, 20, 30] } | ++--------+--------------------------------------------------+ ast : to_string(d) @@ -1705,12 +1705,12 @@ evaluation: | Row 2 | 30 | '30' | +--------+------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------+ -| d | Float64([10, -20, 30]) | -| Output | StringColumn { data: 0x31302d32303330, offsets: [0, 2, 5, 7] } | -+--------+----------------------------------------------------------------+ ++--------+---------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------+ +| d | Float64([10, -20, 30]) | +| Output | StringColumn { data: Utf8ViewArray[10, -20, 30] } | ++--------+---------------------------------------------------+ ast : to_string(d2) @@ -1727,12 +1727,12 @@ evaluation: | Row 2 | 3 | '3' | +--------+------------------+-----------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| d2 | NullableColumn { column: UInt8([1, 0, 3]), validity: [0b_____101] } | -| Output | NullableColumn { column: StringColumn { data: 0x313033, offsets: [0, 1, 2, 3] }, validity: [0b_____101] } | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| d2 | NullableColumn { column: UInt8([1, 0, 3]), validity: [0b_____101] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 0, 3] }, validity: [0b_____101] } | ++--------+--------------------------------------------------------------------------------------------------+ ast : to_string(e) @@ -1749,12 +1749,12 @@ evaluation: | Row 2 | 188.8 | '188.8' | +--------+----------------+---------+ evaluation (internal): -+--------+---------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------+ -| e | Decimal128([3.1, 33.5, 188.8]) | -| Output | StringColumn { data: 0x332e3133332e353138382e38, offsets: [0, 3, 7, 12] } | -+--------+---------------------------------------------------------------------------+ ++--------+--------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------+ +| e | Decimal128([3.1, 33.5, 188.8]) | +| Output | StringColumn { data: Utf8ViewArray[3.1, 33.5, 188.8] } | ++--------+--------------------------------------------------------+ ast : to_string(f) @@ -1771,12 +1771,12 @@ evaluation: | Row 2 | 12.34 | '12.34' | +--------+----------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------+ -| f | Decimal256([0.50, 0.92, 12.34]) | -| Output | StringColumn { data: 0x302e3530302e393231322e3334, offsets: [0, 4, 8, 13] } | -+--------+-----------------------------------------------------------------------------+ ++--------+---------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------+ +| f | Decimal256([0.50, 0.92, 12.34]) | +| Output | StringColumn { data: Utf8ViewArray[0.50, 0.92, 12.34] } | ++--------+---------------------------------------------------------+ ast : a ^ 2 diff --git a/src/query/functions/tests/it/scalars/testdata/array.txt b/src/query/functions/tests/it/scalars/testdata/array.txt index 5fe69a9a48ca..024e45bd3476 100644 --- a/src/query/functions/tests/it/scalars/testdata/array.txt +++ b/src/query/functions/tests/it/scalars/testdata/array.txt @@ -473,12 +473,12 @@ evaluation: | Row 3 | '1234' | false | +--------+-------------+---------+ evaluation (internal): -+------------+-------------------------------------------------------------------+ -| Column | Data | -+------------+-------------------------------------------------------------------+ -| string_col | StringColumn { data: 0x31323531323334, offsets: [0, 1, 2, 3, 7] } | -| Output | Boolean([0b____0000]) | -+------------+-------------------------------------------------------------------+ ++------------+-----------------------------------------------------+ +| Column | Data | ++------------+-----------------------------------------------------+ +| string_col | StringColumn { data: Utf8ViewArray[1, 2, 5, 1234] } | +| Output | Boolean([0b____0000]) | ++------------+-----------------------------------------------------+ ast : contains(['1', '5'], string_col) @@ -497,12 +497,12 @@ evaluation: | Row 3 | '1234' | false | +--------+-------------+---------------+ evaluation (internal): -+------------+-------------------------------------------------------------------+ -| Column | Data | -+------------+-------------------------------------------------------------------+ -| string_col | StringColumn { data: 0x31323531323334, offsets: [0, 1, 2, 3, 7] } | -| Output | Boolean([0b____0101]) | -+------------+-------------------------------------------------------------------+ ++------------+-----------------------------------------------------+ +| Column | Data | ++------------+-----------------------------------------------------+ +| string_col | StringColumn { data: Utf8ViewArray[1, 2, 5, 1234] } | +| Output | Boolean([0b____0101]) | ++------------+-----------------------------------------------------+ ast : contains(['15000', '6000', '7000'], string_col) @@ -521,12 +521,12 @@ evaluation: | Row 3 | '1234' | false | +--------+-------------+---------------+ evaluation (internal): -+------------+-------------------------------------------------------------------+ -| Column | Data | -+------------+-------------------------------------------------------------------+ -| string_col | StringColumn { data: 0x31323531323334, offsets: [0, 1, 2, 3, 7] } | -| Output | Boolean([0b____0000]) | -+------------+-------------------------------------------------------------------+ ++------------+-----------------------------------------------------+ +| Column | Data | ++------------+-----------------------------------------------------+ +| string_col | StringColumn { data: Utf8ViewArray[1, 2, 5, 1234] } | +| Output | Boolean([0b____0000]) | ++------------+-----------------------------------------------------+ ast : contains([1,2,null], nullable_col) diff --git a/src/query/functions/tests/it/scalars/testdata/binary.txt b/src/query/functions/tests/it/scalars/testdata/binary.txt index 2977b76ab21d..84a4e2f3e3fc 100644 --- a/src/query/functions/tests/it/scalars/testdata/binary.txt +++ b/src/query/functions/tests/it/scalars/testdata/binary.txt @@ -78,12 +78,12 @@ evaluation: | Row 1 | '123' | 'MTIz' | +--------+-----------------+--------+ evaluation (internal): -+--------+---------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------+ -| a | StringColumn { data: 0x416263313233, offsets: [0, 3, 6] } | -| Output | StringColumn { data: 0x51574a6a4d54497a, offsets: [0, 4, 8] } | -+--------+---------------------------------------------------------------+ ++--------+--------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, 123] } | +| Output | StringColumn { data: Utf8ViewArray[QWJj, MTIz] } | ++--------+--------------------------------------------------+ ast : to_hex(to_binary('abc')) @@ -109,12 +109,12 @@ evaluation: | Row 2 | 'databend' | '6461746162656e64' | +--------+-----------------+--------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x6162636465666461746162656e64, offsets: [0, 3, 6, 14] } | -| Output | StringColumn { data: 0x36313632363336343635363636343631373436313632363536653634, offsets: [0, 6, 12, 28] } | -+--------+------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[abc, def, databend] } | +| Output | StringColumn { data: Utf8ViewArray[616263, 646566, 6461746162656e64] } | ++--------+------------------------------------------------------------------------+ ast : from_base64('QWJj')::String @@ -148,12 +148,12 @@ evaluation: | Row 1 | 'MTIz' | '123' | +--------+-------------------+---------+ evaluation (internal): -+--------+---------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------+ -| a | StringColumn { data: 0x51574a6a4d54497a, offsets: [0, 4, 8] } | -| Output | StringColumn { data: 0x416263313233, offsets: [0, 3, 6] } | -+--------+---------------------------------------------------------------+ ++--------+--------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[QWJj, MTIz] } | +| Output | StringColumn { data: Utf8ViewArray[Abc, 123] } | ++--------+--------------------------------------------------+ error: @@ -195,12 +195,12 @@ evaluation: | Row 2 | '6461746162656e64' | 'databend' | +--------+-----------------------+------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x36313632363336343635363636343631373436313632363536653634, offsets: [0, 6, 12, 28] } | -| Output | StringColumn { data: 0x6162636465666461746162656e64, offsets: [0, 3, 6, 14] } | -+--------+------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[616263, 646566, 6461746162656e64] } | +| Output | StringColumn { data: Utf8ViewArray[abc, def, databend] } | ++--------+------------------------------------------------------------------------+ ast : TRY_from_base64('QWJj')::String @@ -234,12 +234,12 @@ evaluation: | Row 1 | 'MTIz' | '123' | +--------+-------------------+---------+ evaluation (internal): -+--------+---------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------+ -| a | StringColumn { data: 0x51574a6a4d54497a, offsets: [0, 4, 8] } | -| Output | StringColumn { data: 0x416263313233, offsets: [0, 3, 6] } | -+--------+---------------------------------------------------------------+ ++--------+--------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[QWJj, MTIz] } | +| Output | StringColumn { data: Utf8ViewArray[Abc, 123] } | ++--------+--------------------------------------------------+ ast : TRY_from_base64('!@#') @@ -283,11 +283,11 @@ evaluation: | Row 2 | '6461746162656e64' | 'databend' | +--------+-----------------------+------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x36313632363336343635363636343631373436313632363536653634, offsets: [0, 6, 12, 28] } | -| Output | StringColumn { data: 0x6162636465666461746162656e64, offsets: [0, 3, 6, 14] } | -+--------+------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[616263, 646566, 6461746162656e64] } | +| Output | StringColumn { data: Utf8ViewArray[abc, def, databend] } | ++--------+------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/cast.txt b/src/query/functions/tests/it/scalars/testdata/cast.txt index 345cb91d9f2d..4e17bf4bbf6a 100644 --- a/src/query/functions/tests/it/scalars/testdata/cast.txt +++ b/src/query/functions/tests/it/scalars/testdata/cast.txt @@ -820,12 +820,12 @@ evaluation: | Row 4 | '9223372036854775807' | 9223372036854775807 | +--------+--------------------------------+----------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| str | StringColumn { data: 0x2d393232333337323033363835343737353830382d31303139323233333732303336383534373735383037, offsets: [0, 20, 22, 23, 24, 43] } | -| Output | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------+ +| str | StringColumn { data: Utf8ViewArray[-9223372036854775808, -1, 0, 1, 9223372036854775807] } | +| Output | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | ++--------+-------------------------------------------------------------------------------------------+ error: @@ -852,12 +852,12 @@ evaluation: | Row 4 | 9223372036854775807 | '9223372036854775807' | +--------+----------------------------------------------+------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| num | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | -| Output | StringColumn { data: 0x2d393232333337323033363835343737353830382d31303139323233333732303336383534373735383037, offsets: [0, 20, 22, 23, 24, 43] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------+ +| num | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | +| Output | StringColumn { data: Utf8ViewArray[-9223372036854775808, -1, 0, 1, 9223372036854775807] } | ++--------+-------------------------------------------------------------------------------------------+ ast : CAST(num AS STRING) @@ -874,12 +874,12 @@ evaluation: | Row 2 | 18446744073709551615 | '18446744073709551615' | +--------+----------------------------+------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------+ -| num | UInt64([0, 1, 18446744073709551615]) | -| Output | StringColumn { data: 0x30313138343436373434303733373039353531363135, offsets: [0, 1, 2, 22] } | -+--------+-----------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------+ +| num | UInt64([0, 1, 18446744073709551615]) | +| Output | StringColumn { data: Utf8ViewArray[0, 1, 18446744073709551615] } | ++--------+------------------------------------------------------------------+ error: @@ -963,12 +963,12 @@ evaluation: | Row 1 | true | 'true' | +--------+---------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------+ -| bool | Boolean([0b______10]) | -| Output | StringColumn { data: 0x66616c736574727565, offsets: [0, 5, 9] } | -+--------+-----------------------------------------------------------------+ ++--------+---------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------+ +| bool | Boolean([0b______10]) | +| Output | StringColumn { data: Utf8ViewArray[false, true] } | ++--------+---------------------------------------------------+ ast : CAST('010.010' AS DECIMAL(5,3)) @@ -1482,12 +1482,12 @@ evaluation: | Row 4 | '2022-01-02T01' | '2022-01-02 01:00:00.000000' | +--------+-----------------------------------------------------+------------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x323032322d30312d3032323032322d30312d30325430333a32353a30322e3836383839342d30373a3030323032322d30312d30322030323a30303a3131323032322d30312d30325430313a31323a30302d30373a3030323032322d30312d3032543031, offsets: [0, 10, 42, 61, 86, 99] } | -| Output | [1641081600000000, 1641119102868894, 1641088811000000, 1641111120000000, 1641085200000000] | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] } | +| Output | [1641081600000000, 1641119102868894, 1641088811000000, 1641111120000000, 1641085200000000] | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ast : CAST(TO_TIMESTAMP(-315360000000000) AS VARCHAR) @@ -1580,12 +1580,12 @@ evaluation: | Row 6 | '1979-12-30 00:00:00.000000' | '1979-12-30 00:00:00.000000' | +--------+--------------------------------------+------------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | [-315360000000000, -315360000000, -100, 0, 100, 315360000000, 315360000000000] | -| Output | StringColumn { data: 0x313936302d30312d30342030303a30303a30302e303030303030313936392d31322d32382030383a32343a30302e303030303030313936392d31322d33312032333a35393a35392e393939393030313937302d30312d30312030303a30303a30302e303030303030313937302d30312d30312030303a30303a30302e303030313030313937302d30312d30342031353a33363a30302e303030303030313937392d31322d33302030303a30303a30302e303030303030, offsets: [0, 26, 52, 78, 104, 130, 156, 182] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | [-315360000000000, -315360000000, -100, 0, 100, 315360000000, 315360000000000] | +| Output | StringColumn { data: Utf8ViewArray[1960-01-04 00:00:00.000000, 1969-12-28 08:24:00.000000, 1969-12-31 23:59:59.999900, 1970-01-01 00:00:00.000000, 1970-01-01 00:00:00.000100, 1970-01-04 15:36:00.000000, 1979-12-30 00:00:00.000000] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ error: @@ -1682,12 +1682,12 @@ evaluation: | Row 4 | '2022-01-02T01' | '2022-01-02' | +--------+-----------------------------------------------------+--------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x323032322d30312d3032323032322d30312d30325430333a32353a30322e3836383839342d30373a3030323032322d30312d30322030323a30303a3131323032322d30312d30325430313a31323a30302d30373a3030323032322d30312d3032543031, offsets: [0, 10, 42, 61, 86, 99] } | -| Output | [18994, 18994, 18994, 18994, 18994] | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] } | +| Output | [18994, 18994, 18994, 18994, 18994] | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ast : CAST(TO_DATE(-354285) AS VARCHAR) @@ -1760,12 +1760,12 @@ evaluation: | Row 4 | '9999-12-31' | '9999-12-31' | +--------+---------------------+--------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | [-354285, -100, 0, 100, 2932896] | -| Output | StringColumn { data: 0x313030302d30312d3031313936392d30392d3233313937302d30312d3031313937302d30342d3131393939392d31322d3331, offsets: [0, 10, 20, 30, 40, 50] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| a | [-354285, -100, 0, 100, 2932896] | +| Output | StringColumn { data: Utf8ViewArray[1000-01-01, 1969-09-23, 1970-01-01, 1970-04-11, 9999-12-31] } | ++--------+--------------------------------------------------------------------------------------------------+ error: @@ -1921,7 +1921,7 @@ evaluation (internal): +--------+------------------------------------------------------------------------------------------------+ | Column | Data | +--------+------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | | Output | BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | +--------+------------------------------------------------------------------------------------------------+ @@ -1951,7 +1951,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2014,12 +2014,12 @@ evaluation: | Row 2 | 'ß😀山' | 'ß😀山' | +--------+-------------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| Output | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -+--------+------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| Output | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | ++--------+-------------------------------------------------------------+ error: @@ -2044,12 +2044,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+----------------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | ++--------+----------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(0 AS UINT8) @@ -2539,7 +2539,7 @@ evaluation (internal): +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x747275657b226b223a2276227d5b312c322c335d, offsets: [0, 4, 13, 20] }, validity: [0b_____101] } | +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":"v"}, [1,2,3]] }, validity: [0b_____101] } | | Output | NullableColumn { column: BinaryColumn { data: 0x200000004000000080000003200000022000000220000002500150025003, offsets: [0, 8, 8, 30] }, validity: [0b_____101] } | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3016,12 +3016,12 @@ evaluation: | Row 4 | '9223372036854775807' | 9223372036854775807 | +--------+--------------------------------+-------------------------------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| str | StringColumn { data: 0x2d393232333337323033363835343737353830382d31303139323233333732303336383534373735383037, offsets: [0, 20, 22, 23, 24, 43] } | -| Output | NullableColumn { column: Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]), validity: [0b___11111] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------+ +| str | StringColumn { data: Utf8ViewArray[-9223372036854775808, -1, 0, 1, 9223372036854775807] } | +| Output | NullableColumn { column: Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]), validity: [0b___11111] } | ++--------+-----------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(str AS INT64) @@ -3039,12 +3039,12 @@ evaluation: | Row 3 | NULL | NULL | +--------+------------------------+-------------------------------------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------+ -| str | NullableColumn { column: StringColumn { data: 0x666f6f666f6f3030, offsets: [0, 3, 6, 7, 8] }, validity: [0b____0101] } | -| Output | NullableColumn { column: Int64([0, 0, 0, 0]), validity: [0b____0100] } | -+--------+------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------+ +| str | NullableColumn { column: StringColumn { data: Utf8ViewArray[foo, foo, 0, 0] }, validity: [0b____0101] } | +| Output | NullableColumn { column: Int64([0, 0, 0, 0]), validity: [0b____0100] } | ++--------+---------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(num AS STRING) @@ -3063,12 +3063,12 @@ evaluation: | Row 4 | 9223372036854775807 | '9223372036854775807' | +--------+----------------------------------------------+------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| num | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | -| Output | NullableColumn { column: StringColumn { data: 0x2d393232333337323033363835343737353830382d31303139323233333732303336383534373735383037, offsets: [0, 20, 22, 23, 24, 43] }, validity: [0b___11111] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------+ +| num | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[-9223372036854775808, -1, 0, 1, 9223372036854775807] }, validity: [0b___11111] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(num AS STRING) @@ -3085,12 +3085,12 @@ evaluation: | Row 2 | 18446744073709551615 | '18446744073709551615' | +--------+----------------------------+------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ -| num | UInt64([0, 1, 18446744073709551615]) | -| Output | NullableColumn { column: StringColumn { data: 0x30313138343436373434303733373039353531363135, offsets: [0, 1, 2, 22] }, validity: [0b_____111] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------+ +| num | UInt64([0, 1, 18446744073709551615]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[0, 1, 18446744073709551615] }, validity: [0b_____111] } | ++--------+---------------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST('t' AS BOOLEAN) @@ -3178,12 +3178,12 @@ evaluation: | Row 1 | true | 'true' | +--------+---------------+-----------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------+ -| bool | Boolean([0b______10]) | -| Output | NullableColumn { column: StringColumn { data: 0x66616c736574727565, offsets: [0, 5, 9] }, validity: [0b______11] } | -+--------+--------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------+ +| bool | Boolean([0b______10]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[false, true] }, validity: [0b______11] } | ++--------+------------------------------------------------------------------------------------------------------+ ast : TRY_CAST('010.010' AS DECIMAL(5,3)) @@ -3702,12 +3702,12 @@ evaluation: | Row 4 | '2022-01-02T01' | '2022-01-02 01:00:00.000000' | +--------+-----------------------------------------------------+----------------------------------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x323032322d30312d3032323032322d30312d30325430333a32353a30322e3836383839342d30373a3030323032322d30312d30322030323a30303a3131323032322d30312d30325430313a31323a30302d30373a3030323032322d30312d3032543031, offsets: [0, 10, 42, 61, 86, 99] } | -| Output | NullableColumn { column: [1641081600000000, 1641119102868894, 1641088811000000, 1641111120000000, 1641085200000000], validity: [0b___11111] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] } | +| Output | NullableColumn { column: [1641081600000000, 1641119102868894, 1641088811000000, 1641111120000000, 1641085200000000], validity: [0b___11111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(TO_TIMESTAMP(-315360000000000) AS VARCHAR) @@ -3800,12 +3800,12 @@ evaluation: | Row 6 | '1979-12-30 00:00:00.000000' | '1979-12-30 00:00:00.000000' | +--------+--------------------------------------+------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | [-315360000000000, -315360000000, -100, 0, 100, 315360000000, 315360000000000] | -| Output | NullableColumn { column: StringColumn { data: 0x313936302d30312d30342030303a30303a30302e303030303030313936392d31322d32382030383a32343a30302e303030303030313936392d31322d33312032333a35393a35392e393939393030313937302d30312d30312030303a30303a30302e303030303030313937302d30312d30312030303a30303a30302e303030313030313937302d30312d30342031353a33363a30302e303030303030313937392d31322d33302030303a30303a30302e303030303030, offsets: [0, 26, 52, 78, 104, 130, 156, 182] }, validity: [0b_1111111] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | [-315360000000000, -315360000000, -100, 0, 100, 315360000000, 315360000000000] | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1960-01-04 00:00:00.000000, 1969-12-28 08:24:00.000000, 1969-12-31 23:59:59.999900, 1970-01-01 00:00:00.000000, 1970-01-01 00:00:00.000100, 1970-01-04 15:36:00.000000, 1979-12-30 00:00:00.000000] }, validity: [0b_1111111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : TRY_TO_DATE('2022') @@ -3905,12 +3905,12 @@ evaluation: | Row 4 | '2022-01-02T01' | '2022-01-02' | +--------+-----------------------------------------------------+------------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x323032322d30312d3032323032322d30312d30325430333a32353a30322e3836383839342d30373a3030323032322d30312d30322030323a30303a3131323032322d30312d30325430313a31323a30302d30373a3030323032322d30312d3032543031, offsets: [0, 10, 42, 61, 86, 99] } | -| Output | NullableColumn { column: [18994, 18994, 18994, 18994, 18994], validity: [0b___11111] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] } | +| Output | NullableColumn { column: [18994, 18994, 18994, 18994, 18994], validity: [0b___11111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(TO_DATE(-354285) AS VARCHAR) @@ -3983,12 +3983,12 @@ evaluation: | Row 4 | '9999-12-31' | '9999-12-31' | +--------+---------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | [-354285, -100, 0, 100, 2932896] | -| Output | NullableColumn { column: StringColumn { data: 0x313030302d30312d3031313936392d30392d3233313937302d30312d3031313937302d30342d3131393939392d31322d3331, offsets: [0, 10, 20, 30, 40, 50] }, validity: [0b___11111] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | [-354285, -100, 0, 100, 2932896] | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1000-01-01, 1969-09-23, 1970-01-01, 1970-04-11, 9999-12-31] }, validity: [0b___11111] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ error: @@ -4151,7 +4151,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | | Output | NullableColumn { column: BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____111] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4173,7 +4173,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4195,7 +4195,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4259,12 +4259,12 @@ evaluation: | Row 2 | 'ß😀山' | 'ß😀山' | +--------+-------------------+-----------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| Output | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____111] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____111] } | ++--------+----------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(TRY_CAST(a AS BINARY) AS STRING) @@ -4281,12 +4281,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+----------------------------+-----------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | ++--------+----------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(TRY_CAST(a AS BINARY NULL) AS STRING NULL) @@ -4303,11 +4303,11 @@ evaluation: | Row 2 | NULL | NULL | +--------+----------------------------+-----------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | ++--------+----------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/comparison.txt b/src/query/functions/tests/it/scalars/testdata/comparison.txt index cc5cf9b5d7fc..7dd3ce500bd5 100644 --- a/src/query/functions/tests/it/scalars/testdata/comparison.txt +++ b/src/query/functions/tests/it/scalars/testdata/comparison.txt @@ -223,13 +223,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | true | +--------+------------------------------------------+------------------------------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e353637387b226b223a2276222c2261223a2262227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 59, 80] } | -| rhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e353637387b226b223a2276222c2261223a2264227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 59, 80] } | -| Output | Boolean([0b_1011111]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | +| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] } | +| Output | Boolean([0b_1011111]) | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ ast : lhs = rhs @@ -250,13 +250,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | true | +--------+------------------------------------------+------------------------------------------+---------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e353637387b226b223a2276222c2261223a2262227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 59, 80] } | -| rhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e353637387b226b223a2276222c2261223a2264227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 59, 80] } | -| Output | Boolean([0b_1011111]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | +| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] } | +| Output | Boolean([0b_1011111]) | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ ast : '1'!='2' @@ -405,13 +405,13 @@ evaluation: | Row 3 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | false | +--------+----------------------------------+----------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830375b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 48] } | -| rhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830375b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 48] } | -| Output | Boolean([0b____0000]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] } | +| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] } | +| Output | Boolean([0b____0000]) | ++--------+----------------------------------------------------------------------------------------------+ ast : lhs != rhs @@ -429,13 +429,13 @@ evaluation: | Row 3 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | false | +--------+----------------------------------+----------------------------------+---------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830375b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 48] } | -| rhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830375b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 48] } | -| Output | Boolean([0b____0000]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] } | +| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] } | +| Output | Boolean([0b____0000]) | ++--------+----------------------------------------------------------------------------------------------+ ast : '1'<'2' @@ -573,13 +573,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","c"]]' | '[0,2,3,["a","b","c"]]' | true | +--------+-------------------------+-------------------------+---------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e35363738312e39313265325b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 49, 70] } | -| rhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830302d3333373638313233342e35363738312e39313265325b302c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 49, 70] } | -| Output | Boolean([0b_1111111]) | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, [1,2,3,["a","b","c"]]] } | +| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775800, -33768, 1234.5678, 1.912e2, [0,2,3,["a","b","c"]]] } | +| Output | Boolean([0b_1111111]) | ++--------+--------------------------------------------------------------------------------------------------------------------------+ ast : lhs < rhs @@ -600,13 +600,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","c"]]' | '[0,2,3,["a","b","c"]]' | false | +--------+-------------------------+-------------------------+---------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e35363738312e39313265325b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 49, 70] } | -| rhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830302d3333373638313233342e35363738312e39313265325b302c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 49, 70] } | -| Output | Boolean([0b_0001000]) | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, [1,2,3,["a","b","c"]]] } | +| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775800, -33768, 1234.5678, 1.912e2, [0,2,3,["a","b","c"]]] } | +| Output | Boolean([0b_0001000]) | ++--------+--------------------------------------------------------------------------------------------------------------------------+ ast : '5'<='2' @@ -749,13 +749,13 @@ evaluation: | Row 2 | '[1,2,3,["a","b","c"]]' | '[0,2,3,["a","b","c"]]' | false | +--------+------------------------------------------------+------------------------------------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x226461746162656e64227b226b223a2276222c2261223a2262227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 10, 27, 48] } | -| rhs | StringColumn { data: 0x226461746162656e64227b226b223a2261222c2261223a2264227d5b302c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 10, 27, 48] } | -| Output | Boolean([0b_____011]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray["databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | +| rhs | StringColumn { data: Utf8ViewArray["databend", {"k":"a","a":"d"}, [0,2,3,["a","b","c"]]] } | +| Output | Boolean([0b_____011]) | ++--------+--------------------------------------------------------------------------------------------+ ast : lhs <= rhs @@ -772,13 +772,13 @@ evaluation: | Row 2 | '[1,2,3,["a","b","c"]]' | '[0,2,3,["a","b","c"]]' | false | +--------+------------------------------------------------+------------------------------------------------+---------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x226461746162656e64227b226b223a2276222c2261223a2262227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 10, 27, 48] } | -| rhs | StringColumn { data: 0x226461746162656e64227b226b223a2261222c2261223a2264227d5b302c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 10, 27, 48] } | -| Output | Boolean([0b_____001]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray["databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | +| rhs | StringColumn { data: Utf8ViewArray["databend", {"k":"a","a":"d"}, [0,2,3,["a","b","c"]]] } | +| Output | Boolean([0b_____001]) | ++--------+--------------------------------------------------------------------------------------------+ ast : '3'>'2' @@ -923,13 +923,13 @@ evaluation: | Row 4 | '1234.5678' | '1234.5678' | false | +--------+-----------------------+-----------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e35363738, offsets: [0, 4, 8, 27, 33, 42] } | -| rhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830362d3332373638313233342e35363738, offsets: [0, 4, 8, 27, 33, 42] } | -| Output | Boolean([0b___00100]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678] } | +| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775806, -32768, 1234.5678] } | +| Output | Boolean([0b___00100]) | ++--------+------------------------------------------------------------------------------------------+ ast : lhs > rhs @@ -948,13 +948,13 @@ evaluation: | Row 4 | '1234.5678' | '1234.5678' | false | +--------+-----------------------+-----------------------+---------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e35363738, offsets: [0, 4, 8, 27, 33, 42] } | -| rhs | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830362d3332373638313233342e35363738, offsets: [0, 4, 8, 27, 33, 42] } | -| Output | Boolean([0b___00100]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678] } | +| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775806, -32768, 1234.5678] } | +| Output | Boolean([0b___00100]) | ++--------+------------------------------------------------------------------------------------------+ ast : col > 'efg' @@ -971,12 +971,12 @@ evaluation: | Row 1 | 'efg' | false | +--------+-----------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------+ -| col | StringColumn { data: 0x626364656667, offsets: [0, 3, 6] } | -| Output | Boolean([0b______00]) | -+--------+-----------------------------------------------------------+ ++--------+------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------+ +| col | StringColumn { data: Utf8ViewArray[bcd, efg] } | +| Output | Boolean([0b______00]) | ++--------+------------------------------------------------+ ast : '2'>='1' @@ -1123,13 +1123,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","d"]]' | '[1,2,3,["a","b","c"]]' | true | +--------+-----------------------------------------------------------+-----------------------------------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x393232333337323033363835343737353830372d3332373638313233342e35363738312e3931326532225c5c5c226162635c5c5c22227b226b223a2276222c2261223a2262227d5b312c322c332c5b2261222c2262222c2264225d5d, offsets: [0, 19, 25, 34, 41, 54, 71, 92] } | -| rhs | StringColumn { data: 0x393232333337323033363835343737353830362d3332373638313233342e35363738312e3931326532225c5c5c226162635c5c5c22227b226b223a2276222c2261223a2264227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 19, 25, 34, 41, 54, 71, 92] } | -| Output | Boolean([0b_1011111]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"b"}, [1,2,3,["a","b","d"]]] } | +| rhs | StringColumn { data: Utf8ViewArray[9223372036854775806, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] } | +| Output | Boolean([0b_1011111]) | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ ast : lhs >= rhs @@ -1150,13 +1150,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","d"]]' | '[1,2,3,["a","b","c"]]' | true | +--------+-----------------------------------------------------------+-----------------------------------------------------------+---------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x393232333337323033363835343737353830372d3332373638313233342e35363738312e3931326532225c5c5c226162635c5c5c22227b226b223a2276222c2261223a2262227d5b312c322c332c5b2261222c2262222c2264225d5d, offsets: [0, 19, 25, 34, 41, 54, 71, 92] } | -| rhs | StringColumn { data: 0x393232333337323033363835343737353830362d3332373638313233342e35363738312e3931326532225c5c5c226162635c5c5c22227b226b223a2276222c2261223a2264227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 19, 25, 34, 41, 54, 71, 92] } | -| Output | Boolean([0b_1011111]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"b"}, [1,2,3,["a","b","d"]]] } | +| rhs | StringColumn { data: Utf8ViewArray[9223372036854775806, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] } | +| Output | Boolean([0b_1011111]) | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ ast : '1' like '2' @@ -1223,12 +1223,12 @@ evaluation: | Row 3 | 'abf' | true | +--------+-----------------+---------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x616263616264616265616266, offsets: [0, 3, 6, 9, 12] } | -| Output | Boolean([0b____1111]) | -+--------+------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | +| Output | Boolean([0b____1111]) | ++--------+----------------------------------------------------------+ ast : lhs like 'b%' @@ -1247,12 +1247,35 @@ evaluation: | Row 3 | 'abf' | false | +--------+-----------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x616263616264616265616266, offsets: [0, 3, 6, 9, 12] } | -| Output | Boolean([0b____0000]) | -+--------+------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | +| Output | Boolean([0b____0000]) | ++--------+----------------------------------------------------------+ + + +ast : lhs like 'ab%' +raw expr : like(lhs::String, 'ab%') +checked expr : like(lhs, "ab%") +evaluation: ++--------+-----------------+---------------+ +| | lhs | Output | ++--------+-----------------+---------------+ +| Type | String | Boolean | +| Domain | {"abc"..="abf"} | {FALSE, TRUE} | +| Row 0 | 'abc' | true | +| Row 1 | 'abd' | true | +| Row 2 | 'abe' | true | +| Row 3 | 'abf' | true | ++--------+-----------------+---------------+ +evaluation (internal): ++--------+----------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | +| Output | Boolean([0b____1111]) | ++--------+----------------------------------------------------------+ ast : lhs like 'c' @@ -1271,12 +1294,12 @@ evaluation: | Row 3 | 'abf' | false | +--------+-----------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x616263616264616265616266, offsets: [0, 3, 6, 9, 12] } | -| Output | Boolean([0b____0000]) | -+--------+------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | +| Output | Boolean([0b____0000]) | ++--------+----------------------------------------------------------+ ast : lhs like rhs @@ -1294,13 +1317,13 @@ evaluation: | Row 3 | 'abf' | 'a' | false | +--------+-----------------+-----------------+---------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x616263616264616265616266, offsets: [0, 3, 6, 9, 12] } | -| rhs | StringColumn { data: 0x61255f625f61626561, offsets: [0, 2, 5, 8, 9] } | -| Output | Boolean([0b____0111]) | -+--------+------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | +| rhs | StringColumn { data: Utf8ViewArray[a%, _b_, abe, a] } | +| Output | Boolean([0b____0111]) | ++--------+----------------------------------------------------------+ ast : parse_json('"hello"') like 'h%' @@ -1344,12 +1367,12 @@ evaluation: | Row 2 | '["abe","abf"]' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x22616263227b22616264223a31327d5b22616265222c22616266225d, offsets: [0, 5, 15, 28] } | -| Output | Boolean([0b_____001]) | -+--------+------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray["abc", {"abd":12}, ["abe","abf"]] } | +| Output | Boolean([0b_____001]) | ++--------+------------------------------------------------------------------------+ ast : parse_json(lhs) like '%ab%' @@ -1366,12 +1389,12 @@ evaluation: | Row 2 | '["abe","abf"]' | true | +--------+------------------------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x22616263227b22616264223a31327d5b22616265222c22616266225d, offsets: [0, 5, 15, 28] } | -| Output | Boolean([0b_____111]) | -+--------+------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray["abc", {"abd":12}, ["abe","abf"]] } | +| Output | Boolean([0b_____111]) | ++--------+------------------------------------------------------------------------+ ast : lhs regexp rhs @@ -1391,13 +1414,13 @@ evaluation: | Row 5 | '' | '' | true | +--------+--------------+--------------+---------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x616263616264616265616266616263, offsets: [0, 3, 6, 9, 12, 15, 15] } | -| rhs | StringColumn { data: 0x5e615e6261626561, offsets: [0, 2, 4, 7, 8, 8, 8] } | -| Output | Boolean([0b__101101]) | -+--------+--------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf, abc, ] } | +| rhs | StringColumn { data: Utf8ViewArray[^a, ^b, abe, a, , ] } | +| Output | Boolean([0b__101101]) | ++--------+-----------------------------------------------------------------+ ast : lhs rlike rhs @@ -1417,12 +1440,12 @@ evaluation: | Row 5 | '' | '' | true | +--------+--------------+--------------+---------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: 0x616263616264616265616266616263, offsets: [0, 3, 6, 9, 12, 15, 15] } | -| rhs | StringColumn { data: 0x5e615e6261626561, offsets: [0, 2, 4, 7, 8, 8, 8] } | -| Output | Boolean([0b__101101]) | -+--------+--------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------+ +| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf, abc, ] } | +| rhs | StringColumn { data: Utf8ViewArray[^a, ^b, abe, a, , ] } | +| Output | Boolean([0b__101101]) | ++--------+-----------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/geo_h3.txt b/src/query/functions/tests/it/scalars/testdata/geo_h3.txt index 6a57e63c05ab..3ee681496703 100644 --- a/src/query/functions/tests/it/scalars/testdata/geo_h3.txt +++ b/src/query/functions/tests/it/scalars/testdata/geo_h3.txt @@ -763,12 +763,12 @@ evaluation: | Row 2 | 599686042433355775 | '85283473fffffff' | +--------+-------------------------------------------+-------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| h3 | UInt64([635318325446452991, 644325524701193897, 599686042433355775]) | -| Output | StringColumn { data: 0x386431316161366133383832366666386631316161366133383832366139383532383334373366666666666666, offsets: [0, 15, 30, 45] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------+ +| h3 | UInt64([635318325446452991, 644325524701193897, 599686042433355775]) | +| Output | StringColumn { data: Utf8ViewArray[8d11aa6a38826ff, 8f11aa6a38826a9, 85283473fffffff] } | ++--------+-----------------------------------------------------------------------------------------+ error: @@ -809,12 +809,12 @@ evaluation: | Row 2 | '85283473fffffff' | 599686042433355775 | +--------+-----------------------------------------+----------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| h3_str | StringColumn { data: 0x386431316161366133383832366666386631316161366133383832366139383532383334373366666666666666, offsets: [0, 15, 30, 45] } | -| Output | UInt64([635318325446452991, 644325524701193897, 599686042433355775]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------+ +| h3_str | StringColumn { data: Utf8ViewArray[8d11aa6a38826ff, 8f11aa6a38826a9, 85283473fffffff] } | +| Output | UInt64([635318325446452991, 644325524701193897, 599686042433355775]) | ++--------+-----------------------------------------------------------------------------------------+ error: diff --git a/src/query/functions/tests/it/scalars/testdata/geometry.txt b/src/query/functions/tests/it/scalars/testdata/geometry.txt index 7d148752ef2c..44e7b265c3de 100644 --- a/src/query/functions/tests/it/scalars/testdata/geometry.txt +++ b/src/query/functions/tests/it/scalars/testdata/geometry.txt @@ -257,7 +257,7 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x39713630793630726873753470727579647171766a30, offsets: [0, 10, 22] } | +| a | StringColumn { data: Utf8ViewArray[9q60y60rhs, u4pruydqqvj0] } | | Output | BinaryColumn { data: 0x0103000000010000000500000000000036632a5ec00000001470a6414000000036632a5ec00000004170a6414000000009632a5ec00000004170a6414000000009632a5ec00000001470a6414000000036632a5ec00000001470a6414001030000000100000005000000000000d99bd024400000000916d34c40000000d99bd024400000680a16d34c40000040e49bd024400000680a16d34c40000040e49bd024400000000916d34c40000000d99bd024400000000916d34c40, offsets: [0, 93, 186] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -406,7 +406,7 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x4c494e45535452494e4728302e3020302e302c20312e3020302e302c20312e3020322e302c20302e3020322e302c20302e3020302e30294c494e45535452494e472831302e3120352e322c2031352e3220372e332c2032302e3220382e332c2031302e3920372e372c2031302e3120352e3229, offsets: [0, 55, 115] } | +| a | StringColumn { data: Utf8ViewArray[LINESTRING(0.0 0.0, 1.0 0.0, 1.0 2.0, 0.0 2.0, 0.0 0.0), LINESTRING(10.1 5.2, 15.2 7.3, 20.2 8.3, 10.9 7.7, 10.1 5.2)] } | | Output | BinaryColumn { data: 0x0103000000010000000500000000000000000000000000000000000000000000000000f03f0000000000000000000000000000f03f00000000000000400000000000000000000000000000004000000000000000000000000000000000010300000001000000050000003333333333332440cdcccccccccc14406666666666662e403333333333331d4033333333333334409a99999999992040cdcccccccccc2540cdcccccccccc1e403333333333332440cdcccccccccc1440, offsets: [0, 93, 186] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -704,13 +704,13 @@ evaluation: | Row 2 | 3 | 3 | 'POINT(3 3)' | +--------+---------+---------+--------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------+ -| a | Float64([1, 2, 3]) | -| b | Float64([1, 2, 3]) | -| Output | StringColumn { data: 0x504f494e542831203129504f494e542832203229504f494e542833203329, offsets: [0, 10, 20, 30] } | -+--------+-----------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| a | Float64([1, 2, 3]) | +| b | Float64([1, 2, 3]) | +| Output | StringColumn { data: Utf8ViewArray[POINT(1 1), POINT(2 2), POINT(3 3)] } | ++--------+--------------------------------------------------------------------------+ ast : try_to_geometry(NULL) @@ -935,13 +935,13 @@ evaluation: | Row 2 | '0101000020797f000066666666a9cb17411f85ebc19e325641' | 3857 | 'SRID=3857;POINT(389866.35 5819003.03)' | +--------+---------------------------------------------------------------------------------------------------------------+----------------+------------------------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x303130313030303032303739376630303030363636363636363661396362313734313166383565626331396533323536343130313031303030303230373937663030303036363636363636366139636231373431316638356562633139653332353634313031303130303030323037393766303030303636363636363636613963623137343131663835656263313965333235363431, offsets: [0, 50, 100, 150] } | -| b | Int32([32633, 4326, 3857]) | -| Output | BinaryColumn { data: 0x0101000020797f000066666666a9cb17411f85ebc19e3256410101000020e610000066666666a9cb17411f85ebc19e3256410101000020110f000066666666a9cb17411f85ebc19e325641, offsets: [0, 25, 50, 75] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[0101000020797f000066666666a9cb17411f85ebc19e325641, 0101000020797f000066666666a9cb17411f85ebc19e325641, 0101000020797f000066666666a9cb17411f85ebc19e325641] } | +| b | Int32([32633, 4326, 3857]) | +| Output | BinaryColumn { data: 0x0101000020797f000066666666a9cb17411f85ebc19e3256410101000020e610000066666666a9cb17411f85ebc19e3256410101000020110f000066666666a9cb17411f85ebc19e325641, offsets: [0, 25, 50, 75] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : st_geometryfromwkt('POINT(389866.35 5819003.03)') @@ -968,12 +968,12 @@ evaluation: | Row 2 | 'POINT(389866.35 5819003.03)' | 'POINT(389866.35 5819003.03)' | +--------+-----------------------------------------------------------------+-------------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x504f494e54283338393836362e333520353831393030332e303329504f494e54283338393836362e333520353831393030332e303329504f494e54283338393836362e333520353831393030332e303329, offsets: [0, 27, 54, 81] } | -| Output | BinaryColumn { data: 0x010100000066666666a9cb17411f85ebc19e325641010100000066666666a9cb17411f85ebc19e325641010100000066666666a9cb17411f85ebc19e325641, offsets: [0, 21, 42, 63] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[POINT(389866.35 5819003.03), POINT(389866.35 5819003.03), POINT(389866.35 5819003.03)] } | +| Output | BinaryColumn { data: 0x010100000066666666a9cb17411f85ebc19e325641010100000066666666a9cb17411f85ebc19e325641010100000066666666a9cb17411f85ebc19e325641, offsets: [0, 21, 42, 63] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : st_geometryfromwkt('POINT(389866.35 5819003.03)', 32633) @@ -1000,13 +1000,13 @@ evaluation: | Row 2 | 'POINT(389866.35 5819003.03)' | 3857 | 'SRID=3857;POINT(389866.35 5819003.03)' | +--------+-----------------------------------------------------------------+----------------+------------------------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x504f494e54283338393836362e333520353831393030332e303329504f494e54283338393836362e333520353831393030332e303329504f494e54283338393836362e333520353831393030332e303329, offsets: [0, 27, 54, 81] } | -| b | Int32([32633, 4326, 3857]) | -| Output | BinaryColumn { data: 0x0101000020797f000066666666a9cb17411f85ebc19e3256410101000020e610000066666666a9cb17411f85ebc19e3256410101000020110f000066666666a9cb17411f85ebc19e325641, offsets: [0, 25, 50, 75] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[POINT(389866.35 5819003.03), POINT(389866.35 5819003.03), POINT(389866.35 5819003.03)] } | +| b | Int32([32633, 4326, 3857]) | +| Output | BinaryColumn { data: 0x0101000020797f000066666666a9cb17411f85ebc19e3256410101000020e610000066666666a9cb17411f85ebc19e3256410101000020110f000066666666a9cb17411f85ebc19e325641, offsets: [0, 25, 50, 75] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : st_xmax(to_geometry('POINT(-180 0)')) @@ -1211,14 +1211,14 @@ evaluation: | Row 0 | 'POINT(389866.35 5819003.03)' | 32633 | 3857 | 'SRID=3857;POINT(1489140.093766 6892872.19868)' | +--------+-----------------------------------------------------------------+-----------------+---------------+-------------------------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x504f494e54283338393836362e333520353831393030332e303329, offsets: [0, 27] } | -| b | Int32([32633]) | -| c | Int32([3857]) | -| Output | BinaryColumn { data: 0x0101000020110f00006f0c0118f4b83641522cb70c524b5a41, offsets: [0, 25] } | -+--------+---------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[POINT(389866.35 5819003.03)] } | +| b | Int32([32633]) | +| c | Int32([3857]) | +| Output | BinaryColumn { data: 0x0101000020110f00006f0c0118f4b83641522cb70c524b5a41, offsets: [0, 25] } | ++--------+-----------------------------------------------------------------------------------------------+ ast : st_transform(st_geomfromwkt('POINT(4.500212 52.161170)'), 4326, 28992) @@ -1246,7 +1246,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x504f494e5428342e3530303231322035322e31363131373029, offsets: [0, 25] } | +| a | StringColumn { data: Utf8ViewArray[POINT(4.500212 52.161170)] } | | b | Int32([4326]) | | c | Int32([28992]) | | Output | BinaryColumn { data: 0x0101000020407100005dfe43ba4a06f7402ffce0ac98521c41, offsets: [0, 25] } | diff --git a/src/query/functions/tests/it/scalars/testdata/hash.txt b/src/query/functions/tests/it/scalars/testdata/hash.txt index 14df61111d38..971f377992c4 100644 --- a/src/query/functions/tests/it/scalars/testdata/hash.txt +++ b/src/query/functions/tests/it/scalars/testdata/hash.txt @@ -30,12 +30,12 @@ evaluation: | Row 2 | 'ß😀山' | 'b814c09d48b62faafc315df44a35863e' | +--------+-------------------+------------------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| Output | StringColumn { data: 0x333535393362376365353032306561653363613638666435623666336530333131656630623464616235353838666533663633636438346636663164366162326238313463303964343862363266616166633331356466343461333538363365, offsets: [0, 32, 64, 96] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| Output | StringColumn { data: Utf8ViewArray[35593b7ce5020eae3ca68fd5b6f3e031, 1ef0b4dab5588fe3f63cd84f6f1d6ab2, b814c09d48b62faafc315df44a35863e] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------+ ast : sha('Abc') @@ -70,12 +70,12 @@ evaluation: | Row 2 | 'ß😀山' | 'e978809ba007678383c2db3decbaf02eb0bf72a8' | +--------+-------------------+--------------------------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| Output | StringColumn { data: 0x393135383538616661323237386632353532376631393230333831303833343631363462343766326336366430353663396662326339653663643734623435353566363463306166346137626335393965393738383039626130303736373833383363326462336465636261663032656230626637326138, offsets: [0, 40, 80, 120] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| Output | StringColumn { data: Utf8ViewArray[915858afa2278f25527f192038108346164b47f2, c66d056c9fb2c9e6cd74b4555f64c0af4a7bc599, e978809ba007678383c2db3decbaf02eb0bf72a8] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : blake3('Abc') @@ -110,12 +110,12 @@ evaluation: | Row 2 | 'ß😀山' | '56475d2e89dba36b511ddaa8e4e8e995c094f59f5fbfa0af5929f3f399d9a810' | +--------+-------------------+--------------------------------------------------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| Output | StringColumn { data: 0x316637616133393738393439633432373537393761633630353663303130386538353261356430376334393833386130303739393139343935376238323131313062343439343139383334653564323835643362393566383932623534393464623864373739386331313630326363646531663531633333613135393637376635363437356432653839646261333662353131646461613865346538653939356330393466353966356662666130616635393239663366333939643961383130, offsets: [0, 64, 128, 192] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| Output | StringColumn { data: Utf8ViewArray[1f7aa3978949c4275797ac6056c0108e852a5d07c49838a00799194957b82111, 0b449419834e5d285d3b95f892b5494db8d7798c11602ccde1f51c33a159677f, 56475d2e89dba36b511ddaa8e4e8e995c094f59f5fbfa0af5929f3f399d9a810] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : sha2('Abc',0) @@ -159,13 +159,13 @@ evaluation: | Row 2 | 'ß😀山' | 512 | '3bd4ca36a66c0675e695f3fc44af703cd6c110085adf105138ef56e6768a639f16a9c27b651a0c64f685b24be835e0a62485575477e06d530574865bf1670d30' | +--------+-------------------+-------------+------------------------------------------------------------------------------------------------------------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| b | UInt16([224, 384, 512]) | -| Output | StringColumn { data: 0x31316438363737306635303132393463366233393539343261333966363066653238366131356530363238326162636232323934636661303161356336366339313837313863363237653336306535363833336432663663363338666431613637303836373932363036636665666535303038393238396361333462353261373261383333653666323636316236343431373036383834363362643463613336613636633036373565363935663366633434616637303363643663313130303835616466313035313338656635366536373638613633396631366139633237623635316130633634663638356232346265383335653061363234383535373534373765303664353330353734383635626631363730643330, offsets: [0, 56, 152, 280] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| b | UInt16([224, 384, 512]) | +| Output | StringColumn { data: Utf8ViewArray[11d86770f501294c6b395942a39f60fe286a15e06282abcb2294cfa0, 1a5c66c918718c627e360e56833d2f6c638fd1a67086792606cfefe50089289ca34b52a72a833e6f2661b64417068846, 3bd4ca36a66c0675e695f3fc44af703cd6c110085adf105138ef56e6768a639f16a9c27b651a0c64f685b24be835e0a62485575477e06d530574865bf1670d30] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : city64withseed('Abc',0) @@ -272,13 +272,13 @@ evaluation: | Row 2 | 'ß😀山' | 12 | 14631005279260459058 | +--------+-------------------+-----------+----------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| b | UInt16([10, 11, 12]) | -| Output | UInt64([10385767944629066306, 12123249488783690377, 14631005279260459058]) | -+--------+------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| b | UInt16([10, 11, 12]) | +| Output | UInt64([10385767944629066306, 12123249488783690377, 14631005279260459058]) | ++--------+----------------------------------------------------------------------------+ ast : siphash64('Abc') @@ -366,12 +366,12 @@ evaluation: | Row 1 | 'ß😀山' | 1354619631122873228 | +--------+-------------------------+----------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 10, 19] } | -| Output | UInt64([5782510256878119795, 1354619631122873228]) | -+--------+---------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Dobrý den, ß😀山] } | +| Output | UInt64([5782510256878119795, 1354619631122873228]) | ++--------+--------------------------------------------------------+ ast : xxhash64('Abc') @@ -459,12 +459,12 @@ evaluation: | Row 1 | 'ß😀山' | 656695431091154575 | +--------+-------------------------+----------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 10, 19] } | -| Output | UInt64([314761032262035578, 656695431091154575]) | -+--------+---------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Dobrý den, ß😀山] } | +| Output | UInt64([314761032262035578, 656695431091154575]) | ++--------+--------------------------------------------------------+ ast : xxhash32('Abc') @@ -552,11 +552,11 @@ evaluation: | Row 1 | 'ß😀山' | 1401072642 | +--------+-------------------------+------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 10, 19] } | -| Output | UInt32([19285785, 1401072642]) | -+--------+---------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Dobrý den, ß😀山] } | +| Output | UInt32([19285785, 1401072642]) | ++--------+--------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/map.txt b/src/query/functions/tests/it/scalars/testdata/map.txt index f8618c82aa66..63659a6561ea 100644 --- a/src/query/functions/tests/it/scalars/testdata/map.txt +++ b/src/query/functions/tests/it/scalars/testdata/map.txt @@ -69,17 +69,17 @@ evaluation: | Row 2 | 3 | 6 | 9 | 'c' | NULL | 'g' | {3:'c', 6:NULL, 9:'g'} | +--------+---------+---------+---------+-------------+---------------------+---------------------+------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | Int8([1, 2, 3]) | -| b_col | Int8([4, 5, 6]) | -| c_col | Int8([7, 8, 9]) | -| d_col | NullableColumn { column: StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x6465, offsets: [0, 1, 2, 2] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x6667, offsets: [0, 1, 1, 2] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: Tuple([Int8([1, 4, 7, 2, 5, 8, 3, 6, 9]), NullableColumn { column: StringColumn { data: 0x61646662656367, offsets: [0, 1, 2, 3, 4, 5, 5, 6, 6, 7] }, validity: [0b01011111, 0b_______1] }]), offsets: [0, 3, 6, 9] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | Int8([1, 2, 3]) | +| b_col | Int8([4, 5, 6]) | +| c_col | Int8([7, 8, 9]) | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[d, e, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[f, , g] }, validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([Int8([1, 4, 7, 2, 5, 8, 3, 6, 9]), NullableColumn { column: StringColumn { data: Utf8ViewArray[a, d, f, b, e, , c, , g] }, validity: [0b01011111, 0b_______1] }]), offsets: [0, 3, 6, 9] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map(['k1', 'k2'], [a_col, b_col]) @@ -97,13 +97,13 @@ evaluation: | Row 2 | 3 | 6 | {'k1':3, 'k2':6} | +--------+---------+---------+-------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | Int8([1, 2, 3]) | -| b_col | Int8([4, 5, 6]) | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x6b316b326b316b326b316b32, offsets: [0, 2, 4, 6, 8, 10, 12] }, Int8([1, 4, 2, 5, 3, 6])]), offsets: [0, 2, 4, 6] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | Int8([1, 2, 3]) | +| b_col | Int8([4, 5, 6]) | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[k1, k2, k1, k2, k1, k2] }, Int8([1, 4, 2, 5, 3, 6])]), offsets: [0, 2, 4, 6] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map([],[])[1] @@ -201,15 +201,15 @@ evaluation: | Row 1 | 2 | 4 | 'v2' | 'v4' | NULL | +--------+---------+---------+---------------+---------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------+ -| k1 | Int16([1, 2]) | -| k2 | Int16([3, 4]) | -| v1 | StringColumn { data: 0x76317632, offsets: [0, 2, 4] } | -| v2 | StringColumn { data: 0x76337634, offsets: [0, 2, 4] } | -| Output | NullableColumn { column: StringColumn { data: 0x7631, offsets: [0, 2, 2] }, validity: [0b______01] } | -+--------+------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------+ +| k1 | Int16([1, 2]) | +| k2 | Int16([3, 4]) | +| v1 | StringColumn { data: Utf8ViewArray[v1, v2] } | +| v2 | StringColumn { data: Utf8ViewArray[v3, v4] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, ] }, validity: [0b______01] } | ++--------+-----------------------------------------------------------------------------------------------+ ast : map_keys({}) @@ -262,17 +262,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | ['c', 'f', 'z'] | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+-----------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | -| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | -| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: StringColumn { data: 0x61647862657963667a, offsets: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, offsets: [0, 3, 6, 9] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | +| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | +| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | +| Output | ArrayColumn { values: StringColumn { data: Utf8ViewArray[a, d, x, b, e, y, c, f, z] }, offsets: [0, 3, 6, 9] } | ++--------+----------------------------------------------------------------------------------------------------------------+ ast : map_values({}) @@ -334,17 +334,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | ['v3', NULL, 'v7'] | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | -| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | -| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: NullableColumn { column: StringColumn { data: 0x7631763476367632763576337637, offsets: [0, 2, 4, 6, 8, 10, 10, 12, 12, 14] }, validity: [0b01011111, 0b_______1] }, offsets: [0, 3, 6, 9] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | +| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | +| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | +| Output | ArrayColumn { values: NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v4, v6, v2, v5, , v3, , v7] }, validity: [0b01011111, 0b_______1] }, offsets: [0, 3, 6, 9] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_size({}) @@ -388,17 +388,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | 3 | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | -| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | -| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | -| Output | UInt64([3, 3, 3]) | -+--------+-----------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | +| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | +| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | +| Output | UInt64([3, 3, 3]) | ++--------+-----------------------------------------------------------------------------------------------------+ ast : map_cat({}, {}) @@ -442,17 +442,17 @@ evaluation: | Row 2 | 'a_k3' | 'b_k3' | 'c_k3' | 'aaa3' | 'bbb3' | 'ccc3' | {'a_k3':'aaa3', 'b_k3':'bbb3', 'c_k3':'ccc3'} | +--------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-----------------------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x615f6b31615f6b32615f6b33, offsets: [0, 4, 8, 12] } | -| b_col | StringColumn { data: 0x625f6b31625f6b32625f6b33, offsets: [0, 4, 8, 12] } | -| c_col | StringColumn { data: 0x635f6b31635f6b32635f6b33, offsets: [0, 4, 8, 12] } | -| d_col | StringColumn { data: 0x616161316161613261616133, offsets: [0, 4, 8, 12] } | -| e_col | StringColumn { data: 0x626262316262623262626233, offsets: [0, 4, 8, 12] } | -| f_col | StringColumn { data: 0x636363316363633263636333, offsets: [0, 4, 8, 12] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x615f6b31625f6b31635f6b31615f6b32625f6b32635f6b32615f6b33625f6b33635f6b33, offsets: [0, 4, 8, 12, 16, 20, 24, 28, 32, 36] }, StringColumn { data: 0x616161316262623163636331616161326262623263636332616161336262623363636333, offsets: [0, 4, 8, 12, 16, 20, 24, 28, 32, 36] }]), offsets: [0, 3, 6, 9] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a_k1, a_k2, a_k3] } | +| b_col | StringColumn { data: Utf8ViewArray[b_k1, b_k2, b_k3] } | +| c_col | StringColumn { data: Utf8ViewArray[c_k1, c_k2, c_k3] } | +| d_col | StringColumn { data: Utf8ViewArray[aaa1, aaa2, aaa3] } | +| e_col | StringColumn { data: Utf8ViewArray[bbb1, bbb2, bbb3] } | +| f_col | StringColumn { data: Utf8ViewArray[ccc1, ccc2, ccc3] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a_k1, b_k1, c_k1, a_k2, b_k2, c_k2, a_k3, b_k3, c_k3] }, StringColumn { data: Utf8ViewArray[aaa1, bbb1, ccc1, aaa2, bbb2, ccc2, aaa3, bbb3, ccc3] }]), offsets: [0, 3, 6, 9] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_cat({'k1':'v1','k2':'v2'}, {'k1':'abc'}) @@ -478,17 +478,17 @@ evaluation: | Row 2 | 'c_k3' | 'b_k3' | 'c_k3' | 'aaa3' | 'bbb3' | 'ccc3' | {'c_k3':'ccc3', 'b_k3':'bbb3'} | +--------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-----------------------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x615f6b31615f6b32635f6b33, offsets: [0, 4, 8, 12] } | -| b_col | StringColumn { data: 0x625f6b31635f6b32625f6b33, offsets: [0, 4, 8, 12] } | -| c_col | StringColumn { data: 0x635f6b31635f6b32635f6b33, offsets: [0, 4, 8, 12] } | -| d_col | StringColumn { data: 0x616161316161613261616133, offsets: [0, 4, 8, 12] } | -| e_col | StringColumn { data: 0x626262316262623262626233, offsets: [0, 4, 8, 12] } | -| f_col | StringColumn { data: 0x636363316363633263636333, offsets: [0, 4, 8, 12] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x615f6b31625f6b31635f6b31615f6b32635f6b32635f6b33625f6b33, offsets: [0, 4, 8, 12, 16, 20, 24, 28] }, StringColumn { data: 0x61616131626262316363633161616132636363326363633362626233, offsets: [0, 4, 8, 12, 16, 20, 24, 28] }]), offsets: [0, 3, 5, 7] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a_k1, a_k2, c_k3] } | +| b_col | StringColumn { data: Utf8ViewArray[b_k1, c_k2, b_k3] } | +| c_col | StringColumn { data: Utf8ViewArray[c_k1, c_k2, c_k3] } | +| d_col | StringColumn { data: Utf8ViewArray[aaa1, aaa2, aaa3] } | +| e_col | StringColumn { data: Utf8ViewArray[bbb1, bbb2, bbb3] } | +| f_col | StringColumn { data: Utf8ViewArray[ccc1, ccc2, ccc3] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a_k1, b_k1, c_k1, a_k2, c_k2, c_k3, b_k3] }, StringColumn { data: Utf8ViewArray[aaa1, bbb1, ccc1, aaa2, ccc2, ccc3, bbb3] }]), offsets: [0, 3, 5, 7] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_cat({'k1': 'v1', 'k2': 'v2'}, {'k3': 'v3'}) @@ -602,15 +602,15 @@ evaluation: | Row 2 | 'a_k3' | 'b_k3' | 'aaa3' | 'bbb3' | {'a_k3':'aaa3'} | +--------+-------------------+-------------------+-------------------+-------------------+--------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x615f6b31615f6b32615f6b33, offsets: [0, 4, 8, 12] } | -| b_col | StringColumn { data: 0x625f6b31625f6b32625f6b33, offsets: [0, 4, 8, 12] } | -| d_col | StringColumn { data: 0x616161316161613261616133, offsets: [0, 4, 8, 12] } | -| e_col | StringColumn { data: 0x626262316262623262626233, offsets: [0, 4, 8, 12] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x615f6b31625f6b31625f6b32615f6b33, offsets: [0, 4, 8, 12, 16] }, StringColumn { data: 0x61616131626262316262623261616133, offsets: [0, 4, 8, 12, 16] }]), offsets: [0, 2, 3, 4] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a_k1, a_k2, a_k3] } | +| b_col | StringColumn { data: Utf8ViewArray[b_k1, b_k2, b_k3] } | +| d_col | StringColumn { data: Utf8ViewArray[aaa1, aaa2, aaa3] } | +| e_col | StringColumn { data: Utf8ViewArray[bbb1, bbb2, bbb3] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a_k1, b_k1, b_k2, a_k3] }, StringColumn { data: Utf8ViewArray[aaa1, bbb1, bbb2, aaa3] }]), offsets: [0, 2, 3, 4] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_delete({'k1': 'v1', 'k2': 'v2', 'k3': 'v3', 'k4': 'v4'}, string_key_col) @@ -627,12 +627,12 @@ evaluation: | Row 1 | 'k2' | {'k1':'v1', 'k3':'v3', 'k4':'v4'} | +--------+----------------+------------------------------------+ evaluation (internal): -+----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| string_key_col | StringColumn { data: 0x6b336b32, offsets: [0, 2, 4] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x6b316b326b346b316b336b34, offsets: [0, 2, 4, 6, 8, 10, 12] }, StringColumn { data: 0x763176327634763176337634, offsets: [0, 2, 4, 6, 8, 10, 12] }]), offsets: [0, 3, 6] } | -+----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| string_key_col | StringColumn { data: Utf8ViewArray[k3, k2] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[k1, k2, k4, k1, k3, k4] }, StringColumn { data: Utf8ViewArray[v1, v2, v4, v1, v3, v4] }]), offsets: [0, 3, 6] } | ++----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_delete({'k1': 'v1', 'k2': 'v2', 'k3': 'v3'}, 'k1', 'k2', 'k3') @@ -684,15 +684,15 @@ evaluation: | Row 2 | 'a_k3' | 559 | 'aaa3' | 662 | {'a_k3':559} | +--------+-------------------+-------------+-------------------+-------------+--------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x615f6b31615f6b32615f6b33, offsets: [0, 4, 8, 12] } | -| b_col | Int16([555, 557, 559]) | -| d_col | StringColumn { data: 0x616161316161613261616133, offsets: [0, 4, 8, 12] } | -| e_col | Int16([666, 664, 662]) | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x615f6b316161613161616132615f6b33, offsets: [0, 4, 8, 12, 16] }, Int16([555, 666, 664, 559])]), offsets: [0, 2, 3, 4] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a_k1, a_k2, a_k3] } | +| b_col | Int16([555, 557, 559]) | +| d_col | StringColumn { data: Utf8ViewArray[aaa1, aaa2, aaa3] } | +| e_col | Int16([666, 664, 662]) | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a_k1, aaa1, aaa2, a_k3] }, Int16([555, 666, 664, 559])]), offsets: [0, 2, 3, 4] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ error: @@ -753,17 +753,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | false | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | -| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | -| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | -| Output | Boolean([0b_____001]) | -+--------+-----------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | +| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | +| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | +| Output | Boolean([0b_____001]) | ++--------+-----------------------------------------------------------------------------------------------------+ ast : map_contains_key(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'd') @@ -780,17 +780,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | false | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | -| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | -| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | -| Output | Boolean([0b_____001]) | -+--------+-----------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | +| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | +| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | +| Output | Boolean([0b_____001]) | ++--------+-----------------------------------------------------------------------------------------------------+ ast : map_pick({'a':1,'b':2,'c':3}, 'a', 'b') @@ -861,17 +861,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | {} | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | -| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | -| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x6162, offsets: [0, 1, 2] }, NullableColumn { column: StringColumn { data: 0x76317632, offsets: [0, 2, 4] }, validity: [0b______11] }]), offsets: [0, 1, 2, 2] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | +| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | +| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a, b] }, NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2] }, validity: [0b______11] }]), offsets: [0, 1, 2, 2] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_insert({}, 'k1', 'v1') @@ -924,17 +924,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | {'c':'v3', 'f':NULL, 'z':'v7', 'k1':'v10'} | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | -| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | -| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x6164786b316265796b3163667a6b31, offsets: [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 15] }, NullableColumn { column: StringColumn { data: 0x7631763476367631307632763576313076337637763130, offsets: [0, 2, 4, 6, 9, 11, 13, 13, 16, 18, 18, 20, 23] }, validity: [0b10111111, 0b____1101] }]), offsets: [0, 4, 8, 12] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | +| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | +| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a, d, x, k1, b, e, y, k1, c, f, z, k1] }, NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v4, v6, v10, v2, v5, , v10, v3, , v7, v10] }, validity: [0b10111111, 0b____1101] }]), offsets: [0, 4, 8, 12] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_insert(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'a', 'v10', true) @@ -952,17 +952,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | {'c':'v3', 'f':NULL, 'z':'v7', 'a':'v10'} | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+-------------------------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| b_col | StringColumn { data: 0x646566, offsets: [0, 1, 2, 3] } | -| c_col | StringColumn { data: 0x78797a, offsets: [0, 1, 2, 3] } | -| d_col | NullableColumn { column: StringColumn { data: 0x763176327633, offsets: [0, 2, 4, 6] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: 0x76347635, offsets: [0, 2, 4, 4] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: 0x76367637, offsets: [0, 2, 2, 4] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: 0x6164786265796163667a61, offsets: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] }, NullableColumn { column: StringColumn { data: 0x763130763476367632763576313076337637763130, offsets: [0, 3, 5, 7, 9, 11, 11, 14, 16, 16, 18, 21] }, validity: [0b11011111, 0b_____110] }]), offsets: [0, 3, 7, 11] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | +| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | +| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | +| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a, d, x, b, e, y, a, c, f, z, a] }, NullableColumn { column: StringColumn { data: Utf8ViewArray[v10, v4, v6, v2, v5, , v10, v3, , v7, v10] }, validity: [0b11011111, 0b_____110] }]), offsets: [0, 3, 7, 11] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ error: diff --git a/src/query/functions/tests/it/scalars/testdata/regexp.txt b/src/query/functions/tests/it/scalars/testdata/regexp.txt index beca019c9702..2481b476c7f5 100644 --- a/src/query/functions/tests/it/scalars/testdata/regexp.txt +++ b/src/query/functions/tests/it/scalars/testdata/regexp.txt @@ -56,13 +56,13 @@ evaluation: | Row 2 | '' | '' | 0 | +--------+---------------------------+--------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x646f672063617420646f676161206161612061616161206161206161612061616161, offsets: [0, 11, 34, 34] } | -| pat | StringColumn { data: 0x646f67617b327d, offsets: [0, 3, 7, 7] } | -| Output | UInt64([1, 1, 0]) | -+--------+-------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, ] } | +| pat | StringColumn { data: Utf8ViewArray[dog, a{2}, ] } | +| Output | UInt64([1, 1, 0]) | ++--------+------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos) @@ -79,14 +79,14 @@ evaluation: | Row 2 | '' | '' | 1 | 0 | +--------+---------------------------+--------------+---------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x646f672063617420646f676161206161612061616161206161206161612061616161, offsets: [0, 11, 34, 34] } | -| pat | StringColumn { data: 0x646f67617b327d, offsets: [0, 3, 7, 7] } | -| pos | Int64([1, 2, 1]) | -| Output | UInt64([1, 4, 0]) | -+--------+-------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, ] } | +| pat | StringColumn { data: Utf8ViewArray[dog, a{2}, ] } | +| pos | Int64([1, 2, 1]) | +| Output | UInt64([1, 4, 0]) | ++--------+------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur) @@ -103,15 +103,15 @@ evaluation: | Row 2 | 'aa aa aa aaaa aaaa aaaa' | 'a{4}' | 9 | 2 | 15 | +--------+---------------------------------------------+------------------+---------+---------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x646f672063617420646f6761612061616120616161612061612061616120616161616161206161206161206161616120616161612061616161, offsets: [0, 11, 34, 57] } | -| pat | StringColumn { data: 0x646f67617b327d617b347d, offsets: [0, 3, 7, 11] } | -| pos | Int64([1, 1, 9]) | -| occur | Int64([2, 3, 2]) | -| Output | UInt64([9, 8, 15]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] } | +| pat | StringColumn { data: Utf8ViewArray[dog, a{2}, a{4}] } | +| pos | Int64([1, 1, 9]) | +| occur | Int64([2, 3, 2]) | +| Output | UInt64([9, 8, 15]) | ++--------+-----------------------------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro) @@ -129,16 +129,16 @@ evaluation: | Row 2 | 'aa aa aa aaaa aaaa aaaa' | 'a{4}' | 1 | 2 | 1 | 19 | +--------+---------------------------------------------+------------------+---------+---------+---------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x646f672063617420646f6761612061616120616161612061612061616120616161616161206161206161206161616120616161612061616161, offsets: [0, 11, 34, 57] } | -| pat | StringColumn { data: 0x646f67617b327d617b347d, offsets: [0, 3, 7, 11] } | -| pos | Int64([1, 2, 1]) | -| occur | Int64([2, 2, 2]) | -| ro | Int64([0, 1, 1]) | -| Output | UInt64([9, 10, 19]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] } | +| pat | StringColumn { data: Utf8ViewArray[dog, a{2}, a{4}] } | +| pos | Int64([1, 2, 1]) | +| occur | Int64([2, 2, 2]) | +| ro | Int64([0, 1, 1]) | +| Output | UInt64([9, 10, 19]) | ++--------+-----------------------------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro, mt) @@ -156,17 +156,17 @@ evaluation: | Row 2 | 'aa aa aa aaaa aaaa aaaa' | 'A{4}' | 1 | 2 | 1 | 'i' | 19 | +--------+---------------------------------------------+------------------+---------+---------+---------+-------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x646f672063617420646f6761612061616120616161612061612061616120616161616161206161206161206161616120616161612061616161, offsets: [0, 11, 34, 57] } | -| pat | StringColumn { data: 0x646f67417b327d417b347d, offsets: [0, 3, 7, 11] } | -| pos | Int64([1, 2, 1]) | -| occur | Int64([2, 2, 2]) | -| ro | Int64([0, 1, 1]) | -| mt | StringColumn { data: 0x696369, offsets: [0, 1, 2, 3] } | -| Output | UInt64([9, 0, 19]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] } | +| pat | StringColumn { data: Utf8ViewArray[dog, A{2}, A{4}] } | +| pos | Int64([1, 2, 1]) | +| occur | Int64([2, 2, 2]) | +| ro | Int64([0, 1, 1]) | +| mt | StringColumn { data: Utf8ViewArray[i, c, i] } | +| Output | UInt64([9, 0, 19]) | ++--------+-----------------------------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro) @@ -184,16 +184,16 @@ evaluation: | Row 3 | 'aa aa aa aaaa aaaa aaaa' | 'A{4}' | 1 | 1 | 1 | 14 | +--------+-------------------------------+-----------------------+---------+---------+---------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: 0x646f672063617420646f6761612061616120616161612061612061616120616161616161206161206161206161616120616161612061616161, offsets: [0, 11, 34, 34, 57] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: 0x646f67417b347d, offsets: [0, 3, 3, 3, 7] }, validity: [0b____1001] } | -| pos | Int64([1, 2, 1, 1]) | -| occur | Int64([2, 2, 2, 1]) | -| ro | Int64([0, 1, 1, 1]) | -| Output | NullableColumn { column: UInt64([9, 0, 0, 14]), validity: [0b____1001] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, , aa aa aa aaaa aaaa aaaa] }, validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[dog, , , A{4}] }, validity: [0b____1001] } | +| pos | Int64([1, 2, 1, 1]) | +| occur | Int64([2, 2, 2, 1]) | +| ro | Int64([0, 1, 1, 1]) | +| Output | NullableColumn { column: UInt64([9, 0, 0, 14]), validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro, mt) @@ -211,17 +211,17 @@ evaluation: | Row 3 | 'aa aa aa aaaa aaaa aaaa' | 'A{4}' | 1 | 1 | 1 | 'i' | 14 | +--------+-------------------------------+-----------------------+---------+---------+---------+-------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: 0x646f672063617420646f6761612061616120616161612061612061616120616161616161206161206161206161616120616161612061616161, offsets: [0, 11, 34, 34, 57] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: 0x646f67417b347d, offsets: [0, 3, 3, 3, 7] }, validity: [0b____1001] } | -| pos | Int64([1, 2, 1, 1]) | -| occur | Int64([2, 2, 2, 1]) | -| ro | Int64([0, 1, 1, 1]) | -| mt | StringColumn { data: 0x69636969, offsets: [0, 1, 2, 3, 4] } | -| Output | NullableColumn { column: UInt64([9, 0, 0, 14]), validity: [0b____1001] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, , aa aa aa aaaa aaaa aaaa] }, validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[dog, , , A{4}] }, validity: [0b____1001] } | +| pos | Int64([1, 2, 1, 1]) | +| occur | Int64([2, 2, 2, 1]) | +| ro | Int64([0, 1, 1, 1]) | +| mt | StringColumn { data: Utf8ViewArray[i, c, i, i] } | +| Output | NullableColumn { column: UInt64([9, 0, 0, 14]), validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro) @@ -240,16 +240,16 @@ evaluation: | Row 3 | '周 周周 周周周 周周周周' | '周+' | 5 | 1 | 1 | 9 | +--------+---------------------------------------------------------+-----------------+---------+---------+---------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0xe591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8, offsets: [0, 33, 66, 99, 132] } | -| pat | StringColumn { data: 0xe591a82be591a82be591a82be591a82b, offsets: [0, 4, 8, 12, 16] } | -| pos | Int64([1, 2, 3, 5]) | -| occur | Int64([2, 2, 3, 1]) | -| ro | Int64([0, 1, 1, 1]) | -| Output | UInt64([3, 9, 14, 9]) | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] } | +| pat | StringColumn { data: Utf8ViewArray[周+, 周+, 周+, 周+] } | +| pos | Int64([1, 2, 3, 5]) | +| occur | Int64([2, 2, 3, 1]) | +| ro | Int64([0, 1, 1, 1]) | +| Output | UInt64([3, 9, 14, 9]) | ++--------+------------------------------------------------------------------------------------------------------------------------------------------+ error: @@ -349,13 +349,15 @@ evaluation: | Row 5 | '' | '' | true | +--------+----------------------+-------------------------+---------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x6162636162644162656e65772a0a2a6c696e65666f0a666f, offsets: [0, 3, 6, 9, 19, 24, 24] } | -| pat | StringColumn { data: 0x5e6141626162656e65775c2a2e5c2a6c696e655e666f24, offsets: [0, 2, 4, 7, 19, 23, 23] } | -| Output | Boolean([0b__100111]) | -+--------+--------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[abc, abd, Abe, new* | +| | *line, fo | +| | fo, ] } | +| pat | StringColumn { data: Utf8ViewArray[^a, Ab, abe, new\*.\*line, ^fo$, ] } | +| Output | Boolean([0b__100111]) | ++--------+-------------------------------------------------------------------------+ ast : regexp_like(source, pat, mt) @@ -377,14 +379,16 @@ evaluation: | Row 5 | '' | '' | 'c' | true | +--------+----------------------+-------------------------+------------+---------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x6162636162644162656e65772a0a2a6c696e65666f0a666f, offsets: [0, 3, 6, 9, 19, 24, 24] } | -| pat | StringColumn { data: 0x5e6141626162656e65775c2a2e5c2a6c696e655e666f24, offsets: [0, 2, 4, 7, 19, 23, 23] } | -| mt | StringColumn { data: 0x63696e6d63, offsets: [0, 0, 1, 2, 3, 4, 5] } | -| Output | Boolean([0b__111101]) | -+--------+--------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[abc, abd, Abe, new* | +| | *line, fo | +| | fo, ] } | +| pat | StringColumn { data: Utf8ViewArray[^a, Ab, abe, new\*.\*line, ^fo$, ] } | +| mt | StringColumn { data: Utf8ViewArray[, c, i, n, m, c] } | +| Output | Boolean([0b__111101]) | ++--------+-------------------------------------------------------------------------+ ast : regexp_like(source, pat, mt) @@ -402,14 +406,14 @@ evaluation: | Row 3 | 'abc' | 'abc' | NULL | NULL | +--------+-----------------------+-----------------------+---------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: 0x616263616263616263, offsets: [0, 3, 6, 6, 9] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: 0x616263616263, offsets: [0, 3, 3, 3, 6] }, validity: [0b____1001] } | -| mt | NullableColumn { column: StringColumn { data: 0x6969, offsets: [0, 0, 1, 2, 2] }, validity: [0b____0111] } | -| Output | NullableColumn { column: Boolean([0b____1101]), validity: [0b____0001] } | -+--------+--------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, abc, , abc] }, validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, , , abc] }, validity: [0b____1001] } | +| mt | NullableColumn { column: StringColumn { data: Utf8ViewArray[, i, i, ] }, validity: [0b____0111] } | +| Output | NullableColumn { column: Boolean([0b____1101]), validity: [0b____0001] } | ++--------+----------------------------------------------------------------------------------------------------------+ error: @@ -516,14 +520,14 @@ evaluation: | Row 3 | '' | 'b' | 'X' | '' | +--------+----------------+------------+-------------+---------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x612062206361206220636120622063, offsets: [0, 5, 10, 15, 15] } | -| pat | StringColumn { data: 0x627862, offsets: [0, 1, 2, 2, 3] } | -| repl | StringColumn { data: 0x58585858, offsets: [0, 1, 2, 3, 4] } | -| Output | StringColumn { data: 0x612058206361206220636120622063, offsets: [0, 5, 10, 15, 15] } | -+--------+--------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[a b c, a b c, a b c, ] } | +| pat | StringColumn { data: Utf8ViewArray[b, x, , b] } | +| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | +| Output | StringColumn { data: Utf8ViewArray[a X c, a b c, a b c, ] } | ++--------+-------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos) @@ -542,15 +546,15 @@ evaluation: | Row 3 | 'abc def ghi' | '[a-z]+' | 'X' | 12 | 'abc def ghi' | +--------+---------------------------------+-----------------------+-------------+----------+---------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x6162632064656620676869616263206465662067686961626320646566206768696162632064656620676869, offsets: [0, 11, 22, 33, 44] } | -| pat | StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 12, 18, 24] } | -| repl | StringColumn { data: 0x58585858, offsets: [0, 1, 2, 3, 4] } | -| pos | Int64([1, 4, 8, 12]) | -| Output | StringColumn { data: 0x5820582058616263205820586162632064656620586162632064656620676869, offsets: [0, 5, 12, 21, 32] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, abc def ghi, abc def ghi] } | +| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+, [a-z]+] } | +| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | +| pos | Int64([1, 4, 8, 12]) | +| Output | StringColumn { data: Utf8ViewArray[X X X, abc X X, abc def X, abc def ghi] } | ++--------+------------------------------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur) @@ -569,16 +573,16 @@ evaluation: | Row 3 | 'abc def ghi' | '[a-z]+' | 'X' | 4 | 3 | 'abc def ghi' | +--------+---------------------------------+-----------------------+-------------+---------+---------+---------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x6162632064656620676869616263206465662067686961626320646566206768696162632064656620676869, offsets: [0, 11, 22, 33, 44] } | -| pat | StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 12, 18, 24] } | -| repl | StringColumn { data: 0x58585858, offsets: [0, 1, 2, 3, 4] } | -| pos | Int64([1, 1, 4, 4]) | -| occur | Int64([0, 1, 2, 3]) | -| Output | StringColumn { data: 0x58205820585820646566206768696162632064656620586162632064656620676869, offsets: [0, 5, 14, 23, 34] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, abc def ghi, abc def ghi] } | +| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+, [a-z]+] } | +| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | +| pos | Int64([1, 1, 4, 4]) | +| occur | Int64([0, 1, 2, 3]) | +| Output | StringColumn { data: Utf8ViewArray[X X X, X def ghi, abc def X, abc def ghi] } | ++--------+------------------------------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur, mt) @@ -596,17 +600,17 @@ evaluation: | Row 2 | 'abc DEF ghi' | '[a-z]+' | 'X' | 4 | 1 | 'i' | 'abc X ghi' | +--------+---------------------------------+-----------------------+-------------+---------+---------+------------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x616263206465662067686961626320444546206768696162632044454620676869, offsets: [0, 11, 22, 33] } | -| pat | StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 12, 18] } | -| repl | StringColumn { data: 0x585858, offsets: [0, 1, 2, 3] } | -| pos | Int64([1, 1, 4]) | -| occur | Int64([0, 2, 1]) | -| mt | StringColumn { data: 0x6369, offsets: [0, 0, 1, 2] } | -| Output | StringColumn { data: 0x5820582058616263204445462058616263205820676869, offsets: [0, 5, 14, 23] } | -+--------+-----------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc DEF ghi, abc DEF ghi] } | +| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+] } | +| repl | StringColumn { data: Utf8ViewArray[X, X, X] } | +| pos | Int64([1, 1, 4]) | +| occur | Int64([0, 2, 1]) | +| mt | StringColumn { data: Utf8ViewArray[, c, i] } | +| Output | StringColumn { data: Utf8ViewArray[X X X, abc DEF X, abc X ghi] } | ++--------+-----------------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur) @@ -625,16 +629,16 @@ evaluation: | Row 3 | 'abc DEF ghi' | '[a-z]+' | 'X' | 4 | 1 | 'abc X ghi' | +--------+-------------------------------+--------------------------+-------------+---------+---------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: 0x616263206465662067686961626320444546206768696162632044454620676869, offsets: [0, 11, 22, 22, 33] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 6, 6, 12] }, validity: [0b____1001] } | -| repl | StringColumn { data: 0x58585858, offsets: [0, 1, 2, 3, 4] } | -| pos | Int64([1, 1, 4, 4]) | -| occur | Int64([0, 2, 1, 1]) | -| Output | NullableColumn { column: StringColumn { data: 0x58205820586162632044454620676869616263205820676869, offsets: [0, 5, 16, 16, 25] }, validity: [0b____1001] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc def ghi, abc DEF ghi, , abc DEF ghi] }, validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[[a-z]+, , , [a-z]+] }, validity: [0b____1001] } | +| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | +| pos | Int64([1, 1, 4, 4]) | +| occur | Int64([0, 2, 1, 1]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[X X X, abc DEF ghi, , abc X ghi] }, validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur, mt) @@ -653,17 +657,17 @@ evaluation: | Row 3 | 'abc DEF ghi' | '[a-z]+' | 'X' | 4 | 1 | 'i' | 'abc X ghi' | +--------+-------------------------------+--------------------------+-------------+---------+---------+------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: 0x616263206465662067686961626320444546206768696162632044454620676869, offsets: [0, 11, 22, 22, 33] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 6, 6, 12] }, validity: [0b____1001] } | -| repl | StringColumn { data: 0x58585858, offsets: [0, 1, 2, 3, 4] } | -| pos | Int64([1, 1, 4, 4]) | -| occur | Int64([0, 2, 1, 1]) | -| mt | StringColumn { data: 0x636969, offsets: [0, 0, 1, 2, 3] } | -| Output | NullableColumn { column: StringColumn { data: 0x58205820586162632044454620676869616263205820676869, offsets: [0, 5, 16, 16, 25] }, validity: [0b____1001] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc def ghi, abc DEF ghi, , abc DEF ghi] }, validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[[a-z]+, , , [a-z]+] }, validity: [0b____1001] } | +| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | +| pos | Int64([1, 1, 4, 4]) | +| occur | Int64([0, 2, 1, 1]) | +| mt | StringColumn { data: Utf8ViewArray[, c, i, i] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[X X X, abc DEF ghi, , abc X ghi] }, validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur) @@ -682,16 +686,16 @@ evaluation: | Row 3 | '周 周周 周周周 周周周周' | '周+' | '唐' | 5 | 1 | '周 周周 唐 周周周周' | +--------+---------------------------------------------------------+-----------------+---------------+---------+---------+-------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0xe591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8, offsets: [0, 33, 66, 99, 132] } | -| pat | StringColumn { data: 0xe591a82be591a82be591a82be591a82b, offsets: [0, 4, 8, 12, 16] } | -| repl | StringColumn { data: 0xe59490e59490e59490e59490, offsets: [0, 3, 6, 9, 12] } | -| pos | Int64([1, 2, 3, 5]) | -| occur | Int64([0, 1, 3, 1]) | -| Output | StringColumn { data: 0xe5949020e5949020e5949020e59490e591a820e5949020e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e59490e591a820e591a8e591a820e5949020e591a8e591a8e591a8e591a8, offsets: [0, 15, 45, 69, 96] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] } | +| pat | StringColumn { data: Utf8ViewArray[周+, 周+, 周+, 周+] } | +| repl | StringColumn { data: Utf8ViewArray[唐, 唐, 唐, 唐] } | +| pos | Int64([1, 2, 3, 5]) | +| occur | Int64([0, 1, 3, 1]) | +| Output | StringColumn { data: Utf8ViewArray[唐 唐 唐 唐, 周 唐 周周周 周周周周, 周 周周 周周周 唐, 周 周周 唐 周周周周] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------+ error: @@ -786,13 +790,13 @@ evaluation: | Row 2 | '' | '' | NULL | +--------+----------------------+--------------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x61626320646566206768696162632064656620676869, offsets: [0, 11, 22, 22] } | -| pat | StringColumn { data: 0x5b612d7a5d2b787878, offsets: [0, 6, 9, 9] } | -| Output | NullableColumn { column: StringColumn { data: 0x616263, offsets: [0, 3, 3, 3] }, validity: [0b_____001] } | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, ] } | +| pat | StringColumn { data: Utf8ViewArray[[a-z]+, xxx, ] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, , ] }, validity: [0b_____001] } | ++--------+--------------------------------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos) @@ -810,14 +814,14 @@ evaluation: | Row 2 | 'abc def ghi' | '[a-z]+' | 12 | NULL | +--------+---------------------------------+-----------------------+----------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x616263206465662067686961626320646566206768696162632064656620676869, offsets: [0, 11, 22, 33] } | -| pat | StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 12, 18] } | -| pos | Int64([1, 4, 12]) | -| Output | NullableColumn { column: StringColumn { data: 0x616263646566, offsets: [0, 3, 6, 6] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, abc def ghi] } | +| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+] } | +| pos | Int64([1, 4, 12]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, def, ] }, validity: [0b_____011] } | ++--------+-----------------------------------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos, occur) @@ -835,15 +839,15 @@ evaluation: | Row 2 | 'abc def ghi' | '[a-z]+' | 12 | 3 | NULL | +--------+---------------------------------+-----------------------+----------+---------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x616263206465662067686961626320646566206768696162632064656620676869, offsets: [0, 11, 22, 33] } | -| pat | StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 12, 18] } | -| pos | Int64([1, 4, 12]) | -| occur | Int64([3, 2, 3]) | -| Output | NullableColumn { column: StringColumn { data: 0x676869676869, offsets: [0, 3, 6, 6] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, abc def ghi] } | +| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+] } | +| pos | Int64([1, 4, 12]) | +| occur | Int64([3, 2, 3]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[ghi, ghi, ] }, validity: [0b_____011] } | ++--------+-----------------------------------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos, occur, mt) @@ -861,16 +865,16 @@ evaluation: | Row 2 | 'abc DEF ghi' | '[a-z]+' | 12 | 3 | 'i' | NULL | +--------+---------------------------------+-----------------------+----------+---------+-------------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0x414243206465662067686961626320646566204748496162632044454620676869, offsets: [0, 11, 22, 33] } | -| pat | StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 12, 18] } | -| pos | Int64([1, 4, 12]) | -| occur | Int64([3, 2, 3]) | -| mt | StringColumn { data: 0x636969, offsets: [0, 1, 2, 3] } | -| Output | NullableColumn { column: StringColumn { data: 0x474849, offsets: [0, 0, 3, 3] }, validity: [0b_____010] } | -+--------+-----------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[ABC def ghi, abc def GHI, abc DEF ghi] } | +| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+] } | +| pos | Int64([1, 4, 12]) | +| occur | Int64([3, 2, 3]) | +| mt | StringColumn { data: Utf8ViewArray[c, i, i] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, GHI, ] }, validity: [0b_____010] } | ++--------+--------------------------------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos, occur, mt) @@ -888,16 +892,16 @@ evaluation: | Row 3 | 'abc DEF ghi' | '[a-z]+' | 4 | 1 | 'i' | 'DEF' | +--------+-------------------------------+--------------------------+---------+---------+------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: 0x616263206465662067686961626320444546206768696162632044454620676869, offsets: [0, 11, 22, 22, 33] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: 0x5b612d7a5d2b5b612d7a5d2b, offsets: [0, 6, 6, 6, 12] }, validity: [0b____1001] } | -| pos | Int64([1, 1, 4, 4]) | -| occur | Int64([1, 2, 1, 1]) | -| mt | StringColumn { data: 0x636969, offsets: [0, 0, 1, 2, 3] } | -| Output | NullableColumn { column: StringColumn { data: 0x616263444546, offsets: [0, 3, 3, 3, 6] }, validity: [0b____1001] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc def ghi, abc DEF ghi, , abc DEF ghi] }, validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[[a-z]+, , , [a-z]+] }, validity: [0b____1001] } | +| pos | Int64([1, 1, 4, 4]) | +| occur | Int64([1, 2, 1, 1]) | +| mt | StringColumn { data: Utf8ViewArray[, c, i, i] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, , , DEF] }, validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos, occur) @@ -915,15 +919,15 @@ evaluation: | Row 2 | '周 周周 周周周 周周周周' | '周+' | 14 | 1 | NULL | +--------+---------------------------------------------------------+-----------------+----------+---------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: 0xe591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8e591a820e591a8e591a820e591a8e591a8e591a820e591a8e591a8e591a8e591a8, offsets: [0, 33, 66, 99] } | -| pat | StringColumn { data: 0xe591a82be591a82be591a82b, offsets: [0, 4, 8, 12] } | -| pos | Int64([1, 2, 14]) | -| occur | Int64([1, 2, 1]) | -| Output | NullableColumn { column: StringColumn { data: 0xe591a8e591a8e591a8e591a8, offsets: [0, 3, 12, 12] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------+ +| source | StringColumn { data: Utf8ViewArray[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] } | +| pat | StringColumn { data: Utf8ViewArray[周+, 周+, 周+] } | +| pos | Int64([1, 2, 14]) | +| occur | Int64([1, 2, 1]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[周, 周周周, ] }, validity: [0b_____011] } | ++--------+-----------------------------------------------------------------------------------------------------------------+ error: diff --git a/src/query/functions/tests/it/scalars/testdata/string.txt b/src/query/functions/tests/it/scalars/testdata/string.txt index 20cd5cc4467b..63958666b0e4 100644 --- a/src/query/functions/tests/it/scalars/testdata/string.txt +++ b/src/query/functions/tests/it/scalars/testdata/string.txt @@ -48,12 +48,12 @@ evaluation: | Row 2 | 'ß😀山' | 'SS😀山' | +--------+-------------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| Output | StringColumn { data: 0x414243444f4252c39d2044454e5353f09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -+--------+------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| Output | StringColumn { data: Utf8ViewArray[ABC, DOBRÝ DEN, SS😀山] } | ++--------+--------------------------------------------------------------+ ast : lower('Abc') @@ -106,12 +106,12 @@ evaluation: | Row 2 | 'İ😀山' | 'i̇😀山' | +--------+-------------------+-------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x416263444f4252c39d2044454ec4b0f09f9880e5b1b1, offsets: [0, 3, 13, 22] } | -| Output | StringColumn { data: 0x616263646f6272c3bd2064656e69cc87f09f9880e5b1b1, offsets: [0, 3, 13, 23] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[Abc, DOBRÝ DEN, İ😀山] } | +| Output | StringColumn { data: Utf8ViewArray[abc, dobrý den, i̇😀山] } | ++--------+-------------------------------------------------------------+ ast : bit_length('latin') @@ -182,12 +182,12 @@ evaluation: | Row 2 | 'кириллица and latin' | 224 | +--------+-----------------------------------+----------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x6c6174696ed0bad0b8d180d0b8d0bbd0bbd0b8d186d0b0d0bad0b8d180d0b8d0bbd0bbd0b8d186d0b020616e64206c6174696e, offsets: [0, 5, 23, 51] } | -| Output | UInt64([40, 144, 224]) | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[latin, кириллица, кириллица and latin] } | +| Output | UInt64([40, 144, 224]) | ++--------+-----------------------------------------------------------------------------+ ast : octet_length('latin') @@ -258,12 +258,12 @@ evaluation: | Row 2 | 'кириллица and latin' | 28 | +--------+-----------------------------------+----------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x6c6174696ed0bad0b8d180d0b8d0bbd0bbd0b8d186d0b0d0bad0b8d180d0b8d0bbd0bbd0b8d186d0b020616e64206c6174696e, offsets: [0, 5, 23, 51] } | -| Output | UInt64([5, 18, 28]) | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[latin, кириллица, кириллица and latin] } | +| Output | UInt64([5, 18, 28]) | ++--------+-----------------------------------------------------------------------------+ ast : char_length('latin') @@ -334,12 +334,12 @@ evaluation: | Row 2 | 'кириллица and latin' | 19 | +--------+-----------------------------------+----------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x6c6174696ed0bad0b8d180d0b8d0bbd0bbd0b8d186d0b0d0bad0b8d180d0b8d0bbd0bbd0b8d186d0b020616e64206c6174696e, offsets: [0, 5, 23, 51] } | -| Output | UInt64([5, 9, 19]) | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[latin, кириллица, кириллица and latin] } | +| Output | UInt64([5, 9, 19]) | ++--------+-----------------------------------------------------------------------------+ ast : quote('a\0b') @@ -465,12 +465,12 @@ evaluation: | Row 2 | 'a\nb' | 'a\\nb' | +--------+---------------------+----------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x615c3062615c2762615c6e62, offsets: [0, 4, 8, 12] } | -| Output | StringColumn { data: 0x615c5c3062615c5c5c2762615c5c6e62, offsets: [0, 5, 11, 16] } | -+--------+------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[a\0b, a\'b, a\nb] } | +| Output | StringColumn { data: Utf8ViewArray[a\\0b, a\\\'b, a\\nb] } | ++--------+------------------------------------------------------------+ ast : reverse('abc') @@ -550,12 +550,12 @@ evaluation: | Row 2 | '' | '' | +--------+--------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------+ -| a | StringColumn { data: 0x61626361, offsets: [0, 3, 4, 4] } | -| Output | StringColumn { data: 0x63626161, offsets: [0, 3, 4, 4] } | -+--------+----------------------------------------------------------+ ++--------+------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[abc, a, ] } | +| Output | StringColumn { data: Utf8ViewArray[cba, a, ] } | ++--------+------------------------------------------------+ ast : ascii('1') @@ -636,12 +636,12 @@ evaluation: | Row 3 | '你好' | 228 | +--------+-----------------+------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x313132332d31e4bda0e5a5bd, offsets: [0, 1, 4, 6, 12] } | -| Output | UInt8([49, 49, 45, 228]) | -+--------+------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[1, 123, -1, 你好] } | +| Output | UInt8([49, 49, 45, 228]) | ++--------+--------------------------------------------------------+ ast : ascii(b) @@ -657,12 +657,12 @@ evaluation: | Row 0 | '' | 0 | +--------+-----------+---------+ evaluation (internal): -+--------+--------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------+ -| b | StringColumn { data: 0x, offsets: [0, 0] } | -| Output | UInt8([0]) | -+--------+--------------------------------------------+ ++--------+----------------------------------------+ +| Column | Data | ++--------+----------------------------------------+ +| b | StringColumn { data: Utf8ViewArray[] } | +| Output | UInt8([0]) | ++--------+----------------------------------------+ ast : ltrim(' abc ') @@ -734,12 +734,12 @@ evaluation: | Row 3 | 'abc ' | 'abc ' | +--------+-----------------------+----------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616263202020616263202020616263202020616263202020, offsets: [0, 3, 9, 18, 24] } | -| Output | StringColumn { data: 0x616263616263616263202020616263202020, offsets: [0, 3, 6, 12, 18] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | +| Output | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | ++--------+----------------------------------------------------------------------+ ast : rtrim(' abc ') @@ -811,12 +811,12 @@ evaluation: | Row 3 | 'abc ' | 'abc' | +--------+-----------------------+----------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616263202020616263202020616263202020616263202020, offsets: [0, 3, 9, 18, 24] } | -| Output | StringColumn { data: 0x616263202020616263202020616263616263, offsets: [0, 3, 9, 15, 18] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | +| Output | StringColumn { data: Utf8ViewArray[abc, abc, abc, abc] } | ++--------+----------------------------------------------------------------------+ ast : trim_leading('aaabbaaa', 'a') @@ -915,12 +915,12 @@ evaluation: | Row 3 | 'aabbaa' | 'bbaa' | +--------+-----------------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363616162626161, offsets: [0, 6, 12, 18, 24] } | -| Output | StringColumn { data: 0x6262616162626363626263636464636362626161, offsets: [0, 4, 10, 16, 20] } | -+--------+--------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | +| Output | StringColumn { data: Utf8ViewArray[bbaa, bbccbb, ccddcc, bbaa] } | ++--------+----------------------------------------------------------------------+ ast : trim_leading(a, b) @@ -938,13 +938,13 @@ evaluation: | Row 3 | 'aabbaa' | '' | 'aabbaa' | +--------+-----------------------+------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363616162626161, offsets: [0, 6, 12, 18, 24] } | -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3, 3] } | -| Output | StringColumn { data: 0x626261616363626264646363616162626161, offsets: [0, 4, 8, 12, 18] } | -+--------+--------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | +| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | +| Output | StringColumn { data: Utf8ViewArray[bbaa, ccbb, ddcc, aabbaa] } | ++--------+----------------------------------------------------------------------+ ast : trim_leading('aba', b) @@ -962,12 +962,12 @@ evaluation: | Row 3 | '' | 'aba' | +--------+------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------+ -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3, 3] } | -| Output | StringColumn { data: 0x6261616261616261616261, offsets: [0, 2, 5, 8, 11] } | -+--------+----------------------------------------------------------------------------+ ++--------+---------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------+ +| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | +| Output | StringColumn { data: Utf8ViewArray[ba, aba, aba, aba] } | ++--------+---------------------------------------------------------+ ast : trim_trailing('aaabbaaa', 'a') @@ -1066,12 +1066,12 @@ evaluation: | Row 3 | 'aabbaa' | 'aabbaa' | +--------+-----------------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363616162626161, offsets: [0, 6, 12, 18, 24] } | -| Output | StringColumn { data: 0x61616262616162626363636364646363616162626161, offsets: [0, 6, 10, 16, 22] } | -+--------+--------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | +| Output | StringColumn { data: Utf8ViewArray[aabbaa, bbcc, ccddcc, aabbaa] } | ++--------+----------------------------------------------------------------------+ ast : trim_trailing(a, b) @@ -1089,13 +1089,13 @@ evaluation: | Row 3 | 'aabbaa' | '' | 'aabbaa' | +--------+-----------------------+------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363616162626161, offsets: [0, 6, 12, 18, 24] } | -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3, 3] } | -| Output | StringColumn { data: 0x616162626262636363636464616162626161, offsets: [0, 4, 8, 12, 18] } | -+--------+--------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | +| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | +| Output | StringColumn { data: Utf8ViewArray[aabb, bbcc, ccdd, aabbaa] } | ++--------+----------------------------------------------------------------------+ ast : trim_trailing('aba', b) @@ -1113,12 +1113,12 @@ evaluation: | Row 3 | '' | 'aba' | +--------+------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------+ -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3, 3] } | -| Output | StringColumn { data: 0x6162616261616261616261, offsets: [0, 2, 5, 8, 11] } | -+--------+----------------------------------------------------------------------------+ ++--------+---------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------+ +| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | +| Output | StringColumn { data: Utf8ViewArray[ab, aba, aba, aba] } | ++--------+---------------------------------------------------------+ ast : trim_both('aaabbaaa', 'a') @@ -1226,12 +1226,12 @@ evaluation: | Row 3 | 'aabbaa' | 'bb' | +--------+-----------------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363616162626161, offsets: [0, 6, 12, 18, 24] } | -| Output | StringColumn { data: 0x62626262636362626363646463636262, offsets: [0, 2, 8, 14, 16] } | -+--------+--------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | +| Output | StringColumn { data: Utf8ViewArray[bb, bbccbb, ccddcc, bb] } | ++--------+----------------------------------------------------------------------+ ast : trim_both(a, b) @@ -1249,13 +1249,13 @@ evaluation: | Row 3 | 'aabbaa' | '' | 'aabbaa' | +--------+-----------------------+------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363616162626161, offsets: [0, 6, 12, 18, 24] } | -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3, 3] } | -| Output | StringColumn { data: 0x626263636464616162626161, offsets: [0, 2, 4, 6, 12] } | -+--------+--------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | +| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | +| Output | StringColumn { data: Utf8ViewArray[bb, cc, dd, aabbaa] } | ++--------+----------------------------------------------------------------------+ ast : trim_both('aba', b) @@ -1273,12 +1273,12 @@ evaluation: | Row 3 | '' | 'aba' | +--------+------------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------+ -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3, 3] } | -| Output | StringColumn { data: 0x62616261616261616261, offsets: [0, 1, 4, 7, 10] } | -+--------+--------------------------------------------------------------------------+ ++--------+--------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------+ +| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | +| Output | StringColumn { data: Utf8ViewArray[b, aba, aba, aba] } | ++--------+--------------------------------------------------------+ ast : trim(' abc ') @@ -1350,12 +1350,12 @@ evaluation: | Row 3 | 'abc ' | 'abc' | +--------+-----------------------+--------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616263202020616263202020616263202020616263202020, offsets: [0, 3, 9, 18, 24] } | -| Output | StringColumn { data: 0x616263616263616263616263, offsets: [0, 3, 6, 9, 12] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | +| Output | StringColumn { data: Utf8ViewArray[abc, abc, abc, abc] } | ++--------+----------------------------------------------------------------------+ ast : trim(both 'a' from 'aaabbaaa') @@ -1426,12 +1426,12 @@ evaluation: | Row 2 | 'ccddcc' | 'ccddcc' | +--------+-----------------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| Output | StringColumn { data: 0x6262626263636262636364646363, offsets: [0, 2, 8, 14] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| Output | StringColumn { data: Utf8ViewArray[bb, bbccbb, ccddcc] } | ++--------+--------------------------------------------------------------+ ast : trim(both b from a) @@ -1448,13 +1448,13 @@ evaluation: | Row 2 | 'ccddcc' | 'c' | 'dd' | +--------+-----------------------+-------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| Output | StringColumn { data: 0x626263636464, offsets: [0, 2, 4, 6] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| b | StringColumn { data: Utf8ViewArray[a, b, c] } | +| Output | StringColumn { data: Utf8ViewArray[bb, cc, dd] } | ++--------+--------------------------------------------------------------+ ast : trim(both a from a) @@ -1471,12 +1471,12 @@ evaluation: | Row 2 | 'ccddcc' | '' | +--------+-----------------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| Output | StringColumn { data: 0x, offsets: [0, 0, 0, 0] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| Output | StringColumn { data: Utf8ViewArray[, , ] } | ++--------+--------------------------------------------------------------+ ast : trim(both b from 'aba') @@ -1493,12 +1493,12 @@ evaluation: | Row 2 | 'c' | 'aba' | +--------+-------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------+ -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| Output | StringColumn { data: 0x62616261616261, offsets: [0, 1, 4, 7] } | -+--------+----------------------------------------------------------------+ ++--------+---------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------+ +| b | StringColumn { data: Utf8ViewArray[a, b, c] } | +| Output | StringColumn { data: Utf8ViewArray[b, aba, aba] } | ++--------+---------------------------------------------------+ ast : trim(leading 'a' from 'aaabbaaa') @@ -1569,12 +1569,12 @@ evaluation: | Row 2 | 'ccddcc' | 'ccddcc' | +--------+-----------------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| Output | StringColumn { data: 0x62626161626263636262636364646363, offsets: [0, 4, 10, 16] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| Output | StringColumn { data: Utf8ViewArray[bbaa, bbccbb, ccddcc] } | ++--------+--------------------------------------------------------------+ ast : trim(leading b from a) @@ -1591,13 +1591,13 @@ evaluation: | Row 2 | 'ccddcc' | 'c' | 'ddcc' | +--------+-----------------------+-------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| Output | StringColumn { data: 0x626261616363626264646363, offsets: [0, 4, 8, 12] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| b | StringColumn { data: Utf8ViewArray[a, b, c] } | +| Output | StringColumn { data: Utf8ViewArray[bbaa, ccbb, ddcc] } | ++--------+--------------------------------------------------------------+ ast : trim(leading a from a) @@ -1614,12 +1614,12 @@ evaluation: | Row 2 | 'ccddcc' | '' | +--------+-----------------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| Output | StringColumn { data: 0x, offsets: [0, 0, 0, 0] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| Output | StringColumn { data: Utf8ViewArray[, , ] } | ++--------+--------------------------------------------------------------+ ast : trim(leading b from 'aba') @@ -1636,12 +1636,12 @@ evaluation: | Row 2 | 'c' | 'aba' | +--------+-------------+--------+ evaluation (internal): -+--------+------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------+ -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| Output | StringColumn { data: 0x6261616261616261, offsets: [0, 2, 5, 8] } | -+--------+------------------------------------------------------------------+ ++--------+----------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------+ +| b | StringColumn { data: Utf8ViewArray[a, b, c] } | +| Output | StringColumn { data: Utf8ViewArray[ba, aba, aba] } | ++--------+----------------------------------------------------+ ast : trim(trailing 'a' from 'aaabbaaa') @@ -1712,12 +1712,12 @@ evaluation: | Row 2 | 'ccddcc' | 'ccddcc' | +--------+-----------------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| Output | StringColumn { data: 0x61616262626263636262636364646363, offsets: [0, 4, 10, 16] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| Output | StringColumn { data: Utf8ViewArray[aabb, bbccbb, ccddcc] } | ++--------+--------------------------------------------------------------+ ast : trim(trailing b from a) @@ -1734,13 +1734,13 @@ evaluation: | Row 2 | 'ccddcc' | 'c' | 'ccdd' | +--------+-----------------------+-------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| Output | StringColumn { data: 0x616162626262636363636464, offsets: [0, 4, 8, 12] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| b | StringColumn { data: Utf8ViewArray[a, b, c] } | +| Output | StringColumn { data: Utf8ViewArray[aabb, bbcc, ccdd] } | ++--------+--------------------------------------------------------------+ ast : trim(trailing a from a) @@ -1757,12 +1757,12 @@ evaluation: | Row 2 | 'ccddcc' | '' | +--------+-----------------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616162626161626263636262636364646363, offsets: [0, 6, 12, 18] } | -| Output | StringColumn { data: 0x, offsets: [0, 0, 0, 0] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | +| Output | StringColumn { data: Utf8ViewArray[, , ] } | ++--------+--------------------------------------------------------------+ ast : trim(trailing b from 'aba') @@ -1779,12 +1779,12 @@ evaluation: | Row 2 | 'c' | 'aba' | +--------+-------------+--------+ evaluation (internal): -+--------+------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------+ -| b | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| Output | StringColumn { data: 0x6162616261616261, offsets: [0, 2, 5, 8] } | -+--------+------------------------------------------------------------------+ ++--------+----------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------+ +| b | StringColumn { data: Utf8ViewArray[a, b, c] } | +| Output | StringColumn { data: Utf8ViewArray[ab, aba, aba] } | ++--------+----------------------------------------------------+ ast : concat('5', '3', '4') @@ -1829,12 +1829,12 @@ evaluation: | Row 3 | 'abc ' | 'abc 345' | +--------+-----------------------+----------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616263202020616263202020616263202020616263202020, offsets: [0, 3, 9, 18, 24] } | -| Output | StringColumn { data: 0x616263333435202020616263333435202020616263202020333435616263202020333435, offsets: [0, 6, 15, 27, 36] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | +| Output | StringColumn { data: Utf8ViewArray[abc345, abc345, abc 345, abc 345] } | ++--------+----------------------------------------------------------------------------------+ ast : concat(a, '3') @@ -1853,12 +1853,12 @@ evaluation: | Row 3 | 'd' | 'd3' | +--------+----------------------+------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: 0x6133623363336433, offsets: [0, 2, 4, 6, 8] }, validity: [0b____1011] } | -+--------+------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a3, b3, c3, d3] }, validity: [0b____1011] } | ++--------+---------------------------------------------------------------------------------------------------------+ ast : concat_ws('-', '3', null, '4', null, '5') @@ -1903,12 +1903,12 @@ evaluation: | Row 3 | '-' | '3-4-5' | +--------+-------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x2c2d2c2d, offsets: [0, 1, 2, 3, 4] } | -| Output | StringColumn { data: 0x332c342c35332d342d35332c342c35332d342d35, offsets: [0, 5, 10, 15, 20] } | -+--------+------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[,, -, ,, -] } | +| Output | StringColumn { data: Utf8ViewArray[3,4,5, 3-4-5, 3,4,5, 3-4-5] } | ++--------+------------------------------------------------------------------+ ast : concat_ws(a, '3') @@ -1927,12 +1927,12 @@ evaluation: | Row 3 | 'd' | '3' | +--------+----------------------+-----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: 0x333333, offsets: [0, 1, 2, 2, 3] }, validity: [0b____1011] } | -+--------+----------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[3, 3, , 3] }, validity: [0b____1011] } | ++--------+-----------------------------------------------------------------------------------------------------+ ast : concat_ws(a, '3', '4') @@ -1951,12 +1951,12 @@ evaluation: | Row 3 | 'd' | '3d4' | +--------+----------------------+-----------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: 0x336134336234336434, offsets: [0, 3, 6, 6, 9] }, validity: [0b____1011] } | -+--------+--------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[3a4, 3b4, , 3d4] }, validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------------------------------+ error: @@ -1985,12 +1985,12 @@ evaluation: | Row 2 | 3 | '11' | +--------+----------+--------------------------------------------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | Int8([-1, 2, 3]) | -| Output | StringColumn { data: 0x3131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131303131, offsets: [0, 64, 66, 68] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------+ +| a | Int8([-1, 2, 3]) | +| Output | StringColumn { data: Utf8ViewArray[1111111111111111111111111111111111111111111111111111111111111111, 10, 11] } | ++--------+----------------------------------------------------------------------------------------------------------------+ ast : bin(a2) @@ -2007,12 +2007,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+------------------+-----------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------+ -| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: 0x3131303131, offsets: [0, 1, 3, 5] }, validity: [0b_____011] } | -+--------+---------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------+ +| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 10, 11] }, validity: [0b_____011] } | ++--------+----------------------------------------------------------------------------------------------------+ ast : bin(b) @@ -2029,12 +2029,12 @@ evaluation: | Row 2 | 6 | '110' | +--------+---------+--------+ evaluation (internal): -+--------+------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------+ -| b | Int16([2, 4, 6]) | -| Output | StringColumn { data: 0x3130313030313130, offsets: [0, 2, 5, 8] } | -+--------+------------------------------------------------------------------+ ++--------+----------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------+ +| b | Int16([2, 4, 6]) | +| Output | StringColumn { data: Utf8ViewArray[10, 100, 110] } | ++--------+----------------------------------------------------+ ast : bin(c) @@ -2051,12 +2051,12 @@ evaluation: | Row 2 | 30 | '11110' | +--------+-----------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------+ -| c | UInt32([10, 20, 30]) | -| Output | StringColumn { data: 0x3130313031303130303131313130, offsets: [0, 4, 9, 14] } | -+--------+-------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------+ +| c | UInt32([10, 20, 30]) | +| Output | StringColumn { data: Utf8ViewArray[1010, 10100, 11110] } | ++--------+----------------------------------------------------------+ error: @@ -2093,12 +2093,12 @@ evaluation: | Row 2 | 3 | '3' | +--------+----------+--------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| a | Int8([-1, 2, 3]) | -| Output | StringColumn { data: 0x313737373737373737373737373737373737373737373233, offsets: [0, 22, 23, 24] } | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------+ +| a | Int8([-1, 2, 3]) | +| Output | StringColumn { data: Utf8ViewArray[1777777777777777777777, 2, 3] } | ++--------+--------------------------------------------------------------------+ ast : oct(a2) @@ -2115,12 +2115,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+------------------+-----------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: 0x313233, offsets: [0, 1, 2, 3] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 2, 3] }, validity: [0b_____011] } | ++--------+--------------------------------------------------------------------------------------------------+ ast : oct(b) @@ -2137,12 +2137,12 @@ evaluation: | Row 2 | 6 | '6' | +--------+---------+--------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| b | Int16([2, 4, 6]) | -| Output | StringColumn { data: 0x323436, offsets: [0, 1, 2, 3] } | -+--------+--------------------------------------------------------+ ++--------+-----------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------+ +| b | Int16([2, 4, 6]) | +| Output | StringColumn { data: Utf8ViewArray[2, 4, 6] } | ++--------+-----------------------------------------------+ ast : oct(c) @@ -2159,12 +2159,12 @@ evaluation: | Row 2 | 30 | '36' | +--------+-----------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| c | UInt32([10, 20, 30]) | -| Output | StringColumn { data: 0x313232343336, offsets: [0, 2, 4, 6] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------+ +| c | UInt32([10, 20, 30]) | +| Output | StringColumn { data: Utf8ViewArray[12, 24, 36] } | ++--------+--------------------------------------------------+ error: @@ -2201,12 +2201,12 @@ evaluation: | Row 2 | 3 | '3' | +--------+----------+--------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------+ -| a | Int8([-1, 2, 3]) | -| Output | StringColumn { data: 0x666666666666666666666666666666663233, offsets: [0, 16, 17, 18] } | -+--------+-----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------+ +| a | Int8([-1, 2, 3]) | +| Output | StringColumn { data: Utf8ViewArray[ffffffffffffffff, 2, 3] } | ++--------+--------------------------------------------------------------+ ast : hex(a2) @@ -2223,12 +2223,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+------------------+-----------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: 0x313233, offsets: [0, 1, 2, 3] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 2, 3] }, validity: [0b_____011] } | ++--------+--------------------------------------------------------------------------------------------------+ ast : hex(b) @@ -2245,12 +2245,12 @@ evaluation: | Row 2 | 6 | '6' | +--------+---------+--------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| b | Int16([2, 4, 6]) | -| Output | StringColumn { data: 0x323436, offsets: [0, 1, 2, 3] } | -+--------+--------------------------------------------------------+ ++--------+-----------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------+ +| b | Int16([2, 4, 6]) | +| Output | StringColumn { data: Utf8ViewArray[2, 4, 6] } | ++--------+-----------------------------------------------+ ast : hex(c) @@ -2267,12 +2267,12 @@ evaluation: | Row 2 | 30 | '1e' | +--------+-----------+--------+ evaluation (internal): -+--------+------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------+ -| c | UInt32([10, 20, 30]) | -| Output | StringColumn { data: 0x6131343165, offsets: [0, 1, 3, 5] } | -+--------+------------------------------------------------------------+ ++--------+-------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------+ +| c | UInt32([10, 20, 30]) | +| Output | StringColumn { data: Utf8ViewArray[a, 14, 1e] } | ++--------+-------------------------------------------------+ error: @@ -2303,12 +2303,12 @@ evaluation: | Row 2 | 'databend' | '6461746162656e64' | +--------+-----------------+--------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------+ -| e | StringColumn { data: 0x6162636465666461746162656e64, offsets: [0, 3, 6, 14] } | -| Output | StringColumn { data: 0x36313632363336343635363636343631373436313632363536653634, offsets: [0, 6, 12, 28] } | -+--------+------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------+ +| e | StringColumn { data: Utf8ViewArray[abc, def, databend] } | +| Output | StringColumn { data: Utf8ViewArray[616263, 646566, 6461746162656e64] } | ++--------+------------------------------------------------------------------------+ ast : lpad('hi', 2, '?') @@ -2403,14 +2403,14 @@ evaluation: | Row 2 | 'cc' | 5 | 'bb' | 'bbbcc' | +--------+-----------------+---------+-------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------+ -| a | StringColumn { data: 0x6869746573746363, offsets: [0, 2, 6, 8] } | -| b | UInt8([0, 3, 5]) | -| c | StringColumn { data: 0x3f786262, offsets: [0, 1, 2, 4] } | -| Output | StringColumn { data: 0x7465736262626363, offsets: [0, 0, 3, 8] } | -+--------+------------------------------------------------------------------+ ++--------+----------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[hi, test, cc] } | +| b | UInt8([0, 3, 5]) | +| c | StringColumn { data: Utf8ViewArray[?, x, bb] } | +| Output | StringColumn { data: Utf8ViewArray[, tes, bbbcc] } | ++--------+----------------------------------------------------+ error: @@ -2513,14 +2513,14 @@ evaluation: | Row 2 | 'cc' | 5 | 'bb' | 'ccbbb' | +--------+-----------------+---------+-------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------+ -| a | StringColumn { data: 0x6869746573746363, offsets: [0, 2, 6, 8] } | -| b | UInt8([0, 3, 5]) | -| c | StringColumn { data: 0x3f786262, offsets: [0, 1, 2, 4] } | -| Output | StringColumn { data: 0x7465736363626262, offsets: [0, 0, 3, 8] } | -+--------+------------------------------------------------------------------+ ++--------+----------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[hi, test, cc] } | +| b | UInt8([0, 3, 5]) | +| c | StringColumn { data: Utf8ViewArray[?, x, bb] } | +| Output | StringColumn { data: Utf8ViewArray[, tes, ccbbb] } | ++--------+----------------------------------------------------+ error: @@ -2582,14 +2582,14 @@ evaluation: | Row 3 | 'q' | '' | 'q' | 'q' | +--------+-----------------+-------------+-------------+--------+ evaluation (internal): -+--------+-----------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------+ -| a | StringColumn { data: 0x686974657374636371, offsets: [0, 2, 6, 8, 9] } | -| b | StringColumn { data: 0x6974656363, offsets: [0, 1, 3, 5, 5] } | -| c | StringColumn { data: 0x3f78626271, offsets: [0, 1, 2, 4, 5] } | -| Output | StringColumn { data: 0x683f787374626271, offsets: [0, 2, 5, 7, 8] } | -+--------+-----------------------------------------------------------------------+ ++--------+-------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[hi, test, cc, q] } | +| b | StringColumn { data: Utf8ViewArray[i, te, cc, ] } | +| c | StringColumn { data: Utf8ViewArray[?, x, bb, q] } | +| Output | StringColumn { data: Utf8ViewArray[h?, xst, bb, q] } | ++--------+-------------------------------------------------------+ ast : translate('abcdefabcdef', 'dc', 'zy') @@ -2644,14 +2644,14 @@ evaluation: | Row 3 | 'abcdef' | 'dc' | 'dc' | 'abcdef' | +--------+-----------------------+-------------+-------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x616263646566616263646566616263646566616263646566, offsets: [0, 6, 12, 18, 24] } | -| b | StringColumn { data: 0x646364636463, offsets: [0, 2, 2, 4, 6] } | -| c | StringColumn { data: 0x7a797a796463, offsets: [0, 2, 4, 4, 6] } | -| Output | StringColumn { data: 0x6162797a656661626364656661626566616263646566, offsets: [0, 6, 12, 16, 22] } | -+--------+--------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[abcdef, abcdef, abcdef, abcdef] } | +| b | StringColumn { data: Utf8ViewArray[dc, , dc, dc] } | +| c | StringColumn { data: Utf8ViewArray[zy, zy, , dc] } | +| Output | StringColumn { data: Utf8ViewArray[abyzef, abcdef, abef, abcdef] } | ++--------+----------------------------------------------------------------------+ ast : strcmp('text', 'text2') @@ -2696,13 +2696,13 @@ evaluation: | Row 3 | 'cc' | 'ccb' | -1 | +--------+-----------------+------------------+--------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------+ -| a | StringColumn { data: 0x6968746573746363, offsets: [0, 1, 2, 6, 8] } | -| b | StringColumn { data: 0x6869686974657374636362, offsets: [0, 2, 4, 8, 11] } | -| Output | Int8([1, -1, 0, -1]) | -+--------+----------------------------------------------------------------------------+ ++--------+---------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[i, h, test, cc] } | +| b | StringColumn { data: Utf8ViewArray[hi, hi, test, ccb] } | +| Output | Int8([1, -1, 0, -1]) | ++--------+---------------------------------------------------------+ ast : locate('bar', 'foobarbar') @@ -2837,14 +2837,14 @@ evaluation: | Row 3 | 'q' | '56' | 1 | 0 | +--------+---------------+---------------+---------+----------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x6261726363636371, offsets: [0, 3, 5, 7, 8] } | -| b | StringColumn { data: 0x666f6f6261726261726264636361636378783536, offsets: [0, 9, 16, 18, 20] } | -| c | UInt8([1, 2, 0, 1]) | -| Output | UInt64([4, 3, 0, 0]) | -+--------+------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[bar, cc, cc, q] } | +| b | StringColumn { data: Utf8ViewArray[foobarbar, bdccacc, xx, 56] } | +| c | UInt8([1, 2, 0, 1]) | +| Output | UInt64([4, 3, 0, 0]) | ++--------+------------------------------------------------------------------+ ast : char(65,66,67) @@ -2875,7 +2875,7 @@ evaluation: | Type | UInt8 | UInt8 | UInt8 | Binary | | Domain | {66..=67} | {98..=99} | {68..=69} | Undefined | | Row 0 | 66 | 98 | 68 | 426244 | -| Row 1 | 67 | 99 | 69 | 436345 | +| Row 1 | 67 | 99 | 69 | 426244 | +--------+-----------+-----------+-----------+-----------+ evaluation (internal): +--------+-----------------------------------------------------------+ @@ -2884,7 +2884,7 @@ evaluation (internal): | a | UInt8([66, 67]) | | b | UInt8([98, 99]) | | c | UInt8([68, 69]) | -| Output | BinaryColumn { data: 0x426244436345, offsets: [0, 3, 6] } | +| Output | BinaryColumn { data: 0x426244426244, offsets: [0, 3, 6] } | +--------+-----------------------------------------------------------+ @@ -2907,7 +2907,7 @@ evaluation (internal): | b | UInt8([98, 99]) | | c | UInt8([68, 69]) | | a2 | NullableColumn { column: UInt8([66, 67]), validity: [0b______01] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x426244436345, offsets: [0, 3, 6] }, validity: [0b______01] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x426244426244, offsets: [0, 3, 6] }, validity: [0b______01] } | +--------+--------------------------------------------------------------------------------------------------------------+ @@ -2974,12 +2974,12 @@ evaluation: | Row 3 | 'TEACHER' | 'T260' | +--------+------------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x23f09f9091f09f90916865f09f90916c6c6ff09f9091f09f90916865f09f90916c6c6ff09f90917465616368657254454143484552, offsets: [0, 22, 39, 46, 53] } | -| Output | StringColumn { data: 0xf09f9091343030f09f90913430305432363054323630, offsets: [0, 7, 14, 18, 22] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[#🐑🐑he🐑llo🐑, 🐑he🐑llo🐑, teacher, TEACHER] } | +| Output | StringColumn { data: Utf8ViewArray[🐑400, 🐑400, T260, T260] } | ++--------+-------------------------------------------------------------------------------------+ ast : ord(NULL) @@ -3077,12 +3077,12 @@ evaluation: | Row 2 | 'c' | 'ccc' | +--------+-------------+---------+ evaluation (internal): -+--------+--------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------+ -| a | StringColumn { data: 0x616263, offsets: [0, 1, 2, 3] } | -| Output | StringColumn { data: 0x616161626262636363, offsets: [0, 3, 6, 9] } | -+--------+--------------------------------------------------------------------+ ++--------+-----------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[a, b, c] } | +| Output | StringColumn { data: Utf8ViewArray[aaa, bbb, ccc] } | ++--------+-----------------------------------------------------+ error: @@ -3188,15 +3188,15 @@ evaluation: | Row 3 | 'q' | 1 | 1 | '56' | '56' | +--------+-----------------+---------+---------+---------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------+ -| a | StringColumn { data: 0x686974657374636371, offsets: [0, 2, 6, 8, 9] } | -| b | UInt8([1, 4, 1, 1]) | -| c | UInt8([3, 5, 1, 1]) | -| d | StringColumn { data: 0x78787a6331323536, offsets: [0, 2, 4, 6, 8] } | -| Output | StringColumn { data: 0x78787465737a633132633536, offsets: [0, 2, 7, 10, 12] } | -+--------+-------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------+ +| a | StringColumn { data: Utf8ViewArray[hi, test, cc, q] } | +| b | UInt8([1, 4, 1, 1]) | +| c | UInt8([3, 5, 1, 1]) | +| d | StringColumn { data: Utf8ViewArray[xx, zc, 12, 56] } | +| Output | StringColumn { data: Utf8ViewArray[xx, teszc, 12c, 56] } | ++--------+----------------------------------------------------------+ ast : insert(x, y, z, u) @@ -3214,15 +3214,15 @@ evaluation: | Row 3 | 'q' | 1 | 1 | '56' | '56' | +--------+--------------------------+------------------+------------------+------------------------+-----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| x | NullableColumn { column: StringColumn { data: 0x686974657374636371, offsets: [0, 2, 6, 8, 9] }, validity: [0b____1110] } | -| y | NullableColumn { column: UInt8([1, 4, 1, 1]), validity: [0b____1011] } | -| z | NullableColumn { column: UInt8([3, 5, 1, 1]), validity: [0b____1101] } | -| u | NullableColumn { column: StringColumn { data: 0x78787a6331323536, offsets: [0, 2, 4, 6, 8] }, validity: [0b____1110] } | -| Output | NullableColumn { column: StringColumn { data: 0x78787465737a633132633536, offsets: [0, 2, 7, 10, 12] }, validity: [0b____1000] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------+ +| x | NullableColumn { column: StringColumn { data: Utf8ViewArray[hi, test, cc, q] }, validity: [0b____1110] } | +| y | NullableColumn { column: UInt8([1, 4, 1, 1]), validity: [0b____1011] } | +| z | NullableColumn { column: UInt8([3, 5, 1, 1]), validity: [0b____1101] } | +| u | NullableColumn { column: StringColumn { data: Utf8ViewArray[xx, zc, 12, 56] }, validity: [0b____1110] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[xx, teszc, 12c, 56] }, validity: [0b____1000] } | ++--------+-------------------------------------------------------------------------------------------------------------+ ast : space(0) @@ -3272,12 +3272,12 @@ evaluation: | Row 9 | 9 | ' ' | +--------+---------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) | -| Output | StringColumn { data: 0x202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020, offsets: [0, 0, 1, 3, 6, 10, 15, 21, 28, 36, 45] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------+ +| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) | +| Output | StringColumn { data: Utf8ViewArray[, , , , , , , , , ] } | ++--------+-------------------------------------------------------------------------------------------------------+ ast : left('', 0) @@ -3320,12 +3320,12 @@ evaluation: | Row 10 | 10 | '123456789' | +--------+----------+--------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) | -| Output | StringColumn { data: 0x313132313233313233343132333435313233343536313233343536373132333435363738313233343536373839313233343536373839, offsets: [0, 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 54] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------+ +| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) | +| Output | StringColumn { data: Utf8ViewArray[, 1, 12, 123, 1234, 12345, 123456, 1234567, 12345678, 123456789, 123456789] } | ++--------+------------------------------------------------------------------------------------------------------------------+ ast : right('', 0) @@ -3368,12 +3368,12 @@ evaluation: | Row 10 | 10 | '123456789' | +--------+----------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) | -| Output | StringColumn { data: 0x393839373839363738393536373839343536373839333435363738393233343536373839313233343536373839313233343536373839, offsets: [0, 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 54] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------+ +| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) | +| Output | StringColumn { data: Utf8ViewArray[, 9, 89, 789, 6789, 56789, 456789, 3456789, 23456789, 123456789, 123456789] } | ++--------+------------------------------------------------------------------------------------------------------------------+ ast : mid('1234567890', -3, 3) @@ -3513,13 +3513,13 @@ evaluation: | Row 44 | -4 | 4 | '' | +--------+----------+---------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| pos | Int8([0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -3, -3, -3, -3, -3, -4, -4, -4, -4, -4]) | -| len | UInt8([0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4]) | -| Output | StringColumn { data: 0x61616261626361626362626362636263636363636363636362626362636263616162616263616263, offsets: [0, 0, 0, 0, 0, 0, 0, 1, 3, 6, 9, 9, 10, 12, 14, 16, 16, 17, 18, 19, 20, 20, 20, 20, 20, 20, 20, 21, 22, 23, 24, 24, 25, 27, 29, 31, 31, 32, 34, 37, 40, 40, 40, 40, 40, 40] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| pos | Int8([0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -3, -3, -3, -3, -3, -4, -4, -4, -4, -4]) | +| len | UInt8([0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4]) | +| Output | StringColumn { data: Utf8ViewArray[, , , , , , a, ab, abc, abc, , b, bc, bc, bc, , c, c, c, c, , , , , , , c, c, c, c, , b, bc, bc, bc, , a, ab, abc, abc, , , , , ] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : split('Sakila', 'il') @@ -3582,12 +3582,12 @@ evaluation: | Row 3 | 'aeeceedeef' | 'ee' | ['a', 'c', 'd', 'f'] | +--------+-------------------------------+------------------------+---------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| str | NullableColumn { column: StringColumn { data: 0x3132372e302e302e316161612d2d6262622d4242422d2d636363636361656563656564656566, offsets: [0, 9, 26, 28, 38] }, validity: [0b____1110] } | -| sep | NullableColumn { column: StringColumn { data: 0x2e2d2d63636565, offsets: [0, 1, 3, 5, 7] }, validity: [0b____1110] } | -| Output | NullableColumn { column: ArrayColumn { values: StringColumn { data: 0x3132373030316161616262622d42424263636361636466, offsets: [0, 3, 4, 5, 6, 9, 16, 19, 19, 20, 21, 22, 23] }, offsets: [0, 4, 7, 8, 12] }, validity: [0b____1110] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| str | NullableColumn { column: StringColumn { data: Utf8ViewArray[127.0.0.1, aaa--bbb-BBB--ccc, cc, aeeceedeef] }, validity: [0b____1110] } | +| sep | NullableColumn { column: StringColumn { data: Utf8ViewArray[., --, cc, ee] }, validity: [0b____1110] } | +| Output | NullableColumn { column: ArrayColumn { values: StringColumn { data: Utf8ViewArray[127, 0, 0, 1, aaa, bbb-BBB, ccc, , a, c, d, f] }, offsets: [0, 4, 7, 8, 12] }, validity: [0b____1110] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/tuple.txt b/src/query/functions/tests/it/scalars/testdata/tuple.txt index bda28d987ee2..7ab8f2c58ff8 100644 --- a/src/query/functions/tests/it/scalars/testdata/tuple.txt +++ b/src/query/functions/tests/it/scalars/testdata/tuple.txt @@ -49,12 +49,12 @@ evaluation: | Row 3 | 'd' | ('d', 'd') | +--------+----------------------+----------------------------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____1011] } | -| Output | Tuple([NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____1011] }, NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____1011] }]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | +| Output | Tuple([NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] }, NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] }]) | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ error: @@ -141,12 +141,12 @@ evaluation: | Row 3 | 'd' | 'd' | +--------+----------------------+----------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____1011] } | -+--------+----------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | ++--------+-----------------------------------------------------------------------------------------------------+ ast : col.1 @@ -164,11 +164,11 @@ evaluation: | Row 3 | NULL | NULL | +--------+---------------------------------+----------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| col | NullableColumn { column: Tuple([NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____0011] }]), validity: [0b____0101] } | -| Output | NullableColumn { column: StringColumn { data: 0x61626364, offsets: [0, 1, 2, 3, 4] }, validity: [0b____0001] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| col | NullableColumn { column: Tuple([NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____0011] }]), validity: [0b____0101] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____0001] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/variant.txt b/src/query/functions/tests/it/scalars/testdata/variant.txt index e8545ded6ada..77fcf04b20fa 100644 --- a/src/query/functions/tests/it/scalars/testdata/variant.txt +++ b/src/query/functions/tests/it/scalars/testdata/variant.txt @@ -110,7 +110,7 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e35363738312e3931326532225c5c5c226162635c5c5c2222226461746162656e64227b226b223a2276222c2261223a2262227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 49, 62, 72, 89, 110] } | +| s | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | | Output | BinaryColumn { data: 0x200000000000000020000000400000002000000020000009507fffffffffffffff200000002000000340800020000000200000096040934a456d5cfaad2000000020000009604067e6666666666620000000100000075c226162635c2220000000100000086461746162656e644000000210000001100000011000000110000001616b6276800000042000000220000002200000025000001350015002500380000003100000011000000110000001616263, offsets: [0, 8, 16, 33, 44, 61, 78, 93, 109, 133, 178] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -133,7 +133,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x7472756566616c736531323334, offsets: [0, 4, 9, 9, 13] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, false, , 1234] }, validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x2000000040000000200000003000000020000000200000035004d2, offsets: [0, 8, 16, 16, 27] }, validity: [0b____1011] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -243,7 +243,7 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c74727565393232333337323033363835343737353830372d3332373638313233342e35363738312e3931326532225c5c5c226162635c5c5c2222226461746162656e64227b226b223a2276222c2261223a2262227d5b312c322c332c5b2261222c2262222c2263225d5d, offsets: [0, 4, 8, 27, 33, 42, 49, 62, 72, 89, 110] } | +| s | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | | Output | NullableColumn { column: BinaryColumn { data: 0x200000000000000020000000400000002000000020000009507fffffffffffffff200000002000000340800020000000200000096040934a456d5cfaad2000000020000009604067e6666666666620000000100000075c226162635c2220000000100000086461746162656e644000000210000001100000011000000110000001616b6276800000042000000220000002200000025000001350015002500380000003100000011000000110000001616263, offsets: [0, 8, 16, 33, 44, 61, 78, 93, 109, 133, 178] }, validity: [0b11111111, 0b______11] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -266,7 +266,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x7472756574747431323334, offsets: [0, 4, 7, 7, 11] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, ttt, , 1234] }, validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x200000004000000020000000200000035004d2, offsets: [0, 8, 8, 8, 19] }, validity: [0b____1001] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ @@ -312,12 +312,12 @@ evaluation: | Row 2 | 'true' | NULL | +--------+------------------+-------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c61626374727565, offsets: [0, 4, 7, 11] } | -| Output | NullableColumn { column: StringColumn { data: 0x65787065637465642076616c75652c20706f732031, offsets: [0, 0, 21, 21] }, validity: [0b_____010] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, abc, true] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, expected value, pos 1, ] }, validity: [0b_____010] } | ++--------+--------------------------------------------------------------------------------------------------------------------+ ast : check_json(s) @@ -335,12 +335,12 @@ evaluation: | Row 3 | '1234' | NULL | +--------+-----------------------+-------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x7472756574747431323334, offsets: [0, 4, 7, 7, 11] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: 0x6578706563746564206964656e742c20706f732032, offsets: [0, 0, 21, 21, 21] }, validity: [0b____0010] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, ttt, , 1234] }, validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, expected ident, pos 2, , ] }, validity: [0b____0010] } | ++--------+----------------------------------------------------------------------------------------------------------------------+ ast : length(parse_json('1234')) @@ -384,12 +384,12 @@ evaluation: | Row 2 | '["a","b","c"]' | 3 | +--------+----------------------------------+-------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275655b312c322c332c345d5b2261222c2262222c2263225d, offsets: [0, 4, 13, 26] } | -| Output | NullableColumn { column: UInt32([0, 4, 3]), validity: [0b_____110] } | -+--------+--------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[true, [1,2,3,4], ["a","b","c"]] } | +| Output | NullableColumn { column: UInt32([0, 4, 3]), validity: [0b_____110] } | ++--------+----------------------------------------------------------------------+ ast : length(parse_json(s)) @@ -407,12 +407,12 @@ evaluation: | Row 3 | '["a","b","c"]' | 3 | +--------+------------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275655b312c322c332c345d5b2261222c2262222c2263225d, offsets: [0, 4, 13, 13, 26] }, validity: [0b____1011] } | -| Output | NullableColumn { column: UInt32([0, 4, 0, 3]), validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [1,2,3,4], , ["a","b","c"]] }, validity: [0b____1011] } | +| Output | NullableColumn { column: UInt32([0, 4, 0, 3]), validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------------------------------------+ ast : json_object_keys(parse_json('[1,2,3,4]')) @@ -450,7 +450,7 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x5b312c322c332c345d7b2261223a2262222c2263223a2264227d7b226b31223a227631222c226b32223a227632227d, offsets: [0, 9, 26, 47] } | +| s | StringColumn { data: Utf8ViewArray[[1,2,3,4], {"a":"b","c":"d"}, {"k1":"v1","k2":"v2"}] } | | Output | NullableColumn { column: BinaryColumn { data: 0x80000002100000011000000161638000000210000002100000026b316b32, offsets: [0, 0, 14, 30] }, validity: [0b_____110] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -470,12 +470,12 @@ evaluation: | Row 3 | '{"k1":"v1","k2":"v2"}' | '["k1","k2"]' | +--------+-------------------------------------------------+---------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x5b312c322c332c345d7b2261223a2262222c2263223a2264227d7b226b31223a227631222c226b32223a227632227d, offsets: [0, 9, 26, 26, 47] }, validity: [0b____1011] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x80000002100000011000000161638000000210000002100000026b316b32, offsets: [0, 0, 14, 14, 30] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,4], {"a":"b","c":"d"}, , {"k1":"v1","k2":"v2"}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x80000002100000011000000161638000000210000002100000026b316b32, offsets: [0, 0, 14, 14, 30] }, validity: [0b____1010] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : parse_json('null')[1] @@ -558,7 +558,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275655b312c322c332c345d5b2261222c2262222c2263225d, offsets: [0, 4, 13, 26] } | +| s | StringColumn { data: Utf8ViewArray[true, [1,2,3,4], ["a","b","c"]] } | | i | UInt64([0, 0, 1]) | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 19] }, validity: [0b_____110] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ @@ -579,13 +579,13 @@ evaluation: | Row 3 | '["a","b","c"]' | 1 | '"b"' | +--------+------------------------+------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275655b312c322c332c345d5b2261222c2262222c2263225d, offsets: [0, 4, 13, 13, 26] }, validity: [0b____1011] } | -| i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025003200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [1,2,3,4], , ["a","b","c"]] }, validity: [0b____1011] } | +| i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025003200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ast : parse_json(s)[k] @@ -605,8 +605,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 20] } | -| k | StringColumn { data: 0x6b6b78, offsets: [0, 1, 2, 3] } | +| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, {"a":"b"}] } | +| k | StringColumn { data: Utf8ViewArray[k, k, x] } | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 0, 10, 10] }, validity: [0b_____010] } | +--------+---------------------------------------------------------------------------------------------------------------------------+ @@ -626,13 +626,13 @@ evaluation: | Row 3 | '{"a":"b"}' | 'a' | '"b"' | +--------+-----------------------------+------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 11, 20] }, validity: [0b____1011] } | -| k | StringColumn { data: 0x6b61, offsets: [0, 0, 1, 1, 2] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| k | StringColumn { data: Utf8ViewArray[, k, , a] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ast : get_ignore_case(parse_json('{"Aa":1, "aA":2, "aa":3}'), 'AA') @@ -679,8 +679,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 20] } | -| k | StringColumn { data: 0x6b4b41, offsets: [0, 1, 2, 3] } | +| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, {"a":"b"}] } | +| k | StringColumn { data: Utf8ViewArray[k, K, A] } | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 19] }, validity: [0b_____110] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ @@ -700,13 +700,13 @@ evaluation: | Row 3 | '{"a":"b"}' | 'A' | '"b"' | +--------+-----------------------------+------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 11, 20] }, validity: [0b____1011] } | -| k | StringColumn { data: 0x4b41, offsets: [0, 0, 1, 1, 2] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| k | StringColumn { data: Utf8ViewArray[, K, , A] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ast : get_path(parse_json('[[1,2],3]'), '[0]') @@ -798,8 +798,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275657b226b223a317d5b2261222c2262225d, offsets: [0, 4, 11, 20] } | -| k | StringColumn { data: 0x6b5b226b225d5b2261225d, offsets: [0, 1, 6, 11] } | +| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, ["a","b"]] } | +| k | StringColumn { data: Utf8ViewArray[k, ["k"], ["a"]] } | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 0, 10, 10] }, validity: [0b_____010] } | +--------+---------------------------------------------------------------------------------------------------------------------------+ @@ -819,13 +819,13 @@ evaluation: | Row 3 | '["a","b"]' | '[0]' | '"a"' | +--------+-----------------------------+--------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275657b226b223a317d5b2261222c2262225d, offsets: [0, 4, 11, 11, 20] }, validity: [0b____1011] } | -| k | StringColumn { data: 0x5b305d5b226b225d5b305d, offsets: [0, 3, 8, 8, 11] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000161, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , ["a","b"]] }, validity: [0b____1011] } | +| k | StringColumn { data: Utf8ViewArray[[0], ["k"], , [0]] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000161, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ast : json_extract_path_text('[[1,2],3]', '[0]') @@ -914,13 +914,13 @@ evaluation: | Row 2 | '["a","b"]' | '["a"]' | NULL | +--------+---------------------------------+-------------------+-------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275657b226b223a317d5b2261222c2262225d, offsets: [0, 4, 11, 20] } | -| k | StringColumn { data: 0x6b5b226b225d5b2261225d, offsets: [0, 1, 6, 11] } | -| Output | NullableColumn { column: StringColumn { data: 0x31, offsets: [0, 0, 1, 1] }, validity: [0b_____010] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, ["a","b"]] } | +| k | StringColumn { data: Utf8ViewArray[k, ["k"], ["a"]] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, ] }, validity: [0b_____010] } | ++--------+------------------------------------------------------------------------------------------------+ ast : json_extract_path_text(s, k) @@ -938,13 +938,13 @@ evaluation: | Row 3 | '["a","b"]' | '[0]' | 'a' | +--------+-----------------------------+--------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275657b226b223a317d5b2261222c2262225d, offsets: [0, 4, 11, 11, 20] }, validity: [0b____1011] } | -| k | StringColumn { data: 0x5b305d5b226b225d5b305d, offsets: [0, 3, 8, 8, 11] } | -| Output | NullableColumn { column: StringColumn { data: 0x3161, offsets: [0, 0, 1, 1, 2] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , ["a","b"]] }, validity: [0b____1011] } | +| k | StringColumn { data: Utf8ViewArray[[0], ["k"], , [0]] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, , a] }, validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------------------------------+ ast : as_boolean(parse_json('true')) @@ -1073,12 +1073,12 @@ evaluation: | Row 6 | '{"a":"b"}' | NULL | +--------+------------------------------+--------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | NullableColumn { column: Boolean([0b_0000010]), validity: [0b_0000010] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | NullableColumn { column: Boolean([0b_0000010]), validity: [0b_0000010] } | ++--------+----------------------------------------------------------------------------------------+ ast : as_integer(parse_json(s)) @@ -1099,12 +1099,12 @@ evaluation: | Row 6 | '{"a":"b"}' | NULL | +--------+------------------------------+------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | NullableColumn { column: Int64([0, 0, 123, 0, 0, 0, 0]), validity: [0b_0000100] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | NullableColumn { column: Int64([0, 0, 123, 0, 0, 0, 0]), validity: [0b_0000100] } | ++--------+----------------------------------------------------------------------------------------+ ast : as_float(parse_json(s)) @@ -1125,12 +1125,12 @@ evaluation: | Row 6 | '{"a":"b"}' | NULL | +--------+------------------------------+--------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | NullableColumn { column: Float64([0, 0, 123, 12.34, 0, 0, 0]), validity: [0b_0001100] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | NullableColumn { column: Float64([0, 0, 123, 12.34, 0, 0, 0]), validity: [0b_0001100] } | ++--------+-----------------------------------------------------------------------------------------+ ast : as_string(parse_json(s)) @@ -1151,12 +1151,12 @@ evaluation: | Row 6 | '{"a":"b"}' | NULL | +--------+------------------------------+-------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | NullableColumn { column: StringColumn { data: 0x6162, offsets: [0, 0, 0, 0, 0, 2, 2, 2] }, validity: [0b_0010000] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, , , , ab, , ] }, validity: [0b_0010000] } | ++--------+---------------------------------------------------------------------------------------------------------+ ast : as_array(parse_json(s)) @@ -1180,7 +1180,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | | Output | NullableColumn { column: BinaryColumn { data: 0x80000003200000022000000220000002500150025003, offsets: [0, 0, 0, 0, 0, 0, 22, 22] }, validity: [0b_0100000] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1206,7 +1206,7 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000110000001100000016162, offsets: [0, 0, 0, 0, 0, 0, 0, 14] }, validity: [0b_1000000] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1355,12 +1355,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | Boolean([0b_0000001]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | Boolean([0b_0000001]) | ++--------+----------------------------------------------------------------------------------------+ ast : is_boolean(parse_json(s)) @@ -1381,12 +1381,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | Boolean([0b_0000010]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | Boolean([0b_0000010]) | ++--------+----------------------------------------------------------------------------------------+ ast : is_integer(parse_json(s)) @@ -1407,12 +1407,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | Boolean([0b_0000100]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | Boolean([0b_0000100]) | ++--------+----------------------------------------------------------------------------------------+ ast : is_float(parse_json(s)) @@ -1433,12 +1433,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | Boolean([0b_0001100]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | Boolean([0b_0001100]) | ++--------+----------------------------------------------------------------------------------------+ ast : is_string(parse_json(s)) @@ -1459,12 +1459,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | Boolean([0b_0010000]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | Boolean([0b_0010000]) | ++--------+----------------------------------------------------------------------------------------+ ast : is_array(parse_json(s)) @@ -1485,12 +1485,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | Boolean([0b_0100000]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | Boolean([0b_0100000]) | ++--------+----------------------------------------------------------------------------------------+ ast : is_object(parse_json(s)) @@ -1511,12 +1511,12 @@ evaluation: | Row 6 | '{"a":"b"}' | true | +--------+------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x6e756c6c7472756531323331322e3334226162225b312c322c335d7b2261223a2262227d, offsets: [0, 4, 8, 11, 16, 20, 27, 36] } | -| Output | Boolean([0b_1000000]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| Output | Boolean([0b_1000000]) | ++--------+----------------------------------------------------------------------------------------+ ast : to_boolean(parse_json('true')) @@ -1678,12 +1678,12 @@ evaluation: | Row 2 | 'true' | true | +--------+------------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x7472756574727565, offsets: [0, 4, 4, 8] }, validity: [0b_____101] } | -| Output | NullableColumn { column: Boolean([0b_____101]), validity: [0b_____101] } | -+--------+---------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, , true] }, validity: [0b_____101] } | +| Output | NullableColumn { column: Boolean([0b_____101]), validity: [0b_____101] } | ++--------+-------------------------------------------------------------------------------------------------------+ ast : to_int64(parse_json(s)) @@ -1700,12 +1700,12 @@ evaluation: | Row 2 | '-10' | -10 | +--------+---------------------+------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x312d3130, offsets: [0, 1, 1, 4] }, validity: [0b_____101] } | -| Output | NullableColumn { column: Int64([1, 0, -10]), validity: [0b_____101] } | -+--------+-------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, , -10] }, validity: [0b_____101] } | +| Output | NullableColumn { column: Int64([1, 0, -10]), validity: [0b_____101] } | ++--------+---------------------------------------------------------------------------------------------------+ ast : to_uint64(parse_json(s)) @@ -1722,12 +1722,12 @@ evaluation: | Row 2 | '20' | 20 | +--------+----------------------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x313230, offsets: [0, 1, 1, 3] }, validity: [0b_____101] } | -| Output | NullableColumn { column: UInt64([1, 0, 20]), validity: [0b_____101] } | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, , 20] }, validity: [0b_____101] } | +| Output | NullableColumn { column: UInt64([1, 0, 20]), validity: [0b_____101] } | ++--------+--------------------------------------------------------------------------------------------------+ ast : to_float64(parse_json(s)) @@ -1744,12 +1744,12 @@ evaluation: | Row 2 | '100.2' | 100.2 | +--------+-------------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x312e323130302e32, offsets: [0, 3, 3, 8] }, validity: [0b_____101] } | -| Output | NullableColumn { column: Float64([1.2, 0, 100.2]), validity: [0b_____101] } | -+--------+---------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[1.2, , 100.2] }, validity: [0b_____101] } | +| Output | NullableColumn { column: Float64([1.2, 0, 100.2]), validity: [0b_____101] } | ++--------+-------------------------------------------------------------------------------------------------------+ ast : to_date(parse_json(s)) @@ -1766,12 +1766,12 @@ evaluation: | Row 2 | '"2023-10-01"' | '2023-10-01' | +--------+----------------------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x22323032302d30312d30312222323032332d31302d303122, offsets: [0, 12, 12, 24] }, validity: [0b_____101] } | -| Output | NullableColumn { column: [18262, 0, 19631], validity: [0b_____101] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray["2020-01-01", , "2023-10-01"] }, validity: [0b_____101] } | +| Output | NullableColumn { column: [18262, 0, 19631], validity: [0b_____101] } | ++--------+-----------------------------------------------------------------------------------------------------------------------+ ast : to_timestamp(parse_json(s)) @@ -1788,12 +1788,12 @@ evaluation: | Row 2 | '"2023-10-01 10:11:12"' | '2023-10-01 10:11:12.000000' | +--------+-------------------------------------------+------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x22323032302d30312d30312030303a30303a30302222323032332d31302d30312031303a31313a313222, offsets: [0, 21, 21, 42] }, validity: [0b_____101] } | -| Output | NullableColumn { column: [1577836800000000, 0, 1696155072000000], validity: [0b_____101] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray["2020-01-01 00:00:00", , "2023-10-01 10:11:12"] }, validity: [0b_____101] } | +| Output | NullableColumn { column: [1577836800000000, 0, 1696155072000000], validity: [0b_____101] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------+ ast : to_string(parse_json(s)) @@ -1810,12 +1810,12 @@ evaluation: | Row 2 | '123' | '123' | +--------+-----------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x2261626322313233, offsets: [0, 5, 5, 8] }, validity: [0b_____101] } | -| Output | NullableColumn { column: StringColumn { data: 0x616263313233, offsets: [0, 3, 3, 6] }, validity: [0b_____101] } | -+--------+---------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray["abc", , 123] }, validity: [0b_____101] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, , 123] }, validity: [0b_____101] } | ++--------+-------------------------------------------------------------------------------------------------------+ ast : try_to_boolean(parse_json('true')) @@ -1990,12 +1990,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275653132332d31303031322e333422323032302d30312d30312222323032312d30312d30312032303a30303a3030222261626322, offsets: [0, 4, 7, 11, 16, 16, 28, 49, 54] }, validity: [0b11101111] } | -| Output | NullableColumn { column: Boolean([0b00000001]), validity: [0b00000001] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | +| Output | NullableColumn { column: Boolean([0b00000001]), validity: [0b00000001] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_int64(parse_json(s)) @@ -2017,12 +2017,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275653132332d31303031322e333422323032302d30312d30312222323032312d30312d30312032303a30303a3030222261626322, offsets: [0, 4, 7, 11, 16, 16, 28, 49, 54] }, validity: [0b11101111] } | -| Output | NullableColumn { column: Int64([1, 123, -100, 0, 0, 0, 0, 0]), validity: [0b00000111] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | +| Output | NullableColumn { column: Int64([1, 123, -100, 0, 0, 0, 0, 0]), validity: [0b00000111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_uint64(parse_json(s)) @@ -2044,12 +2044,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+-------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275653132332d31303031322e333422323032302d30312d30312222323032312d30312d30312032303a30303a3030222261626322, offsets: [0, 4, 7, 11, 16, 16, 28, 49, 54] }, validity: [0b11101111] } | -| Output | NullableColumn { column: UInt64([1, 123, 0, 0, 0, 0, 0, 0]), validity: [0b00000011] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | +| Output | NullableColumn { column: UInt64([1, 123, 0, 0, 0, 0, 0, 0]), validity: [0b00000011] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_float64(parse_json(s)) @@ -2071,12 +2071,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275653132332d31303031322e333422323032302d30312d30312222323032312d30312d30312032303a30303a3030222261626322, offsets: [0, 4, 7, 11, 16, 16, 28, 49, 54] }, validity: [0b11101111] } | -| Output | NullableColumn { column: Float64([1, 123, -100, 12.34, 0, 0, 0, 0]), validity: [0b00001111] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | +| Output | NullableColumn { column: Float64([1, 123, -100, 12.34, 0, 0, 0, 0]), validity: [0b00001111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_date(parse_json(s)) @@ -2098,12 +2098,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275653132332d31303031322e333422323032302d30312d30312222323032312d30312d30312032303a30303a3030222261626322, offsets: [0, 4, 7, 11, 16, 16, 28, 49, 54] }, validity: [0b11101111] } | -| Output | NullableColumn { column: [0, 0, 0, 0, 0, 18262, 18628, 0], validity: [0b01100000] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | +| Output | NullableColumn { column: [0, 0, 0, 0, 0, 18262, 18628, 0], validity: [0b01100000] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_timestamp(parse_json(s)) @@ -2125,12 +2125,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275653132332d31303031322e333422323032302d30312d30312222323032312d30312d30312032303a30303a3030222261626322, offsets: [0, 4, 7, 11, 16, 16, 28, 49, 54] }, validity: [0b11101111] } | -| Output | NullableColumn { column: [0, 0, 0, 0, 0, 1577836800000000, 1609531200000000, 0], validity: [0b01100000] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | +| Output | NullableColumn { column: [0, 0, 0, 0, 0, 1577836800000000, 1609531200000000, 0], validity: [0b01100000] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_string(parse_json(s)) @@ -2152,12 +2152,12 @@ evaluation: | Row 7 | '"abc"' | 'abc' | +--------+-------------------------+-----------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275653132332d31303031322e333422323032302d30312d30312222323032312d30312d30312032303a30303a3030222261626322, offsets: [0, 4, 7, 11, 16, 16, 28, 49, 54] }, validity: [0b11101111] } | -| Output | NullableColumn { column: StringColumn { data: 0x747275653132332d31303031322e3334323032302d30312d3031323032312d30312d30312032303a30303a3030616263, offsets: [0, 4, 7, 11, 16, 16, 26, 45, 48] }, validity: [0b11101111] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , 2020-01-01, 2021-01-01 20:00:00, abc] }, validity: [0b11101111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : json_object() @@ -2229,10 +2229,10 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| k1 | NullableColumn { column: StringColumn { data: 0x613162316431, offsets: [0, 2, 4, 4, 6] }, validity: [0b____1011] } | -| v1 | NullableColumn { column: StringColumn { data: 0x6a316b316c31, offsets: [0, 2, 4, 6, 6] }, validity: [0b____0111] } | -| k2 | NullableColumn { column: StringColumn { data: 0x613263326432, offsets: [0, 2, 2, 4, 6] }, validity: [0b____1101] } | -| v2 | NullableColumn { column: StringColumn { data: 0x6a326b326c326d32, offsets: [0, 2, 4, 6, 8] }, validity: [0b____1111] } | +| k1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | +| k2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | +| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j2, k2, l2, m2] }, validity: [0b____1111] } | | Output | BinaryColumn { data: 0x4000000210000002100000021000000210000002613161326a316a3240000001100000021000000262316b3140000001100000021000000263326c3240000001100000021000000264326d32, offsets: [0, 28, 44, 60, 76] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2309,10 +2309,10 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| k1 | NullableColumn { column: StringColumn { data: 0x613162316431, offsets: [0, 2, 4, 4, 6] }, validity: [0b____1011] } | -| v1 | NullableColumn { column: StringColumn { data: 0x6a316b316c31, offsets: [0, 2, 4, 6, 6] }, validity: [0b____0111] } | -| k2 | NullableColumn { column: StringColumn { data: 0x613263326432, offsets: [0, 2, 2, 4, 6] }, validity: [0b____1101] } | -| v2 | NullableColumn { column: StringColumn { data: 0x6a326b326c326d32, offsets: [0, 2, 4, 6, 8] }, validity: [0b____1111] } | +| k1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | +| k2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | +| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j2, k2, l2, m2] }, validity: [0b____1111] } | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000210000002100000021000000210000002613161326a316a3240000001100000021000000262316b3140000001100000021000000263326c3240000001100000021000000264326d32, offsets: [0, 28, 44, 60, 76] }, validity: [0b____1111] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2386,10 +2386,10 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| k1 | NullableColumn { column: StringColumn { data: 0x613162316431, offsets: [0, 2, 4, 4, 6] }, validity: [0b____1011] } | -| v1 | NullableColumn { column: StringColumn { data: 0x6a316b316c31, offsets: [0, 2, 4, 6, 6] }, validity: [0b____0111] } | -| k2 | NullableColumn { column: StringColumn { data: 0x613263326432, offsets: [0, 2, 2, 4, 6] }, validity: [0b____1101] } | -| v2 | NullableColumn { column: StringColumn { data: 0x6a326b326c326d32, offsets: [0, 2, 4, 6, 8] }, validity: [0b____1111] } | +| k1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | +| k2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | +| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j2, k2, l2, m2] }, validity: [0b____1111] } | | Output | BinaryColumn { data: 0x4000000210000002100000021000000210000002613161326a316a3240000001100000021000000262316b3140000001100000021000000263326c324000000210000002100000020000000010000002643164326d32, offsets: [0, 28, 44, 60, 86] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2466,10 +2466,10 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| k1 | NullableColumn { column: StringColumn { data: 0x613162316431, offsets: [0, 2, 4, 4, 6] }, validity: [0b____1011] } | -| v1 | NullableColumn { column: StringColumn { data: 0x6a316b316c31, offsets: [0, 2, 4, 6, 6] }, validity: [0b____0111] } | -| k2 | NullableColumn { column: StringColumn { data: 0x613263326432, offsets: [0, 2, 2, 4, 6] }, validity: [0b____1101] } | -| v2 | NullableColumn { column: StringColumn { data: 0x6a326b326c326d32, offsets: [0, 2, 4, 6, 8] }, validity: [0b____1111] } | +| k1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | +| k2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | +| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j2, k2, l2, m2] }, validity: [0b____1111] } | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000210000002100000021000000210000002613161326a316a3240000001100000021000000262316b3140000001100000021000000263326c324000000210000002100000020000000010000002643164326d32, offsets: [0, 28, 44, 60, 86] }, validity: [0b____1111] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2546,8 +2546,8 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275655b7b226b223a317d2c7b226b223a327d5d5b312c322c332c345d, offsets: [0, 4, 21, 21, 30] }, validity: [0b____1011] } | -| p | StringColumn { data: 0x245b305d245b2a5d2e6b242e61245b302c325d, offsets: [0, 4, 10, 13, 19] } | +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [{"k":1},{"k":2}], , [1,2,3,4]] }, validity: [0b____1011] } | +| p | StringColumn { data: Utf8ViewArray[$[0], $[*].k, $.a, $[0,2]] } | | Output | NullableColumn { column: BinaryColumn { data: 0x800000008000000220000002200000025001500280000002200000022000000250015003, offsets: [0, 4, 20, 20, 36] }, validity: [0b____1011] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2621,13 +2621,13 @@ evaluation: | Row 3 | '[1,2,3,4]' | '$[0,2]' | '1' | +--------+------------------------+------------------+--------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275655b7b226b223a317d2c7b226b223a327d5d5b312c322c332c345d, offsets: [0, 4, 21, 21, 30] }, validity: [0b____1011] } | -| p | StringColumn { data: 0x245b305d245b2a5d2e6b242e61245b302c325d, offsets: [0, 4, 10, 13, 19] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x2000000020000002500120000000200000025001, offsets: [0, 0, 10, 10, 20] }, validity: [0b____1010] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [{"k":1},{"k":2}], , [1,2,3,4]] }, validity: [0b____1011] } | +| p | StringColumn { data: Utf8ViewArray[$[0], $[*].k, $.a, $[0,2]] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x2000000020000002500120000000200000025001, offsets: [0, 0, 10, 10, 20] }, validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ast : json_to_string(parse_json('true')) @@ -2894,9 +2894,9 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v1 | NullableColumn { column: StringColumn { data: 0x613162316431, offsets: [0, 2, 4, 4, 6] }, validity: [0b____1011] } | -| v2 | NullableColumn { column: StringColumn { data: 0x6a316b316c31, offsets: [0, 2, 4, 6, 6] }, validity: [0b____0111] } | -| v3 | NullableColumn { column: StringColumn { data: 0x613263326432, offsets: [0, 2, 2, 4, 6] }, validity: [0b____1101] } | +| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | +| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | +| v3 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | | Output | BinaryColumn { data: 0x8000000310000002100000021000000261316a3161328000000310000002100000020000000062316b31800000030000000010000002100000026c3163328000000310000002000000001000000264316432, offsets: [0, 22, 42, 62, 82] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3062,7 +3062,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275655b312c322c332c345d5b2261222c2262222c2263225d, offsets: [0, 4, 13, 26] } | +| s | StringColumn { data: Utf8ViewArray[true, [1,2,3,4], ["a","b","c"]] } | | i | UInt64([0, 0, 1]) | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 19] }, validity: [0b_____110] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3083,13 +3083,13 @@ evaluation: | Row 3 | '["a","b","c"]' | 1 | '"b"' | +--------+------------------------+------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275655b312c322c332c345d5b2261222c2262222c2263225d, offsets: [0, 4, 13, 13, 26] }, validity: [0b____1011] } | -| i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025003200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [1,2,3,4], , ["a","b","c"]] }, validity: [0b____1011] } | +| i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025003200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ast : parse_json(s)->k @@ -3109,8 +3109,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 20] } | -| k | StringColumn { data: 0x6b6b78, offsets: [0, 1, 2, 3] } | +| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, {"a":"b"}] } | +| k | StringColumn { data: Utf8ViewArray[k, k, x] } | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 0, 10, 10] }, validity: [0b_____010] } | +--------+---------------------------------------------------------------------------------------------------------------------------+ @@ -3130,13 +3130,13 @@ evaluation: | Row 3 | '{"a":"b"}' | 'a' | '"b"' | +--------+-----------------------------+------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 11, 20] }, validity: [0b____1011] } | -| k | StringColumn { data: 0x6b61, offsets: [0, 0, 1, 1, 2] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| k | StringColumn { data: Utf8ViewArray[, k, , a] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ast : parse_json('null')->>1 @@ -3216,13 +3216,13 @@ evaluation: | Row 2 | '["a","b","c"]' | 1 | 'b' | +--------+----------------------------------+---------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275655b312c322c332c345d5b2261222c2262222c2263225d, offsets: [0, 4, 13, 26] } | -| i | UInt64([0, 0, 1]) | -| Output | NullableColumn { column: StringColumn { data: 0x3162, offsets: [0, 0, 1, 2] }, validity: [0b_____110] } | -+--------+---------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[true, [1,2,3,4], ["a","b","c"]] } | +| i | UInt64([0, 0, 1]) | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, b] }, validity: [0b_____110] } | ++--------+-------------------------------------------------------------------------------------------------+ ast : parse_json(s)->>i @@ -3240,13 +3240,13 @@ evaluation: | Row 3 | '["a","b","c"]' | 1 | 'b' | +--------+------------------------+------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275655b312c322c332c345d5b2261222c2262222c2263225d, offsets: [0, 4, 13, 13, 26] }, validity: [0b____1011] } | -| i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | -| Output | NullableColumn { column: StringColumn { data: 0x3362, offsets: [0, 0, 1, 1, 2] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [1,2,3,4], , ["a","b","c"]] }, validity: [0b____1011] } | +| i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 3, , b] }, validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------------------------------------+ ast : parse_json(s)->>k @@ -3263,13 +3263,13 @@ evaluation: | Row 2 | '{"a":"b"}' | 'x' | NULL | +--------+------------------------+-------------+-------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 20] } | -| k | StringColumn { data: 0x6b6b78, offsets: [0, 1, 2, 3] } | -| Output | NullableColumn { column: StringColumn { data: 0x31, offsets: [0, 0, 1, 1] }, validity: [0b_____010] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------+ +| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, {"a":"b"}] } | +| k | StringColumn { data: Utf8ViewArray[k, k, x] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, ] }, validity: [0b_____010] } | ++--------+------------------------------------------------------------------------------------------------+ ast : parse_json(s)->>k @@ -3287,13 +3287,13 @@ evaluation: | Row 3 | '{"a":"b"}' | 'a' | 'b' | +--------+-----------------------------+------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 11, 20] }, validity: [0b____1011] } | -| k | StringColumn { data: 0x6b61, offsets: [0, 0, 1, 1, 2] } | -| Output | NullableColumn { column: StringColumn { data: 0x3162, offsets: [0, 0, 1, 1, 2] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| k | StringColumn { data: Utf8ViewArray[, k, , a] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, , b] }, validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------------------------------+ error: @@ -3427,12 +3427,12 @@ evaluation: | Row 3 | '{"a":"b"}' | NULL | +--------+-----------------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x5b312c322c335d7b226b223a317d7b2261223a2262227d, offsets: [0, 7, 14, 14, 23] }, validity: [0b____1011] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 10, 10, 10, 10] }, validity: [0b____0001] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3], {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 10, 10, 10, 10] }, validity: [0b____0001] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------+ ast : parse_json(s) #> k @@ -3450,13 +3450,13 @@ evaluation: | Row 3 | '{"a":"b"}' | '{a}' | '"b"' | +--------+-----------------------------+-----------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 11, 20] }, validity: [0b____1011] } | -| k | NullableColumn { column: StringColumn { data: 0x7b317d7b6b7d7b617d, offsets: [0, 3, 6, 6, 9] }, validity: [0b____1011] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| k | NullableColumn { column: StringColumn { data: Utf8ViewArray[{1}, {k}, , {a}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ ast : NULL #>> '{0}' @@ -3573,12 +3573,12 @@ evaluation: | Row 3 | '{"a":"b"}' | NULL | +--------+-----------------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x5b312c322c335d7b226b223a317d7b2261223a2262227d, offsets: [0, 7, 14, 14, 23] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: 0x31, offsets: [0, 1, 1, 1, 1] }, validity: [0b____0001] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3], {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, , , ] }, validity: [0b____0001] } | ++--------+------------------------------------------------------------------------------------------------------------------------+ ast : parse_json(s) #>> k @@ -3596,13 +3596,13 @@ evaluation: | Row 3 | '{"a":"b"}' | '{a}' | 'b' | +--------+-----------------------------+-----------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275657b226b223a317d7b2261223a2262227d, offsets: [0, 4, 11, 11, 20] }, validity: [0b____1011] } | -| k | NullableColumn { column: StringColumn { data: 0x7b317d7b6b7d7b617d, offsets: [0, 3, 6, 6, 9] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: 0x3162, offsets: [0, 0, 1, 1, 2] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| k | NullableColumn { column: StringColumn { data: Utf8ViewArray[{1}, {k}, , {a}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, , b] }, validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------------------------------+ ast : parse_json('["1","2","3"]') ? NULL @@ -3675,12 +3675,12 @@ evaluation: | Row 3 | '{"b":1}' | false | +--------+-----------------------------+--------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x5b312c322c335d7b2261223a317d7b2262223a317d, offsets: [0, 7, 14, 14, 21] }, validity: [0b____1011] } | -| Output | NullableColumn { column: Boolean([0b____0010]), validity: [0b____1011] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3], {"a":1}, , {"b":1}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: Boolean([0b____0010]), validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------------------------------------------+ ast : parse_json('["1","2","3"]') ?| NULL @@ -3753,12 +3753,12 @@ evaluation: | Row 3 | '{"c":1}' | false | +--------+-----------------------------+--------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x5b2261222c2265222c2264225d7b2261223a312c2262223a327d7b2263223a317d, offsets: [0, 13, 26, 26, 33] }, validity: [0b____1011] } | -| Output | NullableColumn { column: Boolean([0b____0011]), validity: [0b____1011] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[["a","e","d"], {"a":1,"b":2}, , {"c":1}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: Boolean([0b____0011]), validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ ast : parse_json('["1","2","3"]') ?& NULL @@ -3831,12 +3831,12 @@ evaluation: | Row 3 | '{"a":0,"c":1}' | false | +--------+-------------------------------------+--------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x5b2261222c2265222c2262225d7b2261223a312c2262223a327d7b2261223a302c2263223a317d, offsets: [0, 13, 26, 26, 39] }, validity: [0b____1011] } | -| Output | NullableColumn { column: Boolean([0b____0011]), validity: [0b____1011] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[["a","e","b"], {"a":1,"b":2}, , {"a":0,"c":1}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: Boolean([0b____0011]), validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------+ ast : NULL @> NULL @@ -4295,13 +4295,13 @@ evaluation: | Row 3 | '[1,2,3,4]' | '$[*] > 2' | true | +--------+------------------------+-----------------------------+--------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275655b7b226b223a317d2c7b226b223a327d5d5b312c322c332c345d, offsets: [0, 4, 21, 21, 30] }, validity: [0b____1011] } | -| p | StringColumn { data: 0x242e61203e2030245b2a5d2e6b203d3d2031245b2a5d203e2031245b2a5d203e2032, offsets: [0, 7, 18, 26, 34] } | -| Output | NullableColumn { column: Boolean([0b____1010]), validity: [0b____1011] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [{"k":1},{"k":2}], , [1,2,3,4]] }, validity: [0b____1011] } | +| p | StringColumn { data: Utf8ViewArray[$.a > 0, $[*].k == 1, $[*] > 1, $[*] > 2] } | +| Output | NullableColumn { column: Boolean([0b____1010]), validity: [0b____1011] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------+ ast : parse_json('{"a":1,"b":2}') @@ '$.a == 1' @@ -4400,13 +4400,13 @@ evaluation: | Row 3 | '[1,2,3,4]' | '$[*] > 2' | true | +--------+------------------------+-----------------------------+--------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x747275655b7b226b223a317d2c7b226b223a327d5d5b312c322c332c345d, offsets: [0, 4, 21, 21, 30] }, validity: [0b____1011] } | -| p | StringColumn { data: 0x242e61203e2030245b2a5d2e6b203d3d2031245b2a5d203e2031245b2a5d203e2032, offsets: [0, 7, 18, 26, 34] } | -| Output | NullableColumn { column: Boolean([0b____1010]), validity: [0b____1011] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [{"k":1},{"k":2}], , [1,2,3,4]] }, validity: [0b____1011] } | +| p | StringColumn { data: Utf8ViewArray[$.a > 0, $[*].k == 1, $[*] > 1, $[*] > 2] } | +| Output | NullableColumn { column: Boolean([0b____1010]), validity: [0b____1011] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------+ ast : NULL @? '$.a' @@ -4866,7 +4866,7 @@ evaluation (internal): +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x5b7b2261223a317d2c322c335d5b312c322c335d7b2261223a2262227d, offsets: [0, 13, 20, 20, 29] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[{"a":1},2,3], [1,2,3], , {"a":"b"}] }, validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x800000035000000420000002200000024000000050025003800000032000000220000002200000025001500250034000000110000001100000016162, offsets: [0, 24, 46, 46, 60] }, validity: [0b____1011] } | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4889,8 +4889,8 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: 0x5b312c7b2261223a327d2c335d7b226b223a5b312c322c335d7d7b2261223a2262227d, offsets: [0, 13, 26, 26, 35] }, validity: [0b____1011] } | -| k | NullableColumn { column: StringColumn { data: 0x7b312c617d7b6b2c2d317d7b6b7d7b637d, offsets: [0, 5, 11, 14, 17] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,{"a":2},3], {"k":[1,2,3]}, , {"a":"b"}] }, validity: [0b____1011] } | +| k | NullableColumn { column: StringColumn { data: Utf8ViewArray[{1,a}, {k,-1}, {k}, {c}] }, validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x8000000320000002500000042000000250014000000050034000000110000001500000106b800000022000000220000002500150024000000110000001100000016162, offsets: [0, 24, 53, 53, 67] }, validity: [0b____1011] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4968,8 +4968,8 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: 0x5b312c322c332c6e756c6c5d5b2241222c2242225d7b2261223a2262227d, offsets: [0, 12, 21, 21, 30] }, validity: [0b____1011] } | -| n | NullableColumn { column: StringColumn { data: 0x22686922747275655b312c322c335d, offsets: [0, 4, 4, 8, 15] }, validity: [0b____1101] } | +| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,null], ["A","B"], , {"a":"b"}] }, validity: [0b____1011] } | +| n | NullableColumn { column: StringColumn { data: Utf8ViewArray["hi", , true, [1,2,3]] }, validity: [0b____1101] } | | Output | NullableColumn { column: BinaryColumn { data: 0x8000000520000002200000021000000220000002000000005001500268695003800000025000000e50000016400000011000000110000001616280000003200000022000000220000002500150025003, offsets: [0, 32, 32, 32, 80] }, validity: [0b____1001] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5055,7 +5055,7 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: 0x5b312c312c322c332c332c6e756c6c2c322c312c6e756c6c5d5b2241222c2242222c2241222c2242222c2243225d7b2261223a2262227d, offsets: [0, 25, 46, 46, 55] }, validity: [0b____1011] } | +| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,1,2,3,3,null,2,1,null], ["A","B","A","B","C"], , {"a":"b"}] }, validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x800000042000000220000002200000020000000050015002500380000003100000011000000110000001414243800000015000000e4000000110000001100000016162, offsets: [0, 26, 45, 45, 67] }, validity: [0b____1011] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5168,8 +5168,8 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v1 | NullableColumn { column: StringColumn { data: 0x5b312c322c332c332c6e756c6c2c6e756c6c5d5b2241222c2242222c2241222c2242222c2243225d7b2261223a2262227d, offsets: [0, 19, 40, 40, 49] }, validity: [0b____1011] } | -| v2 | NullableColumn { column: StringColumn { data: 0x5b312c312c322c332c342c352c6e756c6c5d5b2258222c2259222c225a225d7b2261223a2262227d, offsets: [0, 18, 31, 31, 40] }, validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}] }, validity: [0b____1011] } | +| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}] }, validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x800000042000000220000002200000020000000050015002500380000000800000015000000e4000000110000001100000016162, offsets: [0, 26, 30, 30, 52] }, validity: [0b____1011] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5279,13 +5279,13 @@ evaluation: | Row 3 | '{"a":"b"}' | '{"a":"b"}' | '[]' | +--------+---------------------------------+---------------------------------+-------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v1 | NullableColumn { column: StringColumn { data: 0x5b312c322c332c332c6e756c6c2c6e756c6c5d5b2241222c2242222c2241222c2242222c2243225d7b2261223a2262227d, offsets: [0, 19, 40, 40, 49] }, validity: [0b____1011] } | -| v2 | NullableColumn { column: StringColumn { data: 0x5b312c312c322c332c342c352c6e756c6c5d5b2258222c2259222c225a225d7b2261223a2262227d, offsets: [0, 18, 31, 31, 40] }, validity: [0b____1011] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x8000000220000002000000005003800000051000000110000001100000011000000110000001414241424380000000, offsets: [0, 14, 43, 43, 47] }, validity: [0b____1011] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}] }, validity: [0b____1011] } | +| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x8000000220000002000000005003800000051000000110000001100000011000000110000001414241424380000000, offsets: [0, 14, 43, 43, 47] }, validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : json_array_overlap('["A","B","C"]'::variant, '["B","C"]'::variant) @@ -5393,13 +5393,13 @@ evaluation: | Row 3 | '{"a":"b"}' | '{"a":"b"}' | true | +--------+---------------------------------+---------------------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v1 | NullableColumn { column: StringColumn { data: 0x5b312c322c332c332c6e756c6c2c6e756c6c5d5b2241222c2242222c2241222c2242222c2243225d7b2261223a2262227d, offsets: [0, 19, 40, 40, 49] }, validity: [0b____1011] } | -| v2 | NullableColumn { column: StringColumn { data: 0x5b312c312c322c332c342c352c6e756c6c5d5b2258222c2259222c225a225d7b2261223a2262227d, offsets: [0, 18, 31, 31, 40] }, validity: [0b____1011] } | -| Output | NullableColumn { column: Boolean([0b____1001]), validity: [0b____1011] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ +| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}] }, validity: [0b____1011] } | +| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: Boolean([0b____1001]), validity: [0b____1011] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ ast : json_object_insert('{"b":12,"d":34,"m":[1,2],"x":{"k":"v"}}'::variant, 'a', 'hello') @@ -5543,8 +5543,8 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: 0x7b226b223a2276227d7b226d223a226e227d7b2261223a2262222c2263223a2264222c2279223a227a227d, offsets: [0, 9, 18, 18, 43] }, validity: [0b____1011] } | -| n | NullableColumn { column: StringColumn { data: 0x22686922747275655b312c322c335d, offsets: [0, 4, 4, 8, 15] }, validity: [0b____1101] } | +| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}] }, validity: [0b____1011] } | +| n | NullableColumn { column: StringColumn { data: Utf8ViewArray["hi", , true, [1,2,3]] }, validity: [0b____1101] } | | Output | NullableColumn { column: BinaryColumn { data: 0x40000002100000011000000110000001100000026b787668694000000110000001100000016d6e400000041000000110000001100000011000000110000001100000015000001610000001616378796264800000032000000220000002200000025001500250037a, offsets: [0, 25, 39, 39, 104] }, validity: [0b____1011] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5567,8 +5567,8 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: 0x7b226b223a2276227d7b226d223a226e227d7b2261223a2262222c2263223a2264222c2279223a227a227d, offsets: [0, 9, 18, 18, 43] }, validity: [0b____1011] } | -| n | NullableColumn { column: StringColumn { data: 0x22686922747275655b312c322c335d, offsets: [0, 4, 4, 8, 15] }, validity: [0b____1101] } | +| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}] }, validity: [0b____1011] } | +| n | NullableColumn { column: StringColumn { data: Utf8ViewArray["hi", , true, [1,2,3]] }, validity: [0b____1101] } | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000210000001100000011000000210000001636b6869764000000110000001100000016d6e4000000310000001100000011000000110000001500000161000000161637962800000032000000220000002200000025001500250037a, offsets: [0, 25, 39, 39, 94] }, validity: [0b____1011] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5632,12 +5632,12 @@ evaluation: | Row 3 | '{"a":"b","c":"d","y":"z"}' | '{"c":"d","y":"z"}' | +--------+---------------------------------+---------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: 0x7b226b223a2276227d7b226d223a226e227d7b2261223a2262222c2263223a2264222c2279223a227a227d, offsets: [0, 9, 18, 18, 43] }, validity: [0b____1011] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x4000000110000001100000016b764000000040000002100000011000000110000001100000016379647a, offsets: [0, 14, 18, 18, 42] }, validity: [0b____1011] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x4000000110000001100000016b764000000040000002100000011000000110000001100000016379647a, offsets: [0, 14, 18, 18, 42] }, validity: [0b____1011] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : json_object_pick('{"b":12,"d":34,"m":[1,2],"x":{"k":"v"}}'::variant, 'a', 'b', 'c') @@ -5699,11 +5699,11 @@ evaluation: | Row 3 | '{"a":"b","c":"d","y":"z"}' | '{"a":"b"}' | +--------+---------------------------------+--------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: 0x7b226b223a2276227d7b226d223a226e227d7b2261223a2262222c2263223a2264222c2279223a227a227d, offsets: [0, 9, 18, 18, 43] }, validity: [0b____1011] } | -| Output | NullableColumn { column: BinaryColumn { data: 0x400000004000000110000001100000016d6e4000000110000001100000016162, offsets: [0, 4, 18, 18, 32] }, validity: [0b____1011] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}] }, validity: [0b____1011] } | +| Output | NullableColumn { column: BinaryColumn { data: 0x400000004000000110000001100000016d6e4000000110000001100000016162, offsets: [0, 4, 18, 18, 32] }, validity: [0b____1011] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs index 6db4b9188601..f554b6157f38 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs @@ -98,8 +98,10 @@ impl RowConverter for CommonRowConverter { let (_, validity) = c.validity(); let col = c.remove_nullable(); let col = col.as_variant().unwrap(); - let mut builder = - BinaryColumnBuilder::with_capacity(col.len(), col.data().len()); + let mut builder = BinaryColumnBuilder::with_capacity( + col.len(), + col.current_buffer_len(), + ); for (i, val) in col.iter().enumerate() { if let Some(validity) = validity { if unsafe { !validity.get_bit_unchecked(i) } { diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index 958988527b63..d749c81bcc95 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -255,9 +255,9 @@ impl ModifyTableColumnInterpreter { return Ok(PipelineBuildResult::create()); } - // if alter column from string to binary, we don't need to rebuild table - let is_alter_column_string_to_binary = - schema + // if alter column from string to binary in parquet, we don't need to rebuild table + let is_alter_column_string_to_binary = table.storage_format_as_parquet() + && schema .fields() .iter() .zip(new_schema.fields()) diff --git a/src/query/service/src/pipelines/builders/builder_join.rs b/src/query/service/src/pipelines/builders/builder_join.rs index b8833274a1e9..fd5ef8c2f906 100644 --- a/src/query/service/src/pipelines/builders/builder_join.rs +++ b/src/query/service/src/pipelines/builders/builder_join.rs @@ -197,10 +197,6 @@ impl PipelineBuilder { self.main_pipeline.output_len(), barrier, )?); - let mut has_string_column = false; - for field in join.output_schema()?.fields() { - has_string_column |= field.data_type().is_string_column(); - } self.main_pipeline.add_transform(|input, output| { Ok(ProcessorPtr::create(TransformHashJoinProbe::create( @@ -212,7 +208,6 @@ impl PipelineBuilder { self.func_ctx.clone(), &join.join_type, !join.non_equi_conditions.is_empty(), - has_string_column, )?)) })?; diff --git a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_groups_builder.rs b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_groups_builder.rs index 194fe626d46f..d0d5d0c99330 100644 --- a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_groups_builder.rs +++ b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_groups_builder.rs @@ -16,7 +16,7 @@ use std::marker::PhantomData; use databend_common_exception::Result; use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::string::StringColumn; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; @@ -78,26 +78,36 @@ pub struct SerializedKeysGroupColumnsBuilder<'a> { data: Vec<&'a [u8]>, group_data_types: Vec, - single_builder: Option, + single_binary_builder: Option, + single_string_builder: Option, } impl<'a> SerializedKeysGroupColumnsBuilder<'a> { pub fn create(capacity: usize, data_capacity: usize, params: &AggregatorParams) -> Self { - let (single_builder, data) = if params.group_data_types.len() == 1 - && (params.group_data_types[0].is_string() || params.group_data_types[0].is_variant()) - { - ( - Some(BinaryColumnBuilder::with_capacity(capacity, data_capacity)), - vec![], - ) - } else { - (None, Vec::with_capacity(capacity)) - }; + let (single_binary_builder, single_string_builder, data) = + if params.group_data_types.len() == 1 { + if params.group_data_types[0].is_string() { + ( + None, + Some(StringColumnBuilder::with_capacity(capacity)), + vec![], + ) + } else { + ( + Some(BinaryColumnBuilder::with_capacity(capacity, data_capacity)), + None, + vec![], + ) + } + } else { + (None, None, Vec::with_capacity(capacity)) + }; Self { data, group_data_types: params.group_data_types.clone(), - single_builder, + single_binary_builder, + single_string_builder, } } } @@ -106,27 +116,33 @@ impl<'a> GroupColumnsBuilder for SerializedKeysGroupColumnsBuilder<'a> { type T = &'a [u8]; fn append_value(&mut self, v: &'a [u8]) { - match self.single_builder.as_mut() { - Some(builder) => { - builder.put_slice(v); - builder.commit_row(); + match ( + self.single_string_builder.as_mut(), + self.single_binary_builder.as_mut(), + ) { + (Some(s), _) => { + s.put_slice(v); + s.commit_row(); + } + (_, Some(s)) => { + s.put_slice(v); + s.commit_row(); } - None => self.data.push(v), + (_, _) => self.data.push(v), } } fn finish(mut self) -> Result> { - if let Some(builder) = self.single_builder.take() { + if let Some(builder) = self.single_binary_builder.take() { let col = builder.build(); match self.group_data_types[0] { - DataType::String => { - return Ok(vec![Column::String(unsafe { - StringColumn::from_binary_unchecked(col) - })]); - } + DataType::Binary => return Ok(vec![Column::Binary(col)]), DataType::Variant => return Ok(vec![Column::Variant(col)]), _ => {} } + } else if let Some(builder) = self.single_string_builder.take() { + let col = builder.build(); + return Ok(vec![Column::String(col)]); } let rows = self.data.len(); @@ -182,7 +198,7 @@ impl<'a> GroupColumnsBuilder for DictionarySerializedKeysGroupColumnsBuilder<'a> let mut index = 0; let mut res = Vec::with_capacity(self.group_data_types.len()); for data_type in self.group_data_types.iter() { - if data_type.is_string() || data_type.is_variant() { + if data_type.is_variant() { let mut builder = BinaryColumnBuilder::with_capacity(0, 0); for string_type_keys in &self.string_type_data { @@ -191,12 +207,7 @@ impl<'a> GroupColumnsBuilder for DictionarySerializedKeysGroupColumnsBuilder<'a> } index += 1; - res.push(match data_type.is_string() { - true => Column::String(unsafe { - StringColumn::from_binary_unchecked(builder.build()) - }), - false => Column::Variant(builder.build()), - }); + res.push(Column::Variant(builder.build())); } else { let mut column = ColumnBuilder::with_capacity(data_type, rows); diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs index f6694abadd46..3af426c9401a 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs @@ -515,26 +515,30 @@ impl HashJoinBuildState { let space_size = match &keys_state { // safe to unwrap(): offset.len() >= 1. - KeysState::Column(Column::Binary(col) | Column::Variant(col) | Column::Bitmap(col)) => col.offsets().last().unwrap(), - KeysState::Column(Column::String(col) ) => col.offsets().last().unwrap(), - // The function `build_keys_state` of both HashMethodSerializer and HashMethodSingleString - // must return `Column::Binary` | `Column::String` | `Column::Variant` | `Column::Bitmap`. + KeysState::Column(Column::String(col)) => col.current_buffer_len(), + KeysState::Column( + Column::Binary(col) | Column::Variant(col) | Column::Bitmap(col), + ) => col.data().len(), _ => unreachable!(), }; let valid_num = match &$valids { Some(valids) => valids.len() - valids.unset_bits(), None => $chunk.num_rows(), }; - let mut entry_local_space: Vec = - Vec::with_capacity(valid_num * entry_size); - let mut string_local_space: Vec = - Vec::with_capacity(*space_size as usize); - let mut raw_entry_ptr = unsafe { std::mem::transmute::<*mut u8, *mut StringRawEntry>(entry_local_space.as_mut_ptr()) }; + let mut entry_local_space: Vec = Vec::with_capacity(valid_num * entry_size); + let mut string_local_space: Vec = Vec::with_capacity(space_size as usize); + let mut raw_entry_ptr = unsafe { + std::mem::transmute::<*mut u8, *mut StringRawEntry>( + entry_local_space.as_mut_ptr(), + ) + }; let mut string_local_space_ptr = string_local_space.as_mut_ptr(); match $valids { Some(valids) => { - for (row_index, (key, valid)) in build_keys_iter.zip(valids.iter()).enumerate() { + for (row_index, (key, valid)) in + build_keys_iter.zip(valids.iter()).enumerate() + { if !valid { continue; } @@ -557,7 +561,11 @@ impl HashJoinBuildState { (*raw_entry_ptr).early.as_mut_ptr(), std::cmp::min(STRING_EARLY_SIZE, key.len()), ); - std::ptr::copy_nonoverlapping(key.as_ptr(), string_local_space_ptr, key.len()); + std::ptr::copy_nonoverlapping( + key.as_ptr(), + string_local_space_ptr, + key.len(), + ); string_local_space_ptr = string_local_space_ptr.add(key.len()); } @@ -586,7 +594,11 @@ impl HashJoinBuildState { (*raw_entry_ptr).early.as_mut_ptr(), std::cmp::min(STRING_EARLY_SIZE, key.len()), ); - std::ptr::copy_nonoverlapping(key.as_ptr(), string_local_space_ptr, key.len()); + std::ptr::copy_nonoverlapping( + key.as_ptr(), + string_local_space_ptr, + key.len(), + ); string_local_space_ptr = string_local_space_ptr.add(key.len()); } diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs index 0c38cad7d69a..5e27be5eb94a 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs @@ -508,7 +508,6 @@ impl HashJoinProbeState { &generation_state.build_columns, &generation_state.build_columns_data_type, &generation_state.build_num_rows, - &mut probe_state.generation_state.string_items_buf, )?; if self.hash_join_state.hash_join_desc.join_type == JoinType::Full { @@ -597,7 +596,6 @@ impl HashJoinProbeState { &generation_state.build_columns, &generation_state.build_columns_data_type, &generation_state.build_num_rows, - &mut probe_state.generation_state.string_items_buf, )?); build_indexes_idx = 0; } @@ -659,7 +657,6 @@ impl HashJoinProbeState { &generation_state.build_columns, &generation_state.build_columns_data_type, &generation_state.build_num_rows, - &mut probe_state.generation_state.string_items_buf, )?); build_indexes_idx = 0; } @@ -747,7 +744,6 @@ impl HashJoinProbeState { &generation_state.build_columns, &generation_state.build_columns_data_type, &generation_state.build_num_rows, - &mut probe_state.generation_state.string_items_buf, )?; result_blocks.push(self.merge_eq_block( Some(build_block), diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/merge_into_hash_join_optimization.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/merge_into_hash_join_optimization.rs index 26af688afad6..3d5cdbaf0719 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/merge_into_hash_join_optimization.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/merge_into_hash_join_optimization.rs @@ -344,10 +344,7 @@ impl TransformHashJoinProbe { { let end = (interval.1 - chunk_start).min(start + self.max_block_size as u32 - 1); let range = (start..=end).collect::>(); - let data_block = chunk_block.take( - &range, - &mut self.probe_state.generation_state.string_items_buf, - )?; + let data_block = chunk_block.take(&range)?; assert!(!data_block.is_empty()); let (segment_idx, block_idx) = split_prefix(prefix); info!( diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/inner_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/inner_join.rs index 32294afc57b7..51bcbbbb4cc6 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/inner_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/inner_join.rs @@ -210,11 +210,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - Some(DataBlock::take( - input, - &probe_indexes[0..matched_idx], - &mut probe_state.string_items_buf, - )?) + Some(DataBlock::take(input, &probe_indexes[0..matched_idx])?) } else { None }; @@ -224,7 +220,6 @@ impl HashJoinProbeState { &build_state.build_columns, &build_state.build_columns_data_type, &build_state.build_num_rows, - &mut probe_state.string_items_buf, )?) } else { None diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_anti_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_anti_join.rs index 813e5b168c1c..6e42f3c92fb2 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_anti_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_anti_join.rs @@ -79,11 +79,7 @@ impl HashJoinProbeState { )); } - let result_block = DataBlock::take( - &process_state.input, - &probe_indexes[0..count], - &mut probe_state.generation_state.string_items_buf, - )?; + let result_block = DataBlock::take(&process_state.input, &probe_indexes[0..count])?; probe_state.process_state = None; @@ -237,7 +233,6 @@ impl HashJoinProbeState { result_blocks.push(DataBlock::take( &process_state.input, &probe_indexes[0..unmatched_idx], - &mut probe_state.generation_state.string_items_buf, )?); } @@ -266,11 +261,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - Some(DataBlock::take( - input, - &probe_indexes[0..matched_idx], - &mut probe_state.string_items_buf, - )?) + Some(DataBlock::take(input, &probe_indexes[0..matched_idx])?) } else { None }; @@ -280,7 +271,6 @@ impl HashJoinProbeState { &build_state.build_columns, &build_state.build_columns_data_type, &build_state.build_num_rows, - &mut probe_state.string_items_buf, )?) } else { None diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_join.rs index f8dc4a100396..6afb2d80ba34 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_join.rs @@ -372,11 +372,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - let mut probe_block = DataBlock::take( - input, - &probe_indexes[0..unmatched_idx], - &mut probe_state.string_items_buf, - )?; + let mut probe_block = DataBlock::take(input, &probe_indexes[0..unmatched_idx])?; // For full join, wrap nullable for probe block if self.hash_join_state.hash_join_desc.join_type == JoinType::Full { let nullable_probe_columns = probe_block @@ -435,11 +431,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - let mut probe_block = DataBlock::take( - input, - &probe_indexes[0..matched_idx], - &mut probe_state.string_items_buf, - )?; + let mut probe_block = DataBlock::take(input, &probe_indexes[0..matched_idx])?; // For full join, wrap nullable for probe block if self.hash_join_state.hash_join_desc.join_type == JoinType::Full { let nullable_probe_columns = probe_block @@ -459,7 +451,6 @@ impl HashJoinProbeState { &build_state.build_columns, &build_state.build_columns_data_type, &build_state.build_num_rows, - &mut probe_state.string_items_buf, )?; // For left or full join, wrap nullable for build block. let nullable_columns = if build_state.build_num_rows == 0 { diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs index 23f47e26512f..0292ec9eb554 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs @@ -341,11 +341,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - Some(DataBlock::take( - input, - &probe_indexes[0..matched_idx], - &mut probe_state.string_items_buf, - )?) + Some(DataBlock::take(input, &probe_indexes[0..matched_idx])?) } else { None }; @@ -355,7 +351,6 @@ impl HashJoinProbeState { &build_state.build_columns, &build_state.build_columns_data_type, &build_state.build_num_rows, - &mut probe_state.string_items_buf, )?) } else { None diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_semi_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_semi_join.rs index 1f98cc837261..bae30ec7ccd4 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_semi_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_semi_join.rs @@ -82,7 +82,6 @@ impl HashJoinProbeState { result_blocks.push(DataBlock::take( &process_state.input, &probe_indexes[0..matched_idx], - &mut probe_state.generation_state.string_items_buf, )?); } @@ -232,7 +231,6 @@ impl HashJoinProbeState { result_blocks.push(DataBlock::take( &process_state.input, &probe_indexes[0..matched_idx], - &mut probe_state.generation_state.string_items_buf, )?); } @@ -261,11 +259,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - Some(DataBlock::take( - input, - &probe_indexes[0..matched_idx], - &mut probe_state.string_items_buf, - )?) + Some(DataBlock::take(input, &probe_indexes[0..matched_idx])?) } else { None }; @@ -275,7 +269,6 @@ impl HashJoinProbeState { &build_state.build_columns, &build_state.build_columns_data_type, &build_state.build_num_rows, - &mut probe_state.string_items_buf, )?) } else { None diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_join.rs index f5730d477934..cac83b169073 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_join.rs @@ -242,11 +242,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - let probe_block = DataBlock::take( - input, - &probe_indexes[0..matched_idx], - &mut probe_state.string_items_buf, - )?; + let probe_block = DataBlock::take(input, &probe_indexes[0..matched_idx])?; // The join type is right join, we need to wrap nullable for probe side. let nullable_columns = probe_block @@ -264,7 +260,6 @@ impl HashJoinProbeState { &build_state.build_columns, &build_state.build_columns_data_type, &build_state.build_num_rows, - &mut probe_state.string_items_buf, )?) } else { None diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_mark_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_mark_join.rs index 270a67cfe9cb..372e46aa0b03 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_mark_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_mark_join.rs @@ -276,11 +276,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - Some(DataBlock::take( - input, - &probe_indexes[0..matched_idx], - &mut probe_state.string_items_buf, - )?) + Some(DataBlock::take(input, &probe_indexes[0..matched_idx])?) } else { None }; @@ -290,7 +286,6 @@ impl HashJoinProbeState { &build_state.build_columns, &build_state.build_columns_data_type, &build_state.build_num_rows, - &mut probe_state.string_items_buf, )?) } else { None diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_semi_anti_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_semi_anti_join.rs index 95098d589c17..d9d345d5fa49 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_semi_anti_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_semi_anti_join.rs @@ -282,11 +282,7 @@ impl HashJoinProbeState { } let probe_block = if probe_state.is_probe_projected { - Some(DataBlock::take( - input, - &probe_indexes[0..matched_idx], - &mut probe_state.string_items_buf, - )?) + Some(DataBlock::take(input, &probe_indexes[0..matched_idx])?) } else { None }; @@ -296,7 +292,6 @@ impl HashJoinProbeState { &build_state.build_columns, &build_state.build_columns_data_type, &build_state.build_num_rows, - &mut probe_state.string_items_buf, )?) } else { None diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs index 87d7c063b1df..87ac376e47da 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs @@ -85,7 +85,6 @@ impl ProbeState { max_block_size: usize, join_type: &JoinType, with_conjunction: bool, - has_string_column: bool, func_ctx: FunctionContext, other_predicate: Option, ) -> Self { @@ -134,7 +133,7 @@ impl ProbeState { process_state: None, max_block_size, mutable_indexes: MutableIndexes::new(max_block_size), - generation_state: ProbeBlockGenerationState::new(max_block_size, has_string_column), + generation_state: ProbeBlockGenerationState::new(max_block_size), selection: vec![0; max_block_size], hashes: vec![0; max_block_size], selection_count: 0, @@ -185,20 +184,13 @@ pub struct ProbeBlockGenerationState { pub(crate) is_probe_projected: bool, // When we need a bitmap that is all true, we can directly slice it to reduce memory usage. pub(crate) true_validity: Bitmap, - // The string_items_buf is used as a buffer to reduce memory allocation when taking [u8] Columns. - pub(crate) string_items_buf: Option>, } impl ProbeBlockGenerationState { - fn new(size: usize, has_string_column: bool) -> Self { + fn new(size: usize) -> Self { Self { is_probe_projected: false, true_validity: Bitmap::new_constant(true, size), - string_items_buf: if has_string_column { - Some(vec![(0, 0); size]) - } else { - None - }, } } } diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/row.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/row.rs index 776c88dfeb8b..059e4ed2483a 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/row.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/row.rs @@ -65,7 +65,6 @@ impl RowSpace { build_columns: &[ColumnVec], build_columns_data_type: &[DataType], num_rows: &usize, - string_items_buf: &mut Option>, ) -> Result { if *num_rows != 0 { let data_block = DataBlock::take_column_vec( @@ -73,7 +72,6 @@ impl RowSpace { build_columns_data_type, row_ptrs, row_ptrs.len(), - string_items_buf, ); Ok(data_block) } else { diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_probe.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_probe.rs index c478a19f5ebf..cd113272c8b3 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_probe.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_probe.rs @@ -126,7 +126,6 @@ impl TransformHashJoinProbe { func_ctx: FunctionContext, join_type: &JoinType, with_conjunct: bool, - has_string_column: bool, ) -> Result> { join_probe_state.probe_attach(); // Create a hash join spiller. @@ -153,7 +152,6 @@ impl TransformHashJoinProbe { max_block_size, join_type, with_conjunct, - has_string_column, func_ctx, other_predicate, ); diff --git a/src/query/service/src/table_functions/others/suggested_background_compaction_tasks.rs b/src/query/service/src/table_functions/others/suggested_background_compaction_tasks.rs index 3815af7c5443..039df546b5f8 100644 --- a/src/query/service/src/table_functions/others/suggested_background_compaction_tasks.rs +++ b/src/query/service/src/table_functions/others/suggested_background_compaction_tasks.rs @@ -14,12 +14,12 @@ use std::sync::Arc; -use arrow_array::types::UInt64Type; -use arrow_array::BooleanArray; -use arrow_array::LargeBinaryArray; -use arrow_array::PrimitiveArray; -use arrow_array::RecordBatch; use databend_common_exception::Result; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::ValueType; +use databend_common_expression::DataBlock; use databend_common_meta_app::schema::TableStatistics; use databend_enterprise_background_service::Suggestion; use log::info; @@ -80,81 +80,69 @@ impl SuggestedBackgroundTasksSource { let mut suggestions = vec![]; for records in resps { info!(records :? =(&records); "target_tables"); - let db_names = records - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let db_ids = records - .column(1) - .as_any() - .downcast_ref::>() - .unwrap(); - let tb_names = records - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let tb_ids = records - .column(3) - .as_any() - .downcast_ref::>() - .unwrap(); - let segment_advice = records - .column(4) - .as_any() - .downcast_ref::() - .unwrap(); - let block_advice = records - .column(5) - .as_any() - .downcast_ref::() - .unwrap(); - let row_count = records - .column(6) - .as_any() - .downcast_ref::>() - .unwrap(); - let bytes_uncompressed = records - .column(7) - .as_any() - .downcast_ref::>() - .unwrap(); - let bytes_compressed = records - .column(8) - .as_any() - .downcast_ref::>() - .unwrap(); - let index_size = records - .column(9) - .as_any() - .downcast_ref::>() - .unwrap(); - let segment_count = records - .column(10) - .as_any() - .downcast_ref::>() - .unwrap(); - let block_count = records - .column(11) - .as_any() - .downcast_ref::>() - .unwrap(); + + let records = records.consume_convert_to_full(); + let db_names = + StringType::try_downcast_column(records.columns()[0].value.as_column().unwrap()) + .unwrap(); + + let db_ids = + UInt64Type::try_downcast_column(records.columns()[1].value.as_column().unwrap()) + .unwrap(); + + let tb_names = + StringType::try_downcast_column(records.columns()[2].value.as_column().unwrap()) + .unwrap(); + + let tb_ids = + UInt64Type::try_downcast_column(records.columns()[3].value.as_column().unwrap()) + .unwrap(); + + let segment_advice = + BooleanType::try_downcast_column(records.columns()[4].value.as_column().unwrap()) + .unwrap(); + + let block_advice = + BooleanType::try_downcast_column(records.columns()[5].value.as_column().unwrap()) + .unwrap(); + + let row_count = + UInt64Type::try_downcast_column(records.columns()[6].value.as_column().unwrap()) + .unwrap(); + + let bytes_uncompressed = + UInt64Type::try_downcast_column(records.columns()[7].value.as_column().unwrap()) + .unwrap(); + + let bytes_compressed = + UInt64Type::try_downcast_column(records.columns()[8].value.as_column().unwrap()) + .unwrap(); + + let index_size = + UInt64Type::try_downcast_column(records.columns()[9].value.as_column().unwrap()) + .unwrap(); + + let segment_count = + UInt64Type::try_downcast_column(records.columns()[10].value.as_column().unwrap()) + .unwrap(); + + let block_count = + UInt64Type::try_downcast_column(records.columns()[11].value.as_column().unwrap()) + .unwrap(); + for i in 0..records.num_rows() { - let db_name: String = - String::from_utf8_lossy(db_names.value(i).to_vec().as_slice()).to_string(); - let db_id = db_ids.value(i); - let table_name = - String::from_utf8_lossy(tb_names.value(i).to_vec().as_slice()).to_string(); - let table_id = tb_ids.value(i); - let need_compact_segment = segment_advice.value(i); - let need_compact_block = block_advice.value(i); - let number_of_rows = row_count.value(i); - let data_bytes = bytes_uncompressed.value(i); - let compressed_data_bytes = bytes_compressed.value(i); - let index_data_bytes = index_size.value(i); - let number_of_segments = segment_count.value(i); - let number_of_blocks = block_count.value(i); + let db_name: String = db_names.index(i).unwrap().to_string(); + let db_id = db_ids[i]; + let table_name = tb_names.index(i).unwrap().to_string(); + let table_id = tb_ids[i]; + let need_compact_segment = segment_advice.get_bit(i); + let need_compact_block = block_advice.get_bit(i); + let number_of_rows = row_count[i]; + let data_bytes = bytes_uncompressed[i]; + let compressed_data_bytes = bytes_compressed[i]; + let index_data_bytes = index_size[i]; + let number_of_segments = segment_count[i]; + let number_of_blocks = block_count[i]; let suggestion = Suggestion::Compaction { db_id, db_name, @@ -179,7 +167,7 @@ impl SuggestedBackgroundTasksSource { pub async fn do_get_all_suggested_compaction_tables( ctx: Arc, - ) -> Result> { + ) -> Result> { let res = SuggestedBackgroundTasksSource::do_execute_sql( ctx, SUGGEST_TABLES_NEED_COMPACTION.to_string(), diff --git a/src/query/service/src/table_functions/others/suggested_background_tasks.rs b/src/query/service/src/table_functions/others/suggested_background_tasks.rs index a2053c953908..b1654fc23af4 100644 --- a/src/query/service/src/table_functions/others/suggested_background_tasks.rs +++ b/src/query/service/src/table_functions/others/suggested_background_tasks.rs @@ -15,7 +15,6 @@ use std::any::Any; use std::sync::Arc; -use arrow_array::RecordBatch; use chrono::DateTime; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::PartStatistics; @@ -24,7 +23,6 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; -use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::BooleanType; use databend_common_expression::types::StringType; @@ -155,14 +153,10 @@ impl SuggestedBackgroundTasksSource { } #[async_backtrace::framed] - pub async fn do_execute_sql( - ctx: Arc, - sql: String, - ) -> Result> { + pub async fn do_execute_sql(ctx: Arc, sql: String) -> Result> { // Use interpreter_plan_sql, we can write the query log if an error occurs. let (plan, _, _) = interpreter_plan_sql(ctx.clone(), sql.as_str(), false).await?; - let data_schema = plan.schema(); let interpreter = InterpreterFactory::get(ctx.clone(), &plan).await?; let stream = interpreter.execute(ctx.clone()).await?; let blocks = stream.map(|v| v).collect::>().await; @@ -182,10 +176,6 @@ impl SuggestedBackgroundTasksSource { return Ok(None); } let record = DataBlock::concat(&result)?; - let record = record - .to_record_batch_with_dataschema(data_schema.as_ref()) - .map_err(|e| ErrorCode::Internal(format!("{e:?}")))?; - Ok(Some(record)) } diff --git a/src/query/service/src/table_functions/others/udf.rs b/src/query/service/src/table_functions/others/udf.rs index 0abfa5fff800..5d324e7a1338 100644 --- a/src/query/service/src/table_functions/others/udf.rs +++ b/src/query/service/src/table_functions/others/udf.rs @@ -18,6 +18,7 @@ use std::hash::Hash; use std::hash::Hasher; use std::sync::Arc; +use arrow_array::LargeStringArray; use arrow_array::RecordBatch; use arrow_schema::Field; use arrow_schema::Schema; @@ -164,7 +165,7 @@ impl Table for UdfEchoTable { let result = result_batch .column(0) .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); let result = result.value(0).to_string(); let parts = vec![Arc::new(Box::new(StringPart { value: result }) as _)]; diff --git a/src/query/service/src/test_kits/fixture.rs b/src/query/service/src/test_kits/fixture.rs index 2f5f7959e2b5..e46472ab1754 100644 --- a/src/query/service/src/test_kits/fixture.rs +++ b/src/query/service/src/test_kits/fixture.rs @@ -733,17 +733,13 @@ impl TestFixture { schema, (0..num_of_block) .map(|idx| { - let mut title_builder = - StringColumnBuilder::with_capacity(rows_per_block, rows_per_block * 10); - let mut content_builder = - StringColumnBuilder::with_capacity(rows_per_block, rows_per_block * 10); + let mut title_builder = StringColumnBuilder::with_capacity(rows_per_block); + let mut content_builder = StringColumnBuilder::with_capacity(rows_per_block); for i in 0..rows_per_block { let j = (idx * rows_per_block + i) % sample_books.len(); - title_builder.put_str(sample_books[j].0); - title_builder.commit_row(); - content_builder.put_str(sample_books[j].1); - content_builder.commit_row(); + title_builder.put_and_commit(sample_books[j].0); + content_builder.put_and_commit(sample_books[j].1); } let title_column = Column::String(title_builder.build()); let content_column = Column::String(content_builder.build()); diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index 1246d74aae24..35e74afbb312 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -275,7 +275,7 @@ impl DefaultSettings { mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=1)), }), - ("enable_dio", DefaultSettingValue{ + ("enable_dio", DefaultSettingValue{ value: UserSettingValue::UInt64(1), desc: "Enables Direct IO.", mode: SettingMode::Both, @@ -288,7 +288,7 @@ impl DefaultSettings { range: Some(SettingRange::Numeric(0..=1)), }), ("join_spilling_memory_ratio", DefaultSettingValue { - value: UserSettingValue::UInt64(60), + value: UserSettingValue::UInt64(0), desc: "Sets the maximum memory ratio in bytes that hash join can use before spilling data to storage during query execution, 0 is unlimited", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=100)), @@ -463,7 +463,7 @@ impl DefaultSettings { range: Some(SettingRange::Numeric(0..=u64::MAX)), }), ("aggregate_spilling_memory_ratio", DefaultSettingValue { - value: UserSettingValue::UInt64(60), + value: UserSettingValue::UInt64(0), desc: "Sets the maximum memory ratio in bytes that an aggregator can use before spilling data to storage during query execution.", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=100)), @@ -475,7 +475,7 @@ impl DefaultSettings { range: Some(SettingRange::Numeric(0..=u64::MAX)), }), ("window_partition_spilling_memory_ratio", DefaultSettingValue { - value: UserSettingValue::UInt64(60), + value: UserSettingValue::UInt64(0), desc: "Sets the maximum memory ratio in bytes that a window partitioner can use before spilling data to storage during query execution.", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=100)), @@ -511,7 +511,7 @@ impl DefaultSettings { range: Some(SettingRange::Numeric(0..=u64::MAX)), }), ("sort_spilling_memory_ratio", DefaultSettingValue { - value: UserSettingValue::UInt64(60), + value: UserSettingValue::UInt64(0), desc: "Sets the maximum memory ratio in bytes that a sorter can use before spilling data to storage during query execution.", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=100)), @@ -689,8 +689,8 @@ impl DefaultSettings { desc: "Set numeric default_order_by_null mode", mode: SettingMode::Both, range: Some(SettingRange::String(vec![ - "nulls_first".into(), "nulls_last".into(), - "nulls_first_on_asc_last_on_desc".into(), "nulls_last_on_asc_first_on_desc".into(), + "nulls_first".into(), "nulls_last".into(), + "nulls_first_on_asc_last_on_desc".into(), "nulls_last_on_asc_first_on_desc".into(), ])), }), ("ddl_column_type_nullable", DefaultSettingValue { diff --git a/src/query/storages/common/index/benches/build_from_block.rs b/src/query/storages/common/index/benches/build_from_block.rs index bb80b4a3f77d..91f68213aa33 100644 --- a/src/query/storages/common/index/benches/build_from_block.rs +++ b/src/query/storages/common/index/benches/build_from_block.rs @@ -198,7 +198,7 @@ fn rand_str_column(n: i32, len: i32) -> Column { abcdefghijklmnopqrstuvwxyz\ 0123456789)(*&^%$#@!~"; - let mut builder = StringColumnBuilder::with_capacity(n as usize, 0); + let mut builder = StringColumnBuilder::with_capacity(n as usize); for _ in 0..n { for _ in (len / 2)..len { let idx = rng.gen_range(0..CHARSET.len()); diff --git a/src/query/storages/common/index/src/bloom_index.rs b/src/query/storages/common/index/src/bloom_index.rs index a5614b8b2a73..8763f35ae9a5 100644 --- a/src/query/storages/common/index/src/bloom_index.rs +++ b/src/query/storages/common/index/src/bloom_index.rs @@ -545,7 +545,7 @@ impl BloomIndex { /// If it does, the bloom index for the column will not be established. fn check_large_string(column: &Column) -> bool { if let Column::String(v) = &column { - let bytes_per_row = v.data().len() / v.len().max(1); + let bytes_per_row = v.current_buffer_len() / v.len().max(1); if bytes_per_row > 256 { return true; } diff --git a/src/query/storages/fuse/src/fuse_table.rs b/src/query/storages/fuse/src/fuse_table.rs index 521d0a769cbc..c2bbdc4d7755 100644 --- a/src/query/storages/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/src/fuse_table.rs @@ -504,6 +504,10 @@ impl Table for FuseTable { true } + fn storage_format_as_parquet(&self) -> bool { + matches!(self.storage_format, FuseStorageFormat::Parquet) + } + fn cluster_keys(&self, ctx: Arc) -> Vec> { let table_meta = Arc::new(self.clone()); if let Some((_, order)) = &self.cluster_key_meta { diff --git a/src/query/storages/fuse/src/io/read/block/block_reader.rs b/src/query/storages/fuse/src/io/read/block/block_reader.rs index 3d46fdab22bf..975bcd1e8d77 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader.rs @@ -135,6 +135,7 @@ impl BlockReader { .iter() .map(|c| (*c).clone()) .collect(); + let project_indices = Self::build_projection_indices(&project_column_nodes); Ok(Arc::new(BlockReader { diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs index e9ede151a890..d8aada0bdd7b 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs @@ -277,6 +277,7 @@ impl BlockReader { ) -> Result> { let field = column_node.field.clone(); let is_nested = column_node.is_nested; + match self.native_columns_reader.column_iter_to_arrays( readers, &column_node.leaf_indices, diff --git a/src/query/storages/fuse/src/operations/read/runtime_filter_prunner.rs b/src/query/storages/fuse/src/operations/read/runtime_filter_prunner.rs index 4d4674b50d4a..066a418b626d 100644 --- a/src/query/storages/fuse/src/operations/read/runtime_filter_prunner.rs +++ b/src/query/storages/fuse/src/operations/read/runtime_filter_prunner.rs @@ -150,8 +150,8 @@ pub(crate) fn update_bitmap_with_bloom_filter( } idx += 1; }), - KeysState::Column(Column::String(col)) => col.iter_binary().for_each(|key| { - let hash = key.fast_hash(); + KeysState::Column(Column::String(col)) => col.iter().for_each(|key| { + let hash = key.as_bytes().fast_hash(); if filter.contains(&hash) { bitmap.set(idx, true); } diff --git a/src/query/storages/fuse/src/statistics/cluster_statistics.rs b/src/query/storages/fuse/src/statistics/cluster_statistics.rs index 01b2316a70e0..0b72a4794aee 100644 --- a/src/query/storages/fuse/src/statistics/cluster_statistics.rs +++ b/src/query/storages/fuse/src/statistics/cluster_statistics.rs @@ -106,7 +106,7 @@ impl ClusterStatsGenerator { if !self.cluster_key_index.is_empty() { let indices = vec![0u32, block.num_rows() as u32 - 1]; - block = block.take(&indices, &mut None)?; + block = block.take(&indices)?; } block = self diff --git a/src/query/storages/fuse/src/table_functions/clustering_statistics.rs b/src/query/storages/fuse/src/table_functions/clustering_statistics.rs index 113408f2b54c..724630e158d8 100644 --- a/src/query/storages/fuse/src/table_functions/clustering_statistics.rs +++ b/src/query/storages/fuse/src/table_functions/clustering_statistics.rs @@ -159,7 +159,7 @@ impl<'a> ClusteringStatisticsImpl<'a> { let len = std::cmp::min(snapshot.summary.block_count as usize, limit); let mut segment_name = Vec::with_capacity(len); - let mut block_name = StringColumnBuilder::with_capacity(len, len); + let mut block_name = StringColumnBuilder::with_capacity(len); let mut max = Vec::with_capacity(len); let mut min = Vec::with_capacity(len); let mut level = Vec::with_capacity(len); @@ -196,8 +196,7 @@ impl<'a> ClusteringStatisticsImpl<'a> { for block in segment.blocks.iter() { let block = block.as_ref(); - block_name.put_str(&block.location.0); - block_name.commit_row(); + block_name.put_and_commit(&block.location.0); let cluster_stats = block.cluster_stats.as_ref(); let clustered = block diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index fb09f3a98025..7a96a755ba5d 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -79,7 +79,7 @@ impl TableMetaFunc for FuseBlock { let snapshot_id = snapshot.snapshot_id.simple().to_string(); let timestamp = snapshot.timestamp.unwrap_or_default().timestamp_micros(); - let mut block_location = StringColumnBuilder::with_capacity(len, len); + let mut block_location = StringColumnBuilder::with_capacity(len); let mut block_size = Vec::with_capacity(len); let mut file_size = Vec::with_capacity(len); let mut row_count = Vec::with_capacity(len); @@ -102,8 +102,7 @@ impl TableMetaFunc for FuseBlock { for block in segment.blocks.iter() { let block = block.as_ref(); - block_location.put_str(&block.location.0); - block_location.commit_row(); + block_location.put_and_commit(&block.location.0); block_size.push(block.block_size); file_size.push(block.file_size); row_count.push(block.row_count); diff --git a/src/query/storages/fuse/src/table_functions/fuse_column.rs b/src/query/storages/fuse/src/table_functions/fuse_column.rs index 24caf6d4e6ed..7ae42f3025df 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_column.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_column.rs @@ -78,13 +78,13 @@ impl TableMetaFunc for FuseColumn { let snapshot_id = snapshot.snapshot_id.simple().to_string(); let timestamp = snapshot.timestamp.unwrap_or_default().timestamp_micros(); - let mut block_location = StringColumnBuilder::with_capacity(len, len); + let mut block_location = StringColumnBuilder::with_capacity(len); let mut block_size = vec![]; let mut file_size = vec![]; let mut row_count = vec![]; - let mut column_name = StringColumnBuilder::with_capacity(len, len); - let mut column_type = StringColumnBuilder::with_capacity(len, len); + let mut column_name = StringColumnBuilder::with_capacity(len); + let mut column_type = StringColumnBuilder::with_capacity(len); let mut column_id = vec![]; let mut block_offset = vec![]; let mut bytes_compressed = vec![]; @@ -110,17 +110,14 @@ impl TableMetaFunc for FuseColumn { for (id, column) in block.col_metas.iter() { if let Some(f) = leaf_fields.iter().find(|f| f.column_id == *id) { - block_location.put_str(&block.location.0); - block_location.commit_row(); + block_location.put_and_commit(&block.location.0); block_size.push(block.block_size); file_size.push(block.file_size); row_count.push(column.total_rows() as u64); - column_name.put_str(&f.name); - column_name.commit_row(); + column_name.put_and_commit(&f.name); - column_type.put_str(&f.data_type.to_string()); - column_type.commit_row(); + column_type.put_and_commit(f.data_type.to_string()); column_id.push(*id); diff --git a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs index 5e5dda4cc64f..f6ab657277c2 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs @@ -233,11 +233,11 @@ impl<'a> FuseEncodingImpl<'a> { let mut validity_size = Vec::new(); let mut compressed_size = Vec::new(); let mut uncompressed_size = Vec::new(); - let mut l1 = StringColumnBuilder::with_capacity(0, 0); + let mut l1 = StringColumnBuilder::with_capacity(0); let mut l2 = NullableColumnBuilder::::with_capacity(0, &[]); - let mut table_name = StringColumnBuilder::with_capacity(0, 0); - let mut column_name = StringColumnBuilder::with_capacity(0, 0); - let mut column_type = StringColumnBuilder::with_capacity(0, 0); + let mut table_name = StringColumnBuilder::with_capacity(0); + let mut column_name = StringColumnBuilder::with_capacity(0); + let mut column_type = StringColumnBuilder::with_capacity(0); let mut all_num_rows = 0; for (table, columns_info) in info { for (type_str, column_info) in columns_info { @@ -254,8 +254,7 @@ impl<'a> FuseEncodingImpl<'a> { validity_size.push(p.validity_size); compressed_size.push(p.compressed_size); uncompressed_size.push(p.uncompressed_size); - l1.put_str(&encoding_to_string(&p.body)); - l1.commit_row(); + l1.put_and_commit(encoding_to_string(&p.body)); let l2_encoding = match &p.body { PageBody::Dict(dict) => Some(encoding_to_string(&dict.indices.body)), PageBody::Freq(freq) => freq diff --git a/src/query/storages/parquet/Cargo.toml b/src/query/storages/parquet/Cargo.toml index e8b6ef08c014..6cd32a9da075 100644 --- a/src/query/storages/parquet/Cargo.toml +++ b/src/query/storages/parquet/Cargo.toml @@ -44,7 +44,6 @@ thrift = { workspace = true } typetag = { workspace = true } [dev-dependencies] -databend-common-sql = { workspace = true } tempfile = { workspace = true } [lints] diff --git a/src/query/storages/stage/src/read/row_based/formats/csv/block_builder.rs b/src/query/storages/stage/src/read/row_based/formats/csv/block_builder.rs index 199900f4d283..2f979a03a477 100644 --- a/src/query/storages/stage/src/read/row_based/formats/csv/block_builder.rs +++ b/src/query/storages/stage/src/read/row_based/formats/csv/block_builder.rs @@ -83,15 +83,13 @@ impl CsvDecoder { } EmptyFieldAs::String => match builder { ColumnBuilder::String(b) => { - b.put_str(""); - b.commit_row(); + b.put_and_commit(""); } ColumnBuilder::Nullable(box NullableColumnBuilder { builder: ColumnBuilder::String(b), validity, }) => { - b.put_str(""); - b.commit_row(); + b.put_and_commit(""); validity.push(true); } _ => { @@ -188,14 +186,8 @@ impl RowDecoder for CsvDecoder { fn flush(&self, columns: Vec, num_rows: usize) -> Vec { if let Some(projection) = &self.load_context.pos_projection { - let empty_strings = Column::String( - StringColumnBuilder { - need_estimated: false, - data: vec![], - offsets: vec![0; num_rows + 1], - } - .build(), - ); + let empty_strings = + Column::String(StringColumnBuilder::repeat_default(num_rows).build()); columns .into_iter() .enumerate() diff --git a/src/query/storages/system/src/malloc_stats_totals_table.rs b/src/query/storages/system/src/malloc_stats_totals_table.rs index d231d246fe66..4942edaaa69c 100644 --- a/src/query/storages/system/src/malloc_stats_totals_table.rs +++ b/src/query/storages/system/src/malloc_stats_totals_table.rs @@ -92,7 +92,7 @@ impl MallocStatsTotalsTable { } fn build_columns(node_name: &str) -> BuildResult { - let mut names = StringColumnBuilder::with_capacity(6, 6 * 4); + let mut names = StringColumnBuilder::with_capacity(6); let mut values: Vec = vec![]; let e = epoch::mib()?; diff --git a/tests/sqllogictests/src/main.rs b/tests/sqllogictests/src/main.rs index 7345e8cc8039..8a839b145852 100644 --- a/tests/sqllogictests/src/main.rs +++ b/tests/sqllogictests/src/main.rs @@ -79,7 +79,10 @@ pub async fn main() -> Result<()> { // Run mock sources for dictionary test. run_mock_sources(); - + println!( + "Run sqllogictests with args: {}", + std::env::args().skip(1).collect::>().join(" ") + ); let args = SqlLogicTestArgs::parse(); let handlers = match &args.handlers { Some(hs) => hs.iter().map(|s| s.as_str()).collect(), diff --git a/tests/sqllogictests/suites/base/05_ddl/05_0003_ddl_alter_table.test b/tests/sqllogictests/suites/base/05_ddl/05_0003_ddl_alter_table.test index 91f6f02fd83f..4f260ccfc703 100644 --- a/tests/sqllogictests/suites/base/05_ddl/05_0003_ddl_alter_table.test +++ b/tests/sqllogictests/suites/base/05_ddl/05_0003_ddl_alter_table.test @@ -122,7 +122,7 @@ statement ok set hide_options_in_show_create_table=1 statement ok -CREATE TABLE "05_0003_at_t4" ( a string not null, b string null, c array(string) null, d tuple(string, string) null ) ENGINE=FUSE COMPRESSION='zstd' STORAGE_FORMAT='native' +CREATE OR REPLACE TABLE "05_0003_at_t4" ( a string not null, b string null, c array(string) null, d tuple(string, string) null ) ENGINE=FUSE COMPRESSION='zstd' STORAGE_FORMAT='native' statement ok INSERT INTO TABLE `05_0003_at_t4` values('a', 'b', ['c1', 'c2'], ('d1', 'd2')) @@ -154,7 +154,7 @@ SHOW CREATE TABLE `05_0003_at_t4` ---- 05_0003_at_t4 CREATE TABLE "05_0003_at_t4" ( a BINARY NOT NULL, b BINARY NULL, c ARRAY(BINARY) NULL, d TUPLE(1 BINARY, 2 BINARY) NULL ) ENGINE=FUSE -query +query SELECT * FROM `05_0003_at_t4` ---- 61 62 [6331,6332] (6431,6432) diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0027_func_fuse_encoding.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0027_func_fuse_encoding.test index 502bea556252..af4c570443ac 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0027_func_fuse_encoding.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0027_func_fuse_encoding.test @@ -49,7 +49,7 @@ insert into t2 select 'b' as a from numbers(10) limit 10; query III select level_one,level_two,count(*) from fuse_encoding('db_09_0027') where table_name='t2' group by level_one,level_two order by level_one; ---- -OneValue NULL 2 +Common(Lz4) NULL 2 statement ok optimize table t2 compact; @@ -57,12 +57,12 @@ optimize table t2 compact; query III select level_one,level_two,count(*) from fuse_encoding('db_09_0027') where table_name='t2' group by level_one,level_two order by level_one; ---- -Dict Rle 1 +Common(Lz4) NULL 1 query III select level_one,level_two,count(*) from fuse_encoding('db_09_0027') where column_name='d' group by level_one,level_two order by level_one; ---- -Dict Rle 1 +Common(Lz4) NULL 1 query III select level_one,level_two,count(*) from fuse_encoding('db_09_0027') where column_type like '%INT%' group by level_one,level_two order by level_one; @@ -71,4 +71,4 @@ DeltaBitpack NULL 1 OneValue NULL 1 statement ok -DROP DATABASE db_09_0027 \ No newline at end of file +DROP DATABASE db_09_0027 diff --git a/tests/sqllogictests/suites/base/20+_others/20_0013_query_result_cache.test b/tests/sqllogictests/suites/base/20+_others/20_0013_query_result_cache.test index d2fb2e8c3466..6cbba575ac9a 100644 --- a/tests/sqllogictests/suites/base/20+_others/20_0013_query_result_cache.test +++ b/tests/sqllogictests/suites/base/20+_others/20_0013_query_result_cache.test @@ -89,7 +89,7 @@ EXPLAIN SELECT * FROM t1, t2 ORDER BY a, b; ReadQueryResultCache ├── SQL: SELECT * FROM t1, t2 ORDER BY a, b ├── Number of rows: 9 -└── Result size: 125 +└── Result size: 144 @@ -225,7 +225,7 @@ SELECT * FROM t1 ORDER BY a; # Because the cache key is generated from AST. query I -select * FRoM t1 OrDER bY a; +select * FRoM t1 OrDER bY a; ---- 1 2 diff --git a/tests/sqllogictests/suites/mode/cluster/distributed_delete.sql.test b/tests/sqllogictests/suites/mode/cluster/distributed_delete.test similarity index 86% rename from tests/sqllogictests/suites/mode/cluster/distributed_delete.sql.test rename to tests/sqllogictests/suites/mode/cluster/distributed_delete.test index 955a5ffc50a2..8dfb3d9b73d1 100644 --- a/tests/sqllogictests/suites/mode/cluster/distributed_delete.sql.test +++ b/tests/sqllogictests/suites/mode/cluster/distributed_delete.test @@ -6,7 +6,7 @@ drop table if exists t_origin; # make sure there will be multiple blocks there, by shrink the `row_per_block` statement ok -create table t (id int not null, c1 int not null, c2 int not null) row_per_block=10; +create or replace table t (id int not null, c1 int not null, c2 int not null) row_per_block=10; # generate test data statement ok @@ -20,7 +20,7 @@ insert into t select number, number * 10, number * 5 from numbers(1500) where nu # "backup" t statement ok -create table t_origin as select * from t; +create or replace table t_origin as select * from t; # do the deletion (in distributed settings) # two segments are totally rewritten, one segment is reserved @@ -45,7 +45,7 @@ select (select sum(c2) from t_origin where id % 3 != 0 or id <= 500) = (select s # backup t again statement ok -create table t_after_delete as select * from t; +create or replace table t_after_delete as select * from t; # one segment is totally deleted, two segments are reserved statement ok @@ -69,7 +69,7 @@ select (select sum(c2) from t_after_delete where id > 499) = (select sum(c2) fro # backup t again statement ok -create table t_after_delete_2 as select * from t; +create or replace table t_after_delete_2 as select * from t; # some block is totally deleted, some block is reserved, some block is partially reserved statement ok @@ -100,13 +100,13 @@ statement ok drop table if exists del_id; statement ok -create table t (id int not null, c1 int not null, c2 int not null) row_per_block=3; +create or replace table t (id int not null, c1 int not null, c2 int not null) row_per_block=3; statement ok insert into t select number, number * 5, number * 7 from numbers(50); statement ok -create table del_id (id int not null) as select cast(FLOOR(0 + RAND(number) * 50), int) from numbers(10); +create or replace table del_id (id int not null) as select cast(FLOOR(0 + RAND(number) * 50), int) from numbers(10); statement ok delete from t where id in (select id from del_id); diff --git a/tests/sqllogictests/suites/mode/cluster/memo/aggregate_property.test b/tests/sqllogictests/suites/mode/cluster/memo/aggregate_property.test index 26beed75d057..32405dc66997 100644 --- a/tests/sqllogictests/suites/mode/cluster/memo/aggregate_property.test +++ b/tests/sqllogictests/suites/mode/cluster/memo/aggregate_property.test @@ -26,7 +26,7 @@ where t_10.a = t_1000.a and t_100.a = t_1000.a ---- Memo ├── root group: #8 -├── estimated memory: 11.53 KiB +├── estimated memory: 14.91 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -89,7 +89,7 @@ group by t_10.a, t_100.a ---- Memo ├── root group: #8 -├── estimated memory: 28.83 KiB +├── estimated memory: 37.27 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] diff --git a/tests/sqllogictests/suites/mode/cluster/memo/join_property.test b/tests/sqllogictests/suites/mode/cluster/memo/join_property.test index ed0b61f569ae..c6d9effd4cd4 100644 --- a/tests/sqllogictests/suites/mode/cluster/memo/join_property.test +++ b/tests/sqllogictests/suites/mode/cluster/memo/join_property.test @@ -25,7 +25,7 @@ select * from t_10, t_100, t_1000 where t_10.a = t_1000.a and t_100.a = t_1000.a ---- Memo ├── root group: #5 -├── estimated memory: 8.97 KiB +├── estimated memory: 11.59 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -73,7 +73,7 @@ select * from t_1000 left join t_10 on t_1000.a = t_10.a left join t_100 on t_10 ---- Memo ├── root group: #5 -├── estimated memory: 8.33 KiB +├── estimated memory: 10.77 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -119,7 +119,7 @@ select * from t_1000 right join t_10 on t_1000.a = t_10.a right join t_100 on t_ ---- Memo ├── root group: #5 -├── estimated memory: 7.05 KiB +├── estimated memory: 9.11 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -161,7 +161,7 @@ select * from t_1000 full join t_10 on t_1000.a = t_10.a full join t_100 on t_10 ---- Memo ├── root group: #5 -├── estimated memory: 7.05 KiB +├── estimated memory: 9.11 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -203,7 +203,7 @@ select * from t_10, t_100, t_1000 ---- Memo ├── root group: #5 -├── estimated memory: 5.77 KiB +├── estimated memory: 7.45 KiB ├── Group #0 │ ├── Best properties │ │ └── { dist: Any }: expr: #0, cost: 10.000, children: [] diff --git a/tests/sqllogictests/suites/mode/cluster/memo/mix_property.test b/tests/sqllogictests/suites/mode/cluster/memo/mix_property.test index b135f0c753e5..cb8c9ad7328f 100644 --- a/tests/sqllogictests/suites/mode/cluster/memo/mix_property.test +++ b/tests/sqllogictests/suites/mode/cluster/memo/mix_property.test @@ -29,7 +29,7 @@ limit 10 ---- Memo ├── root group: #10 -├── estimated memory: 30.11 KiB +├── estimated memory: 38.92 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] diff --git a/tests/sqllogictests/suites/mode/cluster/shuffle_join.test b/tests/sqllogictests/suites/mode/cluster/shuffle_join.test index 2e9b24766374..6ecd8dc5531e 100644 --- a/tests/sqllogictests/suites/mode/cluster/shuffle_join.test +++ b/tests/sqllogictests/suites/mode/cluster/shuffle_join.test @@ -6,7 +6,7 @@ drop table if exists t1 statement ok -create table t1(a int not null, b int not null) +create or replace table t1(a int not null, b int not null) statement ok @@ -18,7 +18,7 @@ drop table if exists t2 statement ok -create table t2(a int not null, d int not null) +create or replace table t2(a int not null, d int not null) statement ok @@ -77,7 +77,7 @@ statement ok drop table t2 statement ok -create table t1(a int not null, b int not null) +create or replace table t1(a int not null, b int not null) statement ok @@ -85,7 +85,7 @@ insert into t1 values(1, 2), (1, 3), (2, 4) statement ok -create table t2(c int not null, d int not null) +create or replace table t2(c int not null, d int not null) statement ok @@ -108,11 +108,11 @@ statement ok drop table t2 statement ok -create table t1(a int not null, b int not null) +create or replace table t1(a int not null, b int not null) statement ok -create table t2(c int not null, d int not null) +create or replace table t2(c int not null, d int not null) statement ok @@ -219,20 +219,20 @@ statement ok drop table if exists t statement ok -create table t(a int not null) +create or replace table t(a int not null) statement ok insert into t values(1),(2),(3) statement ok -create table t1(b float not null) +create or replace table t1(b float not null) statement ok insert into t1 values(1.0),(2.0),(3.0) statement ok -create table t2(c smallint unsigned null) +create or replace table t2(c smallint unsigned null) statement ok @@ -316,10 +316,10 @@ statement ok drop table t statement ok -CREATE TABLE t3(c0 BIGINT NULL, c1 DOUBLE NULL) +CREATE or replace TABLE t3(c0 BIGINT NULL, c1 DOUBLE NULL) statement ok -CREATE TABLE t4(c0 FLOAT NULL) +CREATE or replace TABLE t4(c0 FLOAT NULL) query I SELECT SUM(count) FROM (SELECT ((false IS NOT NULL AND false) ::INT64)as count FROM t4 NATURAL LEFT JOIN t3) as res @@ -339,11 +339,11 @@ statement ok drop table t4 statement ok -create table t1_null(a int null , b int null) +create or replace table t1_null(a int null , b int null) statement ok -create table t2_null(a int null , b int null) +create or replace table t2_null(a int null , b int null) statement ok @@ -394,13 +394,13 @@ select * from numbers(10) x join (select 1::UInt64 number) y on x.number = y.num 1 1 statement ok -CREATE TABLE onecolumn (x INT NULL) +CREATE or replace TABLE onecolumn (x INT NULL) statement ok INSERT INTO onecolumn(x) VALUES (44), (NULL), (42) statement ok -CREATE TABLE empty (x INT not null) +CREATE or replace TABLE empty (x INT not null) statement ok SELECT * FROM onecolumn AS a(x) CROSS JOIN empty AS b(y) @@ -418,10 +418,10 @@ statement ok drop table if exists z1 statement ok -CREATE TABLE z0(c0BOOLEAN BOOLEAN NULL, c1FLOAT DOUBLE NULL) +CREATE or replace TABLE z0(c0BOOLEAN BOOLEAN NULL, c1FLOAT DOUBLE NULL) statement ok -CREATE TABLE z1(c0BOOLEAN BOOL NULL DEFAULT(true)) +CREATE or replace TABLE z1(c0BOOLEAN BOOL NULL DEFAULT(true)) statement ok INSERT INTO z0(c1float, c0boolean) VALUES (0.27563244104385376, false), (0.7913353443145752, false) @@ -437,13 +437,13 @@ statement ok drop table z1 statement ok -CREATE TABLE t0(c0BOOLEAN BOOLEAN NULL DEFAULT(false)) +CREATE or replace TABLE t0(c0BOOLEAN BOOLEAN NULL DEFAULT(false)) statement ok -CREATE TABLE t1(c0BOOLEAN BOOL NULL, c1FLOAT FLOAT NOT NULL DEFAULT(0.4661566913127899)) +CREATE or replace TABLE t1(c0BOOLEAN BOOL NULL, c1FLOAT FLOAT NOT NULL DEFAULT(0.4661566913127899)) statement ok -CREATE TABLE t2(c0VARCHAR VARCHAR NULL, c1FLOAT DOUBLE NULL DEFAULT(0.954969048500061), c2VARCHAR VARCHAR NULL) +CREATE or replace TABLE t2(c0VARCHAR VARCHAR NULL, c1FLOAT DOUBLE NULL DEFAULT(0.954969048500061), c2VARCHAR VARCHAR NULL) statement ok INSERT INTO t0(c0boolean) VALUES (false), (true) diff --git a/tests/sqllogictests/suites/mode/standalone/explain/explain.test b/tests/sqllogictests/suites/mode/standalone/explain/explain.test index d29b67e5dd6f..0f7686c52c50 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/explain.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/explain.test @@ -5,10 +5,10 @@ statement ok drop table if exists t2 all statement ok -create table t1 as select number as a, number as b from numbers(1) +create OR REPLACE table t1 as select number as a, number as b from numbers(1) statement ok -create table t2 as select number as a, number as b from numbers(5) +create OR REPLACE table t2 as select number as a, number as b from numbers(5) statement error 1005 explain explain select t1.a from t1 where a > 0 @@ -247,9 +247,9 @@ FILE_FORMAT = ( SINGLE = false query T -explain syntax create table t3(a int64, b uint64, c float64, d string, e array(int32), f tuple(f1 bool, f2 string)) engine=fuse cluster by (a, b, c) comment='test' compression='LZ4' +explain syntax create OR REPLACE table t3(a int64, b uint64, c float64, d string, e array(int32), f tuple(f1 bool, f2 string)) engine=fuse cluster by (a, b, c) comment='test' compression='LZ4' ---- -CREATE TABLE t3 ( +CREATE OR REPLACE TABLE t3 ( a Int64, b UInt64, c Float64, @@ -409,9 +409,9 @@ explain syntax create database db1 engine=default CREATE DATABASE db1 ENGINE = DEFAULT query T -explain syntax create table t3(a int64, b uint64, c float64, d string, e array(int32), f tuple(f1 bool, f2 string)) engine=fuse cluster by (a, b, c) comment='test' compression='LZ4' +explain syntax create OR REPLACE table t3(a int64, b uint64, c float64, d string, e array(int32), f tuple(f1 bool, f2 string)) engine=fuse cluster by (a, b, c) comment='test' compression='LZ4' ---- -CREATE TABLE t3 ( +CREATE OR REPLACE TABLE t3 ( a Int64, b UInt64, c Float64, @@ -584,7 +584,7 @@ statement ok drop table if exists t3 statement ok -create table t3 as select number as a, number as b from numbers(10) +create OR REPLACE table t3 as select number as a, number as b from numbers(10) query T explain select * from t1,t2, t3 where (t1.a > 1 and t2.a > 2) or (t1.b < 3 and t2.b < 4) or t3.a = 2 @@ -750,7 +750,7 @@ Limit ├── push downs: [filters: [(t2.a (#2) > 2 OR t2.b (#3) < 4)], limit: NONE] └── estimated rows: 5.00 -query +query explain select * from t1,t2 where (t1.a > 1 or t1.b < 2) and (t1.a > 1 or t1.b < 2) ---- HashJoin @@ -785,7 +785,7 @@ HashJoin ├── push downs: [filters: [], limit: NONE] └── estimated rows: 5.00 -query +query explain select count(distinct a) from t1; ---- AggregateFinal @@ -817,7 +817,7 @@ AggregateFinal ├── push downs: [filters: [], limit: NONE] └── estimated rows: 1.00 -query +query explain select count_distinct(a) from t1; ---- AggregateFinal @@ -849,7 +849,7 @@ AggregateFinal ├── push downs: [filters: [], limit: NONE] └── estimated rows: 1.00 -query +query explain select * from (values(1, 'a'),(2, 'b')) t(c1,c2) ---- ConstantTableScan @@ -863,28 +863,28 @@ drop table t1 statement ok drop table t2 -query +query explain syntax select * from read_parquet('p1', 'p2', 'p3'); ---- SELECT * FROM read_parquet('p1', 'p2', 'p3') -query +query explain syntax select * from read_parquet(prune_page=>true, refresh_meta_cache=>true); ---- SELECT * FROM read_parquet(prune_page=>TRUE, refresh_meta_cache=>TRUE) -query +query explain syntax select * from read_parquet('p1', 'p2', 'p3', prune_page=>true, refresh_meta_cache=>true); ---- SELECT * FROM read_parquet('p1', 'p2', 'p3', prune_page=>TRUE, refresh_meta_cache=>TRUE) -query +query explain syntax select * from read_parquet('p1', 'p2', 'p3', prune_page=>true, refresh_meta_cache=>true); ---- SELECT * @@ -895,9 +895,9 @@ statement ok drop table if exists t4 statement ok -create table t4(a int, b string); +create OR REPLACE table t4(a int, b string); -query +query explain select * from t4 where a = 1 and try_cast(get(try_parse_json(b),'bb') as varchar) = 'xx'; ---- Filter @@ -920,7 +920,7 @@ drop view if exists v4 statement ok create view v4 as select a as a, try_cast(get(try_parse_json(b), 'bb') as varchar) as b from t4; -query +query explain select * from v4 where b = 'xx'; ---- EvalScalar @@ -969,10 +969,10 @@ statement ok drop table if exists b statement ok -create table a(id int, c1 INT NULL) +create OR REPLACE table a(id int, c1 INT NULL) statement ok -create table b(id int, c1 INT NULL) +create OR REPLACE table b(id int, c1 INT NULL) statement ok insert into a values(1, 1683648000) @@ -1095,10 +1095,10 @@ statement ok drop table if exists b statement ok -create table a(id int, c1 INT NULL) +create OR REPLACE table a(id int, c1 INT NULL) statement ok -create table b(id int, c1 INT NULL) +create OR REPLACE table b(id int, c1 INT NULL) statement ok insert into a values (1, 2), (2, 4), (3, 6) @@ -1149,7 +1149,7 @@ statement ok drop table b; statement ok -create table t1(a int, b int); +create OR REPLACE table t1(a int, b int); statement ok insert into t1 values(1, 2), (2, 3), (3, 4); @@ -1234,10 +1234,10 @@ statement ok drop table if exists t2; statement ok -CREATE TABLE t1(i int, j int); +CREATE OR REPLACE TABLE t1(i int, j int); statement ok -CREATE TABLE t2(k int, l int); +CREATE OR REPLACE TABLE t2(k int, l int); statement ok INSERT INTO t1 VALUES (1, 2), (2, 3), (3, 4); @@ -1293,7 +1293,7 @@ statement ok drop table if exists t3; statement ok -CREATE TABLE t3(a int, b map(string, string) null, c map(string, variant) null); +CREATE OR REPLACE TABLE t3(a int, b map(string, string) null, c map(string, variant) null); statement ok INSERT INTO t3 VALUES (1, {'k1':'a', 'k2':'b'}, {'k1':'"a"', 'k2':'100'}), (2, null, null), (3, {'k3':'z'}, {'k3':'"z"'}); @@ -1365,7 +1365,7 @@ statement ok drop table t3; statement ok -CREATE TABLE customers AS SELECT +CREATE OR REPLACE TABLE customers AS SELECT number % 100 AS customer_id, concat('Customer ', to_string(number % 100)) AS customer_name, CASE WHEN (rand() * 10000)::int % 3 = 0 THEN 'Small' @@ -1378,7 +1378,7 @@ FROM numbers(100); statement ok -CREATE TABLE products AS SELECT +CREATE OR REPLACE TABLE products AS SELECT number % 10 AS product_id, concat('Product ', to_string(number % 10)) AS product_name, (rand() * 10000 % 2000 * 0.01)::decimal(10, 2) AS price, @@ -1390,7 +1390,7 @@ CREATE TABLE products AS SELECT FROM numbers(10); statement ok -CREATE TABLE sales AS SELECT +CREATE OR REPLACE TABLE sales AS SELECT number % 500 AS sale_id, number % 100 AS product_id, number % 100 AS customer_id, @@ -1452,10 +1452,10 @@ statement ok drop table sales; statement ok -create table t1 (a int); +create OR REPLACE table t1 (a int); statement ok -create table t2 (b int); +create OR REPLACE table t2 (b int); query T explain select date from (select *, 'year' as date from t1 left join t2 on t1.a = t2.b) where date = ''; @@ -1506,10 +1506,10 @@ statement ok drop table t2; statement ok -create table t1(a int, b int, c varchar(20)); +create OR REPLACE table t1(a int, b int, c varchar(20)); statement ok -create table t2(a int, b int, c varchar(20)); +create OR REPLACE table t2(a int, b int, c varchar(20)); # scalar subquery and sort plan contains count() agg function. query T diff --git a/tests/sqllogictests/suites/mode/standalone/limit.test b/tests/sqllogictests/suites/mode/standalone/limit.test index 656b21f66658..35eafedb7fa5 100644 --- a/tests/sqllogictests/suites/mode/standalone/limit.test +++ b/tests/sqllogictests/suites/mode/standalone/limit.test @@ -2,10 +2,10 @@ # Both results are reasonable statement ok -create table t1 as select number as a from numbers(10); +create or replace table t1 as select number as a from numbers(10); statement ok -create table t2 as select number as b from numbers(100); +create or replace table t2 as select number as b from numbers(100); query I rowsort select * from t1 left join t2 on t1.a = t2.b limit 10 offset 5; @@ -17,10 +17,7 @@ select * from t1 left join t2 on t1.a = t2.b limit 10 offset 5; 9 9 statement ok -drop table if exists t; - -statement ok -create table t(id int, top int); +create or replace table t(id int, top int); statement ok insert into t values(1,10),(2,20),(3,30); diff --git a/tests/sqllogictests/suites/mode/standalone/pr15804.test b/tests/sqllogictests/suites/mode/standalone/pr15804.test index 42c5c5cb56b3..a4e21d5e2492 100644 --- a/tests/sqllogictests/suites/mode/standalone/pr15804.test +++ b/tests/sqllogictests/suites/mode/standalone/pr15804.test @@ -1,8 +1,13 @@ statement ok -create or replace table t2(c varchar); +create or replace TRANSIENT table t2(c varchar) ; statement ok -insert into t2 select repeat('a', 1000000) from numbers(3000); +insert into t2 select repeat('a', 1000000) from numbers(300); + +query R +select avg(length(c)) from t2 ; +---- +1000000.0 statement ok -select * from t2 ignore_result; \ No newline at end of file +drop table t2; diff --git a/tests/sqllogictests/suites/query/functions/02_0005_function_compare.test b/tests/sqllogictests/suites/query/functions/02_0005_function_compare.test index 92558ba55507..c09a7123b538 100644 --- a/tests/sqllogictests/suites/query/functions/02_0005_function_compare.test +++ b/tests/sqllogictests/suites/query/functions/02_0005_function_compare.test @@ -72,6 +72,16 @@ select '\%' not like '\%' ---- 1 +query B +select ('PROMO' || number::string) a, a like 'PROMO%', + a < 'PROMO1', a > 'PROMO1', + a <= 'PROMO1' , a >= 'PROMO1', + a = 'PROMO1' from numbers(3) order by a +---- +PROMO0 1 1 0 1 0 0 +PROMO1 1 0 0 1 1 1 +PROMO2 1 0 1 0 1 0 + statement ok select * from numbers(10) where null = true diff --git a/tests/sqllogictests/suites/query/join/runtime_filter.test b/tests/sqllogictests/suites/query/join/runtime_filter.test index bdb84d3eb936..e8165beb6125 100644 --- a/tests/sqllogictests/suites/query/join/runtime_filter.test +++ b/tests/sqllogictests/suites/query/join/runtime_filter.test @@ -1,12 +1,12 @@ statement ok -CREATE TABLE table1 ( +CREATE OR REPLACE TABLE table1 ( key1 String, key2 String, key3 String ); statement ok -CREATE TABLE table2 ( +CREATE OR REPLACE TABLE table2 ( key1 String, key2 String, key3 String diff --git a/tests/sqllogictests/suites/stage/formats/parquet/options/null_if.test b/tests/sqllogictests/suites/stage/formats/parquet/options/null_if.test index d6acbbda65a1..65ab8a82a226 100644 --- a/tests/sqllogictests/suites/stage/formats/parquet/options/null_if.test +++ b/tests/sqllogictests/suites/stage/formats/parquet/options/null_if.test @@ -26,7 +26,7 @@ remove @data/unload/parquet/null_if/ query copy into @data/unload/parquet/null_if from string ---- -3 40 365 +3 56 365 statement ok drop file format if exists parquet_null_if diff --git a/tests/suites/0_stateless/20+_others/20_0014_sort_spill.result b/tests/suites/0_stateless/20+_others/20_0014_sort_spill.result index 0153132c653c..9a551562af15 100644 --- a/tests/suites/0_stateless/20+_others/20_0014_sort_spill.result +++ b/tests/suites/0_stateless/20+_others/20_0014_sort_spill.result @@ -2,9 +2,9 @@ 0 1 NULL -=================== +==Test if the spill is activated== 2 -=================== +==Enable sort_spilling_bytes_threshold_per_proc== 0 1 NULL @@ -14,12 +14,12 @@ NULL 3 7 2 8 1 9 -=================== +==Test abc== one Two 4 1 -=================== +==Test xy== 2 NULL 2 5 NULL 6 @@ -32,9 +32,9 @@ NULL 6 4 8 NULL 6 2 5 -=================== +==Test a== 16 -=================== +==Test b== 2 NULL 2 5 4 8 @@ -56,11 +56,11 @@ NULL NULL 2 NULL 4 8 ==TEST TOP-N SORT== -=================== +==Test c== 0 -=================== +==Test d== 1 -=================== +==Test e== 2 NULL 2 5 4 8 @@ -70,5 +70,5 @@ NULL NULL NULL 6 NULL NULL 2 5 -=================== +==Test f== 9 diff --git a/tests/suites/0_stateless/20+_others/20_0014_sort_spill.sql b/tests/suites/0_stateless/20+_others/20_0014_sort_spill.sql index da0b0fedce54..2fb47bcee8fe 100644 --- a/tests/suites/0_stateless/20+_others/20_0014_sort_spill.sql +++ b/tests/suites/0_stateless/20+_others/20_0014_sort_spill.sql @@ -14,21 +14,21 @@ SELECT c FROM t ORDER BY c; INSERT INTO temp_files_count SELECT COUNT() as count, 2 as number FROM system.temp_files; -SELECT '==================='; +SELECT '==Test if the spill is activated=='; -- Test if the spill is activated. set sort_spilling_bytes_threshold_per_proc = 0; SELECT any_if(count, number = 2) - any_if(count, number = 1) FROM temp_files_count; set sort_spilling_bytes_threshold_per_proc = 8; -SELECT '==================='; +SELECT '==Enable sort_spilling_bytes_threshold_per_proc=='; INSERT INTO temp_files_count SELECT COUNT() as count, 3 as number FROM system.temp_files; SELECT c FROM t ORDER BY c; SELECT c FROM t ORDER BY c DESC; SELECT a, b FROM t ORDER BY b; -SELECT '==================='; +SELECT '==Test abc=='; drop table if exists abc; CREATE TABLE abc ( a INT, b INT, c INT, d VARCHAR); @@ -36,7 +36,7 @@ INSERT INTO abc VALUES (1, 2, 3, 'one'), (4, 5, 6, 'Two'); SELECT d FROM abc ORDER BY lower(d); SELECT a FROM abc ORDER BY a DESC; -SELECT '==================='; +SELECT '==Test xy=='; drop table if exists xy; CREATE TABLE xy(x INT NULL, y INT NULL); @@ -46,13 +46,13 @@ SELECT x, y FROM xy ORDER BY y NULLS LAST; SELECT x, y FROM xy ORDER BY y DESC NULLS FIRST; INSERT INTO temp_files_count SELECT COUNT() as count, 4 as number FROM system.temp_files; -SELECT '==================='; +SELECT '==Test a=='; set sort_spilling_bytes_threshold_per_proc = 0; SELECT any_if(count, number = 4) - any_if(count, number = 3) FROM temp_files_count; set sort_spilling_bytes_threshold_per_proc = 8; -SELECT '==================='; +SELECT '==Test b=='; -- Test single thread set max_threads = 1; @@ -70,11 +70,11 @@ SELECT '==TEST TOP-N SORT=='; INSERT INTO temp_files_count SELECT COUNT() as count, 5 as number FROM system.temp_files; -SELECT '==================='; +SELECT '==Test c=='; SELECT c FROM t ORDER BY c limit 1; -SELECT '==================='; +SELECT '==Test d=='; INSERT INTO temp_files_count SELECT COUNT() as count, 6 as number FROM system.temp_files; @@ -82,7 +82,7 @@ set sort_spilling_bytes_threshold_per_proc = 0; SELECT any_if(count, number = 6) - any_if(count, number = 5) FROM temp_files_count; set sort_spilling_bytes_threshold_per_proc = 60; -SELECT '==================='; +SELECT '==Test e=='; INSERT INTO temp_files_count SELECT COUNT() as count, 7 as number FROM system.temp_files; @@ -90,7 +90,8 @@ SELECT x, y FROM xy ORDER BY x, y DESC NULLS FIRST LIMIT 3; SELECT x, y FROM xy ORDER BY x NULLS LAST, y DESC NULLS FIRST LIMIT 3; SELECT x, y FROM xy ORDER BY x NULLS FIRST, y DESC NULLS LAST LIMIT 3; -SELECT '==================='; +SELECT '==Test f=='; + INSERT INTO temp_files_count SELECT COUNT() as count, 8 as number FROM system.temp_files; unset max_vacuum_temp_files_after_query; diff --git a/tests/suites/1_stateful/00_stage/00_0001_copy_into_stage.result b/tests/suites/1_stateful/00_stage/00_0001_copy_into_stage.result old mode 100755 new mode 100644 index 45e99d16c182..453a0eb871bf --- a/tests/suites/1_stateful/00_stage/00_0001_copy_into_stage.result +++ b/tests/suites/1_stateful/00_stage/00_0001_copy_into_stage.result @@ -1,4 +1,4 @@ 20 160 160 -20 450 726 +20 530 726 2 20 160 160 diff --git a/tests/suites/1_stateful/05_formats/05_05_parquet/05_05_01_parquet_load_unload.result b/tests/suites/1_stateful/05_formats/05_05_parquet/05_05_01_parquet_load_unload.result old mode 100755 new mode 100644 index 2db90c1b0314..1e08a537ba24 --- a/tests/suites/1_stateful/05_formats/05_05_parquet/05_05_01_parquet_load_unload.result +++ b/tests/suites/1_stateful/05_formats/05_05_parquet/05_05_01_parquet_load_unload.result @@ -28,14 +28,14 @@ a"b 1 ['a"b'] {"k":"v"} 2044-05-06 10:25:02.868894 10.01 ('a',5) ['{"k":"v"}'] [ NULL 2 ['a'b'] [1] 2044-05-06 10:25:02.868894 -10.01 ('b',10) ['[1]'] [('b',10)] <<<< >>>> copy into @s1/unload1/ from test_load_unload -2 362 2703 +2 390 2703 >>>> truncate table test_load_unload >>>> copy into test_load_unload from @s1/unload1.parquet force=true; unload1.parquet 2 0 NULL NULL begin diff select end diff >>>> copy into @s1/unload2/ from test_load_unload -2 362 2703 +2 390 2703 begin diff parquet end diff >>>> truncate table test_load_unload diff --git a/tests/suites/1_stateful/07_stage_attachment/07_0000_insert_with_stage.result b/tests/suites/1_stateful/07_stage_attachment/07_0000_insert_with_stage.result index 3ab5526ac4ad..2ec060c70b1b 100755 --- a/tests/suites/1_stateful/07_stage_attachment/07_0000_insert_with_stage.result +++ b/tests/suites/1_stateful/07_stage_attachment/07_0000_insert_with_stage.result @@ -1,7 +1,7 @@ sample.csv Succeeded 96 -168 +125 null 1 'Beijing' 100 China 2 'Shanghai' 80 China diff --git a/tests/suites/1_stateful/07_stage_attachment/07_0001_replace_with_stage.result b/tests/suites/1_stateful/07_stage_attachment/07_0001_replace_with_stage.result index 22af85296075..98cb421d64e0 100644 --- a/tests/suites/1_stateful/07_stage_attachment/07_0001_replace_with_stage.result +++ b/tests/suites/1_stateful/07_stage_attachment/07_0001_replace_with_stage.result @@ -1,6 +1,6 @@ sample.csv 96 -168 +125 null 1 'Beijing' 100 China 2 'Shanghai' 80 China diff --git a/tests/suites/1_stateful/07_stage_attachment/07_0002_insert_with_stage_deduplicate.result b/tests/suites/1_stateful/07_stage_attachment/07_0002_insert_with_stage_deduplicate.result index fb3372f5e36a..3d6fc1564c68 100644 --- a/tests/suites/1_stateful/07_stage_attachment/07_0002_insert_with_stage_deduplicate.result +++ b/tests/suites/1_stateful/07_stage_attachment/07_0002_insert_with_stage_deduplicate.result @@ -1,6 +1,6 @@ sample.csv 96 -168 +125 null 1 'Beijing' 100 China 2 'Shanghai' 80 China diff --git a/tests/suites/1_stateful/07_stage_attachment/07_0003_insert_with_stage_file_format.result b/tests/suites/1_stateful/07_stage_attachment/07_0003_insert_with_stage_file_format.result index 967e7f28fcd8..03cfbcde73a9 100644 --- a/tests/suites/1_stateful/07_stage_attachment/07_0003_insert_with_stage_file_format.result +++ b/tests/suites/1_stateful/07_stage_attachment/07_0003_insert_with_stage_file_format.result @@ -7,12 +7,12 @@ <<<< Succeeded 20 -71 +51 null >>>> list @s1 <<<< ->>>> select a is null, b is null, c, d from t1 -true false NULL +>>>> select a is null, b is null, c, d = '' from t1 +true false NULL true <<<< >>>> drop table if exists t1 >>>> drop stage if exists s1 diff --git a/tests/suites/1_stateful/07_stage_attachment/07_0003_insert_with_stage_file_format.sh b/tests/suites/1_stateful/07_stage_attachment/07_0003_insert_with_stage_file_format.sh index abe82c6ec3b8..dfdccf3e0c90 100755 --- a/tests/suites/1_stateful/07_stage_attachment/07_0003_insert_with_stage_file_format.sh +++ b/tests/suites/1_stateful/07_stage_attachment/07_0003_insert_with_stage_file_format.sh @@ -15,7 +15,7 @@ curl -s -u root: -XPOST "http://localhost:8000/v1/query" --header 'Content-Type: query "list @s1" -query "select a is null, b is null, c, d from t1" +query "select a is null, b is null, c, d = '' from t1" stmt "drop table if exists t1" stmt "drop stage if exists s1" diff --git a/tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_00_basic.result b/tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_00_basic.result old mode 100755 new mode 100644 index 52714d800bab..069a4e19ea94 --- a/tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_00_basic.result +++ b/tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_00_basic.result @@ -1,8 +1,8 @@ --- named internal stage -2 45 699 +2 53 699 1 2 3 4 5 6 -2 45 699 +2 53 699 --- external stage 1 2 3 4 5 6 From 7e42582b047c0f8b605ec65a430a38f9a388912d Mon Sep 17 00:00:00 2001 From: zhya Date: Fri, 8 Nov 2024 13:59:45 +0800 Subject: [PATCH 10/92] fix: clustering_information panic if string domain is none (#16792) * fix: clustering_information panic if string domain is none * make lint --------- Co-authored-by: Winter Zhang --- .../table_functions/clustering_information.rs | 9 ++++++- ..._func_clustering_information_function.test | 25 ++++++++++++++++++- .../09_http_handler/09_0007_token.py | 9 ++++--- .../09_http_handler/09_0009_cookie.py | 16 ++++++------ 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/query/storages/fuse/src/table_functions/clustering_information.rs b/src/query/storages/fuse/src/table_functions/clustering_information.rs index 879c7964401e..a624d81ffabe 100644 --- a/src/query/storages/fuse/src/table_functions/clustering_information.rs +++ b/src/query/storages/fuse/src/table_functions/clustering_information.rs @@ -296,7 +296,14 @@ impl<'a> ClusteringInformation<'a> { let (keys, values): (Vec<_>, Vec<_>) = points_map.into_iter().unzip(); let cluster_key_types = exprs .into_iter() - .map(|v| v.data_type().clone()) + .map(|v| { + let data_type = v.data_type(); + if matches!(*data_type, DataType::String) { + data_type.wrap_nullable() + } else { + data_type.clone() + } + }) .collect::>(); let indices = compare_scalars(keys, &cluster_key_types)?; for idx in indices.into_iter() { diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0014_func_clustering_information_function.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0014_func_clustering_information_function.test index cbf38e6f59b6..49f9ad3d1e4c 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0014_func_clustering_information_function.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0014_func_clustering_information_function.test @@ -86,4 +86,27 @@ select * exclude(timestamp) from clustering_information('default','t09_0014_1', (SUBSTRING(c FROM 2 FOR 2)) linear 2 0 1.0 2.0 {"00002":2} statement ok -drop table t09_0014_1 +drop table t09_0014_1 all + +##issue https://github.com/databendlabs/databend/issues/16763 +statement ok +create table t09_0014_2(c varchar not null) cluster by(substring(c FROM 1 FOR 4)) + +statement ok +insert into t09_0014_2 values('abcde'),('bcdef') + +statement ok +insert into t09_0014_2 values('bcdff'),('cdefg') + +query TTIIFFT +select * exclude(timestamp) from clustering_information('default','t09_0014_2') +---- +(SUBSTRING(c FROM 1 FOR 4)) linear 2 0 0.0 1.0 {"00001":2} + +query TTIIFFT +select * exclude(timestamp) from clustering_information('default','t09_0014_2', 'substr(c,2,4)') +---- +(SUBSTRING(c FROM 2 FOR 4)) linear 2 0 1.0 2.0 {"00002":2} + +statement ok +drop table t09_0014_2 all diff --git a/tests/suites/1_stateful/09_http_handler/09_0007_token.py b/tests/suites/1_stateful/09_http_handler/09_0007_token.py index 25bb875d3cb4..8a8bcfb4a8e4 100755 --- a/tests/suites/1_stateful/09_http_handler/09_0007_token.py +++ b/tests/suites/1_stateful/09_http_handler/09_0007_token.py @@ -133,10 +133,10 @@ def fake_expired_token(ty): "sid": "", } return ( - "bend-v1-" - + ty - + "-" - + base64.b64encode(json.dumps(expired_claim).encode("utf-8")).decode("utf-8") + "bend-v1-" + + ty + + "-" + + base64.b64encode(json.dumps(expired_claim).encode("utf-8")).decode("utf-8") ) @@ -207,6 +207,7 @@ def main(): if __name__ == "__main__": import logging + try: main() except Exception as e: diff --git a/tests/suites/1_stateful/09_http_handler/09_0009_cookie.py b/tests/suites/1_stateful/09_http_handler/09_0009_cookie.py index 1a424f450e0a..910f0c1cbe2b 100755 --- a/tests/suites/1_stateful/09_http_handler/09_0009_cookie.py +++ b/tests/suites/1_stateful/09_http_handler/09_0009_cookie.py @@ -8,13 +8,14 @@ auth = ("root", "") logging.basicConfig(level=logging.ERROR, format="%(asctime)s %(levelname)s %(message)s") + class GlobalCookieJar(RequestsCookieJar): def __init__(self): super().__init__() def set_cookie(self, cookie: Cookie, *args, **kwargs): - cookie.domain = '' - cookie.path = '/' + cookie.domain = "" + cookie.path = "/" super().set_cookie(cookie, *args, **kwargs) def get_dict(self, domain=None, path=None): @@ -22,11 +23,11 @@ def get_dict(self, domain=None, path=None): return {cookie.name: cookie.value for cookie in self} -def do_query(session_client, query, session_state=None ): +def do_query(session_client, query, session_state=None): url = f"http://localhost:8000/v1/query" query_payload = { "sql": query, - "pagination": {"wait_time_secs": 100, 'max_rows_per_page': 2}, + "pagination": {"wait_time_secs": 100, "max_rows_per_page": 2}, } if session_state: query_payload["session"] = session_state @@ -47,10 +48,10 @@ def test_simple(): assert resp.status_code == 200, resp.text assert resp.json()["data"] == [["1"]], resp.text sid = client.cookies.get("session_id") - #print(sid) + # print(sid) last_access_time1 = int(client.cookies.get("last_access_time")) - #print(last_access_time1) + # print(last_access_time1) assert time.time() - 10 < last_access_time1 < time.time() time.sleep(1.5) @@ -83,7 +84,7 @@ def test_temp_table(): resp = do_query(client, "select * from t1", session_state) assert resp.status_code == 200, resp.text - assert resp.json()['data'] == [["3"], ["4"]] + assert resp.json()["data"] == [["3"], ["4"]] session_state = resp.json()["session"] assert session_state["need_sticky"], resp.text assert session_state["need_keep_alive"] @@ -99,6 +100,7 @@ def main(): test_simple() test_temp_table() + if __name__ == "__main__": try: main() From 21f3bab31442790ffbb52355cd0936cd49bf994d Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 8 Nov 2024 14:39:40 +0800 Subject: [PATCH 11/92] chore: add snapshot location cache for attached table (#16795) To avoid redundant extraction of snapshot location from hint file, a snapshot location cache is added at the table instance level for tables that are attached to another table. --- src/query/storages/fuse/src/fuse_table.rs | 35 ++++++++++++++++------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/query/storages/fuse/src/fuse_table.rs b/src/query/storages/fuse/src/fuse_table.rs index c2bbdc4d7755..8706a8f413ec 100644 --- a/src/query/storages/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/src/fuse_table.rs @@ -23,6 +23,7 @@ use std::sync::Arc; use chrono::Duration; use chrono::TimeDelta; +use databend_common_base::base::tokio; use databend_common_catalog::catalog::StorageDescription; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::PartStatistics; @@ -130,8 +131,11 @@ pub struct FuseTable { table_type: FuseTableType, - // If this is set, reading from fuse_table should only returns the increment blocks + // If this is set, reading from fuse_table should only return the increment blocks pub(crate) changes_desc: Option, + + // A table instance level cache of snapshot_location, if this table is attaching to someone else. + attached_table_location: tokio::sync::OnceCell, } impl FuseTable { @@ -238,6 +242,7 @@ impl FuseTable { table_compression: table_compression.as_str().try_into()?, table_type, changes_desc: None, + attached_table_location: Default::default(), })) } @@ -368,15 +373,25 @@ impl FuseTable { let options = self.table_info.options(); if let Some(storage_prefix) = options.get(OPT_KEY_STORAGE_PREFIX) { - // if table is attached, parse snapshot location from hint file - let hint = format!("{}/{}", storage_prefix, FUSE_TBL_LAST_SNAPSHOT_HINT); - let snapshot_loc = { - let hint_content = self.operator.read(&hint).await?.to_vec(); - let snapshot_full_path = String::from_utf8(hint_content)?; - let operator_info = self.operator.info(); - snapshot_full_path[operator_info.root().len()..].to_string() - }; - Ok(Some(snapshot_loc)) + // If the table is attaching to someone else, + // parse the snapshot location from the hint file. + // + // The snapshot location is allowed + // to be fetched from the table level instance cache. + let snapshot_location = self + .attached_table_location + .get_or_try_init(|| async { + let hint = + format!("{}/{}", storage_prefix, FUSE_TBL_LAST_SNAPSHOT_HINT); + let hint_content = self.operator.read(&hint).await?.to_vec(); + let snapshot_full_path = String::from_utf8(hint_content)?; + let operator_info = self.operator.info(); + Ok::<_, ErrorCode>( + snapshot_full_path[operator_info.root().len()..].to_string(), + ) + }) + .await?; + Ok(Some(snapshot_location.to_owned())) } else { Ok(options .get(OPT_KEY_SNAPSHOT_LOCATION) From 3d76dc68e79db72341c6006808a6e1773ab8b565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Sun, 10 Nov 2024 09:51:02 +0800 Subject: [PATCH 12/92] feat: Implement WAL-based RaftLog storage (#16776) * feat: Implement WAL-based RaftLog storage Replace sled-based storage with a dedicated WAL implementation optimized for Raft Log operations. The new implementation provides: Performance improvements: - Non-blocking batched fdatasync - FILO caching for latest logs (configurable size/count) - ~0.5ms write latency Compatibility: - Backward compatible with existing data format - Auto-upgrades from V003 to V004 format on startup - Supports rolling upgrades (no protocol changes) - databend-metactl supports both V003/V004 import/export Technical details: - New storage format: V004 - Optimized specifically for Raft Log operations - Preserves all existing functionality * refactor: OnDisk ensures dir * refactor: V004 exports more compacted data format * refactor: try not to init sled db if not needed Because sled db is not used by meta-service anymore. It is only used when import old format data. import and export the data to normalize into the latest format to compare. * chore: update version info for V003, fix meta-meta protocol test * chore: reduce startup messages * chore: do not panic when using sled db if it is already initialized as temp * chore: refine startup message --- Cargo.lock | 116 +++-- Cargo.toml | 4 +- src/meta/binaries/Cargo.toml | 1 + src/meta/binaries/meta/ee_main.rs | 1 + src/meta/binaries/meta/entry.rs | 63 ++- src/meta/binaries/meta/oss_main.rs | 2 + src/meta/binaries/metactl/export_from_disk.rs | 2 +- src/meta/binaries/metactl/import.rs | 25 +- src/meta/binaries/metactl/import_v004.rs | 114 +++++ src/meta/binaries/metactl/main.rs | 5 +- src/meta/binaries/metactl/upgrade.rs | 5 +- src/meta/process/src/kv_processor.rs | 10 + src/meta/process/src/process_meta_dir.rs | 7 +- src/meta/raft-store/Cargo.toml | 4 + src/meta/raft-store/src/config.rs | 42 ++ src/meta/raft-store/src/key_spaces.rs | 59 +++ src/meta/raft-store/src/lib.rs | 2 +- src/meta/raft-store/src/log/raft_log.rs | 116 ----- .../raft-store/src/ondisk/data_version.rs | 13 +- src/meta/raft-store/src/ondisk/header.rs | 30 +- src/meta/raft-store/src/ondisk/mod.rs | 275 ++++++------ .../raft-store/src/ondisk/upgrade_to_v003.rs | 12 +- .../raft-store/src/ondisk/upgrade_to_v004.rs | 221 ++++++++++ .../raft-store/src/ondisk/version_info.rs | 16 +- .../raft-store/src/raft_log_v004/callback.rs | 74 ++++ .../mod.rs => raft_log_v004/callback_data.rs} | 13 +- .../src/raft_log_v004/codec_wrapper.rs | 122 ++++++ .../raft-store/src/raft_log_v004/importer.rs | 119 ++++++ .../raft-store/src/raft_log_v004/io_desc.rs | 116 +++++ .../raft-store/src/raft_log_v004/io_phase.rs | 39 ++ .../src/raft_log_v004/log_store_meta.rs | 48 +++ src/meta/raft-store/src/raft_log_v004/mod.rs | 39 ++ .../src/raft_log_v004/raft_log_io_error.rs | 31 ++ .../src/raft_log_v004/raft_log_types.rs | 48 +++ src/meta/raft-store/src/raft_log_v004/util.rs | 32 ++ src/meta/raft-store/src/sm_v003/adapter.rs | 32 +- src/meta/raft-store/src/sm_v003/mod.rs | 1 + .../src/sm_v003/snapshot_store_v003.rs | 35 +- src/meta/raft-store/src/snapshot_config.rs | 7 + src/meta/raft-store/src/state/mod.rs | 6 +- src/meta/raft-store/src/state/raft_state.rs | 163 -------- .../raft-store/src/state/raft_state_kv.rs | 23 + .../raft-store/src/state_machine/log_meta.rs | 8 + src/meta/raft-store/src/state_machine/sm.rs | 4 +- .../raft-store/src/state_machine/testing.rs | 4 + src/meta/raft-store/tests/it/log.rs | 249 ----------- src/meta/raft-store/tests/it/main.rs | 2 - src/meta/raft-store/tests/it/state.rs | 122 ------ src/meta/raft-store/tests/it/testing.rs | 4 - src/meta/service/Cargo.toml | 3 +- src/meta/service/src/configs/outer_v0.rs | 46 ++ src/meta/service/src/export.rs | 32 -- src/meta/service/src/lib.rs | 1 - .../service/src/meta_service/meta_node.rs | 60 ++- .../src/meta_service/raft_service_impl.rs | 4 +- .../src/meta_service/snapshot_receiver_v1.rs | 4 +- .../src/store/raft_log_storage_impl.rs | 395 +++++++----------- .../src/store/raft_state_machine_impl.rs | 6 +- src/meta/service/src/store/store.rs | 8 +- src/meta/service/src/store/store_inner.rs | 219 +++++----- .../service/tests/it/grpc/metasrv_grpc_api.rs | 1 + .../tests/it/grpc/metasrv_grpc_export.rs | 22 +- .../tests/it/meta_node/meta_node_lifecycle.rs | 87 ++-- .../it/meta_node/meta_node_replication.rs | 4 +- src/meta/service/tests/it/store.rs | 20 +- src/meta/service/tests/it/tests/meta_node.rs | 2 +- src/meta/service/tests/it/tests/service.rs | 13 +- src/meta/sled-store/src/db.rs | 18 +- src/meta/sled-store/src/lib.rs | 2 + src/meta/stoerr/src/meta_storage_errors.rs | 6 + src/meta/types/src/raft_types.rs | 3 + tests/compat/meta_meta/test_meta_meta.sh | 65 ++- tests/metactl/.gitignore | 1 + tests/metactl/meta_v003.txt | 3 +- tests/metactl/meta_v004.txt | 167 ++++++++ tests/metactl/test-metactl.sh | 19 +- tests/metactl/want_exported_v003 | 167 -------- tests/metactl/want_exported_v004 | 167 ++++++++ ...{want_snapshot_v003 => want_snapshot_v004} | Bin 79 files changed, 2414 insertions(+), 1617 deletions(-) create mode 100644 src/meta/binaries/metactl/import_v004.rs delete mode 100644 src/meta/raft-store/src/log/raft_log.rs create mode 100644 src/meta/raft-store/src/ondisk/upgrade_to_v004.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/callback.rs rename src/meta/raft-store/src/{log/mod.rs => raft_log_v004/callback_data.rs} (67%) create mode 100644 src/meta/raft-store/src/raft_log_v004/codec_wrapper.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/importer.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/io_desc.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/io_phase.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/log_store_meta.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/mod.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/raft_log_io_error.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/raft_log_types.rs create mode 100644 src/meta/raft-store/src/raft_log_v004/util.rs delete mode 100644 src/meta/raft-store/src/state/raft_state.rs delete mode 100644 src/meta/raft-store/tests/it/log.rs delete mode 100644 src/meta/raft-store/tests/it/state.rs delete mode 100644 src/meta/service/src/export.rs create mode 100644 tests/metactl/meta_v004.txt delete mode 100644 tests/metactl/want_exported_v003 create mode 100644 tests/metactl/want_exported_v004 rename tests/metactl/{want_snapshot_v003 => want_snapshot_v004} (100%) diff --git a/Cargo.lock b/Cargo.lock index f4c02090d96f..29704372018b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,9 +219,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" dependencies = [ "backtrace", ] @@ -1662,6 +1662,17 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "byte-unit" +version = "5.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cd29c3c585209b0cbc7309bfe3ed7efd8c84c21b7af29c8bfae908f8777174" +dependencies = [ + "rust_decimal", + "serde", + "utf8-width", +] + [[package]] name = "bytecheck" version = "0.6.12" @@ -2098,9 +2109,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.15" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -2108,15 +2119,15 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", "clap_lex", "strsim 0.11.1", - "terminal_size 0.3.0", + "terminal_size 0.4.0", ] [[package]] @@ -2162,9 +2173,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -2234,6 +2245,19 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" +[[package]] +name = "codeq" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93a9092f38f07d18434ac49a148db55fdd887b8b852a4655bece6e2f347a8490" +dependencies = [ + "anyhow", + "byteorder", + "crc32fast", + "derive_more", + "serde", +] + [[package]] name = "color-eyre" version = "0.6.2" @@ -3620,7 +3644,7 @@ dependencies = [ "databend-common-io", "databend-common-meta-kvapi", "databend-common-meta-types", - "derive_more 1.0.0", + "derive_more", "enumflags2", "hex", "itertools 0.13.0", @@ -3656,7 +3680,7 @@ dependencies = [ "databend-common-meta-types", "databend-common-metrics", "databend-common-tracing", - "derive_more 1.0.0", + "derive_more", "fastrace", "futures", "itertools 0.13.0", @@ -3745,8 +3769,10 @@ dependencies = [ "databend-common-meta-stoerr", "databend-common-meta-types", "databend-common-tracing", - "derive_more 1.0.0", + "deepsize", + "derive_more", "fastrace", + "fs_extra", "futures", "futures-async-stream", "futures-util", @@ -3758,6 +3784,8 @@ dependencies = [ "openraft", "ordq", "pretty_assertions", + "raft-log", + "rmp-serde", "rotbl", "semver", "serde", @@ -3830,7 +3858,7 @@ dependencies = [ "databend-common-meta-stoerr", "databend-common-tracing", "deepsize", - "derive_more 1.0.0", + "derive_more", "futures-util", "log", "num-derive", @@ -4952,7 +4980,7 @@ dependencies = [ "databend-common-metrics", "databend-common-tracing", "deepsize", - "derive_more 1.0.0", + "derive_more", "env_logger 0.11.5", "fastrace", "feature-set", @@ -4967,13 +4995,13 @@ dependencies = [ "pretty_assertions", "prometheus-client", "prost", + "raft-log", "regex", "reqwest", "semver", "serde", "serde_json", "serfig", - "sled", "temp-env", "tempfile", "test-harness", @@ -5006,6 +5034,7 @@ dependencies = [ "fastrace", "futures", "log", + "raft-log", "rand", "reqwest", "serde", @@ -5038,7 +5067,7 @@ dependencies = [ "base64 0.22.1", "buf-list", "bumpalo", - "byte-unit", + "byte-unit 4.0.19", "byteorder", "bytes", "chrono", @@ -5667,19 +5696,6 @@ dependencies = [ "syn 2.0.58", ] -[[package]] -name = "derive_more" -version = "0.99.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version", - "syn 2.0.58", -] - [[package]] name = "derive_more" version = "1.0.0" @@ -6588,6 +6604,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -10513,13 +10535,13 @@ dependencies = [ [[package]] name = "openraft" version = "0.10.0" -source = "git+https://github.com/databendlabs/openraft?tag=v0.10.0-alpha.6#c6f4a779e0bb563788c4187381065162e76ea996" +source = "git+https://github.com/databendlabs/openraft?tag=v0.10.0-alpha.7#e99cfebbc7a485fe55efe476c083d9decc888ef7" dependencies = [ "anyerror", - "byte-unit", + "byte-unit 5.1.6", "chrono", "clap", - "derive_more 0.99.18", + "derive_more", "futures", "maplit", "openraft-macros", @@ -10535,7 +10557,7 @@ dependencies = [ [[package]] name = "openraft-macros" version = "0.10.0" -source = "git+https://github.com/databendlabs/openraft?tag=v0.10.0-alpha.6#c6f4a779e0bb563788c4187381065162e76ea996" +source = "git+https://github.com/databendlabs/openraft?tag=v0.10.0-alpha.7#e99cfebbc7a485fe55efe476c083d9decc888ef7" dependencies = [ "chrono", "proc-macro2", @@ -12026,6 +12048,20 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "raft-log" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5e7f06d77e694fc984ba36494c091169d9c6695ce2af174e8f1e167a05f588" +dependencies = [ + "byteorder", + "clap", + "codeq", + "fs2", + "log", + "thiserror", +] + [[package]] name = "rand" version = "0.8.5" @@ -14436,12 +14472,12 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +checksum = "4f599bd7ca042cfdf8f4512b277c02ba102247820f9d9d4a9f521f496751a6ef" dependencies = [ "rustix 0.38.37", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -14463,18 +14499,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index b6ce2ec48e71..61c6b9e3b9c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -290,6 +290,7 @@ flagset = "0.4" flatbuffers = "24" # Must use the same version with arrow-ipc flate2 = "1" foreign_vec = "0.1.0" +fs_extra = "1.3.0" futures = "0.3.24" futures-async-stream = { version = "0.2.7" } futures-util = "0.3.24" @@ -409,6 +410,7 @@ prost = { version = "0.13" } prost-build = { version = "0.13" } prqlc = "0.11.3" quanta = "0.11.1" +raft-log = { version = "0.2.2" } rand = { version = "0.8.5", features = ["small_rng"] } rayon = "1.9.0" recursive = "0.1.1" @@ -618,7 +620,7 @@ deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "3038c145" } ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" } jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "ada713c" } openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" } -openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.6" } +openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.7" } orc-rust = { git = "https://github.com/datafusion-contrib/orc-rust", rev = "dfb1ede" } recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" } sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" } diff --git a/src/meta/binaries/Cargo.toml b/src/meta/binaries/Cargo.toml index db188e51d6be..66e746fd7b6b 100644 --- a/src/meta/binaries/Cargo.toml +++ b/src/meta/binaries/Cargo.toml @@ -43,6 +43,7 @@ databend-meta = { workspace = true } fastrace = { workspace = true } futures = { workspace = true } log = { workspace = true } +raft-log = { workspace = true } rand = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } diff --git a/src/meta/binaries/meta/ee_main.rs b/src/meta/binaries/meta/ee_main.rs index 502b16c24a88..0863fc755703 100644 --- a/src/meta/binaries/meta/ee_main.rs +++ b/src/meta/binaries/meta/ee_main.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(stmt_expr_attributes)] #![allow(clippy::uninlined_format_args)] mod entry; mod kvapi; diff --git a/src/meta/binaries/meta/entry.rs b/src/meta/binaries/meta/entry.rs index c24f5b04c483..1bbb062cdd49 100644 --- a/src/meta/binaries/meta/entry.rs +++ b/src/meta/binaries/meta/entry.rs @@ -23,8 +23,6 @@ use databend_common_base::base::Stoppable; use databend_common_grpc::RpcClientConf; use databend_common_meta_raft_store::ondisk::OnDisk; use databend_common_meta_raft_store::ondisk::DATA_VERSION; -use databend_common_meta_sled_store::get_sled_db; -use databend_common_meta_sled_store::init_sled_db; use databend_common_meta_sled_store::openraft::MessageSummary; use databend_common_meta_store::MetaStoreProvider; use databend_common_meta_types::Cmd; @@ -89,15 +87,10 @@ pub async fn entry(conf: Config) -> anyhow::Result<()> { return Ok(()); } - init_sled_db( - conf.raft_config.raft_dir.clone(), - conf.raft_config.sled_cache_size(), - ); - let single_or_join = if conf.raft_config.single { "single".to_string() } else { - format!("join {:#?}", conf.raft_config.join) + format!("join {:?}", conf.raft_config.join) }; let grpc_advertise = if let Some(a) = conf.grpc_api_advertise_address() { @@ -123,38 +116,34 @@ pub async fn entry(conf: Config) -> anyhow::Result<()> { info!("Initialize on-disk data at {}", conf.raft_config.raft_dir); - let db = get_sled_db(); - let mut on_disk = OnDisk::open(&db, &conf.raft_config).await?; + let mut on_disk = OnDisk::open(&conf.raft_config).await?; on_disk.log_stderr(true); - println!("On Disk Data:"); - println!(" Dir: {}", conf.raft_config.raft_dir); - println!(" DataVersion: {:?}", on_disk.header.version); - println!(" In-Upgrading: {:?}", on_disk.header.upgrading); - println!(); - println!("Log:"); - println!(" File: {}", conf.log.file); - println!(" Stderr: {}", conf.log.stderr); - if conf.log.otlp.on { - println!(" OpenTelemetry: {}", conf.log.otlp); - } - if conf.log.tracing.on { - println!(" Tracing: {}", conf.log.tracing); + let h = &on_disk.header; + + #[rustfmt::skip] + { + println!("Disk Data: {:?}; Upgrading: {:?}", h.version, h.upgrading); + println!(" Dir: {}", conf.raft_config.raft_dir); + println!(); + println!("Log File: {}", conf.log.file); + println!(" Stderr: {}", conf.log.stderr); + if conf.log.otlp.on { + println!(" OpenTelemetry: {}", conf.log.otlp); + } + if conf.log.tracing.on { + println!(" Tracing: {}", conf.log.tracing); + } + let r = &conf.raft_config; + println!("Raft Id: {}; Cluster: {}", r.id, r.cluster_name); + println!(" Dir: {}", r.raft_dir); + println!(" Status: {}", single_or_join); + println!(); + println!("HTTP API listen at: {}", conf.admin_api_address); + println!("gRPC API listen at: {} advertise: {}", conf.grpc_api_address, grpc_advertise); + println!("Raft API listen at: {} advertise: {}", raft_listen, raft_advertise,); + println!(); } - println!("Id: {}", conf.raft_config.id); - println!("Raft Cluster Name: {}", conf.raft_config.cluster_name); - println!("Raft Dir: {}", conf.raft_config.raft_dir); - println!("Raft Status: {}", single_or_join); - println!(); - println!("HTTP API"); - println!(" listening at {}", conf.admin_api_address); - println!("gRPC API"); - println!(" listening at {}", conf.grpc_api_address); - println!(" advertise: {}", grpc_advertise); - println!("Raft API"); - println!(" listening at {}", raft_listen,); - println!(" advertise: {}", raft_advertise,); - println!(); on_disk.upgrade().await?; diff --git a/src/meta/binaries/meta/oss_main.rs b/src/meta/binaries/meta/oss_main.rs index 10b836c69eed..6776304670af 100644 --- a/src/meta/binaries/meta/oss_main.rs +++ b/src/meta/binaries/meta/oss_main.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(stmt_expr_attributes)] + mod entry; mod kvapi; diff --git a/src/meta/binaries/metactl/export_from_disk.rs b/src/meta/binaries/metactl/export_from_disk.rs index 1215e3a78a17..9db46e2222a2 100644 --- a/src/meta/binaries/metactl/export_from_disk.rs +++ b/src/meta/binaries/metactl/export_from_disk.rs @@ -36,7 +36,7 @@ pub async fn export_from_dir(args: &ExportArgs) -> anyhow::Result<()> { eprintln!(); eprintln!("Export:"); - let sto_inn = StoreInner::open_create(&raft_config, Some(()), None).await?; + let sto_inn = StoreInner::open(&raft_config).await?; let mut lines = Arc::new(sto_inn).export(); eprintln!(" From: {}", raft_config.raft_dir); diff --git a/src/meta/binaries/metactl/import.rs b/src/meta/binaries/metactl/import.rs index c8a552b8cbaa..bf525f7bfacd 100644 --- a/src/meta/binaries/metactl/import.rs +++ b/src/meta/binaries/metactl/import.rs @@ -29,13 +29,12 @@ use databend_common_meta_raft_store::config::RaftConfig; use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; use databend_common_meta_raft_store::key_spaces::SMEntry; use databend_common_meta_raft_store::ondisk::DataVersion; +use databend_common_meta_raft_store::raft_log_v004; use databend_common_meta_raft_store::sm_v003::adapter::SnapshotUpgradeV002ToV003; use databend_common_meta_raft_store::sm_v003::write_entry::WriteEntry; use databend_common_meta_raft_store::sm_v003::SnapshotStoreV003; -use databend_common_meta_raft_store::state::RaftState; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; -use databend_common_meta_sled_store::get_sled_db; -use databend_common_meta_sled_store::init_sled_db; +use databend_common_meta_sled_store::init_get_sled_db; use databend_common_meta_sled_store::openraft::storage::RaftLogStorageExt; use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; use databend_common_meta_types::raft_types::CommittedLeaderId; @@ -51,6 +50,7 @@ use databend_common_meta_types::Endpoint; use databend_common_meta_types::LogEntry; use databend_common_meta_types::Node; use databend_meta::store::RaftStore; +use raft_log::api::raft_log_writer::RaftLogWriter; use url::Url; use crate::reading; @@ -71,8 +71,6 @@ pub async fn import_data(args: &ImportArgs) -> anyhow::Result<()> { let nodes = build_nodes(args.initial_cluster.clone(), args.id)?; - init_sled_db(raft_dir.clone(), 64 * 1024 * 1024 * 1024); - clear(args)?; let max_log_id = import_from_stdin_or_file(args).await?; @@ -108,6 +106,7 @@ async fn import_lines( } DataVersion::V002 => import_v002(raft_config, it).await?, DataVersion::V003 => import_v003(raft_config, it).await?, + DataVersion::V004 => crate::import_v004::import_v004(raft_config, it).await?, }; Ok(max_log_id) @@ -135,7 +134,7 @@ async fn import_v003( raft_config: RaftConfig, lines: impl IntoIterator>, ) -> anyhow::Result> { - let db = get_sled_db(); + let db = init_get_sled_db(raft_config.raft_dir.clone(), raft_config.sled_cache_size()); let mut n = 0; let mut max_log_id: Option = None; @@ -305,10 +304,9 @@ async fn init_new_cluster( eprintln!(); eprintln!("Initialize Cluster with: {:?}", nodes); - let db = get_sled_db(); let raft_config: RaftConfig = args.clone().into(); - let mut sto = RaftStore::open_create(&raft_config, Some(()), None).await?; + let mut sto = RaftStore::open(&raft_config).await?; let last_applied = { let sm2 = sto.get_state_machine().await; @@ -373,8 +371,13 @@ async fn init_new_cluster( } // Reset node id - let raft_state = RaftState::open_create(&db, &raft_config, Some(()), None).await?; - raft_state.set_node_id(args.id).await?; + { + let mut log = sto.log.write().await; + log.save_user_data(Some(raft_log_v004::LogStoreMeta { + node_id: Some(args.id), + }))?; + raft_log_v004::util::blocking_flush(&mut log).await?; + } Ok(()) } @@ -383,7 +386,7 @@ async fn init_new_cluster( fn clear(args: &ImportArgs) -> anyhow::Result<()> { eprintln!(); eprintln!("Clear All Sled Trees Before Import:"); - let db = get_sled_db(); + let db = init_get_sled_db(args.raft_dir.clone().unwrap(), 64 * 1024 * 1024 * 1024); let tree_names = db.tree_names(); for n in tree_names.iter() { diff --git a/src/meta/binaries/metactl/import_v004.rs b/src/meta/binaries/metactl/import_v004.rs new file mode 100644 index 000000000000..069633551b85 --- /dev/null +++ b/src/meta/binaries/metactl/import_v004.rs @@ -0,0 +1,114 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io; +use std::sync::Arc; +use std::sync::Mutex; + +use databend_common_meta_raft_store::config::RaftConfig; +use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; +use databend_common_meta_raft_store::key_spaces::SMEntry; +use databend_common_meta_raft_store::ondisk::OnDisk; +use databend_common_meta_raft_store::raft_log_v004; +use databend_common_meta_raft_store::raft_log_v004::RaftLogV004; +use databend_common_meta_raft_store::sm_v003::adapter::SnapshotUpgradeV002ToV004; +use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; +use databend_common_meta_raft_store::sm_v003::WriteEntry; +use databend_common_meta_raft_store::state_machine::MetaSnapshotId; +use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::sys_data::SysData; + +/// Import serialized lines for `DataVersion::V004` +/// +/// While importing, the max log id is also returned. +/// +/// It writes logs and related entries to WAL based raft log, and state_machine entries to a snapshot. +pub async fn import_v004( + raft_config: RaftConfig, + lines: impl IntoIterator>, +) -> anyhow::Result> { + OnDisk::ensure_dirs(&raft_config.raft_dir)?; + + let mut n = 0; + + let mut raft_log_importer = { + let raft_log_config = raft_config.clone().to_raft_log_config(); + let raft_log_config = Arc::new(raft_log_config); + let raft_log = RaftLogV004::open(raft_log_config)?; + + raft_log_v004::Importer::new(raft_log) + }; + + let snapshot_store = SnapshotStoreV004::new(raft_config); + let writer = snapshot_store.new_writer()?; + let (tx, join_handle) = writer.spawn_writer_thread("import_v004"); + + let sys_data = Arc::new(Mutex::new(SysData::default())); + + let mut converter = SnapshotUpgradeV002ToV004 { + sys_data: sys_data.clone(), + }; + + for line in lines { + let l = line?; + let (tree_name, kv_entry): (String, RaftStoreEntry) = serde_json::from_str(&l)?; + + if tree_name.starts_with("state_machine/") { + // Write to snapshot + let sm_entry: SMEntry = kv_entry.try_into().map_err(|err_str| { + anyhow::anyhow!("Failed to convert RaftStoreEntry to SMEntry: {}", err_str) + })?; + + let kv = converter.sm_entry_to_rotbl_kv(sm_entry)?; + if let Some(kv) = kv { + tx.send(WriteEntry::Data(kv)).await?; + } + } else { + if let RaftStoreEntry::DataHeader { .. } = kv_entry { + // Data header is not stored in V004 RaftLog + continue; + } + raft_log_importer.import_raft_store_entry(kv_entry)?; + } + + n += 1; + } + + let max_log_id = raft_log_importer.max_log_id; + raft_log_importer.flush().await?; + + let s = { + let r = sys_data.lock().unwrap(); + r.clone() + }; + + tx.send(WriteEntry::Finish(s)).await?; + let temp_snapshot_data = join_handle.await??; + + let last_applied = { + let r = sys_data.lock().unwrap(); + *r.last_applied_ref() + }; + let snapshot_id = MetaSnapshotId::new_with_epoch(last_applied); + let db = temp_snapshot_data.move_to_final_path(snapshot_id.to_string())?; + + eprintln!( + "Imported {} records, snapshot: {}; snapshot_path: {}; snapshot_stat: {}", + n, + snapshot_id, + db.path(), + db.stat() + ); + Ok(max_log_id) +} diff --git a/src/meta/binaries/metactl/main.rs b/src/meta/binaries/metactl/main.rs index 8900ce45ecda..04b9aee59816 100644 --- a/src/meta/binaries/metactl/main.rs +++ b/src/meta/binaries/metactl/main.rs @@ -19,6 +19,7 @@ mod export_from_grpc; pub mod admin; pub mod export_from_disk; pub mod import; +pub mod import_v004; pub(crate) mod reading; pub mod upgrade; @@ -33,7 +34,6 @@ use databend_common_base::base::tokio; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_raft_store::config::RaftConfig; -use databend_common_meta_sled_store::init_sled_db; use databend_common_tracing::init_logging; use databend_common_tracing::Config as LogConfig; use databend_common_tracing::FileConfig; @@ -312,8 +312,7 @@ impl App { None => { export_from_grpc::export_from_running_node(args).await?; } - Some(ref dir) => { - init_sled_db(dir.clone(), 64 * 1024 * 1024 * 1024); + Some(ref _dir) => { export_from_disk::export_from_dir(args).await?; } } diff --git a/src/meta/binaries/metactl/upgrade.rs b/src/meta/binaries/metactl/upgrade.rs index 18678b9a9c48..3943ff0f09ee 100644 --- a/src/meta/binaries/metactl/upgrade.rs +++ b/src/meta/binaries/metactl/upgrade.rs @@ -14,13 +14,10 @@ use databend_common_meta_raft_store::config::RaftConfig; use databend_common_meta_raft_store::ondisk::OnDisk; -use databend_common_meta_sled_store::get_sled_db; /// Upgrade the data in raft_dir to the latest version. pub async fn upgrade(raft_config: &RaftConfig) -> anyhow::Result<()> { - let db = get_sled_db(); - - let mut on_disk = OnDisk::open(&db, raft_config).await?; + let mut on_disk = OnDisk::open(raft_config).await?; on_disk.log_stderr(true); on_disk.upgrade().await?; diff --git a/src/meta/process/src/kv_processor.rs b/src/meta/process/src/kv_processor.rs index c86fb75934d5..0a9dab5f0647 100644 --- a/src/meta/process/src/kv_processor.rs +++ b/src/meta/process/src/kv_processor.rs @@ -82,6 +82,11 @@ where F: Fn(&str, Vec) -> Result, anyhow::Error> Ok(Some(x)) } + RaftStoreEntry::LogEntry(entry) => { + let x = RaftStoreEntry::LogEntry(unwrap_or_return!(self.proc_raft_entry(entry)?)); + Ok(Some(x)) + } + RaftStoreEntry::GenericKV { key, value } => { let data = (self.process_pb)(&key, value.data)?; @@ -104,6 +109,11 @@ where F: Fn(&str, Vec) -> Result, anyhow::Error> RaftStoreEntry::Sequences { .. } => Ok(None), RaftStoreEntry::ClientLastResps { .. } => Ok(None), RaftStoreEntry::LogMeta { .. } => Ok(None), + + RaftStoreEntry::NodeId(_) => Ok(None), + RaftStoreEntry::Vote(_) => Ok(None), + RaftStoreEntry::Committed(_) => Ok(None), + RaftStoreEntry::Purged(_) => Ok(None), } } diff --git a/src/meta/process/src/process_meta_dir.rs b/src/meta/process/src/process_meta_dir.rs index 3ece25d64f3e..0c0d773dd2ba 100644 --- a/src/meta/process/src/process_meta_dir.rs +++ b/src/meta/process/src/process_meta_dir.rs @@ -13,8 +13,7 @@ // limitations under the License. use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; -use databend_common_meta_sled_store::get_sled_db; -use databend_common_meta_sled_store::init_sled_db; +use databend_common_meta_sled_store::init_get_sled_db; use crate::examples::Config; @@ -26,9 +25,7 @@ pub fn process_sled_db(config: &Config, convert: F) -> anyhow::Result<()> where F: Fn(RaftStoreEntry) -> Result, anyhow::Error> { let raft_config = &config.raft_config; - init_sled_db(raft_config.raft_dir.clone(), 64 * 1024 * 1024 * 1024); - - let db = get_sled_db(); + let db = init_get_sled_db(raft_config.raft_dir.clone(), 64 * 1024 * 1024 * 1024); let mut tree_names = db.tree_names(); tree_names.sort(); diff --git a/src/meta/raft-store/Cargo.toml b/src/meta/raft-store/Cargo.toml index 3a94f987cc5e..29d50e79c610 100644 --- a/src/meta/raft-store/Cargo.toml +++ b/src/meta/raft-store/Cargo.toml @@ -29,8 +29,10 @@ databend-common-meta-sled-store = { workspace = true } databend-common-meta-stoerr = { workspace = true } databend-common-meta-types = { workspace = true } databend-common-tracing = { workspace = true } +deepsize = { workspace = true } derive_more = { workspace = true } fastrace = { workspace = true } +fs_extra = { workspace = true } futures = { workspace = true } futures-async-stream = { workspace = true } futures-util = { workspace = true } @@ -41,6 +43,8 @@ maplit = { workspace = true } num = { workspace = true } openraft = { workspace = true } ordq = { workspace = true } +raft-log = { workspace = true } +rmp-serde = { workspace = true } rotbl = { workspace = true } semver = { workspace = true } serde = { workspace = true } diff --git a/src/meta/raft-store/src/config.rs b/src/meta/raft-store/src/config.rs index 24a7ef8dfe14..6fec9de0867f 100644 --- a/src/meta/raft-store/src/config.rs +++ b/src/meta/raft-store/src/config.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::net::Ipv4Addr; +use std::path::Path; use std::sync::LazyLock; use databend_common_exception::Result; @@ -21,6 +22,9 @@ use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::MetaStartupError; +use crate::ondisk::DATA_VERSION; +use crate::raft_log_v004; + pub static DATABEND_COMMIT_VERSION: LazyLock = LazyLock::new(|| { let build_semver = option_env!("VERGEN_BUILD_SEMVER"); let git_sha = option_env!("VERGEN_GIT_SHA"); @@ -61,6 +65,18 @@ pub struct RaftConfig { /// You should only use this in a testing environment, unless YOU KNOW WHAT YOU ARE DOING. pub no_sync: bool, + /// The maximum number of log entries for log entries cache. + pub log_cache_max_items: u64, + + /// The maximum memory in bytes for the log entries cache. + pub log_cache_capacity: u64, + + /// Maximum number of records in a chunk of raft-log WAL. + pub log_wal_chunk_max_records: u64, + + /// Maximum size in bytes for a chunk of raft-log WAL. + pub log_wal_chunk_max_size: u64, + /// The number of logs since the last snapshot to trigger next snapshot. pub snapshot_logs_since_last: u64, @@ -152,6 +168,12 @@ impl Default for RaftConfig { raft_api_port: 28004, raft_dir: "./.databend/meta".to_string(), no_sync: false, + + log_cache_max_items: 1_000_000, + log_cache_capacity: 1024 * 1024 * 1024, + log_wal_chunk_max_records: 100_000, + log_wal_chunk_max_size: 256 * 1024 * 1024, + snapshot_logs_since_last: 1024, heartbeat_interval: 1000, install_snapshot_timeout: 4000, @@ -191,6 +213,26 @@ impl RaftConfig { ) } + /// Build [`RaftLogV004`](crate::raft_log_v004::RaftLogV004) config from [`RaftConfig`]. + pub fn to_raft_log_config(&self) -> raft_log_v004::RaftLogConfig { + let p = Path::new(&self.raft_dir) + .join("df_meta") + .join(format!("{}", DATA_VERSION)) + .join("log"); + + let dir = p.to_str().unwrap().to_string(); + + raft_log_v004::RaftLogConfig { + dir, + log_cache_max_items: Some(self.log_cache_max_items as usize), + log_cache_capacity: Some(self.log_cache_capacity as usize), + chunk_max_records: Some(self.log_wal_chunk_max_records as usize), + chunk_max_size: Some(self.log_wal_chunk_max_size as usize), + read_buffer_size: None, + truncate_incomplete_record: None, + } + } + pub fn raft_api_listen_host_string(&self) -> String { format!("{}:{}", self.raft_listen_host, self.raft_api_port) } diff --git a/src/meta/raft-store/src/key_spaces.rs b/src/meta/raft-store/src/key_spaces.rs index d4e6c4ffa9b2..f4fcb898e461 100644 --- a/src/meta/raft-store/src/key_spaces.rs +++ b/src/meta/raft-store/src/key_spaces.rs @@ -24,6 +24,7 @@ use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::LogIndex; use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::Node; use databend_common_meta_types::SeqNum; @@ -31,6 +32,7 @@ use serde::Deserialize; use serde::Serialize; use crate::ondisk::Header; +use crate::ondisk::OnDisk; use crate::state::RaftStateKey; use crate::state::RaftStateValue; use crate::state_machine::ClientLastRespValue; @@ -240,9 +242,43 @@ pub enum RaftStoreEntry { Sequences { key: ::K, value: ::V, }, ClientLastResps { key: ::K, value: ::V, }, LogMeta { key: ::K, value: ::V, }, + + // V004 log: + LogEntry(Entry), + NodeId(Option), + Vote(Option), + Committed(Option), + Purged(Option), } impl RaftStoreEntry { + /// Upgrade V003 to V004 + pub fn upgrade(self) -> Self { + match self.clone() { + Self::Logs { key: _, value } => Self::LogEntry(value), + Self::RaftStateKV { key: _, value } => match value { + RaftStateValue::NodeId(node_id) => Self::NodeId(Some(node_id)), + RaftStateValue::HardState(vote) => Self::Vote(Some(vote)), + RaftStateValue::Committed(committed) => Self::Committed(committed), + RaftStateValue::StateMachineId(_) => self, + }, + Self::LogMeta { key: _, value } => Self::Purged(Some(value.log_id())), + + RaftStoreEntry::DataHeader { .. } + | RaftStoreEntry::Nodes { .. } + | RaftStoreEntry::StateMachineMeta { .. } + | RaftStoreEntry::Expire { .. } + | RaftStoreEntry::GenericKV { .. } + | RaftStoreEntry::Sequences { .. } + | RaftStoreEntry::ClientLastResps { .. } + | RaftStoreEntry::LogEntry(_) + | RaftStoreEntry::NodeId(_) + | RaftStoreEntry::Vote(_) + | RaftStoreEntry::Committed(_) + | RaftStoreEntry::Purged(_) => self, + } + } + /// Serialize a key-value entry into a two elt vec of vec: `[key, value]`. #[rustfmt::skip] pub fn serialize(kv: &RaftStoreEntry) -> Result<(sled::IVec, sled::IVec), MetaStorageError> { @@ -258,6 +294,14 @@ impl RaftStoreEntry { Self::Sequences { key, value } => serialize_for_sled!(Sequences, key, value), Self::ClientLastResps { key, value } => serialize_for_sled!(ClientLastResps, key, value), Self::LogMeta { key, value } => serialize_for_sled!(LogMeta, key, value), + + RaftStoreEntry::LogEntry(_) | + RaftStoreEntry::NodeId(_)| + RaftStoreEntry::Vote(_) | + RaftStoreEntry::Committed(_)| + RaftStoreEntry::Purged(_) => { + unreachable!("V004 entries should not be serialized for sled") + } } } @@ -288,6 +332,13 @@ impl RaftStoreEntry { unreachable!("unknown prefix: {}", prefix); } + + pub fn new_header(header: Header) -> Self { + RaftStoreEntry::DataHeader { + key: OnDisk::KEY_HEADER.to_string(), + value: header, + } + } } impl TryInto for RaftStoreEntry { @@ -307,6 +358,14 @@ impl TryInto for RaftStoreEntry { Self::RaftStateKV { .. } => {Err("SMEntry does not contain RaftStateKV".to_string())} Self::ClientLastResps { .. } => {Err("SMEntry does not contain ClientLastResps".to_string())} Self::LogMeta { .. } => {Err("SMEntry does not contain LogMeta".to_string())} + + + Self::LogEntry (_) => {Err("SMEntry does not contain LogEntry".to_string())} + Self::Vote (_) => {Err("SMEntry does not contain Vote".to_string())} + Self::NodeId (_) => {Err("SMEntry does not contain NodeId".to_string())} + Self::Committed (_) => {Err("SMEntry does not contain Committed".to_string())} + Self::Purged (_) => {Err("SMEntry does not contain Purged".to_string())} + } } } diff --git a/src/meta/raft-store/src/lib.rs b/src/meta/raft-store/src/lib.rs index 4e82f437ec5b..8e0e9d50f8e7 100644 --- a/src/meta/raft-store/src/lib.rs +++ b/src/meta/raft-store/src/lib.rs @@ -22,9 +22,9 @@ pub mod applier; pub mod config; pub mod key_spaces; pub mod leveled_store; -pub mod log; pub(crate) mod marked; pub mod ondisk; +pub mod raft_log_v004; pub mod sm_v003; pub mod snapshot_config; pub mod state; diff --git a/src/meta/raft-store/src/log/raft_log.rs b/src/meta/raft-store/src/log/raft_log.rs deleted file mode 100644 index 2e7461895bf8..000000000000 --- a/src/meta/raft-store/src/log/raft_log.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::ops::RangeBounds; - -use databend_common_meta_sled_store::sled; -use databend_common_meta_sled_store::AsKeySpace; -use databend_common_meta_sled_store::SledTree; -use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::LogId; -use databend_common_meta_types::raft_types::LogIndex; -use log::info; - -use crate::config::RaftConfig; -use crate::key_spaces::LogMeta; -use crate::key_spaces::Logs; -use crate::state_machine::LogMetaKey; -use crate::state_machine::LogMetaValue; - -pub const TREE_RAFT_LOG: &str = "raft_log"; - -/// RaftLog stores the logs of a raft node. -/// It is part of MetaStore. -#[derive(Clone)] -pub struct RaftLog { - pub inner: SledTree, -} - -impl RaftLog { - /// Open RaftLog - #[fastrace::trace] - pub async fn open(db: &sled::Db, config: &RaftConfig) -> Result { - info!(config :? =(config); "open RaftLog"); - - let tree_name = config.tree_name(TREE_RAFT_LOG); - let inner = SledTree::open(db, tree_name, config.is_sync())?; - let rl = RaftLog { inner }; - Ok(rl) - } - - pub async fn set_last_purged(&self, log_id: LogId) -> Result<(), MetaStorageError> { - self.log_meta() - .insert(&LogMetaKey::LastPurged, &LogMetaValue::LogId(log_id)) - .await?; - Ok(()) - } - - pub fn get_last_purged(&self) -> Result, MetaStorageError> { - let res = self.log_meta().get(&LogMetaKey::LastPurged)?; - match res { - None => Ok(None), - Some(l) => { - let log_id: LogId = l.try_into().unwrap(); - Ok(Some(log_id)) - } - } - } - - /// Delete logs that are in `range`. - /// - /// When this function returns the logs are guaranteed to be fsync-ed. - /// - /// TODO(xp): in raft deleting logs may not need to be fsync-ed. - /// - /// 1. Deleting happens when cleaning applied logs, in which case, these logs will never be read: - /// The logs to clean are all included in a snapshot and state machine. - /// Replication will use the snapshot for sync, or create a new snapshot from the state machine for sync. - /// Thus these logs will never be read. If an un-fsync-ed delete is lost during server crash, it just wait for next delete to clean them up. - /// - /// 2. Overriding uncommitted logs of an old term by some new leader that did not see these logs: - /// In this case, atomic delete is quite enough(to not leave a hole). - /// If the system allows logs hole, non-atomic delete is quite enough(depends on the upper layer). - pub async fn range_remove(&self, range: R) -> Result<(), MetaStorageError> - where R: RangeBounds { - self.logs().range_remove(range, true).await - } - - pub fn range_values(&self, range: R) -> Result, MetaStorageError> - where R: RangeBounds { - self.logs().range_values(range) - } - - /// Append logs into RaftLog. - /// There is no consecutive-ness check. It is the caller's responsibility to leave no holes(if it runs a standard raft:DDD). - /// There is no overriding check either. It always overrides the existent ones. - /// - /// When this function returns the logs are guaranteed to be fsync-ed. - pub async fn append>( - &self, - logs: I, - ) -> Result<(), MetaStorageError> { - self.logs().append(logs).await - } - - /// Returns a borrowed key space in sled::Tree for logs - pub fn logs(&self) -> AsKeySpace { - self.inner.key_space() - } - - /// Returns a borrowed key space in sled::Tree for logs - pub fn log_meta(&self) -> AsKeySpace { - self.inner.key_space() - } -} diff --git a/src/meta/raft-store/src/ondisk/data_version.rs b/src/meta/raft-store/src/ondisk/data_version.rs index 6ced14c6f2cc..6ffd92c2a9c7 100644 --- a/src/meta/raft-store/src/ondisk/data_version.rs +++ b/src/meta/raft-store/src/ondisk/data_version.rs @@ -37,6 +37,9 @@ pub enum DataVersion { /// Store snapshot in rotbl. V003, + + /// WAL based raft-log. + V004, } impl fmt::Debug for DataVersion { @@ -51,8 +54,9 @@ impl fmt::Debug for DataVersion { "V001(2023-05-15: Get rid of compat, use only openraft v08 data types)" ), Self::V002 => write!(f, "V002(2023-07-22: Store snapshot in a file)"), - // TODO(rotbl): udpate the date when merged. - Self::V003 => write!(f, "V003(2024-05-31: Store snapshot in rotbl)"), + Self::V003 => write!(f, "V003(2024-06-27: Store snapshot in rotbl)"), + // TODO(raft-log): udpate the date when merged. + Self::V004 => write!(f, "V004(2024-11-04: WAL based raft-log)"), } } } @@ -64,6 +68,7 @@ impl fmt::Display for DataVersion { Self::V001 => write!(f, "V001"), Self::V002 => write!(f, "V002"), Self::V003 => write!(f, "V003"), + Self::V004 => write!(f, "V004"), } } } @@ -75,7 +80,8 @@ impl DataVersion { Self::V0 => Some(Self::V001), Self::V001 => Some(Self::V002), Self::V002 => Some(Self::V003), - Self::V003 => None, + Self::V003 => Some(Self::V004), + Self::V004 => None, } } @@ -91,6 +97,7 @@ impl DataVersion { Self::V001 => Self::V0, Self::V002 => Self::V001, Self::V003 => Self::V002, + Self::V004 => Self::V002, } } diff --git a/src/meta/raft-store/src/ondisk/header.rs b/src/meta/raft-store/src/ondisk/header.rs index cfbc1b3e8d9a..545c010adc4b 100644 --- a/src/meta/raft-store/src/ondisk/header.rs +++ b/src/meta/raft-store/src/ondisk/header.rs @@ -31,21 +31,30 @@ pub struct Header { /// The target version to upgrade to. /// /// If it is present, the data is upgrading. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] pub upgrading: Option, + + /// The second part of the upgrading process: + /// new version data is ready, and the old version data is cleaning up. + #[serde(default)] + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub cleaning: bool, } impl fmt::Display for Header { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "version: {}, upgrading: {}", - self.version, - if let Some(upgrading) = self.upgrading { - upgrading.to_string() - } else { - "None".to_string() - } - ) + write!(f, "{}", self.version)?; + + if let Some(upgrading) = self.upgrading { + write!(f, " -> {}", upgrading)?; + }; + + if self.cleaning { + write!(f, " (cleaning)")?; + } + + Ok(()) } } @@ -67,6 +76,7 @@ impl Header { Self { version: DATA_VERSION, upgrading: None, + cleaning: false, } } } diff --git a/src/meta/raft-store/src/ondisk/mod.rs b/src/meta/raft-store/src/ondisk/mod.rs index 13e31ace9e2c..950f894e474d 100644 --- a/src/meta/raft-store/src/ondisk/mod.rs +++ b/src/meta/raft-store/src/ondisk/mod.rs @@ -17,31 +17,32 @@ mod data_version; mod header; pub(crate) mod upgrade_to_v003; +pub(crate) mod upgrade_to_v004; pub(crate) mod version_info; -use std::collections::BTreeSet; use std::fmt; use std::fmt::Debug; +use std::fs; +use std::io; +use std::path::Path; +use std::path::PathBuf; pub use data_version::DataVersion; -use databend_common_meta_sled_store::sled; +use databend_common_meta_sled_store::init_get_sled_db; use databend_common_meta_sled_store::SledTree; use databend_common_meta_stoerr::MetaStorageError; pub use header::Header; use log::info; -use openraft::AnyError; +use raft_log::codeq::error_context_ext::ErrorContextExt; use crate::config::RaftConfig; use crate::key_spaces::DataHeader; -use crate::log::TREE_RAFT_LOG; -use crate::sm_v003::SnapshotStoreV002; -use crate::state::TREE_RAFT_STATE; /// The sled tree name to store the data versions. pub const TREE_HEADER: &str = "header"; /// The working data version the program runs on -pub static DATA_VERSION: DataVersion = DataVersion::V003; +pub static DATA_VERSION: DataVersion = DataVersion::V004; /// On disk data descriptor. /// @@ -52,9 +53,6 @@ pub static DATA_VERSION: DataVersion = DataVersion::V003; pub struct OnDisk { pub header: Header, - #[allow(dead_code)] - db: sled::Db, - config: RaftConfig, log_stderr: bool, @@ -71,33 +69,43 @@ impl fmt::Display for OnDisk { } impl OnDisk { - pub(crate) const KEY_HEADER: &'static str = "header"; + pub const KEY_HEADER: &'static str = "header"; + + pub fn ensure_dirs(raft_dir: &str) -> Result<(), io::Error> { + let raft_dir = Path::new(raft_dir); + let version_dir = raft_dir.join("df_meta").join(format!("{}", DATA_VERSION)); + + let log_dir = version_dir.join("log"); + if !log_dir.exists() { + fs::create_dir_all(&log_dir) + .context(|| format!("creating dir {}", log_dir.as_path().display()))?; + info!("Created log dir: {}", log_dir.as_path().display()); + } + + let snapshot_dir = version_dir.join("snapshot"); + if !snapshot_dir.exists() { + fs::create_dir_all(&snapshot_dir) + .context(|| format!("creating dir {}", snapshot_dir.as_path().display()))?; + info!("Created snapshot dir: {}", snapshot_dir.as_path().display()); + } + + Ok(()) + } /// Initialize data version for local store, returns the loaded version. #[fastrace::trace] - pub async fn open(db: &sled::Db, config: &RaftConfig) -> Result { + pub async fn open(config: &RaftConfig) -> Result { info!(config :? =(config); "open and initialize data-version"); - let tree_name = config.tree_name(TREE_HEADER); - let tree = SledTree::open(db, &tree_name, config.is_sync())?; - let ks = tree.key_space::(); + Self::ensure_dirs(&config.raft_dir)?; - let header = ks.get(&Self::KEY_HEADER.to_string()).map_err(|e| { - MetaStorageError::Damaged( - AnyError::error(format!( - "Unable to read meta-service data version from disk; \ - Possible reasons: opening future version meta-service with old version binary, \ - or the on-disk data is damaged. \ - error: {}", - e - )) - .add_context(|| "open on-disk data"), - ) - })?; - info!("Loaded header: {:?}", header); + Self::upgrade_header(config).await?; + + let header = Self::load_header_from_fs(config)?; + info!("Loaded header from fs: {:?}", header); if let Some(v) = header { - return Ok(OnDisk::new(v, db, config)); + return Ok(OnDisk::new(v, config)); } // Without header, by default it is the oldest compatible version: V002. @@ -105,14 +113,15 @@ impl OnDisk { let header = Header { version: DataVersion::V002, upgrading: None, + cleaning: false, }; - ks.insert(&Self::KEY_HEADER.to_string(), &header).await?; + Self::write_header_to_fs(config, &header)?; - Ok(OnDisk::new(header, db, config)) + Ok(OnDisk::new(header, config)) } - fn new(header: Header, db: &sled::Db, config: &RaftConfig) -> Self { + fn new(header: Header, config: &RaftConfig) -> Self { let min_compatible = DATA_VERSION.min_compatible_data_version(); if header.version < min_compatible { @@ -138,12 +147,95 @@ impl OnDisk { Self { header, - db: db.clone(), config: config.clone(), log_stderr: false, } } + async fn upgrade_header(config: &RaftConfig) -> Result<(), io::Error> { + let header_path = Self::header_path(config); + if header_path.exists() { + info!("Header file exists, no need to upgrade"); + return Ok(()); + } + + let db = init_get_sled_db(config.raft_dir.clone(), config.sled_cache_size()); + + let tree_name = config.tree_name(TREE_HEADER); + let tree = SledTree::open(&db, &tree_name, config.is_sync())?; + let ks = tree.key_space::(); + + let header = ks.get(&Self::KEY_HEADER.to_string()).map_err(|e| { + io::Error::new(io::ErrorKind::InvalidData, e).context(|| "open on-disk data") + })?; + info!("Found and loaded header from sled: {:?}", header); + + if let Some(header) = header { + Self::write_header_to_fs(config, &header)?; + + ks.remove_no_return(&Self::KEY_HEADER.to_string(), true) + .await + .map_err(|e| { + io::Error::new(io::ErrorKind::InvalidData, e) + .context(|| "remove header from sled") + })?; + + info!("Removed header from sled"); + } + + Ok(()) + } + + fn header_path(config: &RaftConfig) -> PathBuf { + let raft_dir = Path::new(&config.raft_dir); + raft_dir.join("df_meta").join("VERSION") + } + + pub(crate) fn write_header_to_fs( + config: &RaftConfig, + header: &Header, + ) -> Result<(), io::Error> { + let header_path = Self::header_path(config); + let buf = serde_json::to_vec(header).map_err(|e| { + io::Error::new(io::ErrorKind::InvalidData, e) + .context(|| format!("serializing header at {}", header_path.as_path().display(),)) + })?; + + fs::write(&header_path, &buf).context(|| { + format!( + "writing version file at {}: {}", + header_path.as_path().display(), + String::from_utf8_lossy(&buf) + ) + })?; + + info!( + "Wrote header {:?}; at {}", + header, + header_path.as_path().display() + ); + + Ok(()) + } + + pub(crate) fn load_header_from_fs(config: &RaftConfig) -> Result, io::Error> { + let header_path = Self::header_path(config); + + if !header_path.exists() { + return Ok(None); + } + + let state = fs::read(&header_path) + .context(|| format!("reading version file {}", header_path.as_path().display(),))?; + + let state = serde_json::from_slice::
(&state).map_err(|e| { + io::Error::new(io::ErrorKind::InvalidData, e) + .context(|| format!("parsing version file {}", header_path.as_path().display(),)) + })?; + + Ok(Some(state)) + } + /// Enable or disable logging crucial steps to stderr, when upgrading. pub fn log_stderr(&mut self, log_stderr: bool) { self.log_stderr = log_stderr; @@ -169,33 +261,23 @@ impl OnDisk { unreachable!("Upgrading to V0 is not supported"); } DataVersion::V001 => { - unreachable!("Upgrading to V001 is not supported since 2024-06-13, 1.2.528"); + unreachable!("Upgrading V0 to V001 is not supported since 2024-06-13, 1.2.528"); } DataVersion::V002 => { - let snapshot_store = SnapshotStoreV002::new(self.config.clone()); - - let last_snapshot = snapshot_store.load_last_snapshot().await.map_err(|e| { - let ae = AnyError::new(&e).add_context(|| "load last snapshot"); - MetaStorageError::Damaged(ae) - })?; - - if last_snapshot.is_some() { - self.progress(format_args!( - "There is V002 snapshot, upgrade is done; Finish upgrading" - )); - self.v001_remove_all_state_machine_trees().await?; - - // Note that this will increase `header.version`. - self.finish_upgrading().await?; - } + unreachable!( + "Upgrading V001 to V002 is not supported since 2024-06-13, 1.2.528" + ); } DataVersion::V003 => { self.clean_in_progress_v002_to_v003().await?; } + DataVersion::V004 => { + self.clean_in_progress_v003_to_v004().await?; + } } self.header.upgrading = None; - self.write_header(&self.header).await?; + self.write_header(&self.header)?; self.progress(format_args!("Cleared upgrading flag")); } @@ -219,6 +301,9 @@ impl OnDisk { self.upgrade_v002_to_v003().await?; } DataVersion::V003 => { + self.upgrade_v003_to_v004().await?; + } + DataVersion::V004 => { unreachable!("{} is the latest version", self.header.version) } } @@ -232,51 +317,6 @@ impl OnDisk { Ok(()) } - async fn v001_remove_all_state_machine_trees(&mut self) -> Result<(), MetaStorageError> { - let tree_names = self.tree_names().await?; - - let sm_tree_names = tree_names - .iter() - .filter(|&name| name.starts_with("state_machine/")) - .collect::>(); - - self.progress(format_args!( - "Remove state machine trees: {:?}", - sm_tree_names - )); - - for tree_name in sm_tree_names { - self.db.drop_tree(tree_name)?; - } - - Ok(()) - } - - async fn tree_names(&self) -> Result, MetaStorageError> { - let mut present_tree_names = { - let mut tree_names = BTreeSet::new(); - for n in self.db.tree_names() { - let name = String::from_utf8(n.to_vec())?; - tree_names.insert(name); - } - tree_names - }; - - // Export in header, raft_state, log and other order. - let mut tree_names = vec![]; - - for name in [TREE_HEADER, TREE_RAFT_STATE, TREE_RAFT_LOG] { - if present_tree_names.remove(name) { - tree_names.push(name.to_string()); - } else { - self.progress(format_args!("tree {} not found", name)); - } - } - tree_names.extend(present_tree_names.into_iter().collect::>()); - - Ok(tree_names) - } - /// Set upgrading flag indicating the upgrading is in progress. /// /// When it crashes before upgrading finishes, it can redo the upgrading. @@ -293,44 +333,35 @@ impl OnDisk { assert!(self.header.upgrading.is_none(), "can not upgrade twice"); self.header.upgrading = self.header.version.next(); - self.progress(format_args!("Begin upgrading: {}", self.header)); - - self.write_header(&self.header).await?; - Ok(()) - } - - /// Reset upgrading flag indicating the upgrading is finished, and set header.version to next version. - async fn finish_upgrading(&mut self) -> Result<(), MetaStorageError> { - self.header.version = self.header.upgrading.unwrap(); - self.header.upgrading = None; - self.progress(format_args!("Finished upgrading: {}", self.header)); - self.write_header(&self.header).await?; + self.write_header(&self.header)?; Ok(()) } - async fn write_header(&self, header: &Header) -> Result<(), MetaStorageError> { - let tree = self.header_tree()?; - let ks = tree.key_space::(); + fn clean_upgrading(&mut self) -> Result<(), io::Error> { + assert!(self.header.upgrading.is_some()); - ks.insert(&Self::KEY_HEADER.to_string(), header).await?; + self.header.cleaning = true; + self.progress(format_args!(" Clean upgrading: {}", self.header)); - self.progress(format_args!("Write header: {}", header)); + self.write_header(&self.header)?; Ok(()) } - #[allow(dead_code)] - fn read_header(&self) -> Result, MetaStorageError> { - let tree = self.header_tree()?; - let ks = tree.key_space::(); + /// Reset upgrading flag indicating the upgrading is finished, and set header.version to next version. + fn finish_upgrading(&mut self) -> Result<(), MetaStorageError> { + self.header.version = self.header.upgrading.unwrap(); + self.header.upgrading = None; + self.header.cleaning = false; + self.progress(format_args!(" Finished upgrading: {}", self.header)); - let header = ks.get(&Self::KEY_HEADER.to_string())?; - Ok(header) + self.write_header(&self.header)?; + Ok(()) } - fn header_tree(&self) -> Result { - let tree_name = self.config.tree_name(TREE_HEADER); - SledTree::open(&self.db, tree_name, self.config.is_sync()) + fn write_header(&self, header: &Header) -> Result<(), MetaStorageError> { + Self::write_header_to_fs(&self.config, header)?; + Ok(()) } fn progress(&self, s: impl fmt::Display) { diff --git a/src/meta/raft-store/src/ondisk/upgrade_to_v003.rs b/src/meta/raft-store/src/ondisk/upgrade_to_v003.rs index dbb0bacf5f71..028aa4081f73 100644 --- a/src/meta/raft-store/src/ondisk/upgrade_to_v003.rs +++ b/src/meta/raft-store/src/ondisk/upgrade_to_v003.rs @@ -23,7 +23,7 @@ use openraft::AnyError; use crate::ondisk::DataVersion; use crate::ondisk::OnDisk; -use crate::sm_v003::adapter::upgrade_snapshot_data_v002_to_v003; +use crate::sm_v003::adapter::upgrade_snapshot_data_v002_to_v003_or_v004; use crate::sm_v003::SnapshotStoreV002; use crate::sm_v003::SnapshotStoreV003; use crate::state_machine::MetaSnapshotId; @@ -49,8 +49,8 @@ impl OnDisk { let loaded = ss_store_v002.load_last_snapshot().await?; let Some((snapshot_id, snapshot_data)) = loaded else { - self.progress(format_args!("No V002 snapshot, skip upgrade")); - self.finish_upgrading().await?; + self.progress(format_args!(" No V002 snapshot, skip upgrade")); + self.finish_upgrading()?; return Ok(()); }; @@ -64,7 +64,7 @@ impl OnDisk { self.remove_v002_snapshot().await?; - self.finish_upgrading().await?; + self.finish_upgrading()?; Ok(()) } @@ -87,7 +87,7 @@ impl OnDisk { self.remove_v002_snapshot().await?; // Note that this will increase `header.version`. - self.finish_upgrading().await?; + self.finish_upgrading()?; } Ok(()) @@ -100,7 +100,7 @@ impl OnDisk { ) -> Result<(), io::Error> { let ss_store_v003 = SnapshotStoreV003::new(self.config.clone()); - upgrade_snapshot_data_v002_to_v003( + upgrade_snapshot_data_v002_to_v003_or_v004( &ss_store_v003, Box::new(snapshot_data), snapshot_id.to_string(), diff --git a/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs b/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs new file mode 100644 index 000000000000..9d74de99b900 --- /dev/null +++ b/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs @@ -0,0 +1,221 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Provide upgrading to v003 and cleaning v002 + +use std::fs; +use std::path::Path; +use std::sync::Arc; + +use databend_common_meta_sled_store::drop_sled_db; +use databend_common_meta_sled_store::init_get_sled_db; +use databend_common_meta_sled_store::SledTree; +use databend_common_meta_stoerr::MetaStorageError; +use fs_extra::dir::CopyOptions; +use raft_log::codeq::error_context_ext::ErrorContextExt; +use tokio::io; + +use crate::key_spaces::RaftStoreEntry; +use crate::ondisk::DataVersion; +use crate::ondisk::OnDisk; +use crate::raft_log_v004::importer; +use crate::raft_log_v004::RaftLogV004; +use crate::sm_v003::SnapshotStoreV003; +use crate::sm_v003::SnapshotStoreV004; + +impl OnDisk { + /// Upgrade the on-disk data form [`DataVersion::V003`] to [`DataVersion::V004`]. + /// + /// `V003` saves log in sled db. + /// `V004` saves log in WAL based raft log. + #[fastrace::trace] + pub(crate) async fn upgrade_v003_to_v004(&mut self) -> Result<(), io::Error> { + self.begin_upgrading(DataVersion::V003).await?; + + // 1.1. upgrade raft log + + self.progress(format_args!(" Upgrade V003 raft log in sled db to V004")); + + let raft_log_config = self.config.clone().to_raft_log_config(); + let raft_log_config = Arc::new(raft_log_config); + let raft_log = RaftLogV004::open(raft_log_config)?; + let mut importer = importer::Importer::new(raft_log); + + let db = init_get_sled_db(self.config.raft_dir.clone(), self.config.sled_cache_size()); + + let tree_names = ["raft_state", "raft_log"]; + + for tree_name in tree_names.iter() { + let tree = SledTree::open(&db, tree_name, self.config.is_sync())?; + let kvs = tree.export()?; + for kv in kvs { + let ent = RaftStoreEntry::deserialize(&kv[0], &kv[1])?; + importer.import_raft_store_entry(ent.upgrade())?; + } + } + + importer.flush().await?; + + // 1.2. copy snapshot + + let ss_store_v003 = SnapshotStoreV003::new(self.config.clone()); + let ss_store_v004 = SnapshotStoreV004::new(self.config.clone()); + + let v003_path = ss_store_v003.snapshot_config().snapshot_dir(); + let v004_path = ss_store_v004.snapshot_config().version_dir(); + + if fs::metadata(&v003_path).is_ok() { + let options = CopyOptions::new().overwrite(true).copy_inside(true); + + fs_extra::dir::copy(&v003_path, &v004_path, &options).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!( + "{}; when(copy snapshot from {} to {})", + e, v003_path, v004_path, + ), + ) + })?; + } + + // 2. clean up old version data + + self.clean_upgrading()?; + + self.remove_v003_logs().await?; + self.remove_v003_snapshot().await?; + + // 3. finish upgrading + + self.finish_upgrading()?; + + Ok(()) + } + + /// Revert or finish the unfinished upgrade to v003. + pub(crate) async fn clean_in_progress_v003_to_v004(&mut self) -> Result<(), MetaStorageError> { + assert!(self.header.upgrading.is_some()); + if self.header.cleaning { + self.remove_v003_logs().await?; + self.remove_v003_snapshot().await?; + + // Note that this will increase `header.version`. + self.finish_upgrading()?; + } else { + self.progress(format_args!("to V004 upgrade is in progress; Clean it")); + + let raft_dir = self.config.raft_dir.clone(); + let raft_dir = Path::new(&raft_dir); + + let p = raft_dir.join("df_meta").join("V004"); + + fs::remove_dir_all(&p) + .context(|| format!("remove unfinished upgrade in {}", p.as_path().display()))?; + } + + Ok(()) + } + + async fn remove_v003_snapshot(&mut self) -> Result<(), io::Error> { + let ss_store_v003 = SnapshotStoreV003::new(self.config.clone()); + + let v003_path = ss_store_v003.snapshot_config().snapshot_dir(); + + if fs::metadata(&v003_path).is_ok() { + fs::remove_dir_all(&v003_path).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("{}; when(remove V003 snapshot: {})", e, v003_path,), + ) + })?; + } + + Ok(()) + } + + /// It removes the data from sled db. + /// But not the sled db itself. + async fn remove_v003_logs(&mut self) -> Result<(), MetaStorageError> { + // After upgrading, no sled db is required. + + self.progress(format_args!(" Remove V003 log from sled db",)); + + let db = init_get_sled_db(self.config.raft_dir.clone(), self.config.sled_cache_size()); + for tree_name in db.tree_names() { + if tree_name == "__sled__default" { + continue; + } + + self.progress(format_args!( + " Removing sled tree: {}", + String::from_utf8_lossy(&tree_name) + )); + + db.drop_tree(&tree_name) + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!( + "{}; when(drop sled tree: {})as_str; when(clear sled db after upgrading V003 to V004)", + e, + String::from_utf8_lossy(&tree_name) + ), + ) + }) + ?; + } + + self.progress(format_args!(" Done: Remove V003 log from sled db",)); + + drop_sled_db(); + + // Self { - // TODO(rotbl): update these values when merged. + Self::new( + "af63a77b73ecb4b331cc84cf923007e432021a1c", + "2024-06-27", + new_semver(1, 2, 547), + "Persistent snapshot in rotbl, rotbl backed in-memory state-machine", + ) + } + + const fn v004() -> Self { + // TODO(raft-log): update these values when merged. Self::new( "3694e259c8e7c227fadfac5faa881cd2f2af6bbe", - "2024-05-31", + "2024-11-04", new_semver(1, 2, 53), - "Persistent snapshot in rotbl, rotbl backed in-memory state-machine", + "WAL based raft-log", ) } } @@ -119,6 +128,7 @@ pub static VERSION_INFOS: std::sync::LazyLock DataVersion::V001 => VersionInfo::v001() , DataVersion::V002 => VersionInfo::v002() , DataVersion::V003 => VersionInfo::v003() , + DataVersion::V004 => VersionInfo::v004() , } }); diff --git a/src/meta/raft-store/src/raft_log_v004/callback.rs b/src/meta/raft-store/src/raft_log_v004/callback.rs new file mode 100644 index 000000000000..7cf63c119449 --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/callback.rs @@ -0,0 +1,74 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io; +use std::sync::mpsc::SyncSender; + +use databend_common_meta_types::raft_types; +use log::info; +use log::warn; +use tokio::sync::oneshot; + +use crate::raft_log_v004::callback_data::CallbackData; + +/// The callback to be called when the IO is completed. +/// +/// This is used as a wrapper of Openraft callback or used directly internally in RaftLog. +pub struct Callback { + context: String, + data: CallbackData, +} + +impl Callback { + pub fn new_io_flushed(io_flushed: raft_types::IOFlushed, context: impl ToString) -> Self { + Callback { + context: context.to_string(), + data: CallbackData::IOFlushed(io_flushed), + } + } + + pub fn new_oneshot(tx: oneshot::Sender>, context: impl ToString) -> Self { + Callback { + context: context.to_string(), + data: CallbackData::Oneshot(tx), + } + } + + pub fn new_sync_oneshot(tx: SyncSender>, context: impl ToString) -> Self { + Callback { + context: context.to_string(), + data: CallbackData::SyncOneshot(tx), + } + } +} + +impl raft_log::Callback for Callback { + fn send(self, res: Result<(), io::Error>) { + info!("{}: Callback is called with: {:?}", self.context, res); + + match self.data { + CallbackData::Oneshot(tx) => { + let send_res = tx.send(res); + if send_res.is_err() { + warn!( + "{}: Callback failed to send Oneshot result back to caller", + self.context + ); + } + } + CallbackData::SyncOneshot(tx) => tx.send(res), + CallbackData::IOFlushed(io_flushed) => io_flushed.io_completed(res), + } + } +} diff --git a/src/meta/raft-store/src/log/mod.rs b/src/meta/raft-store/src/raft_log_v004/callback_data.rs similarity index 67% rename from src/meta/raft-store/src/log/mod.rs rename to src/meta/raft-store/src/raft_log_v004/callback_data.rs index e84344a9ee76..443726a284be 100644 --- a/src/meta/raft-store/src/log/mod.rs +++ b/src/meta/raft-store/src/raft_log_v004/callback_data.rs @@ -12,7 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod raft_log; +use std::io; +use std::sync::mpsc::SyncSender; -pub use raft_log::RaftLog; -pub use raft_log::TREE_RAFT_LOG; +use databend_common_meta_types::raft_types; +use tokio::sync::oneshot; + +pub enum CallbackData { + Oneshot(oneshot::Sender>), + SyncOneshot(SyncSender>), + IOFlushed(raft_types::IOFlushed), +} diff --git a/src/meta/raft-store/src/raft_log_v004/codec_wrapper.rs b/src/meta/raft-store/src/raft_log_v004/codec_wrapper.rs new file mode 100644 index 000000000000..292ee9c72289 --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/codec_wrapper.rs @@ -0,0 +1,122 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt; +use std::io; +use std::ops::Deref; +use std::ops::DerefMut; + +use raft_log::codeq::Decode; +use raft_log::codeq::Encode; +use raft_log::codeq::OffsetWriter; +use serde::de::DeserializeOwned; + +/// Codec wrapper to implement Encode/Decode for foreign types. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct Cw(pub T); + +impl Deref for Cw { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Cw { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl fmt::Display for Cw +where T: fmt::Display +{ + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl Encode for Cw +where T: serde::Serialize +{ + fn encode(&self, mut w: W) -> Result { + let mut ow = OffsetWriter::new(&mut w); + + rmp_serde::encode::write_named(&mut ow, &self.0) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + let n = ow.offset(); + + Ok(n) + } +} + +impl Decode for Cw +where T: DeserializeOwned +{ + fn decode(r: R) -> Result { + // rmp_serde::decode::from_read returns when a value is successfully decoded. + let d = rmp_serde::decode::from_read(r) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + Ok(Cw(d)) + } +} + +impl Cw { + pub fn unpack(self) -> T { + self.0 + } + + pub fn to_inner(&self) -> T + where T: Clone { + self.0.clone() + } +} + +#[cfg(test)] +mod tests { + use std::io; + + use databend_common_meta_types::raft_types::new_log_id; + use databend_common_meta_types::raft_types::LogId; + use raft_log::codeq::Decode; + use raft_log::codeq::Encode; + + use super::Cw; + + #[test] + fn test_cw_codec() -> Result<(), io::Error> { + let mut buf = Vec::new(); + + let log_id = Cw(new_log_id(1, 2, 3)); + log_id.encode(&mut buf)?; + + let log_id = Cw(new_log_id(4, 5, 6)); + log_id.encode(&mut buf)?; + + let mut r = buf.as_slice(); + + let l1: Cw = Decode::decode(&mut r)?; + let l2: Cw = Decode::decode(&mut r)?; + assert_eq!(0, r.len()); + assert!(Cw::::decode(&mut r).is_err()); + + assert_eq!(l1.0, new_log_id(1, 2, 3)); + assert_eq!(l2.0, new_log_id(4, 5, 6)); + + Ok(()) + } +} diff --git a/src/meta/raft-store/src/raft_log_v004/importer.rs b/src/meta/raft-store/src/raft_log_v004/importer.rs new file mode 100644 index 000000000000..417d44b43397 --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/importer.rs @@ -0,0 +1,119 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io; + +use databend_common_meta_types::raft_types::LogId; +use raft_log::api::raft_log_writer::RaftLogWriter; + +use crate::key_spaces::RaftStoreEntry; +use crate::raft_log_v004::codec_wrapper::Cw; +use crate::raft_log_v004::log_store_meta::LogStoreMeta; +use crate::raft_log_v004::util; +use crate::raft_log_v004::RaftLogV004; + +/// Import series of [`RaftStoreEntry`] record into [`RaftLogV004`]. +/// +/// [`RaftStoreEntry`] is line-wise format for export or data backup. +pub struct Importer { + pub raft_log: RaftLogV004, + pub max_log_id: Option, +} + +impl Importer { + pub fn new(raft_log: RaftLogV004) -> Self { + Importer { + raft_log, + max_log_id: None, + } + } + + pub async fn flush(mut self) -> Result { + util::blocking_flush(&mut self.raft_log).await?; + Ok(self.raft_log) + } + + pub fn import_raft_store_entry(&mut self, entry: RaftStoreEntry) -> Result<(), io::Error> { + match entry { + RaftStoreEntry::DataHeader { .. } => { + // V004 RaftLog does not store DataHeader + } + + //////////////////////////// V004 log //////////////////////////// + RaftStoreEntry::LogEntry(log_entry) => { + let log_id = log_entry.log_id; + let payload = log_entry.payload; + + self.raft_log.append([(Cw(log_id), Cw(payload))])?; + self.max_log_id = std::cmp::max(self.max_log_id, Some(log_id)); + } + + RaftStoreEntry::NodeId(node_id) => { + self.raft_log + .save_user_data(Some(LogStoreMeta { node_id }))?; + } + + RaftStoreEntry::Vote(vote) => { + if let Some(vote) = vote { + self.raft_log.save_vote(Cw(vote))?; + } + } + + RaftStoreEntry::Committed(committed) => { + if let Some(committed) = committed { + self.raft_log.commit(Cw(committed))?; + } + } + + RaftStoreEntry::Purged(purged) => { + if let Some(purged) = purged { + self.raft_log.purge(Cw(purged))?; + } + } + + ///////////////////////// V003 and before Log //////////////////// + RaftStoreEntry::Logs { .. } => { + unreachable!("V003 Logs should be written to V004 log"); + } + RaftStoreEntry::RaftStateKV { .. } => { + unreachable!("V003 RaftStateKV should be written to V004 log"); + } + RaftStoreEntry::LogMeta { .. } => { + unreachable!("V003 LogMeta should be written to V004 log"); + } + + //////////////////////// State machine entries /////////////////////// + RaftStoreEntry::StateMachineMeta { .. } => { + unreachable!("StateMachineMeta should be written to log"); + } + RaftStoreEntry::Nodes { .. } => { + unreachable!("Nodes should be written to log"); + } + RaftStoreEntry::Expire { .. } => { + unreachable!("Expire should be written to log"); + } + RaftStoreEntry::GenericKV { .. } => { + unreachable!("GenericKV should be written to log"); + } + RaftStoreEntry::Sequences { .. } => { + unreachable!("Sequences should be written to log"); + } + RaftStoreEntry::ClientLastResps { .. } => { + unreachable!("ClientLastResps should be written to log"); + } + } + + Ok(()) + } +} diff --git a/src/meta/raft-store/src/raft_log_v004/io_desc.rs b/src/meta/raft-store/src/raft_log_v004/io_desc.rs new file mode 100644 index 000000000000..119567f4fb87 --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/io_desc.rs @@ -0,0 +1,116 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::error::Error; +use std::fmt; + +use databend_common_meta_types::anyerror::AnyError; +use databend_common_meta_types::raft_types::ErrorSubject; +use databend_common_meta_types::raft_types::ErrorVerb; +use databend_common_meta_types::raft_types::StorageError; +use log::debug; +use log::error; + +use crate::raft_log_v004::io_phase::IOPhase; + +/// Describe an IO operation. +pub struct IODesc { + pub subject: ErrorSubject, + pub verb: ErrorVerb, + pub ctx: String, +} + +impl fmt::Display for IODesc { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}: ({}-{:?})", self.ctx, self.verb, self.subject) + } +} + +impl IODesc { + pub fn start(subject: ErrorSubject, verb: ErrorVerb, ctx: impl ToString) -> Self { + let s = ctx.to_string(); + debug!("{}: ({}-{:?}): start", s, verb, subject); + IODesc { + subject, + verb, + ctx: s, + } + } + + pub fn save_vote(ctx: impl ToString) -> Self { + Self::start(ErrorSubject::Vote, ErrorVerb::Write, ctx) + } + + pub fn save_committed(ctx: impl ToString) -> Self { + Self::start(ErrorSubject::Store, ErrorVerb::Write, ctx) + } + + pub fn append(ctx: impl ToString) -> Self { + Self::start(ErrorSubject::Logs, ErrorVerb::Write, ctx) + } + + pub fn truncate(ctx: impl ToString) -> Self { + Self::start(ErrorSubject::Logs, ErrorVerb::Write, ctx) + } + + pub fn purge(ctx: impl ToString) -> Self { + Self::start(ErrorSubject::Logs, ErrorVerb::Write, ctx) + } + + pub fn read_logs(ctx: impl ToString) -> Self { + Self::start(ErrorSubject::Logs, ErrorVerb::Read, ctx) + } + + pub fn to_storage_error(&self, phase: IOPhase, err: impl Error + 'static) -> StorageError { + error!("{}: failed to {}: error: {}", self, phase, err); + + StorageError::new( + self.subject.clone(), + self.verb, + AnyError::from(&err).add_context(|| self.ctx.clone()), + ) + } + + fn ok_message(&self, phase: IOPhase) -> String { + format!("{}: successfully {}", self, phase) + } + + pub fn ok_submit(&self) -> String { + self.ok_message(IOPhase::Submit) + } + + pub fn ok_submit_flush(&self) -> String { + self.ok_message(IOPhase::SubmitFlush) + } + + pub fn ok_done(&self) -> String { + self.ok_message(IOPhase::Done) + } + + pub fn err_submit(&self, err: impl Error + 'static) -> StorageError { + self.to_storage_error(IOPhase::Submit, err) + } + + pub fn err_submit_flush(&self, err: impl Error + 'static) -> StorageError { + self.to_storage_error(IOPhase::SubmitFlush, err) + } + + pub fn err_await_flush(&self, err: impl Error + 'static) -> StorageError { + self.to_storage_error(IOPhase::AwaitFlush, err) + } + + pub fn err_recv_flush_cb(&self, err: impl Error + 'static) -> StorageError { + self.to_storage_error(IOPhase::ReceiveFlushCallback, err) + } +} diff --git a/src/meta/raft-store/src/raft_log_v004/io_phase.rs b/src/meta/raft-store/src/raft_log_v004/io_phase.rs new file mode 100644 index 000000000000..6821c3184b4f --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/io_phase.rs @@ -0,0 +1,39 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt; + +/// Describe the phase during performing an IO. +/// +/// This is used to provide more context in the error message. +#[derive(Debug)] +pub enum IOPhase { + Submit, + SubmitFlush, + AwaitFlush, + ReceiveFlushCallback, + Done, +} + +impl fmt::Display for IOPhase { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + IOPhase::Submit => write!(f, "submit io"), + IOPhase::SubmitFlush => write!(f, "submit flush"), + IOPhase::AwaitFlush => write!(f, "await flush"), + IOPhase::ReceiveFlushCallback => write!(f, "receive flush callback"), + IOPhase::Done => write!(f, "done"), + } + } +} diff --git a/src/meta/raft-store/src/raft_log_v004/log_store_meta.rs b/src/meta/raft-store/src/raft_log_v004/log_store_meta.rs new file mode 100644 index 000000000000..14901bfb2424 --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/log_store_meta.rs @@ -0,0 +1,48 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io; + +use databend_common_meta_types::raft_types; +use raft_log::codeq::OffsetWriter; +use serde::Deserialize; +use serde::Serialize; + +/// Stores non-Raft data in the log store. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LogStoreMeta { + pub node_id: Option, +} + +impl raft_log::codeq::Encode for LogStoreMeta { + fn encode(&self, mut w: W) -> Result { + let mut ow = OffsetWriter::new(&mut w); + + rmp_serde::encode::write_named(&mut ow, &self) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + let n = ow.offset(); + + Ok(n) + } +} + +impl raft_log::codeq::Decode for LogStoreMeta { + fn decode(r: R) -> Result { + let d = rmp_serde::decode::from_read(r) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + Ok(d) + } +} diff --git a/src/meta/raft-store/src/raft_log_v004/mod.rs b/src/meta/raft-store/src/raft_log_v004/mod.rs new file mode 100644 index 000000000000..f0f50c8a6676 --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/mod.rs @@ -0,0 +1,39 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod callback; +pub mod callback_data; +pub mod codec_wrapper; +pub mod importer; +pub mod io_desc; +pub mod io_phase; +pub mod log_store_meta; +pub mod raft_log_io_error; +pub mod raft_log_types; +pub mod util; + +pub const TREE_RAFT_LOG: &str = "raft_log"; + +pub type RaftLogV004 = raft_log::RaftLog; +pub type RaftLogConfig = raft_log::Config; + +pub use callback::Callback; +pub use callback_data::CallbackData; +pub use codec_wrapper::Cw; +pub use importer::Importer; +pub use io_desc::IODesc; +pub use io_phase::IOPhase; +pub use log_store_meta::LogStoreMeta; +pub use raft_log_io_error::RaftLogIOError; +pub use raft_log_types::RaftLogTypes; diff --git a/src/meta/raft-store/src/raft_log_v004/raft_log_io_error.rs b/src/meta/raft-store/src/raft_log_v004/raft_log_io_error.rs new file mode 100644 index 000000000000..eee3354fd52e --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/raft_log_io_error.rs @@ -0,0 +1,31 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io; + +use databend_common_meta_types::raft_types::ErrorSubject; +use databend_common_meta_types::raft_types::ErrorVerb; + +use crate::raft_log_v004::io_phase::IOPhase; + +/// Describe the error that occurred during IO operations to RaftLog. +#[derive(Debug, thiserror::Error)] +#[error("RaftLogIOError: {verb}-{subject:?}: {ctx}: failed to {phase}; error: {error}")] +pub struct RaftLogIOError { + pub subject: ErrorSubject, + pub verb: ErrorVerb, + pub phase: IOPhase, + pub error: io::Error, + pub ctx: String, +} diff --git a/src/meta/raft-store/src/raft_log_v004/raft_log_types.rs b/src/meta/raft-store/src/raft_log_v004/raft_log_types.rs new file mode 100644 index 000000000000..8e30efe7460d --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/raft_log_types.rs @@ -0,0 +1,48 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ops::Deref; + +use databend_common_meta_types::raft_types; +use deepsize::DeepSizeOf; + +use crate::raft_log_v004::callback::Callback; +use crate::raft_log_v004::codec_wrapper::Cw; +use crate::raft_log_v004::log_store_meta::LogStoreMeta; + +/// Defines the types used by RaftLog implementation +#[derive(PartialEq, Eq, Default, Clone, Debug)] +pub struct RaftLogTypes; + +impl raft_log::Types for RaftLogTypes { + type LogId = Cw; + type LogPayload = Cw; + type Vote = Cw; + type Callback = Callback; + type UserData = LogStoreMeta; + + fn log_index(log_id: &Self::LogId) -> u64 { + log_id.index + } + + fn payload_size(payload: &Self::LogPayload) -> u64 { + let size = match payload.deref() { + raft_types::EntryPayload::Blank => 0, + raft_types::EntryPayload::Normal(log_entry) => log_entry.deep_size_of(), + raft_types::EntryPayload::Membership(_) => size_of::(), + }; + + size as u64 + } +} diff --git a/src/meta/raft-store/src/raft_log_v004/util.rs b/src/meta/raft-store/src/raft_log_v004/util.rs new file mode 100644 index 000000000000..baa39c551dcd --- /dev/null +++ b/src/meta/raft-store/src/raft_log_v004/util.rs @@ -0,0 +1,32 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io; + +use raft_log::api::raft_log_writer::RaftLogWriter; +use tokio::sync::oneshot; + +use crate::raft_log_v004::callback::Callback; +use crate::raft_log_v004::RaftLogV004; + +pub async fn blocking_flush(rl: &mut RaftLogV004) -> Result<(), io::Error> { + let (tx, rx) = oneshot::channel(); + let callback = Callback::new_oneshot(tx, "blocking_flush"); + + rl.flush(callback)?; + rx.await.map_err(|_e| { + io::Error::new(io::ErrorKind::Other, "Failed to receive flush completion") + })??; + Ok(()) +} diff --git a/src/meta/raft-store/src/sm_v003/adapter.rs b/src/meta/raft-store/src/sm_v003/adapter.rs index 883cae089fdb..16df96b79406 100644 --- a/src/meta/raft-store/src/sm_v003/adapter.rs +++ b/src/meta/raft-store/src/sm_v003/adapter.rs @@ -31,19 +31,21 @@ use crate::key_spaces::SMEntry; use crate::leveled_store::rotbl_codec::RotblCodec; use crate::marked::Marked; use crate::sm_v003::write_entry::WriteEntry; -use crate::sm_v003::SnapshotStoreV003; +use crate::sm_v003::SnapshotStoreV004; use crate::state_machine::StateMachineMetaKey; +pub type SnapshotUpgradeV002ToV003 = SnapshotUpgradeV002ToV004; + /// Convert V002 snapshot lines in json of [`SMEntry`] -/// to V003 rotbl key-value pairs. `(String, SeqMarked)`, +/// to V004 rotbl key-value pairs. `(String, SeqMarked)`, /// or update SysData in place. /// /// It holds a lock of SysData because this converter may be run concurrently in multiple threads. -pub struct SnapshotUpgradeV002ToV003 { +pub struct SnapshotUpgradeV002ToV004 { pub sys_data: Arc>, } -impl SnapshotUpgradeV002ToV003 { +impl SnapshotUpgradeV002ToV004 { pub fn convert_line(&mut self, s: &str) -> Result, io::Error> { let ent: SMEntry = serde_json::from_str(s).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; @@ -58,7 +60,7 @@ impl SnapshotUpgradeV002ToV003 { ) -> Result, io::Error> { match ent { SMEntry::DataHeader { .. } => { - // Snapshot V003 in rotbl format does not store data header + // Snapshot V004 in rotbl format does not store data header } SMEntry::Nodes { key, value } => { let mut s = self.sys_data.lock().unwrap(); @@ -108,7 +110,7 @@ impl SnapshotUpgradeV002ToV003 { } } -impl ordq::Work for SnapshotUpgradeV002ToV003 { +impl ordq::Work for SnapshotUpgradeV002ToV004 { type I = WriteEntry>; type O = Result>, io::Error>; @@ -128,11 +130,11 @@ impl ordq::Work for SnapshotUpgradeV002ToV003 { } } -/// Upgrade snapshot V002(ndjson) to V003(rotbl). +/// Upgrade snapshot V002(ndjson) to V004(rotbl). /// /// After install, the state machine has only one level of data. -pub async fn upgrade_snapshot_data_v002_to_v003( - snapshot_store: &SnapshotStoreV003, +pub async fn upgrade_snapshot_data_v002_to_v003_or_v004( + snapshot_store: &SnapshotStoreV004, data: Box, snapshot_id: SnapshotId, ) -> Result { @@ -142,7 +144,7 @@ pub async fn upgrade_snapshot_data_v002_to_v003( let data_size = data.data_size().await?; info!( - "upgrade snapshot from v002 to v003, data len: {}", + "upgrade snapshot from v002 to v004, data len: {}", data_size ); @@ -155,10 +157,10 @@ pub async fn upgrade_snapshot_data_v002_to_v003( let sys_data = Arc::new(Mutex::new(SysData::default())); - // Create a writer to write converted kvs to snapshot v003 + // Create a writer to write converted kvs to snapshot v004 let writer = snapshot_store.new_writer()?; let (writer_tx, writer_join_handle) = - writer.spawn_writer_thread("upgrade_snapshot_data_v002_to_v003"); + writer.spawn_writer_thread("upgrade_snapshot_data_v002_to_v004"); // Create a worker pool to convert the ndjson lines. let (ordq_tx, ordq_rx) = { @@ -167,7 +169,7 @@ pub async fn upgrade_snapshot_data_v002_to_v003( ordq::new( queue_depth, - repeat_with(|| SnapshotUpgradeV002ToV003 { + repeat_with(|| SnapshotUpgradeV002ToV004 { sys_data: sys_data.clone(), }) .take(n_workers), @@ -177,7 +179,7 @@ pub async fn upgrade_snapshot_data_v002_to_v003( // Chain ordq output to writer databend_common_base::runtime::spawn_blocking(move || { // snapshot v002 stores expire index(`exp-/`) after kvs(`kv--/`), - // We need to store expire index before kvs in snapshot v003. + // We need to store expire index before kvs in snapshot v004. let mut kv_cache = Vec::with_capacity(1_000_000); while let Some(res) = ordq_rx.recv() { @@ -242,7 +244,7 @@ pub async fn upgrade_snapshot_data_v002_to_v003( let temp_snapshot = writer_join_handle.await.map_err(closed_err)??; let db = temp_snapshot.move_to_final_path(snapshot_id)?; info!( - "upgraded snapshot from v002 to v003: file_size: {}, db stat: {}, sys_data: {}", + "upgraded snapshot from v002 to v004: file_size: {}, db stat: {}, sys_data: {}", db.inner().file_size(), db.inner().stat(), db.inner().meta().user_data(), diff --git a/src/meta/raft-store/src/sm_v003/mod.rs b/src/meta/raft-store/src/sm_v003/mod.rs index ed918d293c0f..a7b35b4fdf3b 100644 --- a/src/meta/raft-store/src/sm_v003/mod.rs +++ b/src/meta/raft-store/src/sm_v003/mod.rs @@ -38,5 +38,6 @@ pub use sm_v003::SMV003; pub use snapshot_store_v002::SnapshotStoreError; pub use snapshot_store_v002::SnapshotStoreV002; pub use snapshot_store_v003::SnapshotStoreV003; +pub use snapshot_store_v003::SnapshotStoreV004; pub use write_entry::WriteEntry; pub use writer_v003::WriterV003; diff --git a/src/meta/raft-store/src/sm_v003/snapshot_store_v003.rs b/src/meta/raft-store/src/sm_v003/snapshot_store_v003.rs index 588248378dae..609909609a07 100644 --- a/src/meta/raft-store/src/sm_v003/snapshot_store_v003.rs +++ b/src/meta/raft-store/src/sm_v003/snapshot_store_v003.rs @@ -14,6 +14,8 @@ use std::fs; use std::io; +use std::ops::Deref; +use std::ops::DerefMut; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::sys_data::SysData; @@ -27,13 +29,42 @@ use crate::snapshot_config::SnapshotConfig; #[derive(Debug)] pub struct SnapshotStoreV003 { - snapshot_config: SnapshotConfig, + v004: SnapshotStoreV004, } impl SnapshotStoreV003 { pub fn new(config: RaftConfig) -> Self { SnapshotStoreV003 { - snapshot_config: SnapshotConfig::new(DataVersion::V003, config), + v004: SnapshotStoreV004 { + snapshot_config: SnapshotConfig::new(DataVersion::V003, config), + }, + } + } +} + +impl Deref for SnapshotStoreV003 { + type Target = SnapshotStoreV004; + + fn deref(&self) -> &Self::Target { + &self.v004 + } +} + +impl DerefMut for SnapshotStoreV003 { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.v004 + } +} + +#[derive(Debug)] +pub struct SnapshotStoreV004 { + snapshot_config: SnapshotConfig, +} + +impl SnapshotStoreV004 { + pub fn new(config: RaftConfig) -> Self { + SnapshotStoreV004 { + snapshot_config: SnapshotConfig::new(DataVersion::V004, config), } } diff --git a/src/meta/raft-store/src/snapshot_config.rs b/src/meta/raft-store/src/snapshot_config.rs index 9b8994cb1d91..d73b220ef97a 100644 --- a/src/meta/raft-store/src/snapshot_config.rs +++ b/src/meta/raft-store/src/snapshot_config.rs @@ -48,6 +48,13 @@ impl SnapshotConfig { &self.raft_config } + pub fn version_dir(&self) -> String { + format!( + "{}/df_meta/{}", + self.raft_config.raft_dir, self.data_version + ) + } + pub fn snapshot_dir(&self) -> String { format!( "{}/df_meta/{}/snapshot", diff --git a/src/meta/raft-store/src/state/mod.rs b/src/meta/raft-store/src/state/mod.rs index 71330b83dd37..ca6ad818b95f 100644 --- a/src/meta/raft-store/src/state/mod.rs +++ b/src/meta/raft-store/src/state/mod.rs @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Raft state includes some essential information about raft, such as term, voted_for -mod raft_state; pub(crate) mod raft_state_kv; -pub use raft_state::RaftState; -pub use raft_state::TREE_RAFT_STATE; pub use raft_state_kv::RaftStateKey; pub use raft_state_kv::RaftStateValue; + +pub const TREE_RAFT_STATE: &str = "raft_state"; diff --git a/src/meta/raft-store/src/state/raft_state.rs b/src/meta/raft-store/src/state/raft_state.rs deleted file mode 100644 index bdb9dfb41907..000000000000 --- a/src/meta/raft-store/src/state/raft_state.rs +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; - -use databend_common_meta_sled_store::sled; -use databend_common_meta_sled_store::AsKeySpace; -use databend_common_meta_sled_store::SledTree; -use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::raft_types::LogId; -use databend_common_meta_types::raft_types::NodeId; -use databend_common_meta_types::raft_types::Vote; -use databend_common_meta_types::MetaStartupError; -use log::debug; -use log::info; - -use crate::config::RaftConfig; -use crate::key_spaces::RaftStateKV; -use crate::state::RaftStateKey; -use crate::state::RaftStateValue; - -/// Raft state stores everything else other than log and state machine, which includes: -/// id: NodeId, -/// vote: -/// term, -/// node_id, -#[derive(Debug)] -pub struct RaftState { - pub id: NodeId, - - /// If the instance is opened(true) from an existent state(e.g. load from fs) or created(false). - is_open: bool, - - /// A sled tree with key space support. - pub inner: SledTree, -} - -pub const TREE_RAFT_STATE: &str = "raft_state"; - -impl RaftState { - pub fn is_open(&self) -> bool { - self.is_open - } -} - -impl RaftState { - /// Open/create a raft state in a sled db. - /// 1. If `open` is `Some`, it tries to open an existent RaftState if there is one. - /// 2. If `create` is `Some`, it tries to initialize a new RaftState if there is not one. - /// If none of them is `Some`, it is a programming error and will panic. - #[fastrace::trace] - pub async fn open_create( - db: &sled::Db, - config: &RaftConfig, - open: Option<()>, - create: Option<()>, - ) -> Result { - info!(config :? =(config); "open: {:?}, create: {:?}", open, create); - - let tree_name = config.tree_name(TREE_RAFT_STATE); - let inner = SledTree::open(db, &tree_name, config.is_sync())?; - - let state = inner.key_space::(); - let curr_id = state.get(&RaftStateKey::Id)?.map(NodeId::from); - - debug!("get curr_id: {:?}", curr_id); - - let (id, is_open) = if let Some(curr_id) = curr_id { - match (open, create) { - (Some(_), _) => (curr_id, true), - (None, Some(_)) => { - return Err(MetaStartupError::MetaStoreAlreadyExists(curr_id)); - } - (None, None) => panic!("no open no create"), - } - } else { - match (open, create) { - (Some(_), Some(_)) => (config.id, false), - (Some(_), None) => { - return Err(MetaStartupError::MetaStoreNotFound); - } - (None, Some(_)) => (config.id, false), - (None, None) => panic!("no open no create"), - } - }; - - let rs = RaftState { id, is_open, inner }; - - if !rs.is_open() { - rs.init().await?; - } - - Ok(rs) - } - - #[fastrace::trace] - pub async fn set_node_id(&self, id: NodeId) -> Result<(), MetaStorageError> { - let state = self.state(); - state - .insert(&RaftStateKey::Id, &RaftStateValue::NodeId(id)) - .await?; - Ok(()) - } - - /// Initialize a raft state. The only thing to do is to persist the node id - /// so that next time opening it the caller knows it is initialized. - #[fastrace::trace] - async fn init(&self) -> Result<(), MetaStorageError> { - self.set_node_id(self.id).await - } - - pub async fn save_committed(&self, committed: Option) -> Result<(), MetaStorageError> { - let state = self.state(); - state - .insert( - &RaftStateKey::Committed, - &RaftStateValue::Committed(committed), - ) - .await?; - Ok(()) - } - - pub fn read_committed(&self) -> Result, MetaStorageError> { - let state = self.state(); - let committed = state.get(&RaftStateKey::Committed)?; - if let Some(c) = committed { - Ok(Option::::from(c)) - } else { - Ok(None) - } - } - - pub async fn save_vote(&self, vote: &Vote) -> Result<(), MetaStorageError> { - let state = self.state(); - state - .insert(&RaftStateKey::HardState, &RaftStateValue::HardState(*vote)) - .await?; - Ok(()) - } - - pub fn read_vote(&self) -> Result, MetaStorageError> { - let state = self.state(); - let hs = state.get(&RaftStateKey::HardState)?; - let hs = hs.map(Vote::from); - Ok(hs) - } - - /// Returns a borrowed sled tree key space to store meta of raft log - pub fn state(&self) -> AsKeySpace { - self.inner.key_space() - } -} diff --git a/src/meta/raft-store/src/state/raft_state_kv.rs b/src/meta/raft-store/src/state/raft_state_kv.rs index eb4c4fbc776c..3906a5a025a5 100644 --- a/src/meta/raft-store/src/state/raft_state_kv.rs +++ b/src/meta/raft-store/src/state/raft_state_kv.rs @@ -58,6 +58,29 @@ pub enum RaftStateValue { Committed(Option), } +impl RaftStateValue { + pub fn node_id(&self) -> NodeId { + match self { + RaftStateValue::NodeId(x) => *x, + _ => panic!("expect NodeId"), + } + } + + pub fn vote(&self) -> Vote { + match self { + RaftStateValue::HardState(x) => *x, + _ => panic!("expect HardState"), + } + } + + pub fn committed(&self) -> Option { + match self { + RaftStateValue::Committed(x) => *x, + _ => panic!("expect Committed"), + } + } +} + impl fmt::Display for RaftStateKey { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{:?}", self) diff --git a/src/meta/raft-store/src/state_machine/log_meta.rs b/src/meta/raft-store/src/state_machine/log_meta.rs index 4de3977f6fe6..72257981fbd4 100644 --- a/src/meta/raft-store/src/state_machine/log_meta.rs +++ b/src/meta/raft-store/src/state_machine/log_meta.rs @@ -38,6 +38,14 @@ pub enum LogMetaValue { LogId(LogId), } +impl LogMetaValue { + pub fn log_id(&self) -> LogId { + match self { + LogMetaValue::LogId(log_id) => *log_id, + } + } +} + impl fmt::Display for LogMetaKey { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { diff --git a/src/meta/raft-store/src/state_machine/sm.rs b/src/meta/raft-store/src/state_machine/sm.rs index 287d50039482..33d69f5b09e1 100644 --- a/src/meta/raft-store/src/state_machine/sm.rs +++ b/src/meta/raft-store/src/state_machine/sm.rs @@ -19,7 +19,7 @@ use std::time::Duration; use std::time::Instant; use databend_common_base::display::display_unix_epoch::DisplayUnixTimeStampExt; -use databend_common_meta_sled_store::get_sled_db; +use databend_common_meta_sled_store::init_get_sled_db; use databend_common_meta_sled_store::openraft::MessageSummary; use databend_common_meta_sled_store::AsKeySpace; use databend_common_meta_sled_store::SledKeySpace; @@ -140,7 +140,7 @@ impl StateMachine { #[fastrace::trace] pub async fn open(config: &RaftConfig, sm_id: u64) -> Result { - let db = get_sled_db(); + let db = init_get_sled_db(config.raft_dir.clone(), config.sled_cache_size()); let tree_name = StateMachine::tree_name(config, sm_id); debug!("opening tree: {}", &tree_name); diff --git a/src/meta/raft-store/src/state_machine/testing.rs b/src/meta/raft-store/src/state_machine/testing.rs index 32c20a7a1870..c141766047dd 100644 --- a/src/meta/raft-store/src/state_machine/testing.rs +++ b/src/meta/raft-store/src/state_machine/testing.rs @@ -21,6 +21,7 @@ use databend_common_meta_types::LogEntry; use databend_common_meta_types::RaftTxId; use databend_common_meta_types::UpsertKV; use maplit::btreeset; +use openraft::entry::RaftEntry; use openraft::Membership; use crate::key_spaces::RaftStoreEntry; @@ -33,6 +34,8 @@ pub fn snapshot_logs() -> (Vec, Vec) { log_id: new_log_id(1, 0, 1), payload: EntryPayload::Membership(Membership::new(vec![btreeset![1, 2, 3]], ())), }, + Entry::new_blank(new_log_id(1, 0, 2)), + Entry::new_blank(new_log_id(1, 0, 3)), Entry { log_id: new_log_id(1, 0, 4), payload: EntryPayload::Normal(LogEntry { @@ -49,6 +52,7 @@ pub fn snapshot_logs() -> (Vec, Vec) { log_id: new_log_id(1, 0, 6), payload: EntryPayload::Blank, }, + Entry::new_blank(new_log_id(1, 0, 7)), Entry { log_id: new_log_id(1, 0, 8), payload: EntryPayload::Blank, diff --git a/src/meta/raft-store/tests/it/log.rs b/src/meta/raft-store/tests/it/log.rs deleted file mode 100644 index 4aa3141390ef..000000000000 --- a/src/meta/raft-store/tests/it/log.rs +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_raft_store::log::RaftLog; -use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::EntryPayload; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::LogEntry; -use databend_common_meta_types::UpsertKV; -use test_harness::test; - -use crate::testing::new_raft_test_context; -use crate::testing::raft_store_test_harness; - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_log_open() -> anyhow::Result<()> { - let tc = new_raft_test_context(); - let db = &tc.db; - RaftLog::open(db, &tc.raft_config).await?; - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_log_append_and_range_get() -> anyhow::Result<()> { - let tc = new_raft_test_context(); - let db = &tc.db; - let rl = RaftLog::open(db, &tc.raft_config).await?; - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 9), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 10), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 256), - payload: EntryPayload::Blank, - }, - ]; - - rl.append(logs.clone()).await?; - - let got = rl.range_values(0..)?; - assert_eq!(logs, got); - - let got = rl.range_values(0..=2)?; - assert_eq!(logs[0..1], got); - - let got = rl.range_values(0..3)?; - assert_eq!(logs[0..1], got); - - let got = rl.range_values(0..5)?; - assert_eq!(logs[0..2], got); - - let got = rl.range_values(0..10)?; - assert_eq!(logs[0..3], got); - - let got = rl.range_values(0..11)?; - assert_eq!(logs[0..4], got); - - let got = rl.range_values(9..11)?; - assert_eq!(logs[2..4], got); - - let got = rl.range_values(10..256)?; - assert_eq!(logs[3..4], got); - - let got = rl.range_values(10..257)?; - assert_eq!(logs[3..5], got); - - let got = rl.range_values(257..)?; - assert_eq!(logs[5..], got); - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_log_insert() -> anyhow::Result<()> { - let tc = new_raft_test_context(); - let db = &tc.db; - let rl = RaftLog::open(db, &tc.raft_config).await?; - - assert_eq!(None, rl.logs().get(&5)?); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - rl.append(logs.clone()).await?; - - assert_eq!(logs, rl.range_values(..)?); - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_log_get() -> anyhow::Result<()> { - let tc = new_raft_test_context(); - let db = &tc.db; - let rl = RaftLog::open(db, &tc.raft_config).await?; - - assert_eq!(None, rl.logs().get(&5)?); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - rl.append(logs.clone()).await?; - - assert_eq!(None, rl.logs().get(&1)?); - assert_eq!(Some(logs[0].clone()), rl.logs().get(&2)?); - assert_eq!(None, rl.logs().get(&3)?); - assert_eq!(Some(logs[1].clone()), rl.logs().get(&4)?); - assert_eq!(None, rl.logs().get(&5)?); - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_log_last() -> anyhow::Result<()> { - let tc = new_raft_test_context(); - let db = &tc.db; - let rl = RaftLog::open(db, &tc.raft_config).await?; - - assert_eq!(None, rl.logs().last()?); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - rl.append(logs.clone()).await?; - assert_eq!(Some((4, logs[1].clone())), rl.logs().last()?); - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_log_range_remove() -> anyhow::Result<()> { - let tc = new_raft_test_context(); - let db = &tc.db; - let rl = RaftLog::open(db, &tc.raft_config).await?; - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - Entry { - log_id: new_log_id(1, 0, 9), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 10), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 256), - payload: EntryPayload::Blank, - }, - ]; - - rl.append(logs.clone()).await?; - rl.range_remove(0..).await?; - assert_eq!(logs[5..], rl.range_values(0..)?); - - rl.append(logs.clone()).await?; - rl.range_remove(1..).await?; - assert_eq!(logs[5..], rl.range_values(0..)?); - - rl.append(logs.clone()).await?; - rl.range_remove(3..).await?; - assert_eq!(logs[0..1], rl.range_values(0..)?); - - rl.append(logs.clone()).await?; - rl.range_remove(3..10).await?; - assert_eq!(logs[0..1], rl.range_values(0..5)?); - assert_eq!(logs[3..], rl.range_values(5..)?); - - Ok(()) -} diff --git a/src/meta/raft-store/tests/it/main.rs b/src/meta/raft-store/tests/it/main.rs index 45c58ee29019..47f9570c5dea 100644 --- a/src/meta/raft-store/tests/it/main.rs +++ b/src/meta/raft-store/tests/it/main.rs @@ -16,8 +16,6 @@ #![allow(clippy::diverging_sub_expression)] mod config; -mod log; -mod state; mod state_machine; mod testing; mod types; diff --git a/src/meta/raft-store/tests/it/state.rs b/src/meta/raft-store/tests/it/state.rs deleted file mode 100644 index f9577500f34f..000000000000 --- a/src/meta/raft-store/tests/it/state.rs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_raft_store::state::RaftState; -use databend_common_meta_types::raft_types::Vote; -use test_harness::test; - -use crate::testing::new_raft_test_context; -use crate::testing::raft_store_test_harness; - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_state_create() -> anyhow::Result<()> { - // - create a raft state - // - creating another raft state in the same sled db should fail - - let mut tc = new_raft_test_context(); - let db = &tc.db; - tc.raft_config.id = 3; - let rs = RaftState::open_create(db, &tc.raft_config, None, Some(())).await?; - let is_open = rs.is_open(); - - assert_eq!(3, rs.id); - assert!(!is_open); - - tc.raft_config.id = 4; - let res = RaftState::open_create(db, &tc.raft_config, None, Some(())).await; - assert!(res.is_err()); - assert_eq!( - "raft state present id=3, can not create", - res.unwrap_err().to_string() - ); - - tc.raft_config.id = 3; - let res = RaftState::open_create(db, &tc.raft_config, None, Some(())).await; - assert!(res.is_err()); - assert_eq!( - "raft state present id=3, can not create", - res.unwrap_err().to_string() - ); - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_state_open() -> anyhow::Result<()> { - // - create a raft state - // - open it. - - let mut tc = new_raft_test_context(); - let db = &tc.db; - tc.raft_config.id = 3; - let rs = RaftState::open_create(db, &tc.raft_config, None, Some(())).await?; - let is_open = rs.is_open(); - - assert_eq!(3, rs.id); - assert!(!is_open); - - tc.raft_config.id = 1000; - let rs = RaftState::open_create(db, &tc.raft_config, Some(()), None).await?; - let is_open = rs.is_open(); - assert_eq!(3, rs.id); - assert!(is_open); - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_state_open_or_create() -> anyhow::Result<()> { - let mut tc = new_raft_test_context(); - let db = &tc.db; - tc.raft_config.id = 3; - let rs = RaftState::open_create(db, &tc.raft_config, Some(()), Some(())).await?; - let is_open = rs.is_open(); - - assert_eq!(3, rs.id); - assert!(!is_open); - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_raft_state_write_read_vote() -> anyhow::Result<()> { - // - create a raft state - // - write vote and the read it. - - let mut tc = new_raft_test_context(); - let db = &tc.db; - tc.raft_config.id = 3; - let rs = RaftState::open_create(db, &tc.raft_config, None, Some(())).await?; - - assert_eq!(3, rs.id); - - // read got a None - - let got = rs.read_vote()?; - assert_eq!(None, got); - - // write hard state - - let hs = Vote::new(10, 3); - - rs.save_vote(&hs).await?; - - // read the written - - let got = rs.read_vote()?; - assert_eq!(Some(hs), got); - Ok(()) -} diff --git a/src/meta/raft-store/tests/it/testing.rs b/src/meta/raft-store/tests/it/testing.rs index 7d4c40764807..926b3d6816e5 100644 --- a/src/meta/raft-store/tests/it/testing.rs +++ b/src/meta/raft-store/tests/it/testing.rs @@ -17,8 +17,6 @@ use std::sync::Once; use databend_common_base::base::GlobalSequence; use databend_common_meta_raft_store::config::RaftConfig; -use databend_common_meta_sled_store::get_sled_db; -use databend_common_meta_sled_store::sled; use databend_common_tracing::closure_name; use databend_common_tracing::init_logging; use databend_common_tracing::Config; @@ -26,7 +24,6 @@ use fastrace::prelude::*; pub struct RaftTestContext { pub raft_config: RaftConfig, - pub db: sled::Db, } /// Create a new context for testing sled @@ -39,7 +36,6 @@ pub fn new_raft_test_context() -> RaftTestContext { RaftTestContext { raft_config: config, - db: get_sled_db(), } } diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index 8d58305f249b..6b5b2b8fcf91 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -16,7 +16,6 @@ default = ["simd"] memory-profiling = ["databend-common-base/memory-profiling", "databend-common-http/memory-profiling"] simd = ["databend-common-arrow/simd"] io-uring = [ - "sled/io_uring", "databend-common-meta-sled-store/io-uring", "databend-common-meta-raft-store/io-uring", ] @@ -54,11 +53,11 @@ maplit = { workspace = true } poem = { workspace = true } prometheus-client = { workspace = true } prost = { workspace = true } +raft-log = { workspace = true } semver = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serfig = { workspace = true } -sled = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true } tonic-reflection = { workspace = true } diff --git a/src/meta/service/src/configs/outer_v0.rs b/src/meta/service/src/configs/outer_v0.rs index 57eeb0bf047d..c4560284a0f7 100644 --- a/src/meta/service/src/configs/outer_v0.rs +++ b/src/meta/service/src/configs/outer_v0.rs @@ -285,6 +285,12 @@ pub struct ConfigViaEnv { pub kvsrv_api_port: u16, pub kvsrv_raft_dir: String, pub kvsrv_no_sync: bool, + + pub kvsrv_log_cache_max_items: u64, + pub kvsrv_log_cache_capacity: u64, + pub kvsrv_log_wal_chunk_max_records: u64, + pub kvsrv_log_wal_chunk_max_size: u64, + pub kvsrv_snapshot_logs_since_last: u64, pub kvsrv_heartbeat_interval: u64, pub kvsrv_install_snapshot_timeout: u64, @@ -338,6 +344,12 @@ impl From for ConfigViaEnv { kvsrv_api_port: cfg.raft_config.raft_api_port, kvsrv_raft_dir: cfg.raft_config.raft_dir, kvsrv_no_sync: cfg.raft_config.no_sync, + + kvsrv_log_cache_max_items: 1_000_000, + kvsrv_log_cache_capacity: 1024 * 1024 * 1024, + kvsrv_log_wal_chunk_max_records: 100_000, + kvsrv_log_wal_chunk_max_size: 256 * 1024 * 1024, + kvsrv_snapshot_logs_since_last: cfg.raft_config.snapshot_logs_since_last, kvsrv_heartbeat_interval: cfg.raft_config.heartbeat_interval, kvsrv_install_snapshot_timeout: cfg.raft_config.install_snapshot_timeout, @@ -371,6 +383,12 @@ impl Into for ConfigViaEnv { raft_api_port: self.kvsrv_api_port, raft_dir: self.kvsrv_raft_dir, no_sync: self.kvsrv_no_sync, + + log_cache_max_items: self.kvsrv_log_cache_max_items, + log_cache_capacity: self.kvsrv_log_cache_capacity, + log_wal_chunk_max_records: self.kvsrv_log_wal_chunk_max_records, + log_wal_chunk_max_size: self.kvsrv_log_wal_chunk_max_size, + snapshot_logs_since_last: self.kvsrv_snapshot_logs_since_last, heartbeat_interval: self.kvsrv_heartbeat_interval, install_snapshot_timeout: self.kvsrv_install_snapshot_timeout, @@ -471,6 +489,22 @@ pub struct RaftConfig { #[clap(long)] pub no_sync: bool, + /// The maximum number of log entries for log entries cache. Default value is 1_000_000. + #[clap(long, default_value = "1000000")] + pub log_cache_max_items: u64, + + /// The maximum memory in bytes for the log entries cache. Default value is 1G. + #[clap(long, default_value = "1073741824")] + pub log_cache_capacity: u64, + + /// Maximum number of records in a chunk of raft-log WAL. Default value is 100_000. + #[clap(long, default_value = "100000")] + pub log_wal_chunk_max_records: u64, + + /// Maximum size in bytes for a chunk of raft-log WAL. Default value si 256M + #[clap(long, default_value = "268435456")] + pub log_wal_chunk_max_size: u64, + /// The number of logs since the last snapshot to trigger next snapshot. #[clap(long, default_value = "1024")] pub snapshot_logs_since_last: u64, @@ -582,6 +616,12 @@ impl From for InnerRaftConfig { raft_api_port: x.raft_api_port, raft_dir: x.raft_dir, no_sync: x.no_sync, + + log_cache_max_items: x.log_cache_max_items, + log_cache_capacity: x.log_cache_capacity, + log_wal_chunk_max_records: x.log_wal_chunk_max_records, + log_wal_chunk_max_size: x.log_wal_chunk_max_size, + snapshot_logs_since_last: x.snapshot_logs_since_last, heartbeat_interval: x.heartbeat_interval, install_snapshot_timeout: x.install_snapshot_timeout, @@ -615,6 +655,12 @@ impl From for RaftConfig { raft_api_port: inner.raft_api_port, raft_dir: inner.raft_dir, no_sync: inner.no_sync, + + log_cache_max_items: inner.log_cache_max_items, + log_cache_capacity: inner.log_cache_capacity, + log_wal_chunk_max_records: inner.log_wal_chunk_max_records, + log_wal_chunk_max_size: inner.log_wal_chunk_max_size, + snapshot_logs_since_last: inner.snapshot_logs_since_last, heartbeat_interval: inner.heartbeat_interval, install_snapshot_timeout: inner.install_snapshot_timeout, diff --git a/src/meta/service/src/export.rs b/src/meta/service/src/export.rs deleted file mode 100644 index 6064b0269df8..000000000000 --- a/src/meta/service/src/export.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; -use databend_common_meta_stoerr::MetaStorageError; - -/// Convert one line of serialized key-value into json. -/// -/// Exported data is a pair of key with one char prefix and value. -/// Both key and value are in Vec format. -/// The prefix identifies the subtree this record belongs to. See [`SledKeySpace`]. -/// -/// The output json is in form of `(tree_name, {keyspace: {key, value}})`. -/// In this impl the `tree_name` can be one of `state`, `log` and `sm`. See [`MetaRaftStore`]. -pub fn vec_kv_to_json(tree_name: &str, kv: &[Vec]) -> Result { - let kv_entry = RaftStoreEntry::deserialize(&kv[0], &kv[1])?; - - let tree_kv = (tree_name, kv_entry); - let line = serde_json::to_string(&tree_kv)?; - Ok(line) -} diff --git a/src/meta/service/src/lib.rs b/src/meta/service/src/lib.rs index e608deea3edd..d15ad7fc309c 100644 --- a/src/meta/service/src/lib.rs +++ b/src/meta/service/src/lib.rs @@ -18,7 +18,6 @@ pub mod api; pub mod configs; -pub mod export; pub mod message; pub mod meta_service; pub mod metrics; diff --git a/src/meta/service/src/meta_service/meta_node.rs b/src/meta/service/src/meta_service/meta_node.rs index 326d4e715758..81c5d33dca7d 100644 --- a/src/meta/service/src/meta_service/meta_node.rs +++ b/src/meta/service/src/meta_service/meta_node.rs @@ -113,6 +113,16 @@ pub struct MetaNode { pub joined_tasks: AtomicI32, } +impl Drop for MetaNode { + fn drop(&mut self) { + info!( + "MetaNode(id={}, raft={}) is dropping", + self.sto.id, + self.sto.config.raft_api_advertise_host_string() + ); + } +} + impl Opened for MetaNode { fn is_opened(&self) -> bool { self.sto.is_opened() @@ -305,18 +315,9 @@ impl MetaNode { } /// Open or create a meta node. - /// 1. If `open` is `Some`, try to open an existent one. - /// 2. If `create` is `Some`, try to create an one in non-voter mode. #[fastrace::trace] - pub async fn open_create( - config: &RaftConfig, - open: Option<()>, - create: Option<()>, - ) -> Result, MetaStartupError> { - info!( - "open_create_boot, config: {:?}, open: {:?}, create: {:?}", - config, open, create - ); + pub async fn open(config: &RaftConfig) -> Result, MetaStartupError> { + info!("MetaNode::open, config: {:?}", config); let mut config = config.clone(); @@ -329,7 +330,7 @@ impl MetaNode { config.no_sync = true; } - let sto = RaftStore::open_create(&config, open, create).await?; + let sto = RaftStore::open(&config).await?; // config.id only used for the first time let self_node_id = if sto.is_opened() { sto.id } else { config.id }; @@ -346,17 +347,15 @@ impl MetaNode { } /// Open or create a metasrv node. + /// /// Optionally boot a single node cluster. - /// 1. If `open` is `Some`, try to open an existent one. - /// 2. If `create` is `Some`, try to create an one in non-voter mode. + /// If `initialize_cluster` is `Some`, initialize the cluster as a single-node cluster. #[fastrace::trace] - pub async fn open_create_boot( + pub async fn open_boot( config: &RaftConfig, - open: Option<()>, - create: Option<()>, initialize_cluster: Option, ) -> Result, MetaStartupError> { - let mn = Self::open_create(config, open, create).await?; + let mn = Self::open(config).await?; if let Some(node) = initialize_cluster { mn.init_cluster(node).await?; @@ -448,7 +447,7 @@ impl MetaNode { server_metrics::set_last_seq(meta_node.get_last_seq().await); // metrics about server storage - server_metrics::set_db_size(meta_node.get_db_size().unwrap_or_default()); + server_metrics::set_db_size(meta_node.get_db_size().await); server_metrics::set_snapshot_key_num(meta_node.get_key_num().await); last_leader = mm.current_leader; @@ -740,20 +739,12 @@ impl MetaNode { let raft_conf = &conf.raft_config; if raft_conf.single { - let mn = MetaNode::open_create(raft_conf, Some(()), Some(())).await?; + let mn = MetaNode::open(raft_conf).await?; mn.init_cluster(conf.get_node()).await?; return Ok(mn); } - if !raft_conf.join.is_empty() { - // Bring up a new node, join it into a cluster - - let mn = MetaNode::open_create(raft_conf, Some(()), Some(())).await?; - return Ok(mn); - } - // open mode - - let mn = MetaNode::open_create(raft_conf, Some(()), None).await?; + let mn = MetaNode::open(raft_conf).await?; Ok(mn) } @@ -761,7 +752,7 @@ impl MetaNode { /// For every cluster this func should be called exactly once. #[fastrace::trace] pub async fn boot(config: &MetaConfig) -> Result, MetaStartupError> { - let mn = Self::open_create(&config.raft_config, None, Some(())).await?; + let mn = Self::open(&config.raft_config).await?; mn.init_cluster(config.get_node()).await?; Ok(mn) } @@ -826,11 +817,8 @@ impl MetaNode { nodes } - fn get_db_size(&self) -> Result { - self.sto.db.size_on_disk().map_err(|e| { - let se = MetaStorageError::Damaged(AnyError::new(&e).add_context(|| "get db_size")); - MetaError::StorageError(se) - }) + async fn get_db_size(&self) -> u64 { + self.sto.log.read().await.on_disk_size() } async fn get_key_num(&self) -> u64 { @@ -853,7 +841,7 @@ impl MetaNode { let endpoint = self.sto.get_node_raft_endpoint(&self.sto.id).await?; - let db_size = self.get_db_size()?; + let db_size = self.get_db_size().await; let key_num = self.get_key_num().await; let metrics = self.raft.metrics().borrow().clone(); diff --git a/src/meta/service/src/meta_service/raft_service_impl.rs b/src/meta/service/src/meta_service/raft_service_impl.rs index 208e0f3c27d7..0fe2e3f9bd88 100644 --- a/src/meta/service/src/meta_service/raft_service_impl.rs +++ b/src/meta/service/src/meta_service/raft_service_impl.rs @@ -22,7 +22,7 @@ use std::time::Duration; use databend_common_base::base::tokio::sync::Mutex; use databend_common_base::future::TimedFutureExt; use databend_common_meta_client::MetaGrpcReadReq; -use databend_common_meta_raft_store::sm_v003::adapter::upgrade_snapshot_data_v002_to_v003; +use databend_common_meta_raft_store::sm_v003::adapter::upgrade_snapshot_data_v002_to_v003_or_v004; use databend_common_meta_raft_store::sm_v003::open_snapshot::OpenSnapshot; use databend_common_meta_raft_store::sm_v003::received::Received; use databend_common_meta_sled_store::openraft::MessageSummary; @@ -159,7 +159,7 @@ impl RaftServiceImpl { let snapshot_data_v1 = SnapshotData::open_temp(temp_path).map_err(io_err_to_read_snap_err)?; - let db = upgrade_snapshot_data_v002_to_v003( + let db = upgrade_snapshot_data_v002_to_v003_or_v004( &ss_store, Box::new(snapshot_data_v1), snapshot_id, diff --git a/src/meta/service/src/meta_service/snapshot_receiver_v1.rs b/src/meta/service/src/meta_service/snapshot_receiver_v1.rs index a137df539d99..c50bd1968511 100644 --- a/src/meta/service/src/meta_service/snapshot_receiver_v1.rs +++ b/src/meta/service/src/meta_service/snapshot_receiver_v1.rs @@ -18,7 +18,7 @@ use std::io; use std::io::BufWriter; use std::io::Write; -use databend_common_meta_raft_store::sm_v003::SnapshotStoreV003; +use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_sled_store::openraft::error::Fatal; use databend_common_meta_sled_store::openraft::ErrorSubject; use databend_common_meta_sled_store::openraft::ErrorVerb; @@ -100,7 +100,7 @@ impl ReceiverV1 { pub(crate) async fn receive_snapshot_v1( receiver: &mut Option, - ss_store: &SnapshotStoreV003, + ss_store: &SnapshotStoreV004, req: InstallSnapshotRequest, ) -> Result, RaftError> { let snapshot_id = req.meta.snapshot_id.clone(); diff --git a/src/meta/service/src/store/raft_log_storage_impl.rs b/src/meta/service/src/store/raft_log_storage_impl.rs index 743a699400ae..bba4e2ebcf60 100644 --- a/src/meta/service/src/store/raft_log_storage_impl.rs +++ b/src/meta/service/src/store/raft_log_storage_impl.rs @@ -13,37 +13,36 @@ // limitations under the License. use std::fmt::Debug; -use std::io::ErrorKind; +use std::ops::Bound; use std::ops::RangeBounds; -use std::time::Duration; -use databend_common_base::base::tokio; -use databend_common_base::base::tokio::io; +use databend_common_base::base::tokio::sync::oneshot; use databend_common_base::display::display_option::DisplayOptionExt; -use databend_common_meta_sled_store::openraft::storage::IOFlushed; +use databend_common_meta_raft_store::raft_log_v004; +use databend_common_meta_raft_store::raft_log_v004::codec_wrapper::Cw; +use databend_common_meta_raft_store::raft_log_v004::io_desc::IODesc; use databend_common_meta_sled_store::openraft::storage::RaftLogStorage; use databend_common_meta_sled_store::openraft::EntryPayload; -use databend_common_meta_sled_store::openraft::ErrorSubject; -use databend_common_meta_sled_store::openraft::ErrorVerb; use databend_common_meta_sled_store::openraft::LogIdOptionExt; use databend_common_meta_sled_store::openraft::LogState; use databend_common_meta_sled_store::openraft::OptionalSend; use databend_common_meta_sled_store::openraft::RaftLogId; use databend_common_meta_sled_store::openraft::RaftLogReader; use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::IOFlushed; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::Membership; use databend_common_meta_types::raft_types::StorageError; use databend_common_meta_types::raft_types::TypeConfig; use databend_common_meta_types::raft_types::Vote; use deepsize::DeepSizeOf; +use itertools::Itertools; use log::debug; -use log::error; use log::info; +use log::warn; +use raft_log::api::raft_log_writer::RaftLogWriter; -use crate::metrics::raft_metrics; use crate::store::RaftStore; -use crate::store::ToStorageError; impl RaftLogReader for RaftStore { #[fastrace::trace] @@ -102,47 +101,34 @@ impl RaftLogReader for RaftStore { &mut self, range: RB, ) -> Result, StorageError> { - debug!( - "RaftStore::try_get_log_entries: self.id={}, range: {:?}", - self.id, range - ); + let (start, end) = range_boundary(range); - let res = self - .log - .read() - .await - .range_values(range.clone()) - .map_to_sto_err(ErrorSubject::Logs, ErrorVerb::Read); + let io = IODesc::read_logs(format!( + "RaftStore(id={})::try_get_log_entries([{},{})", + self.id, start, end + )); - debug!( - "RaftStore::try_get_log_entries: done: self.id={}, range: {:?}", - self.id, range - ); + let log = self.log.read().await; - match res { - Ok(entries) => Ok(entries), - Err(err) => { - raft_metrics::storage::incr_raft_storage_fail("try_get_log_entries", false); - Err(err) - } - } + let entries = log + .read(start, end) + .map_ok(|(log_id, payload)| Entry { + log_id: log_id.0, + payload: payload.0, + }) + .collect::, _>>() + .map_err(|e| io.err_submit(e))?; + + debug!("{}", io.ok_done()); + Ok(entries) } #[fastrace::trace] async fn read_vote(&mut self) -> Result, StorageError> { - match self - .raft_state - .read() - .await - .read_vote() - .map_to_sto_err(ErrorSubject::Vote, ErrorVerb::Read) - { - Err(err) => { - raft_metrics::storage::incr_raft_storage_fail("read_vote", false); - Err(err) - } - Ok(vote) => Ok(vote), - } + let log = self.log.read().await; + let vote = log.log_state().vote().map(Cw::to_inner); + + Ok(vote) } } @@ -150,48 +136,15 @@ impl RaftLogStorage for RaftStore { type LogReader = RaftStore; async fn get_log_state(&mut self) -> Result, StorageError> { - let last_purged_log_id = match self - .log - .read() - .await - .get_last_purged() - .map_to_sto_err(ErrorSubject::Logs, ErrorVerb::Read) - { - Err(err) => { - raft_metrics::storage::incr_raft_storage_fail("get_log_state", false); - return Err(err); - } - Ok(r) => r, - }; + let log = self.log.read().await; + let state = log.log_state(); - let last = match self - .log - .read() - .await - .logs() - .last() - .map_to_sto_err(ErrorSubject::Logs, ErrorVerb::Read) - { - Err(err) => { - raft_metrics::storage::incr_raft_storage_fail("get_log_state", false); - return Err(err); - } - Ok(r) => r, - }; - - let last_log_id = match last { - None => last_purged_log_id, - Some(x) => Some(x.1.log_id), - }; - - debug!( - "get_log_state: ({:?},{:?}]", - last_purged_log_id, last_log_id - ); + let purged = state.purged().map(Cw::to_inner); + let last = state.last().map(Cw::to_inner); Ok(LogState { - last_purged_log_id, - last_log_id, + last_purged_log_id: purged, + last_log_id: last, }) } @@ -200,204 +153,156 @@ impl RaftLogStorage for RaftStore { } async fn save_committed(&mut self, committed: Option) -> Result<(), StorageError> { - self.raft_state - .write() - .await - .save_committed(committed) - .await - .map_to_sto_err(ErrorSubject::Store, ErrorVerb::Write) + let io = IODesc::save_committed(format!( + "RaftStore(id={})::save_committed({})", + self.id, + committed.display() + )); + + let Some(committed) = committed else { + warn!("{}: skip save_committed(None)", io); + return Ok(()); + }; + + { + let mut log = self.log.write().await; + log.commit(Cw(committed)).map_err(|e| io.err_submit(e))?; + } + + info!( + "{}; No need to flush committed, reversion is acceptable", + io.ok_submit() + ); + Ok(()) } async fn read_committed(&mut self) -> Result, StorageError> { - self.raft_state - .read() - .await - .read_committed() - .map_to_sto_err(ErrorSubject::Store, ErrorVerb::Read) + let log = self.log.read().await; + let committed = log.log_state().committed().map(Cw::to_inner); + + Ok(committed) } #[fastrace::trace] - async fn save_vote(&mut self, hs: &Vote) -> Result<(), StorageError> { - info!(id = self.id; "RaftStore::save_vote({}): start", hs); - - let res = self - .raft_state - .write() - .await - .save_vote(hs) - .await - .map_to_sto_err(ErrorSubject::Vote, ErrorVerb::Write); - - info!(id = self.id; "RaftStore::save_vote({}): done", hs); - - match res { - Err(err) => { - raft_metrics::storage::incr_raft_storage_fail("save_vote", true); - Err(err) - } - Ok(_) => Ok(()), + async fn save_vote(&mut self, vote: &Vote) -> Result<(), StorageError> { + let io = IODesc::save_vote(format!("RaftStore(id={})::save_vote({})", self.id, vote)); + + let (tx, rx) = oneshot::channel(); + + { + let mut log = self.log.write().await; + + log.save_vote(Cw(*vote)).map_err(|e| io.err_submit(e))?; + log.flush(raft_log_v004::Callback::new_oneshot(tx, &io)) + .map_err(|e| io.err_submit_flush(e))?; } + + rx.await + .map_err(|e| io.err_await_flush(e))? + .map_err(|e| io.err_recv_flush_cb(e))?; + + info!("{}: done", io.ok_done()); + Ok(()) } #[fastrace::trace] - async fn append( - &mut self, - entries: I, - callback: IOFlushed, - ) -> Result<(), StorageError> + async fn append(&mut self, entries: I, callback: IOFlushed) -> Result<(), StorageError> where I: IntoIterator + OptionalSend, I::IntoIter: OptionalSend, { - let mut first = None; - let mut last = None; - - let entries = entries + let mut entries = entries .into_iter() - .inspect(|x| { - if first.is_none() { - first = Some(x.log_id); - } - last = Some(x.log_id); - }) - .collect::>(); + .map(|x| (Cw(x.log_id), Cw(x.payload))) + .peekable(); - info!( - "RaftStore::append([{}, {}]): start", - first.display(), - last.display() - ); + let first = entries.peek().map(|x| x.0); - let res = match self.log.write().await.append(entries).await { - Err(err) => { - raft_metrics::storage::incr_raft_storage_fail("append_to_log", true); - Err(err) - } - Ok(_) => Ok(()), - }; + let io = IODesc::append(format!( + "RaftStore(id={})::append([{}, ...])", + self.id, + first.display() + )); - callback.io_completed(res.map_err(|e| io::Error::new(ErrorKind::InvalidData, e))); + let mut log = self.log.write().await; - info!( - "RaftStore::append([{}, {}]): done", - first.display(), - last.display() - ); + log.append(entries).map_err(|e| io.err_submit(e))?; + + debug!("{}", io.ok_submit()); + + log.flush(raft_log_v004::Callback::new_io_flushed(callback, &io)) + .map_err(|e| io.err_submit_flush(e))?; + + info!("{}", io.ok_submit_flush()); Ok(()) } #[fastrace::trace] async fn truncate(&mut self, log_id: LogId) -> Result<(), StorageError> { - info!(id = self.id; "RaftStore::truncate({}): start", log_id); - - let res = self - .log - .write() - .await - .range_remove(log_id.index..) - .await - .map_to_sto_err(ErrorSubject::Log(log_id), ErrorVerb::Delete); - - info!(id = self.id; "RaftStore::truncate({}): done", log_id); - - match res { - Ok(_) => Ok(()), - Err(err) => { - raft_metrics::storage::incr_raft_storage_fail("delete_conflict_logs_since", true); - Err(err) + let io = IODesc::truncate(format!( + "RaftStore(id={})::truncate(since={})", + self.id, log_id + )); + + let mut log = self.log.write().await; + + { + let curr_last = log.log_state().last().map(Cw::to_inner); + if log_id.index >= curr_last.next_index() { + warn!( + "{}: after curr_last({}), skip truncate", + io, + curr_last.display() + ); + return Ok(()); } } + + log.truncate(log_id.index).map_err(|e| io.err_submit(e))?; + + // No need to flush a truncate operation. + info!("{}; No need to flush", io.ok_submit()); + Ok(()) } #[fastrace::trace] async fn purge(&mut self, log_id: LogId) -> Result<(), StorageError> { - let curr_purged = self - .log - .write() - .await - .get_last_purged() - .map_to_sto_err(ErrorSubject::Logs, ErrorVerb::Read)?; + let io = IODesc::purge(format!("RaftStore(id={})::purge(upto={})", self.id, log_id)); - let purge_range = (curr_purged.next_index(), log_id.index); - let purge_range_str = format!("({},{}]", purge_range.0, purge_range.1); + let mut log = self.log.write().await; - info!( - id = self.id, - curr_purged :? =(&curr_purged), - upto_log_id :? =(&log_id); - "RaftStore::purge({}): start", purge_range_str); - - if let Err(err) = self - .log - .write() - .await - .set_last_purged(log_id) - .await - .map_to_sto_err(ErrorSubject::Logs, ErrorVerb::Write) { - raft_metrics::storage::incr_raft_storage_fail("purge_logs_upto", true); - return Err(err); - }; - - info!(id = self.id, log_id :? =(&log_id); "RaftStore::purge({}): Done: set_last_purged()", purge_range_str); - - let log = self.log.write().await.clone(); - - // Purge can be done in another task safely, because: - // - // - Next time when raft starts, it will read last_purged_log_id without examining the actual first log. - // And junk can be removed next time purge_logs_upto() is called. - // - // - Purging operates the start of the logs, and only committed logs are purged; - // while append and truncate operates on the end of the logs, - // it is safe to run purge && (append || truncate) concurrently. - databend_common_base::runtime::spawn({ - let id = self.id; - async move { - info!(id = id, log_id :? =(&log_id); "RaftStore::purge({}): Start: asynchronous one by one remove", purge_range_str); - - let mut removed_cnt = 0; - let mut removed_size = 0; - let curr = curr_purged.next_index(); - for i in curr..=log_id.index { - let res = log.logs().remove_no_return(&i, true).await; - - let removed = match res { - Ok(r) => r, - Err(err) => { - error!(id = id, log_index :% =i; - "RaftStore::purge({}): in asynchronous error: {}", purge_range_str, err); - raft_metrics::storage::incr_raft_storage_fail("purge_logs_upto", true); - return; - } - }; - - if let Some(size) = removed { - removed_cnt += 1; - removed_size += size; - } else { - error!(id = id, log_index :% =i; - "RaftStore::purge({}): in asynchronous error: not found, maybe removed by other thread; quit this thread", purge_range_str); - return; - } - - if i % 100 == 0 { - info!(id = id, log_index :% =i, - removed_cnt = removed_cnt, - removed_size = removed_size, - avg_removed_size = removed_size / (removed_cnt+1); - "RaftStore::purge({}): asynchronous removed log", purge_range_str); - } - - // Do not block for too long if there are many keys to delete. - tokio::time::sleep(Duration::from_millis(2)).await; - } - - info!(id = id, upto_log_id :? =(&log_id); "RaftStore::purge({}): Done: asynchronous one by one remove", purge_range_str); + let curr_purged = log.log_state().purged().map(Cw::to_inner); + if log_id.index < curr_purged.next_index() { + warn!( + "{}: before curr_purged({}), skip purge", + io, + curr_purged.display() + ); + return Ok(()); } - }); + } + + log.purge(Cw(log_id)).map_err(|e| io.err_submit(e))?; + info!("{}; No need to flush", io.ok_submit()); Ok(()) } } + +fn range_boundary>(range: RB) -> (u64, u64) { + let start = match range.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n + 1, + Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + Bound::Included(&n) => n + 1, + Bound::Excluded(&n) => n, + Bound::Unbounded => u64::MAX, + }; + + (start, end) +} diff --git a/src/meta/service/src/store/raft_state_machine_impl.rs b/src/meta/service/src/store/raft_state_machine_impl.rs index 3e3dd2e43d2c..58476898b7b1 100644 --- a/src/meta/service/src/store/raft_state_machine_impl.rs +++ b/src/meta/service/src/store/raft_state_machine_impl.rs @@ -13,7 +13,7 @@ // limitations under the License. use databend_common_meta_raft_store::sm_v003::open_snapshot::OpenSnapshot; -use databend_common_meta_raft_store::sm_v003::SnapshotStoreV003; +use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_sled_store::openraft::storage::RaftStateMachine; use databend_common_meta_sled_store::openraft::OptionalSend; use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; @@ -75,7 +75,7 @@ impl RaftStateMachine for RaftStore { // This method is not used #[fastrace::trace] async fn begin_receiving_snapshot(&mut self) -> Result, StorageError> { - let ss_store = SnapshotStoreV003::new(self.inner.config.clone()); + let ss_store = SnapshotStoreV004::new(self.inner.config.clone()); let db = ss_store .new_temp() .map_err(|e| StorageError::write_snapshot(None, &e))?; @@ -98,7 +98,7 @@ impl RaftStateMachine for RaftStore { let sig = meta.signature(); - let ss_store = SnapshotStoreV003::new(self.inner.config.clone()); + let ss_store = SnapshotStoreV004::new(self.inner.config.clone()); let final_path = ss_store .snapshot_config() .move_to_final_path(&snapshot.path, meta.snapshot_id.clone()) diff --git a/src/meta/service/src/store/store.rs b/src/meta/service/src/store/store.rs index 9669aebc202e..c908db4edb4b 100644 --- a/src/meta/service/src/store/store.rs +++ b/src/meta/service/src/store/store.rs @@ -36,12 +36,8 @@ impl RaftStore { } #[fastrace::trace] - pub async fn open_create( - config: &RaftConfig, - open: Option<()>, - create: Option<()>, - ) -> Result { - let sto = StoreInner::open_create(config, open, create).await?; + pub async fn open(config: &RaftConfig) -> Result { + let sto = StoreInner::open(config).await?; Ok(Self::new(sto)) } diff --git a/src/meta/service/src/store/store_inner.rs b/src/meta/service/src/store/store_inner.rs index 86a84fb6a6a2..7ba4b33c24f2 100644 --- a/src/meta/service/src/store/store_inner.rs +++ b/src/meta/service/src/store/store_inner.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fs; use std::io; use std::io::ErrorKind; use std::sync::Arc; @@ -22,28 +23,25 @@ use databend_common_base::base::tokio; use databend_common_base::base::tokio::sync::RwLock; use databend_common_base::base::tokio::sync::RwLockWriteGuard; use databend_common_meta_raft_store::config::RaftConfig; -use databend_common_meta_raft_store::key_spaces::RaftStateKV; use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; use databend_common_meta_raft_store::leveled_store::db_exporter::DBExporter; -use databend_common_meta_raft_store::log::RaftLog; +use databend_common_meta_raft_store::ondisk::Header; use databend_common_meta_raft_store::ondisk::TREE_HEADER; +use databend_common_meta_raft_store::raft_log_v004; +use databend_common_meta_raft_store::raft_log_v004::util; +use databend_common_meta_raft_store::raft_log_v004::Cw; +use databend_common_meta_raft_store::raft_log_v004::RaftLogV004; use databend_common_meta_raft_store::sm_v003::write_entry::WriteEntry; -use databend_common_meta_raft_store::sm_v003::SnapshotStoreV003; +use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_raft_store::sm_v003::SMV003; -use databend_common_meta_raft_store::state::RaftState; -use databend_common_meta_raft_store::state::RaftStateKey; -use databend_common_meta_raft_store::state::RaftStateValue; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; -use databend_common_meta_sled_store::get_sled_db; -use databend_common_meta_sled_store::SledTree; use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::Membership; use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::raft_types::Snapshot; use databend_common_meta_types::raft_types::SnapshotMeta; use databend_common_meta_types::raft_types::StorageError; -use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::Endpoint; use databend_common_meta_types::MetaNetworkError; @@ -53,22 +51,21 @@ use futures::TryStreamExt; use log::debug; use log::error; use log::info; +use raft_log::api::raft_log_writer::RaftLogWriter; use tokio::time::sleep; -use crate::export::vec_kv_to_json; use crate::Opened; /// This is the inner store that provides support utilities for implementing the raft storage API. /// -/// This store is backed by a sled db, contents are stored in 3 trees: -/// state: -/// id -/// vote -/// log +/// This store include two parts: +/// log(including id, vote, committed, and purged) /// state_machine pub struct StoreInner { /// The ID of the Raft node for which this storage instances is configured. - /// ID is also stored in raft_state. Since `id` never changes, this is a cache for fast access. + /// ID is also stored in raft-log. + /// + /// `id` never changes, this is a cache for fast access. pub id: NodeId, pub(crate) config: RaftConfig, @@ -76,17 +73,8 @@ pub struct StoreInner { /// If the instance is opened from an existent state(e.g. load from fs) or created. is_opened: bool, - /// The sled db for log, raft_state and state machine. - pub(crate) db: sled::Db, - - /// Raft state includes: - /// id: NodeId, - /// vote, // the last `Vote` - /// committed, // last `LogId` that is known committed - pub raft_state: RwLock, - /// A series of raft logs. - pub log: RwLock, + pub log: Arc>, /// The Raft state machine. pub state_machine: Arc>, @@ -106,26 +94,13 @@ impl Opened for StoreInner { } impl StoreInner { - /// Open an existent `metasrv` instance or create an new one: - /// 1. If `open` is `Some`, try to open an existent one. - /// 2. If `create` is `Some`, try to create one. - /// Otherwise it panic + /// Open an existent raft-store or create a new one. #[fastrace::trace] - pub async fn open_create( - config: &RaftConfig, - open: Option<()>, - create: Option<()>, - ) -> Result { - info!(config_id :% =(&config.config_id); "open: {:?}, create: {:?}", open, create); - - let db = get_sled_db(); - - let raft_state = RaftState::open_create(&db, config, open, create).await?; - let is_open = raft_state.is_open(); - info!("RaftState opened is_open: {}", is_open); - - let log = RaftLog::open(&db, config).await?; - info!("RaftLog opened"); + pub async fn open(config: &RaftConfig) -> Result { + info!( + "open_or_create StoreInner: id={}, config_id={}", + config.id, config.config_id + ); fn to_startup_err(e: impl std::error::Error + 'static) -> MetaStartupError { let ae = AnyError::new(&e); @@ -133,7 +108,41 @@ impl StoreInner { MetaStartupError::StoreOpenError(store_err) } - let ss_store = SnapshotStoreV003::new(config.clone()); + let raft_log_config = Arc::new(config.to_raft_log_config()); + + let dir = &raft_log_config.dir; + + fs::create_dir_all(dir).map_err(|e| { + let err = io::Error::new( + e.kind(), + format!("{}; when:(create raft log dir: {}", e, dir), + ); + to_startup_err(err) + })?; + + let mut log = RaftLogV004::open(raft_log_config.clone()).map_err(to_startup_err)?; + info!("RaftLog opened at: {}", raft_log_config.dir); + + let state = log.log_state(); + let stored_node_id = state.user_data.as_ref().and_then(|x| x.node_id); + + let is_open = stored_node_id.is_some(); + + // If id is stored, ignore the id in config. + let id = stored_node_id.unwrap_or(config.id); + + if !is_open { + log.save_user_data(Some(raft_log_v004::log_store_meta::LogStoreMeta { + node_id: Some(config.id), + })) + .map_err(to_startup_err)?; + + util::blocking_flush(&mut log) + .await + .map_err(to_startup_err)?; + } + + let ss_store = SnapshotStoreV004::new(config.clone()); let loader = ss_store.new_loader(); let last = loader.load_last_snapshot().await.map_err(to_startup_err)?; @@ -157,12 +166,10 @@ impl StoreInner { }; let store = Self { - id: raft_state.id, + id, config: config.clone(), is_opened: is_open, - db, - raft_state: RwLock::new(raft_state), - log: RwLock::new(log), + log: Arc::new(RwLock::new(log)), state_machine: Arc::new(RwLock::new(sm)), }; @@ -170,8 +177,8 @@ impl StoreInner { } /// Return a snapshot store of this instance. - pub fn snapshot_store(&self) -> SnapshotStoreV003 { - SnapshotStoreV003::new(self.config.clone()) + pub fn snapshot_store(&self) -> SnapshotStoreV004 { + SnapshotStoreV004::new(self.config.clone()) } async fn rebuild_state_machine(id: &MetaSnapshotId, snapshot: DB) -> Result { @@ -316,6 +323,14 @@ impl StoreInner { io::Error::new(ErrorKind::InvalidData, e) } + fn encode_entry(tree_name: &str, ent: &RaftStoreEntry) -> Result { + let name_entry = (tree_name, ent); + + let line = serde_json::to_string(&name_entry) + .map_err(|e| io::Error::new(ErrorKind::InvalidData, e))?; + Ok(line) + } + // Lock all data components so that we have a consistent view. // // Hold the singleton compactor to prevent snapshot from being replaced until exporting finished. @@ -344,71 +359,59 @@ impl StoreInner { } }; - let raft_state = self.raft_state.read().await; - let log = self.log.read().await; + let mut dump = { + let log = self.log.read().await; + log.dump_data() + }; + + // Log is dumped thus there won't be a gap between sm and log. + // It is now safe to release the compactor. + let db = compactor.db().cloned(); + drop(compactor); // Export data header first { - let header_tree = SledTree::open(&self.db, TREE_HEADER, false).map_err(invalid_data)?; - - let header_kvs = header_tree.export()?; - - for kv in header_kvs.iter() { - let line = vec_kv_to_json(TREE_HEADER, kv)?; - yield line; - } + let entry = RaftStoreEntry::new_header(Header::this_version()); + yield encode_entry(TREE_HEADER, &entry)?; } + let state = dump.state(); + // Export raft state { - let tree_name = &raft_state.inner.name; - - let ks = raft_state.inner.key_space::(); - - let id = ks.get(&RaftStateKey::Id)?.map(NodeId::from); - - if let Some(id) = id { - let ent_id = RaftStoreEntry::RaftStateKV { - key: RaftStateKey::Id, - value: RaftStateValue::NodeId(id), - }; - - let s = serde_json::to_string(&(tree_name, ent_id)).map_err(invalid_data)?; - yield s; - } + let tree_name = "raft_log"; - let vote = ks.get(&RaftStateKey::HardState)?.map(Vote::from); + let node_id = state.user_data.as_ref().and_then(|ud| ud.node_id); + let entry = RaftStoreEntry::NodeId(node_id); + yield encode_entry(tree_name, &entry)?; - if let Some(vote) = vote { - let ent_vote = RaftStoreEntry::RaftStateKV { - key: RaftStateKey::HardState, - value: RaftStateValue::HardState(vote), - }; - - let s = serde_json::to_string(&(tree_name, ent_vote)).map_err(invalid_data)?; - yield s; - } + let vote = state.vote().map(Cw::to_inner); + let entry = RaftStoreEntry::Vote(vote); + yield encode_entry(tree_name, &entry)?; - let committed = ks - .get(&RaftStateKey::Committed)? - .and_then(Option::::from); + let committed = state.committed().map(Cw::to_inner); + let entry = RaftStoreEntry::Committed(committed); + yield encode_entry(tree_name, &entry)?; + }; - let ent_committed = RaftStoreEntry::RaftStateKV { - key: RaftStateKey::Committed, - value: RaftStateValue::Committed(committed), - }; + { + let tree_name = "raft_log"; - let s = serde_json::to_string(&(tree_name, ent_committed)).map_err(invalid_data)?; - yield s; - }; + let purged = state.purged().map(Cw::to_inner); + let entry = RaftStoreEntry::Purged(purged); + yield encode_entry(tree_name, &entry)?; - drop(raft_state); + for res in dump.iter() { + let (log_id, payload) = res?; + let log_id = log_id.unpack(); + let payload = payload.unpack(); - // Dump logs that has smaller or equal leader id as `vote` - let log_tree_name = log.inner.name.clone(); - let log_kvs = log.inner.export()?; + let log_entry = Entry { log_id, payload }; - drop(log); + let entry = RaftStoreEntry::LogEntry(log_entry); + yield encode_entry(tree_name, &entry)?; + } + } // Dump snapshot of state machine @@ -416,16 +419,6 @@ impl StoreInner { // The name in form of "state_machine/[0-9]+" had been used by the sled tree based sm. // Do not change it for keeping compatibility. let sm_tree_name = "state_machine/0"; - let db = compactor.db().cloned(); - drop(compactor); - - for kv in log_kvs.iter() { - let kv_entry = RaftStoreEntry::deserialize(&kv[0], &kv[1])?; - - let tree_kv = (&log_tree_name, kv_entry); - let line = serde_json::to_string(&tree_kv).map_err(invalid_data)?; - yield line; - } info!("StoreInner::export db: {:?}", db); diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs index b121cb784812..c125903aa299 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs @@ -71,6 +71,7 @@ async fn test_restart() -> anyhow::Result<()> { srv.stop(None).await?; drop(client); + drop(srv); tokio::time::sleep(Duration::from_millis(1000)).await; diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index 814ee6b6282d..a81646be9af7 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -76,16 +76,18 @@ async fn test_export() -> anyhow::Result<()> { } let want = vec![ - r#"["test-29000-raft_state",{"RaftStateKV":{"key":"Id","value":{"NodeId":0}}}]"#, - r#"["test-29000-raft_state",{"RaftStateKV":{"key":"HardState","value":{"HardState":{"leader_id":{"term":1,"node_id":0},"committed":true}}}}]"#, - r#"["test-29000-raft_state",{"RaftStateKV":{"key":"Committed","value":{"Committed":{"leader_id":{"term":1,"node_id":0},"index":6}}}}]"#, - r#"["test-29000-raft_log",{"Logs":{"key":0,"value":{"log_id":{"leader_id":{"term":0,"node_id":0},"index":0},"payload":{"Membership":{"configs":[[0]],"nodes":{"0":{}}}}}}}]"#, - r#"["test-29000-raft_log",{"Logs":{"key":1,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":1},"payload":"Blank"}}}]"#, - r#"["test-29000-raft_log",{"Logs":{"key":2,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":2},"payload":{"Normal":{"txid":null,"time_ms":1111111111111,"cmd":{"AddNode":{"node_id":0,"node":{"name":"0","endpoint":{"addr":"localhost","port":29000},"grpc_api_advertise_address":"127.0.0.1:29000"},"overriding":false}}}}}}}]"#, - r#"["test-29000-raft_log",{"Logs":{"key":3,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":3},"payload":{"Membership":{"configs":[[0]],"nodes":{"0":{}}}}}}}]"#, - r#"["test-29000-raft_log",{"Logs":{"key":4,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":4},"payload":{"Normal":{"txid":null,"time_ms":1111111111111,"cmd":{"UpsertKV":{"key":"foo","seq":{"GE":0},"value":{"Update":[102,111,111]},"value_meta":null}}}}}}}]"#, - r#"["test-29000-raft_log",{"Logs":{"key":5,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":5},"payload":{"Normal":{"txid":null,"time_ms":1111111111111,"cmd":{"UpsertKV":{"key":"bar","seq":{"GE":0},"value":{"Update":[98,97,114]},"value_meta":null}}}}}}}]"#, - r#"["test-29000-raft_log",{"Logs":{"key":6,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":6},"payload":{"Normal":{"txid":null,"time_ms":1111111111111,"cmd":{"UpsertKV":{"key":"wow","seq":{"GE":0},"value":{"Update":[119,111,119]},"value_meta":null}}}}}}}]"#, + r#"["header",{"DataHeader":{"key":"header","value":{"version":"V004"}}}]"#, + r#"["raft_log",{"NodeId":0}]"#, + r#"["raft_log",{"Vote":{"leader_id":{"term":1,"node_id":0},"committed":true}}]"#, + r#"["raft_log",{"Committed":{"leader_id":{"term":1,"node_id":0},"index":6}}]"#, + r#"["raft_log",{"Purged":null}]"#, + r#"["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":0,"node_id":0},"index":0},"payload":{"Membership":{"configs":[[0]],"nodes":{"0":{}}}}}}]"#, + r#"["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":1},"payload":"Blank"}}]"#, + r#"["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":2},"payload":{"Normal":{"txid":null,"time_ms":1111111111111,"cmd":{"AddNode":{"node_id":0,"node":{"name":"0","endpoint":{"addr":"localhost","port":29000},"grpc_api_advertise_address":"127.0.0.1:29000"},"overriding":false}}}}}}]"#, + r#"["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":3},"payload":{"Membership":{"configs":[[0]],"nodes":{"0":{}}}}}}]"#, + r#"["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":4},"payload":{"Normal":{"txid":null,"time_ms":1111111111111,"cmd":{"UpsertKV":{"key":"foo","seq":{"GE":0},"value":{"Update":[102,111,111]},"value_meta":null}}}}}}]"#, + r#"["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":5},"payload":{"Normal":{"txid":null,"time_ms":1111111111111,"cmd":{"UpsertKV":{"key":"bar","seq":{"GE":0},"value":{"Update":[98,97,114]},"value_meta":null}}}}}}]"#, + r#"["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":6},"payload":{"Normal":{"txid":null,"time_ms":1111111111111,"cmd":{"UpsertKV":{"key":"wow","seq":{"GE":0},"value":{"Update":[119,111,119]},"value_meta":null}}}}}}]"#, r#"["state_machine/0",{"Sequences":{"key":"generic-kv","value":3}}]"#, r#"["state_machine/0",{"StateMachineMeta":{"key":"LastApplied","value":{"LogId":{"leader_id":{"term":1,"node_id":0},"index":6}}}}]"#, r#"["state_machine/0",{"StateMachineMeta":{"key":"LastMembership","value":{"Membership":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":3},"membership":{"configs":[[0]],"nodes":{"0":{}}}}}}}]"#, diff --git a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs index 0dcebfe27872..4eba750aa52a 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs @@ -18,6 +18,7 @@ use std::time::Duration; use databend_common_base::base::tokio::time::sleep; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_sled_store::openraft::LogIdOptionExt; +use databend_common_meta_sled_store::openraft::RaftLogReader; use databend_common_meta_sled_store::openraft::ServerState; use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; use databend_common_meta_types::raft_types::new_log_id; @@ -98,30 +99,31 @@ async fn test_meta_node_join() -> anyhow::Result<()> { let (mut _nlog, mut tcs) = start_meta_node_cluster(btreeset![0], btreeset![1]).await?; let mut all = test_context_nodes(&tcs); - let tc0 = tcs.remove(0); - let tc1 = tcs.remove(0); + let mut tc0 = tcs.remove(0); + let mut tc1 = tcs.remove(0); info!("--- bring up non-voter 2"); let node_id = 2; - let tc2 = MetaSrvTestContext::new(node_id); - - let mn2 = MetaNode::open_create(&tc2.config.raft_config, None, Some(())).await?; + let mut tc2 = MetaSrvTestContext::new(node_id); + { + let mn2 = MetaNode::open(&tc2.config.raft_config).await?; + all.push(mn2); + } info!("--- join non-voter 2 to cluster by leader"); + { + let leader_id = all[0].get_leader().await?.unwrap(); + let leader = all[leader_id as usize].clone(); - let leader_id = all[0].get_leader().await?.unwrap(); - let leader = all[leader_id as usize].clone(); - - let admin_req = join_req( - node_id, - tc2.config.raft_config.raft_api_addr().await?, - tc2.config.grpc_api_advertise_address(), - 0, - ); - leader.handle_forwardable_request(admin_req).await?; - - all.push(mn2.clone()); + let admin_req = join_req( + node_id, + tc2.config.raft_config.raft_api_addr().await?, + tc2.config.grpc_api_advertise_address(), + 0, + ); + leader.handle_forwardable_request(admin_req).await?; + } info!("--- check all nodes has node-3 joined"); { @@ -136,8 +138,11 @@ async fn test_meta_node_join() -> anyhow::Result<()> { info!("--- bring up non-voter 3"); let node_id = 3; - let tc3 = MetaSrvTestContext::new(node_id); - let mn3 = MetaNode::open_create(&tc3.config.raft_config, None, Some(())).await?; + let mut tc3 = MetaSrvTestContext::new(node_id); + { + let mn3 = MetaNode::open(&tc3.config.raft_config).await?; + all.push(mn3.clone()); + } info!("--- join node-3 by sending rpc `join` to a non-leader"); { @@ -155,7 +160,6 @@ async fn test_meta_node_join() -> anyhow::Result<()> { info!("--- check all nodes has node-3 joined"); - all.push(mn3.clone()); for mn in all.iter() { mn.raft .wait(timeout()) @@ -171,13 +175,21 @@ async fn test_meta_node_join() -> anyhow::Result<()> { for mn in all.drain(..) { mn.stop().await?; } + drop(all); info!("--- re-open all meta node"); - let mn0 = MetaNode::open_create(&tc0.config.raft_config, Some(()), None).await?; - let mn1 = MetaNode::open_create(&tc1.config.raft_config, Some(()), None).await?; - let mn2 = MetaNode::open_create(&tc2.config.raft_config, Some(()), None).await?; - let mn3 = MetaNode::open_create(&tc3.config.raft_config, Some(()), None).await?; + tc0.drop_meta_node(); + tc1.drop_meta_node(); + tc2.drop_meta_node(); + tc3.drop_meta_node(); + + sleep(Duration::from_secs(1)).await; + + let mn0 = MetaNode::open(&tc0.config.raft_config).await?; + let mn1 = MetaNode::open(&tc1.config.raft_config).await?; + let mn2 = MetaNode::open(&tc2.config.raft_config).await?; + let mn3 = MetaNode::open(&tc3.config.raft_config).await?; let all = [mn0, mn1, mn2, mn3]; @@ -209,7 +221,7 @@ async fn test_meta_node_join_rejoin() -> anyhow::Result<()> { let node_id = 1; let tc1 = MetaSrvTestContext::new(node_id); - let mn1 = MetaNode::open_create(&tc1.config.raft_config, None, Some(())).await?; + let mn1 = MetaNode::open(&tc1.config.raft_config).await?; info!("--- join non-voter 1 to cluster"); @@ -240,7 +252,7 @@ async fn test_meta_node_join_rejoin() -> anyhow::Result<()> { let node_id = 2; let tc2 = MetaSrvTestContext::new(node_id); - let mn2 = MetaNode::open_create(&tc2.config.raft_config, None, Some(())).await?; + let mn2 = MetaNode::open(&tc2.config.raft_config).await?; info!("--- join node-2 by sending rpc `join` to a non-leader"); { @@ -385,7 +397,8 @@ async fn test_meta_node_leave() -> anyhow::Result<()> { // - Leave a non-voter node by sending a Leave request to a non-voter. // - Restart all nodes and check if states are restored. - let (mut log_index, tcs) = start_meta_node_cluster(btreeset![0, 1, 2], btreeset![3]).await?; + let (mut log_index, mut tcs) = + start_meta_node_cluster(btreeset![0, 1, 2], btreeset![3]).await?; let mut all = test_context_nodes(&tcs); let leader_id = 0; @@ -467,11 +480,15 @@ async fn test_meta_node_leave() -> anyhow::Result<()> { // restart the cluster and check membership info!("--- re-open all meta node"); + drop(leader); + tcs[0].drop_meta_node(); + tcs[2].drop_meta_node(); + let tc0 = &tcs[0]; let tc2 = &tcs[2]; - let mn0 = MetaNode::open_create(&tc0.config.raft_config, Some(()), None).await?; - let mn2 = MetaNode::open_create(&tc2.config.raft_config, Some(()), None).await?; + let mn0 = MetaNode::open(&tc0.config.raft_config).await?; + let mn2 = MetaNode::open(&tc2.config.raft_config).await?; let all = [mn0, mn2]; @@ -678,7 +695,7 @@ async fn test_meta_node_restart_single_node() -> anyhow::Result<()> { // - TODO(xp): A new snapshot will be created and transferred on demand. let mut log_index: u64 = 0; - let (_id, tc) = start_meta_node_leader().await?; + let (_id, mut tc) = start_meta_node_leader().await?; // initial membership, leader blank, add node log_index += 2; @@ -697,16 +714,18 @@ async fn test_meta_node_restart_single_node() -> anyhow::Result<()> { .await?; log_index += 1; - want_hs = leader.sto.raft_state.read().await.read_vote()?; + want_hs = leader.sto.clone().read_vote().await?; leader.stop().await?; } info!("--- reopen MetaNode"); + tc.drop_meta_node(); + let raft_conf = &tc.config.raft_config; - let leader = MetaNode::open_create(raft_conf, Some(()), None).await?; + let leader = MetaNode::open(raft_conf).await?; log_index += 1; @@ -729,13 +748,13 @@ async fn test_meta_node_restart_single_node() -> anyhow::Result<()> { info!("--- check hard state"); { - let hs = leader.sto.raft_state.read().await.read_vote()?; + let hs = leader.sto.clone().read_vote().await?; assert_eq!(want_hs, hs); } info!("--- check logs"); { - let logs = leader.sto.log.read().await.range_values(..)?; + let logs = leader.sto.clone().try_get_log_entries(..).await?; info!("logs: {:?}", logs); assert_eq!(log_index as usize + 1, logs.len()); } diff --git a/src/meta/service/tests/it/meta_node/meta_node_replication.rs b/src/meta/service/tests/it/meta_node/meta_node_replication.rs index 73d8e45bd94f..68e752edc9de 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_replication.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_replication.rs @@ -16,7 +16,7 @@ use std::fs; use std::io::Read; use databend_common_arrow::arrow::array::ViewType; -use databend_common_meta_raft_store::sm_v003::SnapshotStoreV003; +use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; use databend_common_meta_sled_store::openraft::error::SnapshotMismatch; use databend_common_meta_sled_store::openraft::testing::log_id; @@ -288,7 +288,7 @@ async fn test_raft_service_install_snapshot_v003() -> anyhow::Result<()> { }; // build a temp snapshot data - let ss_store = SnapshotStoreV003::new(tc0.config.raft_config.clone()); + let ss_store = SnapshotStoreV004::new(tc0.config.raft_config.clone()); let writer = ss_store.new_writer()?; let snapshot_data = { diff --git a/src/meta/service/tests/it/store.rs b/src/meta/service/tests/it/store.rs index 92565d0df8ec..d64400495c3c 100644 --- a/src/meta/service/tests/it/store.rs +++ b/src/meta/service/tests/it/store.rs @@ -52,7 +52,7 @@ struct MetaStoreBuilder {} impl StoreBuilder for MetaStoreBuilder { async fn build(&self) -> Result<(MetaSrvTestContext, LogStore, SMStore), StorageError> { let tc = MetaSrvTestContext::new(555); - let sto = RaftStore::open_create(&tc.config.raft_config, None, Some(())) + let sto = RaftStore::open(&tc.config.raft_config) .await .expect("fail to create store"); Ok((tc, sto.clone(), sto)) @@ -81,7 +81,7 @@ async fn test_meta_store_restart() -> anyhow::Result<()> { info!("--- new meta store"); { - let mut sto = RaftStore::open_create(&tc.config.raft_config, None, Some(())).await?; + let mut sto = RaftStore::open(&tc.config.raft_config).await?; assert_eq!(id, sto.id); assert!(!sto.is_opened()); assert_eq!(None, sto.read_vote().await?); @@ -100,7 +100,7 @@ async fn test_meta_store_restart() -> anyhow::Result<()> { info!("--- reopen meta store"); { - let mut sto = RaftStore::open_create(&tc.config.raft_config, Some(()), None).await?; + let mut sto = RaftStore::open(&tc.config.raft_config).await?; assert_eq!(id, sto.id); assert!(sto.is_opened()); assert_eq!(Some(Vote::new(10, 5)), sto.read_vote().await?); @@ -126,13 +126,13 @@ async fn test_meta_store_build_snapshot() -> anyhow::Result<()> { let id = 3; let tc = MetaSrvTestContext::new(id); - let mut sto = RaftStore::open_create(&tc.config.raft_config, None, Some(())).await?; + let mut sto = RaftStore::open(&tc.config.raft_config).await?; info!("--- feed logs and state machine"); let (logs, want) = snapshot_logs(); - sto.log.write().await.append(logs.clone()).await?; + sto.blocking_append(logs.clone()).await?; sto.state_machine.write().await.apply_entries(logs).await?; let curr_snap = sto.build_snapshot().await?; @@ -174,13 +174,13 @@ async fn test_meta_store_current_snapshot() -> anyhow::Result<()> { let id = 3; let tc = MetaSrvTestContext::new(id); - let mut sto = RaftStore::open_create(&tc.config.raft_config, None, Some(())).await?; + let mut sto = RaftStore::open(&tc.config.raft_config).await?; info!("--- feed logs and state machine"); let (logs, want) = snapshot_logs(); - sto.log.write().await.append(logs.clone()).await?; + sto.blocking_append(logs.clone()).await?; { let mut sm = sto.state_machine.write().await; sm.apply_entries(logs).await?; @@ -221,11 +221,11 @@ async fn test_meta_store_install_snapshot() -> anyhow::Result<()> { { let tc = MetaSrvTestContext::new(id); - let mut sto = RaftStore::open_create(&tc.config.raft_config, None, Some(())).await?; + let mut sto = RaftStore::open(&tc.config.raft_config).await?; info!("--- feed logs and state machine"); - sto.log.write().await.append(logs.clone()).await?; + sto.blocking_append(logs.clone()).await?; sto.state_machine.write().await.apply_entries(logs).await?; snap = sto.build_snapshot().await?; @@ -237,7 +237,7 @@ async fn test_meta_store_install_snapshot() -> anyhow::Result<()> { { let tc = MetaSrvTestContext::new(id); - let mut sto = RaftStore::open_create(&tc.config.raft_config, None, Some(())).await?; + let mut sto = RaftStore::open(&tc.config.raft_config).await?; info!("--- install snapshot"); { diff --git a/src/meta/service/tests/it/tests/meta_node.rs b/src/meta/service/tests/it/tests/meta_node.rs index 27fd4bc4c139..fec6ea5e5b03 100644 --- a/src/meta/service/tests/it/tests/meta_node.rs +++ b/src/meta/service/tests/it/tests/meta_node.rs @@ -193,7 +193,7 @@ pub(crate) async fn start_meta_node_non_voter( let raft_conf = &tc.config.raft_config; - let mn = MetaNode::open_create(raft_conf, None, Some(())).await?; + let mn = MetaNode::open(raft_conf).await?; // // Disable heartbeat, because in openraft v0.8 heartbeat is a blank log. // // Log index becomes non-deterministic. diff --git a/src/meta/service/tests/it/tests/service.rs b/src/meta/service/tests/it/tests/service.rs index e145626fef3b..ee59f4846538 100644 --- a/src/meta/service/tests/it/tests/service.rs +++ b/src/meta/service/tests/it/tests/service.rs @@ -110,6 +110,8 @@ pub fn next_port() -> u16 { /// It holds a reference to a MetaNode or a GrpcServer, for testing MetaNode or GrpcServer. pub struct MetaSrvTestContext { + pub _temp_dir: tempfile::TempDir, + pub config: configs::Config, pub meta_node: Option>, @@ -126,6 +128,8 @@ impl Drop for MetaSrvTestContext { impl MetaSrvTestContext { /// Create a new Config for test, with unique port assigned pub fn new(id: u64) -> MetaSrvTestContext { + let temp_dir = tempfile::tempdir().unwrap(); + let config_id = next_port(); let mut config = configs::Config::default(); @@ -141,7 +145,8 @@ impl MetaSrvTestContext { config.raft_config.config_id = config_id.to_string(); // Use a unique dir for each test case. - config.raft_config.raft_dir = format!("{}-{}", config.raft_config.raft_dir, config_id); + config.raft_config.raft_dir = + format!("{}/{}/raft_dir", temp_dir.path().display(), config_id); // By default, create a meta node instead of open an existent one. config.raft_config.single = true; @@ -175,6 +180,7 @@ impl MetaSrvTestContext { config, meta_node: None, grpc_srv: None, + _temp_dir: temp_dir, }; c.rm_raft_dir("new MetaSrvTestContext"); @@ -244,6 +250,11 @@ impl MetaSrvTestContext { client.forward(req).await?; Ok(()) } + + pub fn drop_meta_node(&mut self) { + self.meta_node.take(); + self.grpc_srv.take(); + } } /// Build metasrv or metasrv cluster, returns the clients diff --git a/src/meta/sled-store/src/db.rs b/src/meta/sled-store/src/db.rs index 23dccf960fa6..976581630ebf 100644 --- a/src/meta/sled-store/src/db.rs +++ b/src/meta/sled-store/src/db.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use std::sync::LazyLock; use std::sync::Mutex; +use log::warn; use tempfile::TempDir; pub(crate) struct GlobalSledDb { @@ -94,7 +95,7 @@ pub fn init_sled_db(path: String, cache_size: u64) { }; if inited_as_temp { - panic!( + warn!( "sled db is already initialized with temp dir: {}, can not re-init with path {}", curr_path, path ); @@ -113,3 +114,18 @@ pub fn get_sled_db() -> sled::Db { panic!("init_sled_db() or init_temp_sled_db() has to be called before using get_sled_db()"); } + +pub fn init_get_sled_db(path: String, cache_size: u64) -> sled::Db { + init_sled_db(path, cache_size); + get_sled_db() +} + +/// Drop the global sled db. +/// +/// Which means this program will not use sled db anymore. +pub fn drop_sled_db() { + { + let mut guard = GLOBAL_SLED.as_ref().lock().unwrap(); + *guard = None; + } +} diff --git a/src/meta/sled-store/src/lib.rs b/src/meta/sled-store/src/lib.rs index 60d41bc44d18..af488668f521 100644 --- a/src/meta/sled-store/src/lib.rs +++ b/src/meta/sled-store/src/lib.rs @@ -18,7 +18,9 @@ //! //! It is used by raft for log and state machine storage. pub use bytes_error::SledBytesError; +pub use db::drop_sled_db; pub use db::get_sled_db; +pub use db::init_get_sled_db; pub use db::init_sled_db; pub use db::init_temp_sled_db; pub use openraft; diff --git a/src/meta/stoerr/src/meta_storage_errors.rs b/src/meta/stoerr/src/meta_storage_errors.rs index 9259f0b21e2b..a80c953fb93e 100644 --- a/src/meta/stoerr/src/meta_storage_errors.rs +++ b/src/meta/stoerr/src/meta_storage_errors.rs @@ -76,6 +76,12 @@ impl From for MetaStorageError { } } +impl From for MetaStorageError { + fn from(error: io::Error) -> Self { + MetaStorageError::Damaged(AnyError::new(&error)) + } +} + impl From for io::Error { fn from(e: MetaStorageError) -> Self { io::Error::new(io::ErrorKind::InvalidData, e) diff --git a/src/meta/types/src/raft_types.rs b/src/meta/types/src/raft_types.rs index e17c27ff4bde..40037180ddb6 100644 --- a/src/meta/types/src/raft_types.rs +++ b/src/meta/types/src/raft_types.rs @@ -40,6 +40,8 @@ impl RaftTypeConfig for TypeConfig { type Responder = OneshotResponder; } +pub type IOFlushed = openraft::storage::IOFlushed; + pub type CommittedLeaderId = openraft::CommittedLeaderId; pub type LogId = openraft::LogId; pub type Vote = openraft::Vote; @@ -58,6 +60,7 @@ pub type SnapshotSegmentId = openraft::SnapshotSegmentId; pub type RaftMetrics = openraft::RaftMetrics; pub type ErrorSubject = openraft::ErrorSubject; +pub type ErrorVerb = openraft::ErrorVerb; pub type RPCError = openraft::error::RPCError; pub type RemoteError = openraft::error::RemoteError; diff --git a/tests/compat/meta_meta/test_meta_meta.sh b/tests/compat/meta_meta/test_meta_meta.sh index aca77e628523..1867f1649d09 100755 --- a/tests/compat/meta_meta/test_meta_meta.sh +++ b/tests/compat/meta_meta/test_meta_meta.sh @@ -134,28 +134,71 @@ curl -qs $(admin_addr 2)/v1/cluster/status echo " === Check consistency between leader and follower" echo "" -echo " === Export leader meta data to ./.databend/leader" -# skip DataHeader that contains distinguished version info -# skip NodeId -# sort because newer version export `Sequence` in different order +echo " === Export leader meta data to ./.databend/leader-tmp" ./bins/$leader_meta_ver/bin/databend-metactl \ --export \ --grpc-api-address $(grpc_addr 1) \ - | grep -v 'NodeId\|DataHeader' \ - | sort \ - > ./.databend/leader + > ./.databend/leader-tmp -echo " === Export follower meta data to ./.databend/follower" +echo " === Export follower meta data to ./.databend/follower-tmp" ./bins/$follower_meta_ver/bin/databend-metactl \ --export \ --grpc-api-address $(grpc_addr 2) \ + > ./.databend/follower-tmp + +echo " === Shutdown databend-meta servers" +killall databend-meta +sleep 3 + +# Old version SM exported data contains DataHeader + +cat ./.databend/leader-tmp | grep 'state_machine' | grep -v DataHeader | sort > ./.databend/leader-sm +cat ./.databend/follower-tmp | grep 'state_machine' | grep -v DataHeader | sort > ./.databend/follower-sm + +echo " === diff SM data between Leader and Follower" +diff ./.databend/leader-sm ./.databend/follower-sm + + + +echo " === mkdir to import with latest datbend-metactl" +mkdir -p ./.databend/_upgrade_meta_1 +mkdir -p ./.databend/_upgrade_meta_2 + + +# Exported log data format has changed, re-import them and compare. +# +# SM data in V002 does not output in correct order: exp- is after kv-, +# which is out of order when import to rotbl. +# +# Thus we skip all state machine data, but keeps log data and SM meta. + +echo " === Import Leader's log data" +cat ./.databend/leader-tmp \ + | grep -v '"Expire":\|"GenericKV":' \ + | ./bins/current/bin/databend-metactl --import --raft-dir ./.databend/_upgrade_meta_1 + +echo " === Import Follower's log data" +cat ./.databend/follower-tmp \ + | grep -v '"Expire":\|"GenericKV":' \ + | ./bins/current/bin/databend-metactl --import --raft-dir ./.databend/_upgrade_meta_2 + +# skip DataHeader that contains distinguished version info +# skip NodeId +# sort because newer version export `Sequence` in different order + +echo " === Export Leader's data" +./bins/current/bin/databend-metactl --export --raft-dir ./.databend/_upgrade_meta_1 \ + | grep -v 'NodeId\|DataHeader' \ + | sort \ + > ./.databend/leader + +echo " === Export Follower's data" +./bins/current/bin/databend-metactl --export --raft-dir ./.databend/_upgrade_meta_2 \ | grep -v 'NodeId\|DataHeader' \ | sort \ > ./.databend/follower + echo " === diff leader exported and follower exported" diff ./.databend/leader ./.databend/follower - - - diff --git a/tests/metactl/.gitignore b/tests/metactl/.gitignore index 26732633add0..74997cebf209 100644 --- a/tests/metactl/.gitignore +++ b/tests/metactl/.gitignore @@ -1,2 +1,3 @@ /_meta_dir /exported +/grpc_exported diff --git a/tests/metactl/meta_v003.txt b/tests/metactl/meta_v003.txt index ae7ce1edc849..ed10555457d8 100644 --- a/tests/metactl/meta_v003.txt +++ b/tests/metactl/meta_v003.txt @@ -1,8 +1,6 @@ ["header",{"DataHeader":{"key":"header","value":{"version":"V003","upgrading":null}}}] ["raft_state",{"RaftStateKV":{"key":"Id","value":{"NodeId":1}}}] ["raft_state",{"RaftStateKV":{"key":"HardState","value":{"HardState":{"leader_id":{"term":1,"node_id":1},"committed":false}}}}] -["raft_log",{"Logs":{"key":0,"value":{"log_id":{"leader_id":{"term":0,"node_id":0},"index":0},"payload":{"Membership":{"configs":[[1]],"nodes":{"1":{}}}}}}}] -["raft_log",{"Logs":{"key":1,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":1},"payload":"Blank"}}}] ["raft_log",{"Logs":{"key":2,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":2},"payload":{"Normal":{"txid":null,"time_ms":1667290820099,"cmd":{"AddNode":{"node_id":1,"node":{"name":"1","endpoint":{"addr":"localhost","port":28103},"grpc_api_advertise_address":"0.0.0.0:9191"},"overriding":false}}}}}}}] ["raft_log",{"Logs":{"key":3,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":3},"payload":{"Normal":{"txid":null,"time_ms":1667290820429,"cmd":{"AddNode":{"node_id":2,"node":{"name":"2","endpoint":{"addr":"localhost","port":28203},"grpc_api_advertise_address":"0.0.0.0:28202"},"overriding":false}}}}}}}] ["raft_log",{"Logs":{"key":4,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":4},"payload":{"Membership":{"configs":[[1],[1,2]],"nodes":{"1":{},"2":{}}}}}}}] @@ -85,6 +83,7 @@ ["raft_log",{"Logs":{"key":81,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":81},"payload":{"Normal":{"txid":null,"time_ms":1667290965904,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291025}}}}}}}}] ["raft_log",{"Logs":{"key":82,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":82},"payload":{"Normal":{"txid":null,"time_ms":1667290971893,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291031}}}}}}}}] ["raft_log",{"Logs":{"key":83,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":83},"payload":{"Normal":{"txid":null,"time_ms":1667290974891,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291034}}}}}}}}] +["raft_log",{"LogMeta":{"key":"LastPurged","value":{"LogId":{"leader_id":{"term":1,"node_id":0},"index":1}}}}] ["state_machine/0",{"Nodes":{"key":1,"value":{"name":"1","endpoint":{"addr":"localhost","port":28103},"grpc_api_advertise_address":"0.0.0.0:9191"}}}] ["state_machine/0",{"Nodes":{"key":2,"value":{"name":"2","endpoint":{"addr":"localhost","port":28203},"grpc_api_advertise_address":"0.0.0.0:28202"}}}] ["state_machine/0",{"Nodes":{"key":3,"value":{"name":"3","endpoint":{"addr":"localhost","port":28303},"grpc_api_advertise_address":"0.0.0.0:28302"}}}] diff --git a/tests/metactl/meta_v004.txt b/tests/metactl/meta_v004.txt new file mode 100644 index 000000000000..72fedaf22c29 --- /dev/null +++ b/tests/metactl/meta_v004.txt @@ -0,0 +1,167 @@ +["header",{"DataHeader":{"key":"header","value":{"version":"V004"}}}] +["raft_log",{"NodeId":1}] +["raft_log",{"Vote":{"leader_id":{"term":1,"node_id":1},"committed":false}}] +["raft_log",{"Committed":null}] +["raft_log",{"Purged":{"leader_id":{"term":1,"node_id":0},"index":1}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":2},"payload":{"Normal":{"txid":null,"time_ms":1667290820099,"cmd":{"AddNode":{"node_id":1,"node":{"name":"1","endpoint":{"addr":"localhost","port":28103},"grpc_api_advertise_address":"0.0.0.0:9191"},"overriding":false}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":3},"payload":{"Normal":{"txid":null,"time_ms":1667290820429,"cmd":{"AddNode":{"node_id":2,"node":{"name":"2","endpoint":{"addr":"localhost","port":28203},"grpc_api_advertise_address":"0.0.0.0:28202"},"overriding":false}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":4},"payload":{"Membership":{"configs":[[1],[1,2]],"nodes":{"1":{},"2":{}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":5},"payload":{"Membership":{"configs":[[1,2]],"nodes":{"1":{},"2":{}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":6},"payload":{"Normal":{"txid":null,"time_ms":1667290821018,"cmd":{"AddNode":{"node_id":3,"node":{"name":"3","endpoint":{"addr":"localhost","port":28303},"grpc_api_advertise_address":"0.0.0.0:28302"},"overriding":false}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":7},"payload":{"Membership":{"configs":[[1,2],[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":8},"payload":{"Membership":{"configs":[[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":9},"payload":{"Normal":{"txid":null,"time_ms":1667290824580,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":10},"payload":{"Normal":{"txid":null,"time_ms":1667290824586,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/default","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/1","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/default","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/default","value":[49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/default","value":[10,1,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/1","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,100,101,102,97,117,108,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":11},"payload":{"Normal":{"txid":null,"time_ms":1667290824603,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,82,112,117,87,110,100,84,102,53,74,108,103,121,74,67,112,65,65,116,81,88,54,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,48,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290884}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":12},"payload":{"Normal":{"txid":null,"time_ms":1667290826333,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,75,77,90,52,86,118,113,68,70,86,69,120,108,90,70,84,104,75,68,122,90,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,50,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290886}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":13},"payload":{"Normal":{"txid":null,"time_ms":1667290826867,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,55,71,86,80,49,71,115,81,50,106,112,68,77,117,49,116,105,56,85,110,70,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,51,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290886}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":14},"payload":{"Normal":{"txid":null,"time_ms":1667290836132,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":15},"payload":{"Normal":{"txid":null,"time_ms":1667290836137,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/9","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db1","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db1","value":[57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db1","value":[10,1,9,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/9","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":16},"payload":{"Normal":{"txid":null,"time_ms":1667290838209,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":17},"payload":{"Normal":{"txid":null,"time_ms":1667290838215,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/9","expected":0,"target":{"Seq":11}},{"key":"__fd_table/9/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/9/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_to_name/14","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/9/t1","value":[49,52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/9/t1","value":[10,1,14,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/14","value":[8,9,18,2,116,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":18},"payload":{"Normal":{"txid":null,"time_ms":1667290838764,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/14","expected":0,"target":{"Seq":17}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,53,99,55,49,49,56,55,98,49,53,97,52,52,53,54,100,98,49,55,50,48,50,98,100,51,97,55,101,99,102,56,51,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,2,16,170,2,24,228,7,32,184,12,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":19},"payload":{"Normal":{"txid":null,"time_ms":1667290843099,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/14","expected":0,"target":{"Seq":21}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,99,99,51,56,51,102,51,52,57,50,54,54,52,50,53,99,98,55,51,56,100,50,99,99,49,49,54,53,53,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,4,16,212,4,24,200,15,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":20},"payload":{"Normal":{"txid":null,"time_ms":1667290846841,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":10}},{"key":"__fd_database_by_id/9","expected":0,"target":{"Seq":15}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db1","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,54,46,56,52,48,55,51,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":21},"payload":{"Normal":{"txid":null,"time_ms":1667290848281,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290908}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":22},"payload":{"Normal":{"txid":null,"time_ms":1667290849028,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":23},"payload":{"Normal":{"txid":null,"time_ms":1667290849033,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/25","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db1","expected":0,"target":{"Seq":12}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db1","value":[50,53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db1","value":[10,2,9,25,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/25","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":24},"payload":{"Normal":{"txid":null,"time_ms":1667290851094,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":25},"payload":{"Normal":{"txid":null,"time_ms":1667290851100,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/25","expected":0,"target":{"Seq":27}},{"key":"__fd_table/25/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/25/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":19}},{"key":"__fd_table_id_to_name/30","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/25/t1","value":[51,48],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/25/t1","value":[10,1,30,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/30","value":[8,25,18,2,116,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":26},"payload":{"Normal":{"txid":null,"time_ms":1667290851947,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290911}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":27},"payload":{"Normal":{"txid":null,"time_ms":1667290852139,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/30","expected":0,"target":{"Seq":33}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,56,100,97,99,49,50,51,97,97,54,101,55,52,51,53,102,97,55,49,52,50,99,55,100,56,54,98,98,49,98,54,55,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,1,16,148,1,24,140,7,32,184,12,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":28},"payload":{"Normal":{"txid":null,"time_ms":1667290853220,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/30","expected":0,"target":{"Seq":38}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,50,97,57,100,99,97,53,55,52,51,54,54,52,53,101,97,97,53,97,51,52,100,57,100,102,97,50,51,55,98,50,54,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,2,16,168,2,24,152,14,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":29},"payload":{"Normal":{"txid":null,"time_ms":1667290861626,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290921}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":30},"payload":{"Normal":{"txid":null,"time_ms":1667290863578,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":26}},{"key":"__fd_database_by_id/25","expected":0,"target":{"Seq":31}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db1","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,51,46,53,55,55,49,55,50,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":31},"payload":{"Normal":{"txid":null,"time_ms":1667290864962,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":32},"payload":{"Normal":{"txid":null,"time_ms":1667290864967,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/book_db","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/42","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/book_db","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/book_db","value":[52,50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/book_db","value":[10,1,42,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/42","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,98,111,111,107,95,100,98,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":33},"payload":{"Normal":{"txid":null,"time_ms":1667290867024,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":34},"payload":{"Normal":{"txid":null,"time_ms":1667290867030,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/42","expected":0,"target":{"Seq":44}},{"key":"__fd_table/42/books","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/42/books","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":35}},{"key":"__fd_table_id_to_name/47","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/42/books","value":[52,55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/47","value":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/42/books","value":[10,1,47,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/47","value":[8,42,18,5,98,111,111,107,115,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":35},"payload":{"Normal":{"txid":null,"time_ms":1667290867598,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/47","expected":0,"target":{"Seq":50}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/47","value":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,52,50,47,52,55,47,95,115,115,47,55,48,98,53,48,55,53,97,54,98,51,99,52,55,102,51,57,57,100,51,48,102,57,100,52,97,54,98,98,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,16,8,1,16,64,24,180,3,32,176,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":36},"payload":{"Normal":{"txid":null,"time_ms":1667290870377,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/book_db","expected":0,"target":{"Seq":43}},{"key":"__fd_database_by_id/42","expected":0,"target":{"Seq":48}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/book_db","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,48,46,51,55,54,53,56,53,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":37},"payload":{"Normal":{"txid":null,"time_ms":1667290872604,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":38},"payload":{"Normal":{"txid":null,"time_ms":1667290872609,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db12_0002","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/56","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db12_0002","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db12_0002","value":[53,54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db12_0002","value":[10,1,56,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/56","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,9,100,98,49,50,95,48,48,48,50,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":39},"payload":{"Normal":{"txid":null,"time_ms":1667290874661,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":40},"payload":{"Normal":{"txid":null,"time_ms":1667290874666,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":58}},{"key":"__fd_table/56/t","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/56/t","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":52}},{"key":"__fd_table_id_to_name/61","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/56/t","value":[54,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/61","value":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/56/t","value":[10,1,61,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/61","value":[8,56,18,1,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":41},"payload":{"Normal":{"txid":null,"time_ms":1667290875684,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":62}},{"key":"__fd_table/56/t","expected":0,"target":{"Seq":63}},{"key":"__fd_table_by_id/61","expected":0,"target":{"Seq":64}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":66}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Delete":{"key":"__fd_table/56/t","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_table_by_id/61","value":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,53,46,54,56,51,49,52,51,32,85,84,67,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":42},"payload":{"Normal":{"txid":null,"time_ms":1667290880720,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db12_0002","expected":0,"target":{"Seq":57}},{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":68}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db12_0002","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,48,46,55,49,56,55,54,51,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":43},"payload":{"Normal":{"txid":null,"time_ms":1667290881744,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290941}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":44},"payload":{"Normal":{"txid":null,"time_ms":1667290883474,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":45},"payload":{"Normal":{"txid":null,"time_ms":1667290883479,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":3}},{"key":"__fd_table/1/t12_0004","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/t12_0004","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":70}},{"key":"__fd_table_id_to_name/73","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/t12_0004","value":[55,51],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/t12_0004","value":[10,1,73,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/73","value":[8,1,18,8,116,49,50,95,48,48,48,52,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":46},"payload":{"Normal":{"txid":null,"time_ms":1667290885036,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":76}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,48,97,99,51,52,98,49,56,52,48,56,50,52,57,97,99,98,97,101,51,50,98,53,56,101,51,101,57,55,50,102,98,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,2,16,8,24,188,1,32,148,2,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":47},"payload":{"Normal":{"txid":null,"time_ms":1667290886657,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":80}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":48},"payload":{"Normal":{"txid":null,"time_ms":1667290889731,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290949}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":49},"payload":{"Normal":{"txid":null,"time_ms":1667290890274,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290950}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":50},"payload":{"Normal":{"txid":null,"time_ms":1667290907158,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":74}},{"key":"__fd_table/1/t12_0004","expected":0,"target":{"Seq":75}},{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":81}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":78}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Delete":{"key":"__fd_table/1/t12_0004","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,55,46,49,53,55,50,51,55,32,85,84,67,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":51},"payload":{"Normal":{"txid":null,"time_ms":1667290909908,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":52},"payload":{"Normal":{"txid":null,"time_ms":1667290909915,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":84}},{"key":"__fd_table/1/nation","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/nation","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":86}},{"key":"__fd_table_id_to_name/87","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/nation","value":[56,55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/87","value":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/nation","value":[10,1,87,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/87","value":[8,1,18,6,110,97,116,105,111,110,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":53},"payload":{"Normal":{"txid":null,"time_ms":1667290912442,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":54},"payload":{"Normal":{"txid":null,"time_ms":1667290912448,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":88}},{"key":"__fd_table/1/region","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/region","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":92}},{"key":"__fd_table_id_to_name/94","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/region","value":[57,52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/94","value":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/region","value":[10,1,94,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/94","value":[8,1,18,6,114,101,103,105,111,110,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":55},"payload":{"Normal":{"txid":null,"time_ms":1667290914988,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":56},"payload":{"Normal":{"txid":null,"time_ms":1667290914994,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":95}},{"key":"__fd_table/1/part","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/part","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":99}},{"key":"__fd_table_id_to_name/101","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/part","value":[49,48,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/101","value":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/part","value":[10,1,101,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/101","value":[8,1,18,4,112,97,114,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":57},"payload":{"Normal":{"txid":null,"time_ms":1667290915186,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290975}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":58},"payload":{"Normal":{"txid":null,"time_ms":1667290917533,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":59},"payload":{"Normal":{"txid":null,"time_ms":1667290917538,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":102}},{"key":"__fd_table/1/supplier","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/supplier","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":106}},{"key":"__fd_table_id_to_name/109","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/supplier","value":[49,48,57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/109","value":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/supplier","value":[10,1,109,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/109","value":[8,1,18,8,115,117,112,112,108,105,101,114,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":60},"payload":{"Normal":{"txid":null,"time_ms":1667290920075,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":61},"payload":{"Normal":{"txid":null,"time_ms":1667290920080,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":110}},{"key":"__fd_table/1/partsupp","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/partsupp","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":114}},{"key":"__fd_table_id_to_name/116","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/partsupp","value":[49,49,54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/116","value":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/partsupp","value":[10,1,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[56],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/116","value":[8,1,18,8,112,97,114,116,115,117,112,112,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":62},"payload":{"Normal":{"txid":null,"time_ms":1667290920676,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290980}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":63},"payload":{"Normal":{"txid":null,"time_ms":1667290921608,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290981}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":64},"payload":{"Normal":{"txid":null,"time_ms":1667290922624,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":65},"payload":{"Normal":{"txid":null,"time_ms":1667290922629,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":117}},{"key":"__fd_table/1/customer","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/customer","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":121}},{"key":"__fd_table_id_to_name/125","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/customer","value":[49,50,53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/125","value":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/customer","value":[10,1,125,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/125","value":[8,1,18,8,99,117,115,116,111,109,101,114,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":66},"payload":{"Normal":{"txid":null,"time_ms":1667290925169,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":67},"payload":{"Normal":{"txid":null,"time_ms":1667290925175,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":126}},{"key":"__fd_table/1/orders","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/orders","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":130}},{"key":"__fd_table_id_to_name/132","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/orders","value":[49,51,50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/132","value":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/orders","value":[10,2,132,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49,48],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/132","value":[8,1,18,6,111,114,100,101,114,115,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":68},"payload":{"Normal":{"txid":null,"time_ms":1667290927714,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":69},"payload":{"Normal":{"txid":null,"time_ms":1667290927723,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":133}},{"key":"__fd_table/1/lineitem","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/lineitem","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":137}},{"key":"__fd_table_id_to_name/139","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/lineitem","value":[49,51,57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/139","value":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/lineitem","value":[10,2,139,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/139","value":[8,1,18,8,108,105,110,101,105,116,101,109,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":70},"payload":{"Normal":{"txid":null,"time_ms":1667290933209,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/125","expected":0,"target":{"Seq":128}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/125","value":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,50,53,47,95,115,115,47,100,50,57,48,56,57,50,102,54,54,53,98,52,49,51,51,97,53,101,97,102,97,55,53,48,100,50,99,50,52,53,53,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,22,8,152,117,16,169,141,183,1,24,143,225,84,32,190,196,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":71},"payload":{"Normal":{"txid":null,"time_ms":1667290939149,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/139","expected":0,"target":{"Seq":142}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/139","value":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,57,47,95,115,115,47,99,49,48,97,51,56,97,48,53,48,97,98,52,97,100,50,98,51,49,51,48,50,56,97,98,97,100,56,54,101,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,24,8,252,211,36,16,221,134,134,46,24,168,150,134,15,32,171,222,7,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":72},"payload":{"Normal":{"txid":null,"time_ms":1667290940280,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/87","expected":0,"target":{"Seq":90}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/87","value":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,56,55,47,95,115,115,47,102,101,52,52,102,102,53,48,55,101,54,56,52,52,49,56,57,55,54,101,102,53,100,101,99,57,97,98,51,97,49,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,17,8,25,16,218,20,24,235,16,32,142,8,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":73},"payload":{"Normal":{"txid":null,"time_ms":1667290942403,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/132","expected":0,"target":{"Seq":135}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/132","value":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,50,47,95,115,115,47,50,52,53,100,98,57,101,55,57,53,101,54,52,53,54,57,56,100,55,48,97,54,51,50,57,53,102,101,56,49,57,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,24,8,240,147,9,16,184,204,229,9,24,218,217,169,3,32,178,202,3,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":74},"payload":{"Normal":{"txid":null,"time_ms":1667290942655,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291002}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":75},"payload":{"Normal":{"txid":null,"time_ms":1667290944081,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/116","expected":0,"target":{"Seq":119}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/116","value":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,49,54,47,95,115,115,47,57,56,56,50,53,55,56,53,102,51,49,48,52,51,98,55,98,53,57,51,57,56,48,51,100,102,54,102,48,55,55,101,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,24,8,128,241,4,16,201,161,158,6,24,148,192,168,2,32,252,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":76},"payload":{"Normal":{"txid":null,"time_ms":1667290944493,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291004}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":77},"payload":{"Normal":{"txid":null,"time_ms":1667290945499,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/101","expected":0,"target":{"Seq":104}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/101","value":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,49,47,95,115,115,47,57,51,53,49,56,97,52,97,55,102,98,52,52,99,56,101,57,54,49,51,51,50,54,56,57,57,50,48,99,55,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,23,8,160,156,1,16,166,214,200,1,24,209,232,62,32,180,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":78},"payload":{"Normal":{"txid":null,"time_ms":1667290946629,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/94","expected":0,"target":{"Seq":97}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/94","value":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,57,52,47,95,115,115,47,53,102,57,55,100,48,99,49,98,56,99,53,52,55,102,55,97,97,49,102,97,55,99,102,48,55,99,101,57,57,51,100,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,17,8,5,16,224,3,24,168,6,32,227,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":79},"payload":{"Normal":{"txid":null,"time_ms":1667290947121,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291007}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":80},"payload":{"Normal":{"txid":null,"time_ms":1667290947751,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/109","expected":0,"target":{"Seq":112}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/109","value":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,57,47,95,115,115,47,53,48,100,54,101,53,55,57,48,51,49,101,52,54,51,53,97,54,97,97,50,99,48,48,54,99,50,51,48,101,98,49,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,20,8,232,7,16,247,193,10,24,134,172,5,32,246,58,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":81},"payload":{"Normal":{"txid":null,"time_ms":1667290965904,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291025}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":82},"payload":{"Normal":{"txid":null,"time_ms":1667290971893,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291031}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":83},"payload":{"Normal":{"txid":null,"time_ms":1667290974891,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291034}}}}}}}] +["state_machine/0",{"Sequences":{"key":"generic-kv","value":159}}] +["state_machine/0",{"StateMachineMeta":{"key":"LastApplied","value":{"LogId":{"leader_id":{"term":1,"node_id":0},"index":83}}}}] +["state_machine/0",{"StateMachineMeta":{"key":"LastMembership","value":{"Membership":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":8},"membership":{"configs":[[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}}] +["state_machine/0",{"Nodes":{"key":1,"value":{"name":"1","endpoint":{"addr":"localhost","port":28103},"grpc_api_advertise_address":"0.0.0.0:9191"}}}] +["state_machine/0",{"Nodes":{"key":2,"value":{"name":"2","endpoint":{"addr":"localhost","port":28203},"grpc_api_advertise_address":"0.0.0.0:28202"}}}] +["state_machine/0",{"Nodes":{"key":3,"value":{"name":"3","endpoint":{"addr":"localhost","port":28303},"grpc_api_advertise_address":"0.0.0.0:28302"}}}] +["state_machine/0",{"Expire":{"key":{"time_ms":1667291025000,"seq":157},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1"}}}] +["state_machine/0",{"Expire":{"key":{"time_ms":1667291031000,"seq":158},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6"}}}] +["state_machine/0",{"Expire":{"key":{"time_ms":1667291034000,"seq":159},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1"}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","value":{"seq":157,"meta":{"expire_at":1667291025},"data":[123,34,105,100,34,58,34,55,71,86,80,49,71,115,81,50,106,112,68,77,117,49,116,105,56,85,110,70,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,51,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","value":{"seq":159,"meta":{"expire_at":1667291034},"data":[123,34,105,100,34,58,34,75,77,90,52,86,118,113,68,70,86,69,120,108,90,70,84,104,75,68,122,90,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,50,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","value":{"seq":158,"meta":{"expire_at":1667291031},"data":[123,34,105,100,34,58,34,82,112,117,87,110,100,84,102,53,74,108,103,121,74,67,112,65,65,116,81,88,54,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,48,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database/test_tenant/default","value":{"seq":2,"meta":{"expire_at":null},"data":[49]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/1","value":{"seq":140,"meta":{"expire_at":null},"data":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/25","value":{"seq":41,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,51,46,53,55,55,49,55,50,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/42","value":{"seq":55,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,48,46,51,55,54,53,56,53,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/56","value":{"seq":71,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,48,46,55,49,56,55,54,51,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/9","value":{"seq":23,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,54,46,56,52,48,55,51,54,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/1","value":{"seq":5,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,100,101,102,97,117,108,116,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/25","value":{"seq":29,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/42","value":{"seq":46,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,98,111,111,107,95,100,98,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/56","value":{"seq":60,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,9,100,98,49,50,95,48,48,48,50,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/9","value":{"seq":13,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/book_db","value":{"seq":45,"meta":{"expire_at":null},"data":[10,1,42,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/db1","value":{"seq":28,"meta":{"expire_at":null},"data":[10,2,9,25,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/db12_0002","value":{"seq":59,"meta":{"expire_at":null},"data":[10,1,56,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/default","value":{"seq":4,"meta":{"expire_at":null},"data":[10,1,1,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_id_gen/database_id","value":{"seq":56,"meta":null,"data":[]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_id_gen/table_id","value":{"seq":139,"meta":null,"data":[]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/customer","value":{"seq":127,"meta":{"expire_at":null},"data":[49,50,53]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/lineitem","value":{"seq":141,"meta":{"expire_at":null},"data":[49,51,57]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/nation","value":{"seq":89,"meta":{"expire_at":null},"data":[56,55]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/orders","value":{"seq":134,"meta":{"expire_at":null},"data":[49,51,50]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/part","value":{"seq":103,"meta":{"expire_at":null},"data":[49,48,49]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/partsupp","value":{"seq":118,"meta":{"expire_at":null},"data":[49,49,54]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/region","value":{"seq":96,"meta":{"expire_at":null},"data":[57,52]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/supplier","value":{"seq":111,"meta":{"expire_at":null},"data":[49,48,57]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/25/t1","value":{"seq":32,"meta":{"expire_at":null},"data":[51,48]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/42/books","value":{"seq":49,"meta":{"expire_at":null},"data":[52,55]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/9/t1","value":{"seq":16,"meta":{"expire_at":null},"data":[49,52]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/101","value":{"seq":153,"meta":{"expire_at":null},"data":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,49,47,95,115,115,47,57,51,53,49,56,97,52,97,55,102,98,52,52,99,56,101,57,54,49,51,51,50,54,56,57,57,50,48,99,55,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,23,8,160,156,1,16,166,214,200,1,24,209,232,62,32,180,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/109","value":{"seq":156,"meta":{"expire_at":null},"data":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,57,47,95,115,115,47,53,48,100,54,101,53,55,57,48,51,49,101,52,54,51,53,97,54,97,97,50,99,48,48,54,99,50,51,48,101,98,49,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,20,8,232,7,16,247,193,10,24,134,172,5,32,246,58,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/116","value":{"seq":151,"meta":{"expire_at":null},"data":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,49,54,47,95,115,115,47,57,56,56,50,53,55,56,53,102,51,49,48,52,51,98,55,98,53,57,51,57,56,48,51,100,102,54,102,48,55,55,101,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,24,8,128,241,4,16,201,161,158,6,24,148,192,168,2,32,252,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/125","value":{"seq":146,"meta":{"expire_at":null},"data":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,50,53,47,95,115,115,47,100,50,57,48,56,57,50,102,54,54,53,98,52,49,51,51,97,53,101,97,102,97,55,53,48,100,50,99,50,52,53,53,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,22,8,152,117,16,169,141,183,1,24,143,225,84,32,190,196,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/132","value":{"seq":149,"meta":{"expire_at":null},"data":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,50,47,95,115,115,47,50,52,53,100,98,57,101,55,57,53,101,54,52,53,54,57,56,100,55,48,97,54,51,50,57,53,102,101,56,49,57,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,24,8,240,147,9,16,184,204,229,9,24,218,217,169,3,32,178,202,3,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/139","value":{"seq":147,"meta":{"expire_at":null},"data":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,57,47,95,115,115,47,99,49,48,97,51,56,97,48,53,48,97,98,52,97,100,50,98,51,49,51,48,50,56,97,98,97,100,56,54,101,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,24,8,252,211,36,16,221,134,134,46,24,168,150,134,15,32,171,222,7,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/14","value":{"seq":22,"meta":{"expire_at":null},"data":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,99,99,51,56,51,102,51,52,57,50,54,54,52,50,53,99,98,55,51,56,100,50,99,99,49,49,54,53,53,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,4,16,212,4,24,200,15,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/30","value":{"seq":39,"meta":{"expire_at":null},"data":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,50,97,57,100,99,97,53,55,52,51,54,54,52,53,101,97,97,53,97,51,52,100,57,100,102,97,50,51,55,98,50,54,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,2,16,168,2,24,152,14,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/47","value":{"seq":54,"meta":{"expire_at":null},"data":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,52,50,47,52,55,47,95,115,115,47,55,48,98,53,48,55,53,97,54,98,51,99,52,55,102,51,57,57,100,51,48,102,57,100,52,97,54,98,98,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,16,8,1,16,64,24,180,3,32,176,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/61","value":{"seq":69,"meta":{"expire_at":null},"data":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,53,46,54,56,51,49,52,51,32,85,84,67,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/73","value":{"seq":85,"meta":{"expire_at":null},"data":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,55,46,49,53,55,50,51,55,32,85,84,67,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/87","value":{"seq":148,"meta":{"expire_at":null},"data":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,56,55,47,95,115,115,47,102,101,52,52,102,102,53,48,55,101,54,56,52,52,49,56,57,55,54,101,102,53,100,101,99,57,97,98,51,97,49,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,17,8,25,16,218,20,24,235,16,32,142,8,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/94","value":{"seq":154,"meta":{"expire_at":null},"data":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,57,52,47,95,115,115,47,53,102,57,55,100,48,99,49,98,56,99,53,52,55,102,55,97,97,49,102,97,55,99,102,48,55,99,101,57,57,51,100,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,17,8,5,16,224,3,24,168,6,32,227,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_count/test_tenant","value":{"seq":144,"meta":{"expire_at":null},"data":[49,49]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/customer","value":{"seq":129,"meta":{"expire_at":null},"data":[10,1,125,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/lineitem","value":{"seq":143,"meta":{"expire_at":null},"data":[10,2,139,1,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/nation","value":{"seq":91,"meta":{"expire_at":null},"data":[10,1,87,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/orders","value":{"seq":136,"meta":{"expire_at":null},"data":[10,2,132,1,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/part","value":{"seq":105,"meta":{"expire_at":null},"data":[10,1,101,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/partsupp","value":{"seq":120,"meta":{"expire_at":null},"data":[10,1,116,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/region","value":{"seq":98,"meta":{"expire_at":null},"data":[10,1,94,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/supplier","value":{"seq":113,"meta":{"expire_at":null},"data":[10,1,109,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/t12_0004","value":{"seq":77,"meta":{"expire_at":null},"data":[10,1,73,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/25/t1","value":{"seq":34,"meta":{"expire_at":null},"data":[10,1,30,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/42/books","value":{"seq":51,"meta":{"expire_at":null},"data":[10,1,47,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/56/t","value":{"seq":65,"meta":{"expire_at":null},"data":[10,1,61,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/9/t1","value":{"seq":18,"meta":{"expire_at":null},"data":[10,1,14,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/101","value":{"seq":107,"meta":{"expire_at":null},"data":[8,1,18,4,112,97,114,116,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/109","value":{"seq":115,"meta":{"expire_at":null},"data":[8,1,18,8,115,117,112,112,108,105,101,114,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/116","value":{"seq":122,"meta":{"expire_at":null},"data":[8,1,18,8,112,97,114,116,115,117,112,112,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/125","value":{"seq":131,"meta":{"expire_at":null},"data":[8,1,18,8,99,117,115,116,111,109,101,114,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/132","value":{"seq":138,"meta":{"expire_at":null},"data":[8,1,18,6,111,114,100,101,114,115,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/139","value":{"seq":145,"meta":{"expire_at":null},"data":[8,1,18,8,108,105,110,101,105,116,101,109,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/14","value":{"seq":20,"meta":{"expire_at":null},"data":[8,9,18,2,116,49,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/30","value":{"seq":36,"meta":{"expire_at":null},"data":[8,25,18,2,116,49,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/47","value":{"seq":53,"meta":{"expire_at":null},"data":[8,42,18,5,98,111,111,107,115,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/61","value":{"seq":67,"meta":{"expire_at":null},"data":[8,56,18,1,116,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/73","value":{"seq":79,"meta":{"expire_at":null},"data":[8,1,18,8,116,49,50,95,48,48,48,52,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/87","value":{"seq":93,"meta":{"expire_at":null},"data":[8,1,18,6,110,97,116,105,111,110,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/94","value":{"seq":100,"meta":{"expire_at":null},"data":[8,1,18,6,114,101,103,105,111,110,160,6,19,168,6,1]}}}] diff --git a/tests/metactl/test-metactl.sh b/tests/metactl/test-metactl.sh index c08446ae1acf..dd2a71ea9b61 100755 --- a/tests/metactl/test-metactl.sh +++ b/tests/metactl/test-metactl.sh @@ -7,8 +7,10 @@ BUILD_PROFILE="${BUILD_PROFILE:-debug}" meta_dir="$SCRIPT_PATH/_meta_dir" meta_json_v002="$SCRIPT_PATH/meta_v002.txt" -want_exported_v003="$SCRIPT_PATH/want_exported_v003" -want_snapshot_v003="$SCRIPT_PATH/want_snapshot_v003" +meta_json_v003="$SCRIPT_PATH/meta_v003.txt" +meta_json_v004="$SCRIPT_PATH/meta_v004.txt" +want_exported_v004="$SCRIPT_PATH/want_exported_v004" +want_snapshot_v004="$SCRIPT_PATH/want_snapshot_v004" exported="$SCRIPT_PATH/exported" grpc_exported="$SCRIPT_PATH/grpc_exported" @@ -40,7 +42,7 @@ metactl_import_export () { echo " === ${title} 1.1. Check snapshot data" echo " === " - snapshot_path="$(ls $meta_dir/df_meta/V003/snapshot/1-0-83-*.snap)" + snapshot_path="$(ls $meta_dir/df_meta/V004/snapshot/1-0-83-*.snap)" echo "=== snapshot path:" ls $snapshot_path @@ -83,7 +85,9 @@ metactl_import_export () { sleep 1 } -metactl_import_export 'V003' "$meta_json_v002" "$want_exported_v003" "$want_snapshot_v003" +metactl_import_export 'V002' "$meta_json_v002" "$want_exported_v004" "$want_snapshot_v004" +metactl_import_export 'V003' "$meta_json_v003" "$want_exported_v004" "$want_snapshot_v004" +metactl_import_export 'V004' "$meta_json_v004" "$want_exported_v004" "$want_snapshot_v004" echo " === " @@ -108,7 +112,7 @@ cat $grpc_exported echo " === grpc_exported file data end" echo " === check if there is a node record in it" -if grep -Fxq '["raft_state",{"RaftStateKV":{"key":"Id","value":{"NodeId":0}}}]' $grpc_exported; then +if grep -Fxq '["raft_log",{"NodeId":0}]' $grpc_exported; then echo " === Node record found, good!" else echo " === No Node record found!!!" @@ -116,7 +120,7 @@ else fi echo " === check if there is a header record in it" -if grep -Fxq '["header",{"DataHeader":{"key":"header","value":{"version":"V003","upgrading":null}}}]' $grpc_exported; then +if grep -Fxq '["header",{"DataHeader":{"key":"header","value":{"version":"V004"}}}]' $grpc_exported; then echo " === Header record found, good!" else echo " === No Header record found!!!" @@ -128,12 +132,13 @@ kill $METASRV_PID sleep 3 + echo " === " echo " === 5. Test import data with incompatible header $grpc_exported to dir $meta_dir" echo " === " -echo '["header",{"DataHeader":{"key":"header","value":{"version":"V100","upgrading":null}}}]' > $grpc_exported +echo '["header",{"DataHeader":{"key":"header","value":{"version":"V100"}}}]' > $grpc_exported echo " === import into $meta_dir" cat $grpc_exported | diff --git a/tests/metactl/want_exported_v003 b/tests/metactl/want_exported_v003 deleted file mode 100644 index 1c8d6b51f68a..000000000000 --- a/tests/metactl/want_exported_v003 +++ /dev/null @@ -1,167 +0,0 @@ -["header",{"DataHeader":{"key":"header","value":{"version":"V003","upgrading":null}}}] -["raft_state",{"RaftStateKV":{"key":"Id","value":{"NodeId":1}}}] -["raft_state",{"RaftStateKV":{"key":"HardState","value":{"HardState":{"leader_id":{"term":1,"node_id":1},"committed":false}}}}] -["raft_state",{"RaftStateKV":{"key":"Committed","value":{"Committed":null}}}] -["raft_log",{"Logs":{"key":2,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":2},"payload":{"Normal":{"txid":null,"time_ms":1667290820099,"cmd":{"AddNode":{"node_id":1,"node":{"name":"1","endpoint":{"addr":"localhost","port":28103},"grpc_api_advertise_address":"0.0.0.0:9191"},"overriding":false}}}}}}}] -["raft_log",{"Logs":{"key":3,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":3},"payload":{"Normal":{"txid":null,"time_ms":1667290820429,"cmd":{"AddNode":{"node_id":2,"node":{"name":"2","endpoint":{"addr":"localhost","port":28203},"grpc_api_advertise_address":"0.0.0.0:28202"},"overriding":false}}}}}}}] -["raft_log",{"Logs":{"key":4,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":4},"payload":{"Membership":{"configs":[[1],[1,2]],"nodes":{"1":{},"2":{}}}}}}}] -["raft_log",{"Logs":{"key":5,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":5},"payload":{"Membership":{"configs":[[1,2]],"nodes":{"1":{},"2":{}}}}}}}] -["raft_log",{"Logs":{"key":6,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":6},"payload":{"Normal":{"txid":null,"time_ms":1667290821018,"cmd":{"AddNode":{"node_id":3,"node":{"name":"3","endpoint":{"addr":"localhost","port":28303},"grpc_api_advertise_address":"0.0.0.0:28302"},"overriding":false}}}}}}}] -["raft_log",{"Logs":{"key":7,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":7},"payload":{"Membership":{"configs":[[1,2],[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}}] -["raft_log",{"Logs":{"key":8,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":8},"payload":{"Membership":{"configs":[[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}}] -["raft_log",{"Logs":{"key":9,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":9},"payload":{"Normal":{"txid":null,"time_ms":1667290824580,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":10,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":10},"payload":{"Normal":{"txid":null,"time_ms":1667290824586,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/default","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/1","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/default","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/default","value":[49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/default","value":[10,1,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/1","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,100,101,102,97,117,108,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":11,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":11},"payload":{"Normal":{"txid":null,"time_ms":1667290824603,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,82,112,117,87,110,100,84,102,53,74,108,103,121,74,67,112,65,65,116,81,88,54,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,48,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290884}}}}}}}}] -["raft_log",{"Logs":{"key":12,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":12},"payload":{"Normal":{"txid":null,"time_ms":1667290826333,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,75,77,90,52,86,118,113,68,70,86,69,120,108,90,70,84,104,75,68,122,90,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,50,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290886}}}}}}}}] -["raft_log",{"Logs":{"key":13,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":13},"payload":{"Normal":{"txid":null,"time_ms":1667290826867,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,55,71,86,80,49,71,115,81,50,106,112,68,77,117,49,116,105,56,85,110,70,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,51,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290886}}}}}}}}] -["raft_log",{"Logs":{"key":14,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":14},"payload":{"Normal":{"txid":null,"time_ms":1667290836132,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":15,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":15},"payload":{"Normal":{"txid":null,"time_ms":1667290836137,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/9","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db1","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db1","value":[57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db1","value":[10,1,9,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/9","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":16,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":16},"payload":{"Normal":{"txid":null,"time_ms":1667290838209,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":17,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":17},"payload":{"Normal":{"txid":null,"time_ms":1667290838215,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/9","expected":0,"target":{"Seq":11}},{"key":"__fd_table/9/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/9/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_to_name/14","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/9/t1","value":[49,52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/9/t1","value":[10,1,14,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/14","value":[8,9,18,2,116,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":18,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":18},"payload":{"Normal":{"txid":null,"time_ms":1667290838764,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/14","expected":0,"target":{"Seq":17}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,53,99,55,49,49,56,55,98,49,53,97,52,52,53,54,100,98,49,55,50,48,50,98,100,51,97,55,101,99,102,56,51,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,2,16,170,2,24,228,7,32,184,12,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":19,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":19},"payload":{"Normal":{"txid":null,"time_ms":1667290843099,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/14","expected":0,"target":{"Seq":21}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,99,99,51,56,51,102,51,52,57,50,54,54,52,50,53,99,98,55,51,56,100,50,99,99,49,49,54,53,53,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,4,16,212,4,24,200,15,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":20,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":20},"payload":{"Normal":{"txid":null,"time_ms":1667290846841,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":10}},{"key":"__fd_database_by_id/9","expected":0,"target":{"Seq":15}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db1","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,54,46,56,52,48,55,51,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":21,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":21},"payload":{"Normal":{"txid":null,"time_ms":1667290848281,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290908}}}}}}}}] -["raft_log",{"Logs":{"key":22,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":22},"payload":{"Normal":{"txid":null,"time_ms":1667290849028,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":23,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":23},"payload":{"Normal":{"txid":null,"time_ms":1667290849033,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/25","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db1","expected":0,"target":{"Seq":12}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db1","value":[50,53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db1","value":[10,2,9,25,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/25","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":24,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":24},"payload":{"Normal":{"txid":null,"time_ms":1667290851094,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":25,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":25},"payload":{"Normal":{"txid":null,"time_ms":1667290851100,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/25","expected":0,"target":{"Seq":27}},{"key":"__fd_table/25/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/25/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":19}},{"key":"__fd_table_id_to_name/30","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/25/t1","value":[51,48],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/25/t1","value":[10,1,30,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/30","value":[8,25,18,2,116,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":26,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":26},"payload":{"Normal":{"txid":null,"time_ms":1667290851947,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290911}}}}}}}}] -["raft_log",{"Logs":{"key":27,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":27},"payload":{"Normal":{"txid":null,"time_ms":1667290852139,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/30","expected":0,"target":{"Seq":33}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,56,100,97,99,49,50,51,97,97,54,101,55,52,51,53,102,97,55,49,52,50,99,55,100,56,54,98,98,49,98,54,55,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,1,16,148,1,24,140,7,32,184,12,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":28,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":28},"payload":{"Normal":{"txid":null,"time_ms":1667290853220,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/30","expected":0,"target":{"Seq":38}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,50,97,57,100,99,97,53,55,52,51,54,54,52,53,101,97,97,53,97,51,52,100,57,100,102,97,50,51,55,98,50,54,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,2,16,168,2,24,152,14,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":29,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":29},"payload":{"Normal":{"txid":null,"time_ms":1667290861626,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290921}}}}}}}}] -["raft_log",{"Logs":{"key":30,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":30},"payload":{"Normal":{"txid":null,"time_ms":1667290863578,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":26}},{"key":"__fd_database_by_id/25","expected":0,"target":{"Seq":31}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db1","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,51,46,53,55,55,49,55,50,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":31,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":31},"payload":{"Normal":{"txid":null,"time_ms":1667290864962,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":32,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":32},"payload":{"Normal":{"txid":null,"time_ms":1667290864967,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/book_db","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/42","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/book_db","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/book_db","value":[52,50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/book_db","value":[10,1,42,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/42","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,98,111,111,107,95,100,98,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":33,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":33},"payload":{"Normal":{"txid":null,"time_ms":1667290867024,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":34,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":34},"payload":{"Normal":{"txid":null,"time_ms":1667290867030,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/42","expected":0,"target":{"Seq":44}},{"key":"__fd_table/42/books","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/42/books","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":35}},{"key":"__fd_table_id_to_name/47","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/42/books","value":[52,55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/47","value":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/42/books","value":[10,1,47,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/47","value":[8,42,18,5,98,111,111,107,115,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":35,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":35},"payload":{"Normal":{"txid":null,"time_ms":1667290867598,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/47","expected":0,"target":{"Seq":50}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/47","value":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,52,50,47,52,55,47,95,115,115,47,55,48,98,53,48,55,53,97,54,98,51,99,52,55,102,51,57,57,100,51,48,102,57,100,52,97,54,98,98,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,16,8,1,16,64,24,180,3,32,176,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":36,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":36},"payload":{"Normal":{"txid":null,"time_ms":1667290870377,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/book_db","expected":0,"target":{"Seq":43}},{"key":"__fd_database_by_id/42","expected":0,"target":{"Seq":48}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/book_db","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,48,46,51,55,54,53,56,53,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":37,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":37},"payload":{"Normal":{"txid":null,"time_ms":1667290872604,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":38,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":38},"payload":{"Normal":{"txid":null,"time_ms":1667290872609,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db12_0002","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/56","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db12_0002","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db12_0002","value":[53,54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db12_0002","value":[10,1,56,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/56","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,9,100,98,49,50,95,48,48,48,50,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":39,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":39},"payload":{"Normal":{"txid":null,"time_ms":1667290874661,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":40,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":40},"payload":{"Normal":{"txid":null,"time_ms":1667290874666,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":58}},{"key":"__fd_table/56/t","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/56/t","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":52}},{"key":"__fd_table_id_to_name/61","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/56/t","value":[54,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/61","value":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/56/t","value":[10,1,61,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/61","value":[8,56,18,1,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":41,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":41},"payload":{"Normal":{"txid":null,"time_ms":1667290875684,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":62}},{"key":"__fd_table/56/t","expected":0,"target":{"Seq":63}},{"key":"__fd_table_by_id/61","expected":0,"target":{"Seq":64}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":66}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Delete":{"key":"__fd_table/56/t","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_table_by_id/61","value":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,53,46,54,56,51,49,52,51,32,85,84,67,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":42,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":42},"payload":{"Normal":{"txid":null,"time_ms":1667290880720,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db12_0002","expected":0,"target":{"Seq":57}},{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":68}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db12_0002","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,48,46,55,49,56,55,54,51,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":43,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":43},"payload":{"Normal":{"txid":null,"time_ms":1667290881744,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290941}}}}}}}}] -["raft_log",{"Logs":{"key":44,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":44},"payload":{"Normal":{"txid":null,"time_ms":1667290883474,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":45,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":45},"payload":{"Normal":{"txid":null,"time_ms":1667290883479,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":3}},{"key":"__fd_table/1/t12_0004","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/t12_0004","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":70}},{"key":"__fd_table_id_to_name/73","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/t12_0004","value":[55,51],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/t12_0004","value":[10,1,73,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/73","value":[8,1,18,8,116,49,50,95,48,48,48,52,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":46,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":46},"payload":{"Normal":{"txid":null,"time_ms":1667290885036,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":76}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,48,97,99,51,52,98,49,56,52,48,56,50,52,57,97,99,98,97,101,51,50,98,53,56,101,51,101,57,55,50,102,98,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,2,16,8,24,188,1,32,148,2,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":47,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":47},"payload":{"Normal":{"txid":null,"time_ms":1667290886657,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":80}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":48,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":48},"payload":{"Normal":{"txid":null,"time_ms":1667290889731,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290949}}}}}}}}] -["raft_log",{"Logs":{"key":49,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":49},"payload":{"Normal":{"txid":null,"time_ms":1667290890274,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290950}}}}}}}}] -["raft_log",{"Logs":{"key":50,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":50},"payload":{"Normal":{"txid":null,"time_ms":1667290907158,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":74}},{"key":"__fd_table/1/t12_0004","expected":0,"target":{"Seq":75}},{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":81}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":78}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Delete":{"key":"__fd_table/1/t12_0004","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,55,46,49,53,55,50,51,55,32,85,84,67,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":51,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":51},"payload":{"Normal":{"txid":null,"time_ms":1667290909908,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":52,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":52},"payload":{"Normal":{"txid":null,"time_ms":1667290909915,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":84}},{"key":"__fd_table/1/nation","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/nation","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":86}},{"key":"__fd_table_id_to_name/87","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/nation","value":[56,55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/87","value":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/nation","value":[10,1,87,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/87","value":[8,1,18,6,110,97,116,105,111,110,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":53,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":53},"payload":{"Normal":{"txid":null,"time_ms":1667290912442,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":54,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":54},"payload":{"Normal":{"txid":null,"time_ms":1667290912448,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":88}},{"key":"__fd_table/1/region","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/region","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":92}},{"key":"__fd_table_id_to_name/94","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/region","value":[57,52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/94","value":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/region","value":[10,1,94,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/94","value":[8,1,18,6,114,101,103,105,111,110,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":55,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":55},"payload":{"Normal":{"txid":null,"time_ms":1667290914988,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":56,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":56},"payload":{"Normal":{"txid":null,"time_ms":1667290914994,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":95}},{"key":"__fd_table/1/part","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/part","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":99}},{"key":"__fd_table_id_to_name/101","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/part","value":[49,48,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/101","value":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/part","value":[10,1,101,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/101","value":[8,1,18,4,112,97,114,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":57,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":57},"payload":{"Normal":{"txid":null,"time_ms":1667290915186,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290975}}}}}}}}] -["raft_log",{"Logs":{"key":58,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":58},"payload":{"Normal":{"txid":null,"time_ms":1667290917533,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":59,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":59},"payload":{"Normal":{"txid":null,"time_ms":1667290917538,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":102}},{"key":"__fd_table/1/supplier","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/supplier","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":106}},{"key":"__fd_table_id_to_name/109","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/supplier","value":[49,48,57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/109","value":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/supplier","value":[10,1,109,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/109","value":[8,1,18,8,115,117,112,112,108,105,101,114,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":60,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":60},"payload":{"Normal":{"txid":null,"time_ms":1667290920075,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":61,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":61},"payload":{"Normal":{"txid":null,"time_ms":1667290920080,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":110}},{"key":"__fd_table/1/partsupp","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/partsupp","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":114}},{"key":"__fd_table_id_to_name/116","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/partsupp","value":[49,49,54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/116","value":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/partsupp","value":[10,1,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[56],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/116","value":[8,1,18,8,112,97,114,116,115,117,112,112,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":62,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":62},"payload":{"Normal":{"txid":null,"time_ms":1667290920676,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290980}}}}}}}}] -["raft_log",{"Logs":{"key":63,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":63},"payload":{"Normal":{"txid":null,"time_ms":1667290921608,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290981}}}}}}}}] -["raft_log",{"Logs":{"key":64,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":64},"payload":{"Normal":{"txid":null,"time_ms":1667290922624,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":65,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":65},"payload":{"Normal":{"txid":null,"time_ms":1667290922629,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":117}},{"key":"__fd_table/1/customer","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/customer","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":121}},{"key":"__fd_table_id_to_name/125","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/customer","value":[49,50,53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/125","value":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/customer","value":[10,1,125,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/125","value":[8,1,18,8,99,117,115,116,111,109,101,114,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":66,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":66},"payload":{"Normal":{"txid":null,"time_ms":1667290925169,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":67,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":67},"payload":{"Normal":{"txid":null,"time_ms":1667290925175,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":126}},{"key":"__fd_table/1/orders","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/orders","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":130}},{"key":"__fd_table_id_to_name/132","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/orders","value":[49,51,50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/132","value":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/orders","value":[10,2,132,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49,48],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/132","value":[8,1,18,6,111,114,100,101,114,115,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":68,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":68},"payload":{"Normal":{"txid":null,"time_ms":1667290927714,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}}] -["raft_log",{"Logs":{"key":69,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":69},"payload":{"Normal":{"txid":null,"time_ms":1667290927723,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":133}},{"key":"__fd_table/1/lineitem","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/lineitem","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":137}},{"key":"__fd_table_id_to_name/139","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/lineitem","value":[49,51,57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/139","value":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/lineitem","value":[10,2,139,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/139","value":[8,1,18,8,108,105,110,101,105,116,101,109,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":70,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":70},"payload":{"Normal":{"txid":null,"time_ms":1667290933209,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/125","expected":0,"target":{"Seq":128}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/125","value":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,50,53,47,95,115,115,47,100,50,57,48,56,57,50,102,54,54,53,98,52,49,51,51,97,53,101,97,102,97,55,53,48,100,50,99,50,52,53,53,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,22,8,152,117,16,169,141,183,1,24,143,225,84,32,190,196,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":71,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":71},"payload":{"Normal":{"txid":null,"time_ms":1667290939149,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/139","expected":0,"target":{"Seq":142}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/139","value":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,57,47,95,115,115,47,99,49,48,97,51,56,97,48,53,48,97,98,52,97,100,50,98,51,49,51,48,50,56,97,98,97,100,56,54,101,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,24,8,252,211,36,16,221,134,134,46,24,168,150,134,15,32,171,222,7,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":72,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":72},"payload":{"Normal":{"txid":null,"time_ms":1667290940280,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/87","expected":0,"target":{"Seq":90}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/87","value":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,56,55,47,95,115,115,47,102,101,52,52,102,102,53,48,55,101,54,56,52,52,49,56,57,55,54,101,102,53,100,101,99,57,97,98,51,97,49,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,17,8,25,16,218,20,24,235,16,32,142,8,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":73,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":73},"payload":{"Normal":{"txid":null,"time_ms":1667290942403,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/132","expected":0,"target":{"Seq":135}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/132","value":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,50,47,95,115,115,47,50,52,53,100,98,57,101,55,57,53,101,54,52,53,54,57,56,100,55,48,97,54,51,50,57,53,102,101,56,49,57,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,24,8,240,147,9,16,184,204,229,9,24,218,217,169,3,32,178,202,3,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":74,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":74},"payload":{"Normal":{"txid":null,"time_ms":1667290942655,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291002}}}}}}}}] -["raft_log",{"Logs":{"key":75,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":75},"payload":{"Normal":{"txid":null,"time_ms":1667290944081,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/116","expected":0,"target":{"Seq":119}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/116","value":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,49,54,47,95,115,115,47,57,56,56,50,53,55,56,53,102,51,49,48,52,51,98,55,98,53,57,51,57,56,48,51,100,102,54,102,48,55,55,101,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,24,8,128,241,4,16,201,161,158,6,24,148,192,168,2,32,252,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":76,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":76},"payload":{"Normal":{"txid":null,"time_ms":1667290944493,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291004}}}}}}}}] -["raft_log",{"Logs":{"key":77,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":77},"payload":{"Normal":{"txid":null,"time_ms":1667290945499,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/101","expected":0,"target":{"Seq":104}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/101","value":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,49,47,95,115,115,47,57,51,53,49,56,97,52,97,55,102,98,52,52,99,56,101,57,54,49,51,51,50,54,56,57,57,50,48,99,55,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,23,8,160,156,1,16,166,214,200,1,24,209,232,62,32,180,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":78,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":78},"payload":{"Normal":{"txid":null,"time_ms":1667290946629,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/94","expected":0,"target":{"Seq":97}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/94","value":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,57,52,47,95,115,115,47,53,102,57,55,100,48,99,49,98,56,99,53,52,55,102,55,97,97,49,102,97,55,99,102,48,55,99,101,57,57,51,100,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,17,8,5,16,224,3,24,168,6,32,227,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":79,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":79},"payload":{"Normal":{"txid":null,"time_ms":1667290947121,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291007}}}}}}}}] -["raft_log",{"Logs":{"key":80,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":80},"payload":{"Normal":{"txid":null,"time_ms":1667290947751,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/109","expected":0,"target":{"Seq":112}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/109","value":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,57,47,95,115,115,47,53,48,100,54,101,53,55,57,48,51,49,101,52,54,51,53,97,54,97,97,50,99,48,48,54,99,50,51,48,101,98,49,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,20,8,232,7,16,247,193,10,24,134,172,5,32,246,58,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}}] -["raft_log",{"Logs":{"key":81,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":81},"payload":{"Normal":{"txid":null,"time_ms":1667290965904,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291025}}}}}}}}] -["raft_log",{"Logs":{"key":82,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":82},"payload":{"Normal":{"txid":null,"time_ms":1667290971893,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291031}}}}}}}}] -["raft_log",{"Logs":{"key":83,"value":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":83},"payload":{"Normal":{"txid":null,"time_ms":1667290974891,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291034}}}}}}}}] -["raft_log",{"LogMeta":{"key":"LastPurged","value":{"LogId":{"leader_id":{"term":1,"node_id":0},"index":1}}}}] -["state_machine/0",{"Sequences":{"key":"generic-kv","value":159}}] -["state_machine/0",{"StateMachineMeta":{"key":"LastApplied","value":{"LogId":{"leader_id":{"term":1,"node_id":0},"index":83}}}}] -["state_machine/0",{"StateMachineMeta":{"key":"LastMembership","value":{"Membership":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":8},"membership":{"configs":[[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}}] -["state_machine/0",{"Nodes":{"key":1,"value":{"name":"1","endpoint":{"addr":"localhost","port":28103},"grpc_api_advertise_address":"0.0.0.0:9191"}}}] -["state_machine/0",{"Nodes":{"key":2,"value":{"name":"2","endpoint":{"addr":"localhost","port":28203},"grpc_api_advertise_address":"0.0.0.0:28202"}}}] -["state_machine/0",{"Nodes":{"key":3,"value":{"name":"3","endpoint":{"addr":"localhost","port":28303},"grpc_api_advertise_address":"0.0.0.0:28302"}}}] -["state_machine/0",{"Expire":{"key":{"time_ms":1667291025000,"seq":157},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1"}}}] -["state_machine/0",{"Expire":{"key":{"time_ms":1667291031000,"seq":158},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6"}}}] -["state_machine/0",{"Expire":{"key":{"time_ms":1667291034000,"seq":159},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1"}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","value":{"seq":157,"meta":{"expire_at":1667291025},"data":[123,34,105,100,34,58,34,55,71,86,80,49,71,115,81,50,106,112,68,77,117,49,116,105,56,85,110,70,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,51,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","value":{"seq":159,"meta":{"expire_at":1667291034},"data":[123,34,105,100,34,58,34,75,77,90,52,86,118,113,68,70,86,69,120,108,90,70,84,104,75,68,122,90,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,50,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","value":{"seq":158,"meta":{"expire_at":1667291031},"data":[123,34,105,100,34,58,34,82,112,117,87,110,100,84,102,53,74,108,103,121,74,67,112,65,65,116,81,88,54,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,48,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database/test_tenant/default","value":{"seq":2,"meta":{"expire_at":null},"data":[49]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/1","value":{"seq":140,"meta":{"expire_at":null},"data":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/25","value":{"seq":41,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,51,46,53,55,55,49,55,50,32,85,84,67,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/42","value":{"seq":55,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,48,46,51,55,54,53,56,53,32,85,84,67,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/56","value":{"seq":71,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,48,46,55,49,56,55,54,51,32,85,84,67,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/9","value":{"seq":23,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,54,46,56,52,48,55,51,54,32,85,84,67,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/1","value":{"seq":5,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,100,101,102,97,117,108,116,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/25","value":{"seq":29,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/42","value":{"seq":46,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,98,111,111,107,95,100,98,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/56","value":{"seq":60,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,9,100,98,49,50,95,48,48,48,50,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/9","value":{"seq":13,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/book_db","value":{"seq":45,"meta":{"expire_at":null},"data":[10,1,42,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/db1","value":{"seq":28,"meta":{"expire_at":null},"data":[10,2,9,25,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/db12_0002","value":{"seq":59,"meta":{"expire_at":null},"data":[10,1,56,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/default","value":{"seq":4,"meta":{"expire_at":null},"data":[10,1,1,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_id_gen/database_id","value":{"seq":56,"meta":null,"data":[]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_id_gen/table_id","value":{"seq":139,"meta":null,"data":[]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/1/customer","value":{"seq":127,"meta":{"expire_at":null},"data":[49,50,53]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/1/lineitem","value":{"seq":141,"meta":{"expire_at":null},"data":[49,51,57]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/1/nation","value":{"seq":89,"meta":{"expire_at":null},"data":[56,55]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/1/orders","value":{"seq":134,"meta":{"expire_at":null},"data":[49,51,50]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/1/part","value":{"seq":103,"meta":{"expire_at":null},"data":[49,48,49]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/1/partsupp","value":{"seq":118,"meta":{"expire_at":null},"data":[49,49,54]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/1/region","value":{"seq":96,"meta":{"expire_at":null},"data":[57,52]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/1/supplier","value":{"seq":111,"meta":{"expire_at":null},"data":[49,48,57]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/25/t1","value":{"seq":32,"meta":{"expire_at":null},"data":[51,48]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/42/books","value":{"seq":49,"meta":{"expire_at":null},"data":[52,55]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table/9/t1","value":{"seq":16,"meta":{"expire_at":null},"data":[49,52]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/101","value":{"seq":153,"meta":{"expire_at":null},"data":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,49,47,95,115,115,47,57,51,53,49,56,97,52,97,55,102,98,52,52,99,56,101,57,54,49,51,51,50,54,56,57,57,50,48,99,55,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,23,8,160,156,1,16,166,214,200,1,24,209,232,62,32,180,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/109","value":{"seq":156,"meta":{"expire_at":null},"data":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,57,47,95,115,115,47,53,48,100,54,101,53,55,57,48,51,49,101,52,54,51,53,97,54,97,97,50,99,48,48,54,99,50,51,48,101,98,49,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,20,8,232,7,16,247,193,10,24,134,172,5,32,246,58,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/116","value":{"seq":151,"meta":{"expire_at":null},"data":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,49,54,47,95,115,115,47,57,56,56,50,53,55,56,53,102,51,49,48,52,51,98,55,98,53,57,51,57,56,48,51,100,102,54,102,48,55,55,101,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,24,8,128,241,4,16,201,161,158,6,24,148,192,168,2,32,252,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/125","value":{"seq":146,"meta":{"expire_at":null},"data":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,50,53,47,95,115,115,47,100,50,57,48,56,57,50,102,54,54,53,98,52,49,51,51,97,53,101,97,102,97,55,53,48,100,50,99,50,52,53,53,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,22,8,152,117,16,169,141,183,1,24,143,225,84,32,190,196,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/132","value":{"seq":149,"meta":{"expire_at":null},"data":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,50,47,95,115,115,47,50,52,53,100,98,57,101,55,57,53,101,54,52,53,54,57,56,100,55,48,97,54,51,50,57,53,102,101,56,49,57,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,24,8,240,147,9,16,184,204,229,9,24,218,217,169,3,32,178,202,3,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/139","value":{"seq":147,"meta":{"expire_at":null},"data":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,57,47,95,115,115,47,99,49,48,97,51,56,97,48,53,48,97,98,52,97,100,50,98,51,49,51,48,50,56,97,98,97,100,56,54,101,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,24,8,252,211,36,16,221,134,134,46,24,168,150,134,15,32,171,222,7,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/14","value":{"seq":22,"meta":{"expire_at":null},"data":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,99,99,51,56,51,102,51,52,57,50,54,54,52,50,53,99,98,55,51,56,100,50,99,99,49,49,54,53,53,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,4,16,212,4,24,200,15,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/30","value":{"seq":39,"meta":{"expire_at":null},"data":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,50,97,57,100,99,97,53,55,52,51,54,54,52,53,101,97,97,53,97,51,52,100,57,100,102,97,50,51,55,98,50,54,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,2,16,168,2,24,152,14,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/47","value":{"seq":54,"meta":{"expire_at":null},"data":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,52,50,47,52,55,47,95,115,115,47,55,48,98,53,48,55,53,97,54,98,51,99,52,55,102,51,57,57,100,51,48,102,57,100,52,97,54,98,98,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,16,8,1,16,64,24,180,3,32,176,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/61","value":{"seq":69,"meta":{"expire_at":null},"data":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,53,46,54,56,51,49,52,51,32,85,84,67,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/73","value":{"seq":85,"meta":{"expire_at":null},"data":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,55,46,49,53,55,50,51,55,32,85,84,67,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/87","value":{"seq":148,"meta":{"expire_at":null},"data":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,56,55,47,95,115,115,47,102,101,52,52,102,102,53,48,55,101,54,56,52,52,49,56,57,55,54,101,102,53,100,101,99,57,97,98,51,97,49,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,17,8,25,16,218,20,24,235,16,32,142,8,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/94","value":{"seq":154,"meta":{"expire_at":null},"data":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,57,52,47,95,115,115,47,53,102,57,55,100,48,99,49,98,56,99,53,52,55,102,55,97,97,49,102,97,55,99,102,48,55,99,101,57,57,51,100,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,17,8,5,16,224,3,24,168,6,32,227,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_count/test_tenant","value":{"seq":144,"meta":{"expire_at":null},"data":[49,49]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/customer","value":{"seq":129,"meta":{"expire_at":null},"data":[10,1,125,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/lineitem","value":{"seq":143,"meta":{"expire_at":null},"data":[10,2,139,1,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/nation","value":{"seq":91,"meta":{"expire_at":null},"data":[10,1,87,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/orders","value":{"seq":136,"meta":{"expire_at":null},"data":[10,2,132,1,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/part","value":{"seq":105,"meta":{"expire_at":null},"data":[10,1,101,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/partsupp","value":{"seq":120,"meta":{"expire_at":null},"data":[10,1,116,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/region","value":{"seq":98,"meta":{"expire_at":null},"data":[10,1,94,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/supplier","value":{"seq":113,"meta":{"expire_at":null},"data":[10,1,109,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/t12_0004","value":{"seq":77,"meta":{"expire_at":null},"data":[10,1,73,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/25/t1","value":{"seq":34,"meta":{"expire_at":null},"data":[10,1,30,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/42/books","value":{"seq":51,"meta":{"expire_at":null},"data":[10,1,47,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/56/t","value":{"seq":65,"meta":{"expire_at":null},"data":[10,1,61,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/9/t1","value":{"seq":18,"meta":{"expire_at":null},"data":[10,1,14,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/101","value":{"seq":107,"meta":{"expire_at":null},"data":[8,1,18,4,112,97,114,116,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/109","value":{"seq":115,"meta":{"expire_at":null},"data":[8,1,18,8,115,117,112,112,108,105,101,114,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/116","value":{"seq":122,"meta":{"expire_at":null},"data":[8,1,18,8,112,97,114,116,115,117,112,112,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/125","value":{"seq":131,"meta":{"expire_at":null},"data":[8,1,18,8,99,117,115,116,111,109,101,114,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/132","value":{"seq":138,"meta":{"expire_at":null},"data":[8,1,18,6,111,114,100,101,114,115,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/139","value":{"seq":145,"meta":{"expire_at":null},"data":[8,1,18,8,108,105,110,101,105,116,101,109,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/14","value":{"seq":20,"meta":{"expire_at":null},"data":[8,9,18,2,116,49,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/30","value":{"seq":36,"meta":{"expire_at":null},"data":[8,25,18,2,116,49,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/47","value":{"seq":53,"meta":{"expire_at":null},"data":[8,42,18,5,98,111,111,107,115,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/61","value":{"seq":67,"meta":{"expire_at":null},"data":[8,56,18,1,116,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/73","value":{"seq":79,"meta":{"expire_at":null},"data":[8,1,18,8,116,49,50,95,48,48,48,52,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/87","value":{"seq":93,"meta":{"expire_at":null},"data":[8,1,18,6,110,97,116,105,111,110,160,6,19,168,6,1]}}}] -["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/94","value":{"seq":100,"meta":{"expire_at":null},"data":[8,1,18,6,114,101,103,105,111,110,160,6,19,168,6,1]}}}] diff --git a/tests/metactl/want_exported_v004 b/tests/metactl/want_exported_v004 new file mode 100644 index 000000000000..72fedaf22c29 --- /dev/null +++ b/tests/metactl/want_exported_v004 @@ -0,0 +1,167 @@ +["header",{"DataHeader":{"key":"header","value":{"version":"V004"}}}] +["raft_log",{"NodeId":1}] +["raft_log",{"Vote":{"leader_id":{"term":1,"node_id":1},"committed":false}}] +["raft_log",{"Committed":null}] +["raft_log",{"Purged":{"leader_id":{"term":1,"node_id":0},"index":1}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":2},"payload":{"Normal":{"txid":null,"time_ms":1667290820099,"cmd":{"AddNode":{"node_id":1,"node":{"name":"1","endpoint":{"addr":"localhost","port":28103},"grpc_api_advertise_address":"0.0.0.0:9191"},"overriding":false}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":3},"payload":{"Normal":{"txid":null,"time_ms":1667290820429,"cmd":{"AddNode":{"node_id":2,"node":{"name":"2","endpoint":{"addr":"localhost","port":28203},"grpc_api_advertise_address":"0.0.0.0:28202"},"overriding":false}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":4},"payload":{"Membership":{"configs":[[1],[1,2]],"nodes":{"1":{},"2":{}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":5},"payload":{"Membership":{"configs":[[1,2]],"nodes":{"1":{},"2":{}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":6},"payload":{"Normal":{"txid":null,"time_ms":1667290821018,"cmd":{"AddNode":{"node_id":3,"node":{"name":"3","endpoint":{"addr":"localhost","port":28303},"grpc_api_advertise_address":"0.0.0.0:28302"},"overriding":false}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":7},"payload":{"Membership":{"configs":[[1,2],[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":8},"payload":{"Membership":{"configs":[[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":9},"payload":{"Normal":{"txid":null,"time_ms":1667290824580,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":10},"payload":{"Normal":{"txid":null,"time_ms":1667290824586,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/default","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/1","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/default","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/default","value":[49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/default","value":[10,1,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/1","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,100,101,102,97,117,108,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":11},"payload":{"Normal":{"txid":null,"time_ms":1667290824603,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,82,112,117,87,110,100,84,102,53,74,108,103,121,74,67,112,65,65,116,81,88,54,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,48,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290884}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":12},"payload":{"Normal":{"txid":null,"time_ms":1667290826333,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,75,77,90,52,86,118,113,68,70,86,69,120,108,90,70,84,104,75,68,122,90,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,50,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290886}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":13},"payload":{"Normal":{"txid":null,"time_ms":1667290826867,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"Exact":0},"value":{"Update":[123,34,105,100,34,58,34,55,71,86,80,49,71,115,81,50,106,112,68,77,117,49,116,105,56,85,110,70,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,51,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]},"value_meta":{"expire_at":1667290886}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":14},"payload":{"Normal":{"txid":null,"time_ms":1667290836132,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":15},"payload":{"Normal":{"txid":null,"time_ms":1667290836137,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/9","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db1","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db1","value":[57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db1","value":[10,1,9,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/9","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":16},"payload":{"Normal":{"txid":null,"time_ms":1667290838209,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":17},"payload":{"Normal":{"txid":null,"time_ms":1667290838215,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/9","expected":0,"target":{"Seq":11}},{"key":"__fd_table/9/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/9/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_to_name/14","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/9/t1","value":[49,52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/9/t1","value":[10,1,14,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/14","value":[8,9,18,2,116,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":18},"payload":{"Normal":{"txid":null,"time_ms":1667290838764,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/14","expected":0,"target":{"Seq":17}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,53,99,55,49,49,56,55,98,49,53,97,52,52,53,54,100,98,49,55,50,48,50,98,100,51,97,55,101,99,102,56,51,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,2,16,170,2,24,228,7,32,184,12,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":19},"payload":{"Normal":{"txid":null,"time_ms":1667290843099,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/14","expected":0,"target":{"Seq":21}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/14","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,99,99,51,56,51,102,51,52,57,50,54,54,52,50,53,99,98,55,51,56,100,50,99,99,49,49,54,53,53,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,4,16,212,4,24,200,15,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":20},"payload":{"Normal":{"txid":null,"time_ms":1667290846841,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":10}},{"key":"__fd_database_by_id/9","expected":0,"target":{"Seq":15}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db1","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/9","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,54,46,56,52,48,55,51,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":21},"payload":{"Normal":{"txid":null,"time_ms":1667290848281,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290908}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":22},"payload":{"Normal":{"txid":null,"time_ms":1667290849028,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":23},"payload":{"Normal":{"txid":null,"time_ms":1667290849033,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/25","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db1","expected":0,"target":{"Seq":12}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db1","value":[50,53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db1","value":[10,2,9,25,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/25","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":24},"payload":{"Normal":{"txid":null,"time_ms":1667290851094,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":25},"payload":{"Normal":{"txid":null,"time_ms":1667290851100,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/25","expected":0,"target":{"Seq":27}},{"key":"__fd_table/25/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/25/t1","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":19}},{"key":"__fd_table_id_to_name/30","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/25/t1","value":[51,48],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/25/t1","value":[10,1,30,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/30","value":[8,25,18,2,116,49,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":26},"payload":{"Normal":{"txid":null,"time_ms":1667290851947,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290911}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":27},"payload":{"Normal":{"txid":null,"time_ms":1667290852139,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/30","expected":0,"target":{"Seq":33}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,56,100,97,99,49,50,51,97,97,54,101,55,52,51,53,102,97,55,49,52,50,99,55,100,56,54,98,98,49,98,54,55,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,1,16,148,1,24,140,7,32,184,12,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":28},"payload":{"Normal":{"txid":null,"time_ms":1667290853220,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/30","expected":0,"target":{"Seq":38}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/30","value":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,50,97,57,100,99,97,53,55,52,51,54,54,52,53,101,97,97,53,97,51,52,100,57,100,102,97,50,51,55,98,50,54,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,2,16,168,2,24,152,14,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":29},"payload":{"Normal":{"txid":null,"time_ms":1667290861626,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290921}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":30},"payload":{"Normal":{"txid":null,"time_ms":1667290863578,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db1","expected":0,"target":{"Seq":26}},{"key":"__fd_database_by_id/25","expected":0,"target":{"Seq":31}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db1","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/25","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,51,46,53,55,55,49,55,50,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":31},"payload":{"Normal":{"txid":null,"time_ms":1667290864962,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":32},"payload":{"Normal":{"txid":null,"time_ms":1667290864967,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/book_db","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/42","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/book_db","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/book_db","value":[52,50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/book_db","value":[10,1,42,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/42","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,98,111,111,107,95,100,98,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":33},"payload":{"Normal":{"txid":null,"time_ms":1667290867024,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":34},"payload":{"Normal":{"txid":null,"time_ms":1667290867030,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/42","expected":0,"target":{"Seq":44}},{"key":"__fd_table/42/books","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/42/books","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":35}},{"key":"__fd_table_id_to_name/47","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/42/books","value":[52,55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/47","value":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/42/books","value":[10,1,47,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/47","value":[8,42,18,5,98,111,111,107,115,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":35},"payload":{"Normal":{"txid":null,"time_ms":1667290867598,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/47","expected":0,"target":{"Seq":50}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/47","value":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,52,50,47,52,55,47,95,115,115,47,55,48,98,53,48,55,53,97,54,98,51,99,52,55,102,51,57,57,100,51,48,102,57,100,52,97,54,98,98,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,16,8,1,16,64,24,180,3,32,176,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":36},"payload":{"Normal":{"txid":null,"time_ms":1667290870377,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/book_db","expected":0,"target":{"Seq":43}},{"key":"__fd_database_by_id/42","expected":0,"target":{"Seq":48}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/book_db","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/42","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,48,46,51,55,54,53,56,53,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":37},"payload":{"Normal":{"txid":null,"time_ms":1667290872604,"cmd":{"UpsertKV":{"key":"__fd_id_gen/database_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":38},"payload":{"Normal":{"txid":null,"time_ms":1667290872609,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db12_0002","expected":0,"target":{"Seq":0}},{"key":"__fd_database_id_to_name/56","expected":0,"target":{"Seq":0}},{"key":"__fd_db_id_list/test_tenant/db12_0002","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database/test_tenant/db12_0002","value":[53,54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_db_id_list/test_tenant/db12_0002","value":[10,1,56,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_database_id_to_name/56","value":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,9,100,98,49,50,95,48,48,48,50,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":39},"payload":{"Normal":{"txid":null,"time_ms":1667290874661,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":40},"payload":{"Normal":{"txid":null,"time_ms":1667290874666,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":58}},{"key":"__fd_table/56/t","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/56/t","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":52}},{"key":"__fd_table_id_to_name/61","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/56/t","value":[54,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/61","value":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/56/t","value":[10,1,61,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/61","value":[8,56,18,1,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":41},"payload":{"Normal":{"txid":null,"time_ms":1667290875684,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":62}},{"key":"__fd_table/56/t","expected":0,"target":{"Seq":63}},{"key":"__fd_table_by_id/61","expected":0,"target":{"Seq":64}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":66}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Delete":{"key":"__fd_table/56/t","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_table_by_id/61","value":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,53,46,54,56,51,49,52,51,32,85,84,67,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":42},"payload":{"Normal":{"txid":null,"time_ms":1667290880720,"cmd":{"Transaction":{"condition":[{"key":"__fd_database/test_tenant/db12_0002","expected":0,"target":{"Seq":57}},{"key":"__fd_database_by_id/56","expected":0,"target":{"Seq":68}}],"if_then":[{"request":{"Delete":{"key":"__fd_database/test_tenant/db12_0002","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_database_by_id/56","value":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,48,46,55,49,56,55,54,51,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":43},"payload":{"Normal":{"txid":null,"time_ms":1667290881744,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290941}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":44},"payload":{"Normal":{"txid":null,"time_ms":1667290883474,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":45},"payload":{"Normal":{"txid":null,"time_ms":1667290883479,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":3}},{"key":"__fd_table/1/t12_0004","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/t12_0004","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":70}},{"key":"__fd_table_id_to_name/73","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/t12_0004","value":[55,51],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/t12_0004","value":[10,1,73,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/73","value":[8,1,18,8,116,49,50,95,48,48,48,52,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":46},"payload":{"Normal":{"txid":null,"time_ms":1667290885036,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":76}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,48,97,99,51,52,98,49,56,52,48,56,50,52,57,97,99,98,97,101,51,50,98,53,56,101,51,101,57,55,50,102,98,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,2,16,8,24,188,1,32,148,2,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":47},"payload":{"Normal":{"txid":null,"time_ms":1667290886657,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":80}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":48},"payload":{"Normal":{"txid":null,"time_ms":1667290889731,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290949}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":49},"payload":{"Normal":{"txid":null,"time_ms":1667290890274,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290950}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":50},"payload":{"Normal":{"txid":null,"time_ms":1667290907158,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":74}},{"key":"__fd_table/1/t12_0004","expected":0,"target":{"Seq":75}},{"key":"__fd_table_by_id/73","expected":0,"target":{"Seq":81}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":78}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Delete":{"key":"__fd_table/1/t12_0004","prev_value":true,"match_seq":null}}},{"request":{"Put":{"key":"__fd_table_by_id/73","value":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,55,46,49,53,55,50,51,55,32,85,84,67,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[51],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":51},"payload":{"Normal":{"txid":null,"time_ms":1667290909908,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":52},"payload":{"Normal":{"txid":null,"time_ms":1667290909915,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":84}},{"key":"__fd_table/1/nation","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/nation","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":86}},{"key":"__fd_table_id_to_name/87","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/nation","value":[56,55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/87","value":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/nation","value":[10,1,87,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/87","value":[8,1,18,6,110,97,116,105,111,110,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":53},"payload":{"Normal":{"txid":null,"time_ms":1667290912442,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":54},"payload":{"Normal":{"txid":null,"time_ms":1667290912448,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":88}},{"key":"__fd_table/1/region","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/region","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":92}},{"key":"__fd_table_id_to_name/94","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/region","value":[57,52],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/94","value":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/region","value":[10,1,94,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/94","value":[8,1,18,6,114,101,103,105,111,110,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":55},"payload":{"Normal":{"txid":null,"time_ms":1667290914988,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":56},"payload":{"Normal":{"txid":null,"time_ms":1667290914994,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":95}},{"key":"__fd_table/1/part","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/part","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":99}},{"key":"__fd_table_id_to_name/101","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/part","value":[49,48,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/101","value":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/part","value":[10,1,101,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/101","value":[8,1,18,4,112,97,114,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":57},"payload":{"Normal":{"txid":null,"time_ms":1667290915186,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290975}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":58},"payload":{"Normal":{"txid":null,"time_ms":1667290917533,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":59},"payload":{"Normal":{"txid":null,"time_ms":1667290917538,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":102}},{"key":"__fd_table/1/supplier","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/supplier","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":106}},{"key":"__fd_table_id_to_name/109","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/supplier","value":[49,48,57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/109","value":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/supplier","value":[10,1,109,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[55],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/109","value":[8,1,18,8,115,117,112,112,108,105,101,114,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":60},"payload":{"Normal":{"txid":null,"time_ms":1667290920075,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":61},"payload":{"Normal":{"txid":null,"time_ms":1667290920080,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":110}},{"key":"__fd_table/1/partsupp","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/partsupp","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":114}},{"key":"__fd_table_id_to_name/116","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/partsupp","value":[49,49,54],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/116","value":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/partsupp","value":[10,1,116,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[56],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/116","value":[8,1,18,8,112,97,114,116,115,117,112,112,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":62},"payload":{"Normal":{"txid":null,"time_ms":1667290920676,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290980}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":63},"payload":{"Normal":{"txid":null,"time_ms":1667290921608,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667290981}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":64},"payload":{"Normal":{"txid":null,"time_ms":1667290922624,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":65},"payload":{"Normal":{"txid":null,"time_ms":1667290922629,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":117}},{"key":"__fd_table/1/customer","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/customer","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":121}},{"key":"__fd_table_id_to_name/125","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/customer","value":[49,50,53],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/125","value":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/customer","value":[10,1,125,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/125","value":[8,1,18,8,99,117,115,116,111,109,101,114,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":66},"payload":{"Normal":{"txid":null,"time_ms":1667290925169,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":67},"payload":{"Normal":{"txid":null,"time_ms":1667290925175,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":126}},{"key":"__fd_table/1/orders","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/orders","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":130}},{"key":"__fd_table_id_to_name/132","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/orders","value":[49,51,50],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/132","value":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/orders","value":[10,2,132,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49,48],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/132","value":[8,1,18,6,111,114,100,101,114,115,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":68},"payload":{"Normal":{"txid":null,"time_ms":1667290927714,"cmd":{"UpsertKV":{"key":"__fd_id_gen/table_id","seq":"Any","value":{"Update":[]},"value_meta":null}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":69},"payload":{"Normal":{"txid":null,"time_ms":1667290927723,"cmd":{"Transaction":{"condition":[{"key":"__fd_database_by_id/1","expected":0,"target":{"Seq":133}},{"key":"__fd_table/1/lineitem","expected":0,"target":{"Seq":0}},{"key":"__fd_table_id_list/1/lineitem","expected":0,"target":{"Seq":0}},{"key":"__fd_table_count/test_tenant","expected":0,"target":{"Seq":137}},{"key":"__fd_table_id_to_name/139","expected":0,"target":{"Seq":0}}],"if_then":[{"request":{"Put":{"key":"__fd_database_by_id/1","value":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table/1/lineitem","value":[49,51,57],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_by_id/139","value":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,6,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_list/1/lineitem","value":[10,2,139,1,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_count/test_tenant","value":[49,49],"prev_value":true,"expire_at":null}}},{"request":{"Put":{"key":"__fd_table_id_to_name/139","value":[8,1,18,8,108,105,110,101,105,116,101,109,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":70},"payload":{"Normal":{"txid":null,"time_ms":1667290933209,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/125","expected":0,"target":{"Seq":128}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/125","value":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,50,53,47,95,115,115,47,100,50,57,48,56,57,50,102,54,54,53,98,52,49,51,51,97,53,101,97,102,97,55,53,48,100,50,99,50,52,53,53,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,22,8,152,117,16,169,141,183,1,24,143,225,84,32,190,196,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":71},"payload":{"Normal":{"txid":null,"time_ms":1667290939149,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/139","expected":0,"target":{"Seq":142}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/139","value":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,57,47,95,115,115,47,99,49,48,97,51,56,97,48,53,48,97,98,52,97,100,50,98,51,49,51,48,50,56,97,98,97,100,56,54,101,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,24,8,252,211,36,16,221,134,134,46,24,168,150,134,15,32,171,222,7,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":72},"payload":{"Normal":{"txid":null,"time_ms":1667290940280,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/87","expected":0,"target":{"Seq":90}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/87","value":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,56,55,47,95,115,115,47,102,101,52,52,102,102,53,48,55,101,54,56,52,52,49,56,57,55,54,101,102,53,100,101,99,57,97,98,51,97,49,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,17,8,25,16,218,20,24,235,16,32,142,8,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":73},"payload":{"Normal":{"txid":null,"time_ms":1667290942403,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/132","expected":0,"target":{"Seq":135}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/132","value":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,50,47,95,115,115,47,50,52,53,100,98,57,101,55,57,53,101,54,52,53,54,57,56,100,55,48,97,54,51,50,57,53,102,101,56,49,57,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,24,8,240,147,9,16,184,204,229,9,24,218,217,169,3,32,178,202,3,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":74},"payload":{"Normal":{"txid":null,"time_ms":1667290942655,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291002}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":75},"payload":{"Normal":{"txid":null,"time_ms":1667290944081,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/116","expected":0,"target":{"Seq":119}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/116","value":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,49,54,47,95,115,115,47,57,56,56,50,53,55,56,53,102,51,49,48,52,51,98,55,98,53,57,51,57,56,48,51,100,102,54,102,48,55,55,101,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,24,8,128,241,4,16,201,161,158,6,24,148,192,168,2,32,252,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":76},"payload":{"Normal":{"txid":null,"time_ms":1667290944493,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291004}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":77},"payload":{"Normal":{"txid":null,"time_ms":1667290945499,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/101","expected":0,"target":{"Seq":104}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/101","value":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,49,47,95,115,115,47,57,51,53,49,56,97,52,97,55,102,98,52,52,99,56,101,57,54,49,51,51,50,54,56,57,57,50,48,99,55,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,23,8,160,156,1,16,166,214,200,1,24,209,232,62,32,180,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":78},"payload":{"Normal":{"txid":null,"time_ms":1667290946629,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/94","expected":0,"target":{"Seq":97}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/94","value":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,57,52,47,95,115,115,47,53,102,57,55,100,48,99,49,98,56,99,53,52,55,102,55,97,97,49,102,97,55,99,102,48,55,99,101,57,57,51,100,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,17,8,5,16,224,3,24,168,6,32,227,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":79},"payload":{"Normal":{"txid":null,"time_ms":1667290947121,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291007}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":80},"payload":{"Normal":{"txid":null,"time_ms":1667290947751,"cmd":{"Transaction":{"condition":[{"key":"__fd_table_by_id/109","expected":0,"target":{"Seq":112}}],"if_then":[{"request":{"Put":{"key":"__fd_table_by_id/109","value":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,57,47,95,115,115,47,53,48,100,54,101,53,55,57,48,51,49,101,52,54,51,53,97,54,97,97,50,99,48,48,54,99,50,51,48,101,98,49,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,20,8,232,7,16,247,193,10,24,134,172,5,32,246,58,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1],"prev_value":true,"expire_at":null}}}],"else_then":[]}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":81},"payload":{"Normal":{"txid":null,"time_ms":1667290965904,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291025}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":82},"payload":{"Normal":{"txid":null,"time_ms":1667290971893,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291031}}}}}}}] +["raft_log",{"LogEntry":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":83},"payload":{"Normal":{"txid":null,"time_ms":1667290974891,"cmd":{"UpsertKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","seq":{"GE":1},"value":"AsIs","value_meta":{"expire_at":1667291034}}}}}}}] +["state_machine/0",{"Sequences":{"key":"generic-kv","value":159}}] +["state_machine/0",{"StateMachineMeta":{"key":"LastApplied","value":{"LogId":{"leader_id":{"term":1,"node_id":0},"index":83}}}}] +["state_machine/0",{"StateMachineMeta":{"key":"LastMembership","value":{"Membership":{"log_id":{"leader_id":{"term":1,"node_id":0},"index":8},"membership":{"configs":[[1,2,3]],"nodes":{"1":{},"2":{},"3":{}}}}}}}] +["state_machine/0",{"Nodes":{"key":1,"value":{"name":"1","endpoint":{"addr":"localhost","port":28103},"grpc_api_advertise_address":"0.0.0.0:9191"}}}] +["state_machine/0",{"Nodes":{"key":2,"value":{"name":"2","endpoint":{"addr":"localhost","port":28203},"grpc_api_advertise_address":"0.0.0.0:28202"}}}] +["state_machine/0",{"Nodes":{"key":3,"value":{"name":"3","endpoint":{"addr":"localhost","port":28303},"grpc_api_advertise_address":"0.0.0.0:28302"}}}] +["state_machine/0",{"Expire":{"key":{"time_ms":1667291025000,"seq":157},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1"}}}] +["state_machine/0",{"Expire":{"key":{"time_ms":1667291031000,"seq":158},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6"}}}] +["state_machine/0",{"Expire":{"key":{"time_ms":1667291034000,"seq":159},"value":{"seq":1,"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1"}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/7GVP1GsQ2jpDMu1ti8UnF1","value":{"seq":157,"meta":{"expire_at":1667291025},"data":[123,34,105,100,34,58,34,55,71,86,80,49,71,115,81,50,106,112,68,77,117,49,116,105,56,85,110,70,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,51,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/KMZ4VvqDFVExlZFThKDzZ1","value":{"seq":159,"meta":{"expire_at":1667291034},"data":[123,34,105,100,34,58,34,75,77,90,52,86,118,113,68,70,86,69,120,108,90,70,84,104,75,68,122,90,49,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,50,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_clusters/test_tenant/test_cluster/databend_query/RpuWndTf5JlgyJCpAAtQX6","value":{"seq":158,"meta":{"expire_at":1667291031},"data":[123,34,105,100,34,58,34,82,112,117,87,110,100,84,102,53,74,108,103,121,74,67,112,65,65,116,81,88,54,34,44,34,99,112,117,95,110,117,109,115,34,58,48,44,34,118,101,114,115,105,111,110,34,58,48,44,34,102,108,105,103,104,116,95,97,100,100,114,101,115,115,34,58,34,49,50,55,46,48,46,48,46,49,58,57,48,57,48,34,44,34,98,105,110,97,114,121,95,118,101,114,115,105,111,110,34,58,34,118,48,46,56,46,57,53,45,110,105,103,104,116,108,121,45,97,52,49,56,55,100,51,40,114,117,115,116,45,49,46,54,54,46,48,45,110,105,103,104,116,108,121,45,50,48,50,50,45,49,49,45,48,49,84,48,56,58,49,55,58,51,54,46,55,48,49,55,54,53,90,41,34,125]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database/test_tenant/default","value":{"seq":2,"meta":{"expire_at":null},"data":[49]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/1","value":{"seq":140,"meta":{"expire_at":null},"data":[162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,52,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,50,52,46,53,55,48,53,56,54,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/25","value":{"seq":41,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,56,46,53,51,48,52,57,48,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,51,46,53,55,55,49,55,50,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/42","value":{"seq":55,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,54,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,52,46,52,54,48,51,56,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,48,46,51,55,54,53,56,53,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/56","value":{"seq":71,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,50,46,49,48,55,48,48,57,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,48,46,55,49,56,55,54,51,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_by_id/9","value":{"seq":23,"meta":{"expire_at":null},"data":[42,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,53,46,54,51,52,49,50,54,32,85,84,67,186,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,52,54,46,56,52,48,55,51,54,32,85,84,67,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/1","value":{"seq":5,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,100,101,102,97,117,108,116,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/25","value":{"seq":29,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/42","value":{"seq":46,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,7,98,111,111,107,95,100,98,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/56","value":{"seq":60,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,9,100,98,49,50,95,48,48,48,50,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_database_id_to_name/9","value":{"seq":13,"meta":{"expire_at":null},"data":[10,11,116,101,115,116,95,116,101,110,97,110,116,18,3,100,98,49,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/book_db","value":{"seq":45,"meta":{"expire_at":null},"data":[10,1,42,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/db1","value":{"seq":28,"meta":{"expire_at":null},"data":[10,2,9,25,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/db12_0002","value":{"seq":59,"meta":{"expire_at":null},"data":[10,1,56,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_db_id_list/test_tenant/default","value":{"seq":4,"meta":{"expire_at":null},"data":[10,1,1,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_id_gen/database_id","value":{"seq":56,"meta":null,"data":[]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_id_gen/table_id","value":{"seq":139,"meta":null,"data":[]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/customer","value":{"seq":127,"meta":{"expire_at":null},"data":[49,50,53]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/lineitem","value":{"seq":141,"meta":{"expire_at":null},"data":[49,51,57]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/nation","value":{"seq":89,"meta":{"expire_at":null},"data":[56,55]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/orders","value":{"seq":134,"meta":{"expire_at":null},"data":[49,51,50]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/part","value":{"seq":103,"meta":{"expire_at":null},"data":[49,48,49]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/partsupp","value":{"seq":118,"meta":{"expire_at":null},"data":[49,49,54]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/region","value":{"seq":96,"meta":{"expire_at":null},"data":[57,52]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/1/supplier","value":{"seq":111,"meta":{"expire_at":null},"data":[49,48,57]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/25/t1","value":{"seq":32,"meta":{"expire_at":null},"data":[51,48]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/42/books","value":{"seq":49,"meta":{"expire_at":null},"data":[52,55]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table/9/t1","value":{"seq":16,"meta":{"expire_at":null},"data":[49,52]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/101","value":{"seq":153,"meta":{"expire_at":null},"data":[10,131,2,10,27,10,9,112,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,109,102,103,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,112,95,98,114,97,110,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,116,121,112,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,112,95,115,105,122,101,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,95,99,111,110,116,97,105,110,101,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,95,114,101,116,97,105,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,112,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,49,47,95,115,115,47,57,51,53,49,56,97,52,97,55,102,98,52,52,99,56,101,57,54,49,51,51,50,54,56,57,57,50,48,99,55,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,52,46,52,56,48,57,48,56,32,85,84,67,186,1,23,8,160,156,1,16,166,214,200,1,24,209,232,62,32,180,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/109","value":{"seq":156,"meta":{"expire_at":null},"data":[10,206,1,10,27,10,9,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,115,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,115,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,115,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,48,57,47,95,115,115,47,53,48,100,54,101,53,55,57,48,51,49,101,52,54,51,53,97,54,97,97,50,99,48,48,54,99,50,51,48,101,98,49,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,51,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,55,46,48,50,53,50,54,52,32,85,84,67,186,1,20,8,232,7,16,247,193,10,24,134,172,5,32,246,58,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/116","value":{"seq":151,"meta":{"expire_at":null},"data":[10,160,1,10,28,10,10,112,115,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,112,115,95,97,118,97,105,108,113,116,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,112,115,95,115,117,112,112,108,121,99,111,115,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,112,115,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,49,54,47,95,115,115,47,57,56,56,50,53,55,56,53,102,51,49,48,52,51,98,55,98,53,57,51,57,56,48,51,100,102,54,102,48,55,55,101,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,57,46,53,54,53,50,55,50,32,85,84,67,186,1,24,8,128,241,4,16,201,161,158,6,24,148,192,168,2,32,252,134,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/125","value":{"seq":146,"meta":{"expire_at":null},"data":[10,238,1,10,27,10,9,99,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,99,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,100,100,114,101,115,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,99,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,99,95,112,104,111,110,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,97,99,99,116,98,97,108,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,99,95,109,107,116,115,101,103,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,99,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,50,53,47,95,115,115,47,100,50,57,48,56,57,50,102,54,54,53,98,52,49,51,51,97,53,101,97,102,97,55,53,48,100,50,99,50,52,53,53,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,50,46,49,49,52,54,48,55,32,85,84,67,186,1,22,8,152,117,16,169,141,183,1,24,143,225,84,32,190,196,2,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/132","value":{"seq":149,"meta":{"expire_at":null},"data":[10,158,2,10,28,10,10,111,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,117,115,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,111,95,111,114,100,101,114,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,111,95,116,111,116,97,108,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,111,95,111,114,100,101,114,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,111,95,111,114,100,101,114,112,114,105,111,114,105,116,121,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,25,10,7,111,95,99,108,101,114,107,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,111,95,115,104,105,112,112,114,105,111,114,105,116,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,111,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,50,47,95,115,115,47,50,52,53,100,98,57,101,55,57,53,101,54,52,53,54,57,56,100,55,48,97,54,51,50,57,53,102,101,56,49,57,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,52,46,54,54,50,54,57,48,32,85,84,67,186,1,24,8,240,147,9,16,184,204,229,9,24,218,217,169,3,32,178,202,3,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/139","value":{"seq":147,"meta":{"expire_at":null},"data":[10,242,3,10,28,10,10,108,95,111,114,100,101,114,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,112,97,114,116,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,115,117,112,112,107,101,121,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,110,117,109,98,101,114,26,8,50,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,113,117,97,110,116,105,116,121,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,33,10,15,108,95,101,120,116,101,110,100,101,100,112,114,105,99,101,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,100,105,115,99,111,117,110,116,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,23,10,5,108,95,116,97,120,26,8,98,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,114,101,116,117,114,110,102,108,97,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,108,105,110,101,115,116,97,116,117,115,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,30,10,12,108,95,99,111,109,109,105,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,31,10,13,108,95,114,101,99,101,105,112,116,100,97,116,101,26,8,106,0,160,6,19,168,6,1,160,6,19,168,6,1,10,32,10,14,108,95,115,104,105,112,105,110,115,116,114,117,99,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,28,10,10,108,95,115,104,105,112,109,111,100,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,108,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,49,47,49,51,57,47,95,115,115,47,99,49,48,97,51,56,97,48,53,48,97,98,52,97,100,50,98,51,49,51,48,50,56,97,98,97,100,56,54,101,102,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,50,58,48,55,46,50,48,57,50,48,56,32,85,84,67,186,1,24,8,252,211,36,16,221,134,134,46,24,168,150,134,15,32,171,222,7,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/14","value":{"seq":22,"meta":{"expire_at":null},"data":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,57,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,57,47,49,52,47,95,115,115,47,99,99,51,56,51,102,51,52,57,50,54,54,52,50,53,99,98,55,51,56,100,50,99,99,49,49,54,53,53,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,51,55,46,54,57,52,56,56,56,32,85,84,67,186,1,17,8,4,16,212,4,24,200,15,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/30","value":{"seq":39,"meta":{"expire_at":null},"data":[10,174,1,10,19,10,1,97,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,98,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,99,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,100,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,102,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,103,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,19,10,1,104,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,50,53,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,50,53,47,51,48,47,95,115,115,47,50,97,57,100,99,97,53,55,52,51,54,54,52,53,101,97,97,53,97,51,52,100,57,100,102,97,50,51,55,98,50,54,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,52,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,48,58,53,48,46,53,56,54,57,53,48,32,85,84,67,186,1,17,8,2,16,168,2,24,152,14,32,240,24,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/47","value":{"seq":54,"meta":{"expire_at":null},"data":[10,81,10,23,10,5,116,105,116,108,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,97,117,116,104,111,114,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,22,10,4,100,97,116,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,52,50,42,71,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,50,52,50,47,52,55,47,95,115,115,47,55,48,98,53,48,55,53,97,54,98,51,99,52,55,102,51,57,57,100,51,48,102,57,100,52,97,54,98,98,55,53,48,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,48,54,46,53,49,52,49,52,50,32,85,84,67,186,1,16,8,1,16,64,24,180,3,32,176,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/61","value":{"seq":69,"meta":{"expire_at":null},"data":[10,28,10,20,10,2,99,49,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,17,10,11,100,97,116,97,98,97,115,101,95,105,100,18,2,53,54,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,56,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,52,46,49,53,53,55,53,57,32,85,84,67,186,1,6,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,49,53,46,54,56,51,49,52,51,32,85,84,67,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/73","value":{"seq":85,"meta":{"expire_at":null},"data":[10,27,10,19,10,1,99,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,55,51,47,95,115,115,47,57,52,97,101,51,55,99,52,100,100,56,49,52,56,56,101,56,53,97,101,50,101,98,99,100,100,52,54,56,52,53,50,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,49,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,50,50,46,57,54,51,49,57,51,32,85,84,67,186,1,16,8,3,16,12,24,244,2,32,168,4,160,6,19,168,6,1,194,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,55,46,49,53,55,50,51,55,32,85,84,67,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/87","value":{"seq":148,"meta":{"expire_at":null},"data":[10,123,10,29,10,11,110,95,110,97,116,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,110,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,29,10,11,110,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,110,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,56,55,47,95,115,115,47,102,101,52,52,102,102,53,48,55,101,54,56,52,52,49,56,57,55,54,101,102,53,100,101,99,57,97,98,51,97,49,52,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,55,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,52,57,46,52,48,48,49,52,57,32,85,84,67,186,1,17,8,25,16,218,20,24,235,16,32,142,8,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_by_id/94","value":{"seq":154,"meta":{"expire_at":null},"data":[10,92,10,29,10,11,114,95,114,101,103,105,111,110,107,101,121,26,8,42,0,160,6,19,168,6,1,160,6,19,168,6,1,10,24,10,6,114,95,110,97,109,101,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,10,27,10,9,114,95,99,111,109,109,101,110,116,26,8,122,0,160,6,19,168,6,1,160,6,19,168,6,1,160,6,19,168,6,1,42,16,10,11,100,97,116,97,98,97,115,101,95,105,100,18,1,49,42,70,10,17,115,110,97,112,115,104,111,116,95,108,111,99,97,116,105,111,110,18,49,49,47,57,52,47,95,115,115,47,53,102,57,55,100,48,99,49,98,56,99,53,52,55,102,55,97,97,49,102,97,55,99,102,48,55,99,101,57,57,51,100,95,118,49,46,106,115,111,110,50,4,70,85,83,69,82,7,100,101,102,97,117,108,116,162,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,170,1,30,50,48,50,50,45,49,49,45,48,49,32,48,56,58,50,49,58,53,49,46,57,51,52,54,49,57,32,85,84,67,186,1,17,8,5,16,224,3,24,168,6,32,227,5,160,6,19,168,6,1,202,1,0,202,1,0,202,1,0,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_count/test_tenant","value":{"seq":144,"meta":{"expire_at":null},"data":[49,49]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/customer","value":{"seq":129,"meta":{"expire_at":null},"data":[10,1,125,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/lineitem","value":{"seq":143,"meta":{"expire_at":null},"data":[10,2,139,1,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/nation","value":{"seq":91,"meta":{"expire_at":null},"data":[10,1,87,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/orders","value":{"seq":136,"meta":{"expire_at":null},"data":[10,2,132,1,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/part","value":{"seq":105,"meta":{"expire_at":null},"data":[10,1,101,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/partsupp","value":{"seq":120,"meta":{"expire_at":null},"data":[10,1,116,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/region","value":{"seq":98,"meta":{"expire_at":null},"data":[10,1,94,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/supplier","value":{"seq":113,"meta":{"expire_at":null},"data":[10,1,109,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/1/t12_0004","value":{"seq":77,"meta":{"expire_at":null},"data":[10,1,73,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/25/t1","value":{"seq":34,"meta":{"expire_at":null},"data":[10,1,30,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/42/books","value":{"seq":51,"meta":{"expire_at":null},"data":[10,1,47,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/56/t","value":{"seq":65,"meta":{"expire_at":null},"data":[10,1,61,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_list/9/t1","value":{"seq":18,"meta":{"expire_at":null},"data":[10,1,14,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/101","value":{"seq":107,"meta":{"expire_at":null},"data":[8,1,18,4,112,97,114,116,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/109","value":{"seq":115,"meta":{"expire_at":null},"data":[8,1,18,8,115,117,112,112,108,105,101,114,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/116","value":{"seq":122,"meta":{"expire_at":null},"data":[8,1,18,8,112,97,114,116,115,117,112,112,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/125","value":{"seq":131,"meta":{"expire_at":null},"data":[8,1,18,8,99,117,115,116,111,109,101,114,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/132","value":{"seq":138,"meta":{"expire_at":null},"data":[8,1,18,6,111,114,100,101,114,115,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/139","value":{"seq":145,"meta":{"expire_at":null},"data":[8,1,18,8,108,105,110,101,105,116,101,109,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/14","value":{"seq":20,"meta":{"expire_at":null},"data":[8,9,18,2,116,49,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/30","value":{"seq":36,"meta":{"expire_at":null},"data":[8,25,18,2,116,49,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/47","value":{"seq":53,"meta":{"expire_at":null},"data":[8,42,18,5,98,111,111,107,115,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/61","value":{"seq":67,"meta":{"expire_at":null},"data":[8,56,18,1,116,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/73","value":{"seq":79,"meta":{"expire_at":null},"data":[8,1,18,8,116,49,50,95,48,48,48,52,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/87","value":{"seq":93,"meta":{"expire_at":null},"data":[8,1,18,6,110,97,116,105,111,110,160,6,19,168,6,1]}}}] +["state_machine/0",{"GenericKV":{"key":"__fd_table_id_to_name/94","value":{"seq":100,"meta":{"expire_at":null},"data":[8,1,18,6,114,101,103,105,111,110,160,6,19,168,6,1]}}}] diff --git a/tests/metactl/want_snapshot_v003 b/tests/metactl/want_snapshot_v004 similarity index 100% rename from tests/metactl/want_snapshot_v003 rename to tests/metactl/want_snapshot_v004 From ce75153a87a9cbd4838e149894f05b8129288018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Sun, 10 Nov 2024 18:08:43 +0800 Subject: [PATCH 13/92] chore: update meta-service version info and compatibility doc (#16800) --- src/meta/README.md | 8 +++++++- src/meta/raft-store/src/ondisk/data_version.rs | 3 +-- src/meta/raft-store/src/ondisk/version_info.rs | 7 +++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/meta/README.md b/src/meta/README.md index 03674ed0c366..2140f7bc8990 100644 --- a/src/meta/README.md +++ b/src/meta/README.md @@ -100,7 +100,8 @@ History versions that are not included in the above chart: |:--------------------|:-------------------------| | [0.9.41, 1.2.212) | [0.9.41, 1.2.212) | | [1.2.212, 1.2.479) | [0.9.41, 1.2.479) | -| [1.2.479, +∞) | [1.2.288, +∞) | +| [1.2.479, 1.2.655) | [1.2.288, 1.2.655) | +| [1.2.655, +∞) | [1.2.288, +∞) | - `1.2.53` Incompatible, rolling upgrade is allowed without snapshot transmitting. @@ -123,6 +124,9 @@ History versions that are not included in the above chart: - `1.2.552` 2024-07-02 Introduce on-disk `V003`, using `rotbl` format snapshot, which is compatible with `V002`. The oldest compatible version is `1.2.288`(`1.2.212~1.2.287` are removed). + +- `1.2.655` 2024-11-11 Introduce on-disk `V004`, using WAL based Raft log storage, + which is compatible with `V002`. The oldest compatible version is `1.2.288`(`1.2.212~1.2.287` are removed). ## Compatibility of databend-meta on-disk data @@ -131,6 +135,8 @@ The on-disk data of Databend-meta evolves over time while maintaining backward c | DataVersion | Databend-version | Min Compatible with | |:------------|:-----------------|:--------------------| +| V004 | 1.2.655 | V002 | +| V003 | 1.2.547 | V002 | | V002 | 1.2.53 | V001 | | V001 | 1.1.40 | V0 | diff --git a/src/meta/raft-store/src/ondisk/data_version.rs b/src/meta/raft-store/src/ondisk/data_version.rs index 6ffd92c2a9c7..f8e068e6efc9 100644 --- a/src/meta/raft-store/src/ondisk/data_version.rs +++ b/src/meta/raft-store/src/ondisk/data_version.rs @@ -55,8 +55,7 @@ impl fmt::Debug for DataVersion { ), Self::V002 => write!(f, "V002(2023-07-22: Store snapshot in a file)"), Self::V003 => write!(f, "V003(2024-06-27: Store snapshot in rotbl)"), - // TODO(raft-log): udpate the date when merged. - Self::V004 => write!(f, "V004(2024-11-04: WAL based raft-log)"), + Self::V004 => write!(f, "V004(2024-11-11: WAL based raft-log)"), } } } diff --git a/src/meta/raft-store/src/ondisk/version_info.rs b/src/meta/raft-store/src/ondisk/version_info.rs index d9bdecf28473..e39a355bf90a 100644 --- a/src/meta/raft-store/src/ondisk/version_info.rs +++ b/src/meta/raft-store/src/ondisk/version_info.rs @@ -111,11 +111,10 @@ impl VersionInfo { } const fn v004() -> Self { - // TODO(raft-log): update these values when merged. Self::new( - "3694e259c8e7c227fadfac5faa881cd2f2af6bbe", - "2024-11-04", - new_semver(1, 2, 53), + "3d76dc68e79db72341c6006808a6e1773ab8b565", + "2024-11-11", + new_semver(1, 2, 655), "WAL based raft-log", ) } From fc0c4e0f7d3c3a2ce18ad8219bbc09b9a90513f5 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Mon, 11 Nov 2024 14:18:08 +0800 Subject: [PATCH 14/92] fix(query): fix broken file log layout (#16803) --- src/common/tracing/src/init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/tracing/src/init.rs b/src/common/tracing/src/init.rs index b406f4ca4733..734f4b093edb 100644 --- a/src/common/tracing/src/init.rs +++ b/src/common/tracing/src/init.rs @@ -188,7 +188,7 @@ pub fn init_logging( .parse(&cfg.file.level), )) .filter(make_log_filter(&cfg.file.prefix_filter)) - .append(normal_log_file); + .append(normal_log_file.with_layout(get_layout(&cfg.file.format))); logger = logger.dispatch(dispatch); } From c3688cbc2945b91d5b9cd1c221c43c9984d42256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Mon, 11 Nov 2024 15:05:24 +0800 Subject: [PATCH 15/92] fix: re-ensure dir before upgrading (#16805) Because upgrade-cleaning would remove the entire dir. --- src/meta/raft-store/src/ondisk/upgrade_to_v004.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs b/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs index 9d74de99b900..e036892b8554 100644 --- a/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs +++ b/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs @@ -41,6 +41,9 @@ impl OnDisk { /// `V004` saves log in WAL based raft log. #[fastrace::trace] pub(crate) async fn upgrade_v003_to_v004(&mut self) -> Result<(), io::Error> { + // The previous cleaning step may remove the dir + Self::ensure_dirs(&self.config.raft_dir)?; + self.begin_upgrading(DataVersion::V003).await?; // 1.1. upgrade raft log From 1f712dc8ecfa04ff3bc2e8e167395bccab17d251 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Mon, 11 Nov 2024 21:54:19 +0800 Subject: [PATCH 16/92] refactor(base): add stacktrace to replace backtrace (#16643) * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): make lint for macos * refactor(base): make lint for macos * refactor(base): make lint for linux * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): add display text cache * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * refactor(base): lock free backtrace capture * fix(base): fix test failure * fix(base): fix test failure --- Cargo.lock | 42 +- Cargo.toml | 4 + src/common/arrow/src/arrow/ffi/schema.rs | 1 + src/common/base/src/mem_allocator/mmap.rs | 1 + src/common/exception/Cargo.toml | 7 +- src/common/exception/src/elf/dwarf.rs | 205 +++ .../src/elf/dwarf_inline_functions.rs | 374 +++++ .../exception/src/elf/dwarf_subprogram.rs | 103 ++ src/common/exception/src/elf/dwarf_unit.rs | 302 ++++ .../exception/src/elf/library_loader.rs | 227 +++ .../exception/src/elf/library_manager.rs | 197 +++ .../exception/src/elf/library_symbol.rs | 46 + src/common/exception/src/elf/mod.rs | 29 + src/common/exception/src/exception.rs | 105 +- .../exception/src/exception_backtrace.rs | 252 ++- src/common/exception/src/exception_into.rs | 60 +- src/common/exception/src/lib.rs | 5 + .../exception/tests/it/exception_flight.rs | 7 +- .../pipeline/core/src/processors/profile.rs | 5 +- .../09_fuse_engine/09_0026_merge_into.test | 1433 ++++++++++++++++- .../1_stateful/02_query/02_0000_kill_query.py | 2 +- 21 files changed, 3225 insertions(+), 182 deletions(-) create mode 100644 src/common/exception/src/elf/dwarf.rs create mode 100644 src/common/exception/src/elf/dwarf_inline_functions.rs create mode 100644 src/common/exception/src/elf/dwarf_subprogram.rs create mode 100644 src/common/exception/src/elf/dwarf_unit.rs create mode 100644 src/common/exception/src/elf/library_loader.rs create mode 100644 src/common/exception/src/elf/library_manager.rs create mode 100644 src/common/exception/src/elf/library_symbol.rs create mode 100644 src/common/exception/src/elf/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 29704372018b..7b737905655a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3307,12 +3307,17 @@ dependencies = [ "databend-common-ast", "geos", "geozero 0.14.0", + "gimli 0.31.1", "http 1.1.0", + "libc", + "object", + "once_cell", "opendal", "parquet", "paste", "prost", "reqwest", + "rustc-demangle", "serde", "serde_json", "sqlx", @@ -6476,6 +6481,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -7010,6 +7021,17 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +dependencies = [ + "fallible-iterator", + "indexmap 2.6.0", + "stable_deref_trait", +] + [[package]] name = "gix" version = "0.63.0" @@ -7992,6 +8014,9 @@ name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" @@ -10410,14 +10435,16 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.3" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "crc32fast", - "hashbrown 0.14.5", + "flate2", + "hashbrown 0.15.0", "indexmap 2.6.0", "memchr", + "ruzstd", ] [[package]] @@ -12947,6 +12974,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ruzstd" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99c3938e133aac070997ddc684d4b393777d293ba170f2988c8fd5ea2ad4ce21" +dependencies = [ + "twox-hash", +] + [[package]] name = "ryu" version = "1.0.18" diff --git a/Cargo.toml b/Cargo.toml index 61c6b9e3b9c3..7d68a2405e59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -299,6 +299,7 @@ geo-types = "0.7.13" geohash = "0.13.0" geos = { version = "9.0.0", features = ["static", "geo", "geo-types"] } geozero = { version = "0.14.0", features = ["default", "with-wkb", "with-geos", "with-geojson"] } +gimli = "0.31.0" globiter = "0.1" goldenfile = "1.4" h3o = "0.4.0" @@ -354,6 +355,7 @@ num-bigint = "0.4.6" num-derive = "0.3.3" num-traits = "0.2.19" num_cpus = "1.13.1" +object = "0.36.5" object_store_opendal = "0.48.1" once_cell = "1.15.0" openai_api_rust = "0.1" @@ -514,6 +516,7 @@ nom-rule = "0.4" pratt = "0.4.0" pretty = "0.11.3" rspack-codespan-reporting = "0.11" +rustc-demangle = "0.1" strsim = "0.10" strum_macros = "0.24" vergen = { version = "8.3.1", default-features = false, features = ["build", "cargo", "git", "gix", "rustc"] } @@ -597,6 +600,7 @@ gimli = { opt-level = 3 } miniz_oxide = { opt-level = 3 } object = { opt-level = 3 } rustc-demangle = { opt-level = 3 } +databend-common-exception = { opt-level = 3 } [profile.test] opt-level = 0 diff --git a/src/common/arrow/src/arrow/ffi/schema.rs b/src/common/arrow/src/arrow/ffi/schema.rs index b97923c2e414..7c47a49f205c 100644 --- a/src/common/arrow/src/arrow/ffi/schema.rs +++ b/src/common/arrow/src/arrow/ffi/schema.rs @@ -528,6 +528,7 @@ unsafe fn read_bytes(ptr: *const u8, len: usize) -> &'static str { } unsafe fn metadata_from_bytes(data: *const ::std::os::raw::c_char) -> (Metadata, Extension) { + #[allow(clippy::unnecessary_cast)] let mut data = data as *const u8; // u8 = i8 if data.is_null() { return (Metadata::default(), None); diff --git a/src/common/base/src/mem_allocator/mmap.rs b/src/common/base/src/mem_allocator/mmap.rs index 759c3679fb8b..8224c832ea84 100644 --- a/src/common/base/src/mem_allocator/mmap.rs +++ b/src/common/base/src/mem_allocator/mmap.rs @@ -286,6 +286,7 @@ pub mod linux { } // fallback to (5.13.0) let fallback_version = 5u32 << 16 | 13u32 << 8; + #[allow(clippy::unnecessary_cast)] let slice = unsafe { &*(&uname.release[..length] as *const _ as *const [u8]) }; let result = match std::str::from_utf8(slice) { Ok(ver) => match semver::Version::parse(ver) { diff --git a/src/common/exception/Cargo.toml b/src/common/exception/Cargo.toml index 48ff7f20f330..74eb2c38e5ae 100644 --- a/src/common/exception/Cargo.toml +++ b/src/common/exception/Cargo.toml @@ -17,16 +17,21 @@ databend-common-ast = { workspace = true } anyhow = { workspace = true } arrow-flight = { workspace = true } arrow-schema = { workspace = true } -backtrace = { workspace = true } +backtrace = { workspace = true, features = ["std", "serialize-serde"] } bincode = { workspace = true } geos = { workspace = true } geozero = { workspace = true } +gimli = { workspace = true } http = { workspace = true } +libc = { workspace = true } +object = { workspace = true } +once_cell = { workspace = true } opendal = { workspace = true } parquet = { workspace = true } paste = { workspace = true } prost = { workspace = true } reqwest = { workspace = true } +rustc-demangle = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } diff --git a/src/common/exception/src/elf/dwarf.rs b/src/common/exception/src/elf/dwarf.rs new file mode 100644 index 000000000000..1dafec886c9c --- /dev/null +++ b/src/common/exception/src/elf/dwarf.rs @@ -0,0 +1,205 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use gimli::DebugAbbrev; +use gimli::DebugAddr; +use gimli::DebugAranges; +use gimli::DebugInfo; +use gimli::DebugInfoOffset; +use gimli::DebugLine; +use gimli::DebugLineStr; +use gimli::DebugRanges; +use gimli::DebugRngLists; +use gimli::DebugStr; +use gimli::DebugStrOffsets; +use gimli::EndianSlice; +use gimli::NativeEndian; +use gimli::RangeLists; +use gimli::Reader; +use gimli::UnitHeader; +use gimli::UnitType; +use object::CompressionFormat; +use object::Object; +use object::ObjectSection; + +use crate::elf::dwarf_unit::Unit; +use crate::elf::dwarf_unit::UnitAttrs; +use crate::elf::ElfFile; + +#[derive(Debug)] +pub struct CallLocation { + pub symbol: Option, + pub file: Option, + pub line: Option, + pub column: Option, + pub is_inlined: bool, +} + +pub struct Dwarf { + #[allow(unused)] + elf: Arc, + debug_str: DebugStr>, + debug_info: DebugInfo>, + debug_line: DebugLine>, + debug_line_str: DebugLineStr>, + debug_str_offsets: DebugStrOffsets>, + debug_aranges: DebugAranges>, + debug_abbrev: DebugAbbrev>, + debug_addr: DebugAddr>, + debug_range_list: RangeLists>, +} + +static EMPTY_BYTES: &[u8] = &[]; + +impl Dwarf { + pub fn create(elf: Arc) -> Option { + fn get_debug_section(elf: &ElfFile, name: &str) -> EndianSlice<'static, NativeEndian> { + let Some(section) = elf.section_by_name(name) else { + return EndianSlice::new(EMPTY_BYTES, NativeEndian); + }; + + // Unsupported compress debug info + let Ok(compressed) = section.compressed_file_range() else { + return EndianSlice::new(EMPTY_BYTES, NativeEndian); + }; + + #[allow(clippy::missing_transmute_annotations)] + unsafe { + match compressed.format != CompressionFormat::None { + true => EndianSlice::new(EMPTY_BYTES, NativeEndian), + false => match section.data() { + Err(_) => EndianSlice::new(EMPTY_BYTES, NativeEndian), + Ok(data) => EndianSlice::new(std::mem::transmute(data), NativeEndian), + }, + } + } + } + + for name in [".debug_info", ".debug_abbrev", ".debug_line"] { + if get_debug_section(&elf, name).is_empty() { + return None; + } + } + + Some(Dwarf { + debug_str: DebugStr::from(get_debug_section(&elf, ".debug_str")), + debug_info: DebugInfo::from(get_debug_section(&elf, ".debug_info")), + debug_line: DebugLine::from(get_debug_section(&elf, ".debug_line")), + debug_line_str: DebugLineStr::from(get_debug_section(&elf, ".debug_line_str")), + debug_str_offsets: DebugStrOffsets::from(get_debug_section(&elf, ".debug_str_offsets")), + debug_aranges: DebugAranges::from(get_debug_section(&elf, ".debug_aranges")), + debug_abbrev: DebugAbbrev::from(get_debug_section(&elf, ".debug_abbrev")), + debug_range_list: RangeLists::new( + DebugRanges::from(get_debug_section(&elf, ".debug_ranges")), + DebugRngLists::from(get_debug_section(&elf, ".debug_rnglists")), + ), + debug_addr: DebugAddr::from(get_debug_section(&elf, ".debug_addr")), + elf, + }) + } + + fn find_debug_info_offset(&self, probe: u64) -> Option> { + let mut heads = self.debug_aranges.headers(); + while let Some(head) = heads.next().ok()? { + let mut entries = head.entries(); + while let Some(entry) = entries.next().ok()? { + if probe >= entry.address() && probe <= entry.address() + entry.length() { + return Some(head.debug_info_offset()); + } + } + } + + None + } + + fn get_unit( + &self, + head: UnitHeader>, + ) -> gimli::Result>>> { + let abbrev_offset = head.debug_abbrev_offset(); + let Ok(abbreviations) = self.debug_abbrev.abbreviations(abbrev_offset) else { + return Ok(None); + }; + + let mut cursor = head.entries(&abbreviations); + let (_idx, root) = cursor.next_dfs()?.unwrap(); + + let mut attrs = root.attrs(); + let mut unit_attrs = UnitAttrs::create(); + + while let Some(attr) = attrs.next()? { + unit_attrs.set_attr(&self.debug_str, attr); + } + + Ok(Some(Unit { + head, + abbreviations, + attrs: unit_attrs, + debug_str: self.debug_str, + debug_info: self.debug_info, + debug_abbrev: self.debug_abbrev, + debug_line: self.debug_line, + debug_line_str: self.debug_line_str, + debug_str_offsets: self.debug_str_offsets, + debug_addr: self.debug_addr, + range_list: self.debug_range_list, + })) + } + + fn fast_find_frames(&self, probe: u64) -> gimli::Result>> { + if let Some(debug_info_offset) = self.find_debug_info_offset(probe) { + let head = self.debug_info.header_from_offset(debug_info_offset)?; + + let type_ = head.type_(); + if matches!(type_, UnitType::Compilation | UnitType::Skeleton(_)) { + if let Some(unit) = self.get_unit(head)? { + return Ok(Some(unit.find_frames(probe)?)); + } + } + } + + Ok(None) + } + + fn slow_find_frames(&self, probe: u64) -> gimli::Result> { + let mut units = self.debug_info.units(); + while let Some(head) = units.next()? { + if matches!(head.type_(), UnitType::Compilation | UnitType::Skeleton(_)) { + if let Some(unit) = self.get_unit(head)? { + if unit.match_pc(probe) { + return unit.find_frames(probe); + } + } + } + } + + Ok(vec![]) + } + + pub fn find_frames(&self, probe: u64) -> gimli::Result> { + match self.fast_find_frames(probe)? { + Some(location) => Ok(location), + None => self.slow_find_frames(probe), + } + } +} + +// #[cfg(target_os = "linux")] +#[derive(Copy, Clone, Debug)] +pub enum HighPc { + Addr(u64), + Offset(u64), +} diff --git a/src/common/exception/src/elf/dwarf_inline_functions.rs b/src/common/exception/src/elf/dwarf_inline_functions.rs new file mode 100644 index 000000000000..2a95b0172365 --- /dev/null +++ b/src/common/exception/src/elf/dwarf_inline_functions.rs @@ -0,0 +1,374 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use gimli::Attribute; +use gimli::AttributeValue; +use gimli::EntriesRaw; +use gimli::RangeListsOffset; +use gimli::Reader; +use gimli::Result; +use gimli::UnitOffset; + +use crate::elf::dwarf::CallLocation; +use crate::elf::dwarf::HighPc; +use crate::elf::dwarf_unit::Unit; +use crate::elf::dwarf_unit::UnitAttrs; + +pub struct SubroutineAttrs { + high_pc: Option, + low_pc: Option, + ranges_offset: Option>, + + name: Option, + line: Option, + file: Option, + column: Option, +} + +impl SubroutineAttrs { + pub fn create() -> SubroutineAttrs { + SubroutineAttrs { + line: None, + file: None, + name: None, + column: None, + low_pc: None, + high_pc: None, + ranges_offset: None, + } + } + + pub fn set_attr(&mut self, attr: Attribute, unit: &Unit) { + match attr.name() { + gimli::DW_AT_low_pc => match attr.value() { + AttributeValue::DebugAddrIndex(idx) => self.low_pc = Some(unit.get_address(idx)), + AttributeValue::Addr(value) => self.low_pc = Some(value), + _ => {} + }, + gimli::DW_AT_high_pc => match attr.value() { + AttributeValue::Addr(val) => self.high_pc = Some(HighPc::Addr(val)), + AttributeValue::Udata(val) => self.high_pc = Some(HighPc::Offset(val)), + AttributeValue::DebugAddrIndex(idx) => { + self.high_pc = Some(HighPc::Addr(unit.get_address(idx))) + } + _ => {} + }, + gimli::DW_AT_ranges => { + if let AttributeValue::RangeListsRef(v) = attr.value() { + self.ranges_offset = Some(RangeListsOffset(v.0)); + } + } + gimli::DW_AT_linkage_name | gimli::DW_AT_MIPS_linkage_name => { + if let Some(val) = unit.attr_str(attr.value()) { + self.name = Some(val); + } + } + gimli::DW_AT_name => { + if self.name.is_none() { + self.name = unit.attr_str(attr.value()); + } + } + gimli::DW_AT_abstract_origin | gimli::DW_AT_specification => { + if self.name.is_none() { + if let Ok(Some(v)) = unit.name_attr(attr.value(), 16) { + self.name = Some(v); + } + } + } + gimli::DW_AT_call_file => { + if let AttributeValue::FileIndex(idx) = attr.value() { + if let Ok(filename) = unit.find_file(idx) { + self.file = filename; + } + } + } + gimli::DW_AT_call_line => { + self.line = attr.udata_value().map(|x| x as u32); + } + gimli::DW_AT_call_column => { + self.column = attr.udata_value().map(|x| x as u32); + } + _ => {} + } + } + + pub fn match_pc(&self, probe: u64) -> bool { + match (self.low_pc, self.high_pc) { + (Some(low), Some(high)) => { + probe >= low + && match high { + HighPc::Addr(high) => probe < high, + HighPc::Offset(size) => probe < low + size, + } + } + _ => false, + } + } +} + +impl Unit { + pub(crate) fn attr_str(&self, value: AttributeValue) -> Option { + match value { + AttributeValue::String(string) => Some(string), + AttributeValue::DebugStrRef(offset) => self.debug_str.get_str(offset).ok(), + AttributeValue::DebugLineStrRef(offset) => self.debug_line_str.get_str(offset).ok(), + AttributeValue::DebugStrOffsetsIndex(index) => { + let offset = self + .debug_str_offsets + .get_str_offset(self.head.format(), self.attrs.str_offsets_base, index) + .ok()?; + self.debug_str.get_str(offset).ok() + } + _ => None, + } + } + + fn name_entry(&self, offset: UnitOffset, recursion: usize) -> Result> { + let mut entries = self.head.entries_raw(&self.abbreviations, Some(offset))?; + let abbrev = if let Some(abbrev) = entries.read_abbreviation()? { + abbrev + } else { + return Err(gimli::Error::NoEntryAtGivenOffset); + }; + + let mut name = None; + let mut next = None; + for spec in abbrev.attributes() { + let attr = entries.read_attribute(*spec)?; + match attr.name() { + gimli::DW_AT_linkage_name | gimli::DW_AT_MIPS_linkage_name => { + if let Some(val) = self.attr_str(attr.value()) { + return Ok(Some(val)); + } + } + gimli::DW_AT_name => name = self.attr_str(attr.value()), + gimli::DW_AT_abstract_origin | gimli::DW_AT_specification => { + next = Some(attr.value()) + } + _ => {} + }; + } + + if name.is_some() { + return Ok(name); + } + + if let Some(next) = next { + return self.name_attr(next, recursion - 1); + } + + Ok(None) + } + + pub(crate) fn name_attr(&self, v: AttributeValue, recursion: usize) -> Result> { + if recursion == 0 { + return Ok(None); + } + + match v { + AttributeValue::UnitRef(offset) => self.name_entry(offset, recursion), + AttributeValue::DebugInfoRef(dr) => { + let mut head = None; + let mut units = self.debug_info.units(); + + while let Some(unit_head) = units + .next() + .map_err(|_| gimli::Error::NoEntryAtGivenOffset)? + { + if unit_head.offset().as_debug_info_offset().unwrap() > dr { + break; + } + + head = Some(unit_head); + } + + if let Some(head) = head { + let unit_offset = dr + .to_unit_offset(&head) + .ok_or(gimli::Error::NoEntryAtGivenOffset)?; + + let abbrev_offset = head.debug_abbrev_offset(); + let Ok(abbreviations) = self.debug_abbrev.abbreviations(abbrev_offset) else { + return Ok(None); + }; + + let mut cursor = head.entries(&abbreviations); + let (_idx, root) = cursor.next_dfs()?.unwrap(); + + let mut attrs = root.attrs(); + let mut unit_attrs = UnitAttrs::create(); + + while let Some(attr) = attrs.next()? { + unit_attrs.set_attr(&self.debug_str, attr); + } + + let unit = Unit { + head, + abbreviations, + attrs: unit_attrs, + debug_str: self.debug_str.clone(), + debug_info: self.debug_info.clone(), + debug_abbrev: self.debug_abbrev.clone(), + debug_line: self.debug_line.clone(), + debug_line_str: self.debug_line_str.clone(), + debug_str_offsets: self.debug_str_offsets.clone(), + debug_addr: self.debug_addr.clone(), + range_list: self.range_list.clone(), + }; + + return unit.name_entry(unit_offset, recursion); + } + + Ok(None) + } + _ => Ok(None), + } + } + + fn inlined_functions( + &self, + mut entries: EntriesRaw, + probe: u64, + depth: isize, + inlined_functions: &mut Vec, + ) -> Result<()> { + loop { + let next_depth = entries.next_depth(); + + if next_depth <= depth { + return Ok(()); + } + + if let Some(abbrev) = entries.read_abbreviation()? { + match abbrev.tag() { + gimli::DW_TAG_subprogram => { + entries.skip_attributes(abbrev.attributes())?; + while entries.next_depth() > next_depth { + if let Some(abbrev) = entries.read_abbreviation()? { + entries.skip_attributes(abbrev.attributes())?; + } + } + } + gimli::DW_TAG_inlined_subroutine => { + let mut attrs = SubroutineAttrs::create(); + for spec in abbrev.attributes() { + let attr = entries.read_attribute(*spec)?; + attrs.set_attr(attr, self); + } + + let match_range = match attrs.ranges_offset { + None => false, + Some(range_offset) => self.match_range(probe, range_offset), + }; + + if !match_range && !attrs.match_pc(probe) { + continue; + } + + let name = match attrs.name { + None => None, + Some(name) => match name.to_string_lossy() { + Err(_) => None, + Ok(name) => { + Some(format!("{:#}", rustc_demangle::demangle(name.as_ref()))) + } + }, + }; + + inlined_functions.push(CallLocation { + symbol: name, + file: attrs.file, + line: attrs.line, + column: attrs.column, + is_inlined: true, + }); + + self.inlined_functions(entries, probe, next_depth, inlined_functions)?; + + return Ok(()); + } + _ => { + entries.skip_attributes(abbrev.attributes())?; + } + } + } + } + } + + pub fn find_function( + &self, + offset: UnitOffset, + probe: u64, + functions: &mut Vec, + ) -> Result<()> { + let mut entries = self.head.entries_raw(&self.abbreviations, Some(offset))?; + let depth = entries.next_depth(); + let abbrev = entries.read_abbreviation()?.unwrap(); + debug_assert_eq!(abbrev.tag(), gimli::DW_TAG_subprogram); + + let mut name = None; + for spec in abbrev.attributes() { + let attr = entries.read_attribute(*spec)?; + match attr.name() { + gimli::DW_AT_linkage_name | gimli::DW_AT_MIPS_linkage_name => { + if let Some(val) = self.attr_str(attr.value()) { + name = Some(val); + } + } + gimli::DW_AT_name => { + if name.is_none() { + name = self.attr_str(attr.value()); + } + } + gimli::DW_AT_abstract_origin | gimli::DW_AT_specification => { + if name.is_none() { + name = self.name_attr(attr.value(), 16)?; + } + } + _ => {} + }; + } + + self.inlined_functions(entries, probe, depth, functions)?; + + let symbol = match name { + None => None, + Some(name) => match name.to_string_lossy() { + Err(_) => None, + Ok(name) => Some(format!("{:#}", rustc_demangle::demangle(name.as_ref()))), + }, + }; + + let (mut file, mut line, mut column) = self.find_location(probe)?; + + functions.reverse(); + + #[allow(clippy::needless_range_loop)] + for index in 0..functions.len() { + std::mem::swap(&mut functions[index].file, &mut file); + std::mem::swap(&mut functions[index].line, &mut line); + std::mem::swap(&mut functions[index].column, &mut column); + } + + functions.push(CallLocation { + symbol, + file, + line, + column, + is_inlined: false, + }); + + Ok(()) + } +} diff --git a/src/common/exception/src/elf/dwarf_subprogram.rs b/src/common/exception/src/elf/dwarf_subprogram.rs new file mode 100644 index 000000000000..e83bc8b9d7b5 --- /dev/null +++ b/src/common/exception/src/elf/dwarf_subprogram.rs @@ -0,0 +1,103 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use gimli::Attribute; +use gimli::AttributeValue; +use gimli::RangeListsOffset; +use gimli::Reader; +use gimli::Result; +use gimli::UnitOffset; + +use crate::elf::dwarf::HighPc; +use crate::elf::dwarf_unit::Unit; + +pub struct SubprogramAttrs { + high_pc: Option, + low_pc: Option, + ranges_offset: Option>, +} + +impl SubprogramAttrs { + pub fn create() -> SubprogramAttrs { + SubprogramAttrs { + high_pc: None, + low_pc: None, + ranges_offset: None, + } + } + + pub fn set_attr(&mut self, attr: Attribute) { + match (attr.name(), attr.value()) { + (gimli::DW_AT_high_pc, AttributeValue::Addr(addr)) => { + self.high_pc = Some(HighPc::Addr(addr)); + } + (gimli::DW_AT_high_pc, AttributeValue::Udata(offset)) => { + self.high_pc = Some(HighPc::Offset(offset)); + } + (gimli::DW_AT_low_pc, AttributeValue::Addr(v)) => { + self.low_pc = Some(v); + } + (gimli::DW_AT_ranges, AttributeValue::RangeListsRef(v)) => { + self.ranges_offset = Some(RangeListsOffset(v.0)); + } + _ => {} + } + } + + pub fn match_pc(&self, probe: u64) -> bool { + match (self.low_pc, self.high_pc) { + (Some(low), Some(high)) => { + probe >= low + && match high { + HighPc::Addr(high) => probe < high, + HighPc::Offset(size) => probe < low + size, + } + } + _ => false, + } + } +} + +impl Unit { + pub fn find_subprogram(&self, probe: u64) -> Result>> { + let mut entries = self.head.entries_raw(&self.abbreviations, None)?; + + while !entries.is_empty() { + let dw_die_offset = entries.next_offset(); + if let Some(abbrev) = entries.read_abbreviation()? { + if abbrev.tag() == gimli::DW_TAG_subprogram { + let mut attrs = SubprogramAttrs::create(); + + for spec in abbrev.attributes() { + let attr = entries.read_attribute(*spec)?; + attrs.set_attr(attr); + } + + let range_match = match attrs.ranges_offset { + None => false, + Some(range_offset) => self.match_range(probe, range_offset), + }; + + if range_match || attrs.match_pc(probe) { + return Ok(Some(dw_die_offset)); + } + } else { + entries.skip_attributes(abbrev.attributes())?; + } + } + } + + Ok(None) + } +} diff --git a/src/common/exception/src/elf/dwarf_unit.rs b/src/common/exception/src/elf/dwarf_unit.rs new file mode 100644 index 000000000000..ac4fae8d05cd --- /dev/null +++ b/src/common/exception/src/elf/dwarf_unit.rs @@ -0,0 +1,302 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::num::NonZeroU64; +use std::path::PathBuf; + +use gimli::Abbreviations; +use gimli::Attribute; +use gimli::AttributeValue; +use gimli::DebugAbbrev; +use gimli::DebugAddr; +use gimli::DebugAddrBase; +use gimli::DebugAddrIndex; +use gimli::DebugInfo; +use gimli::DebugLine; +use gimli::DebugLineOffset; +use gimli::DebugLineStr; +use gimli::DebugLocListsBase; +use gimli::DebugRngListsBase; +use gimli::DebugStr; +use gimli::DebugStrOffsets; +use gimli::DebugStrOffsetsBase; +use gimli::RangeLists; +use gimli::RangeListsOffset; +use gimli::Reader; +use gimli::ReaderOffset; +use gimli::UnitHeader; + +use crate::elf::dwarf::CallLocation; +use crate::elf::dwarf::HighPc; + +// Unit first die attrs +pub struct UnitAttrs { + pub high_pc: Option, + pub low_pc: Option, + pub base_addr: Option, + pub name: Option, + pub comp_dir: Option, + + pub ranges_offset: Option>, + + pub addr_base: DebugAddrBase, + loclists_base: DebugLocListsBase, + rnglists_base: DebugRngListsBase, + pub str_offsets_base: DebugStrOffsetsBase, + debug_line_offset: Option>, +} + +impl UnitAttrs { + pub fn create() -> UnitAttrs { + UnitAttrs { + high_pc: None, + low_pc: None, + base_addr: None, + name: None, + comp_dir: None, + ranges_offset: None, + + debug_line_offset: None, + addr_base: DebugAddrBase(R::Offset::from_u8(0)), + loclists_base: DebugLocListsBase(R::Offset::from_u8(0)), + rnglists_base: DebugRngListsBase(R::Offset::from_u8(0)), + str_offsets_base: DebugStrOffsetsBase(R::Offset::from_u8(0)), + } + } + + pub fn set_attr(&mut self, debug_str: &DebugStr, attr: Attribute) { + match (attr.name(), attr.value()) { + (gimli::DW_AT_high_pc, v) => { + self.high_pc = match v { + AttributeValue::Addr(v) => Some(HighPc::Addr(v)), + AttributeValue::Udata(v) => Some(HighPc::Offset(v)), + _ => unreachable!(), + }; + } + (gimli::DW_AT_low_pc, AttributeValue::Addr(v)) => { + self.low_pc = Some(v); + self.base_addr = Some(v); + } + (gimli::DW_AT_name, v) => { + self.name = v.string_value(debug_str); + } + (gimli::DW_AT_entry_pc, AttributeValue::Addr(v)) => { + self.base_addr = Some(v); + } + (gimli::DW_AT_comp_dir, v) => { + self.comp_dir = v.string_value(debug_str); + } + (gimli::DW_AT_ranges, AttributeValue::RangeListsRef(v)) => { + self.ranges_offset = Some(RangeListsOffset(v.0)); + } + (gimli::DW_AT_stmt_list, AttributeValue::DebugLineRef(v)) => { + self.debug_line_offset = Some(v); + } + (gimli::DW_AT_addr_base, AttributeValue::DebugAddrBase(base)) => { + self.addr_base = base; + } + (gimli::DW_AT_GNU_addr_base, AttributeValue::DebugAddrBase(base)) => { + self.addr_base = base; + } + (gimli::DW_AT_loclists_base, AttributeValue::DebugLocListsBase(base)) => { + self.loclists_base = base; + } + (gimli::DW_AT_rnglists_base, AttributeValue::DebugRngListsBase(base)) => { + self.rnglists_base = base; + } + (gimli::DW_AT_GNU_ranges_base, AttributeValue::DebugRngListsBase(base)) => { + self.rnglists_base = base; + } + (gimli::DW_AT_str_offsets_base, AttributeValue::DebugStrOffsetsBase(base)) => { + self.str_offsets_base = base; + } + _ => {} + } + } +} + +pub struct Unit { + pub(crate) head: UnitHeader, + + pub(crate) debug_str: DebugStr, + pub(crate) debug_info: DebugInfo, + pub(crate) debug_abbrev: DebugAbbrev, + pub(crate) debug_line: DebugLine, + pub(crate) debug_line_str: DebugLineStr, + pub(crate) debug_addr: DebugAddr, + pub(crate) range_list: RangeLists, + pub(crate) debug_str_offsets: DebugStrOffsets, + + pub(crate) abbreviations: Abbreviations, + pub(crate) attrs: UnitAttrs, +} + +impl Unit { + pub fn match_range(&self, probe: u64, offset: RangeListsOffset) -> bool { + if let Ok(mut ranges) = self.range_list.ranges( + offset, + self.head.encoding(), + self.attrs.base_addr.unwrap_or(0), + &self.debug_addr, + self.attrs.addr_base, + ) { + while let Ok(Some(range)) = ranges.next() { + if probe >= range.begin && probe < range.end { + return true; + } + } + } + + false + } + + pub fn find_file(&self, file_idx: u64) -> gimli::Result> { + if let Some(offset) = &self.attrs.debug_line_offset { + let program = self.debug_line.program( + *offset, + self.head.address_size(), + self.attrs.comp_dir.clone(), + self.attrs.name.clone(), + )?; + + let header = program.header(); + + if let Some(file) = header.file(file_idx) { + let mut path_buf = PathBuf::new(); + + if let Some(dir) = &self.attrs.comp_dir { + path_buf.push(dir.to_string_lossy().unwrap().into_owned()); + } + + if file.directory_index() != 0 { + if let Some(v) = file.directory(header) { + if let Some(v) = v.string_value(&self.debug_str) { + path_buf.push(v.to_string_lossy().unwrap().into_owned()); + } + } + } + + if let Some(v) = file.path_name().string_value(&self.debug_str) { + path_buf.push(v.to_string_lossy().unwrap().into_owned()); + } + + return Ok(Some(format!("{}", path_buf.display()))); + } + } + + Ok(None) + } + + pub fn find_location( + &self, + probe: u64, + ) -> gimli::Result<(Option, Option, Option)> { + if let Some(offset) = &self.attrs.debug_line_offset { + let program = self.debug_line.program( + *offset, + self.head.address_size(), + self.attrs.comp_dir.clone(), + self.attrs.name.clone(), + )?; + + let mut is_candidate: bool = false; + let mut file_idx = 0; + let mut line = 0; + let mut column = 0; + + let mut rows = program.rows(); + while let Some((_, row)) = rows.next_row()? { + if !is_candidate && !row.end_sequence() && row.address() <= probe { + is_candidate = true; + } + + if is_candidate { + if row.address() > probe { + let mut path_buf = PathBuf::new(); + + if let Some(dir) = &self.attrs.comp_dir { + path_buf.push(dir.to_string_lossy().unwrap().into_owned()); + } + + let header = rows.header(); + if let Some(file) = header.file(file_idx) { + if file.directory_index() != 0 { + if let Some(v) = file.directory(header) { + if let Some(v) = v.string_value(&self.debug_str) { + path_buf.push(v.to_string_lossy().unwrap().into_owned()); + } + } + } + + if let Some(v) = file.path_name().string_value(&self.debug_str) { + path_buf.push(v.to_string_lossy().unwrap().into_owned()); + } + } + + return Ok(( + Some(format!("{}", path_buf.display())), + Some(line), + Some(column), + )); + } + + file_idx = row.file_index(); + line = row.line().map(NonZeroU64::get).unwrap_or(0) as u32; + column = match row.column() { + gimli::ColumnType::LeftEdge => 0, + gimli::ColumnType::Column(x) => x.get() as u32, + }; + } + + if row.end_sequence() { + is_candidate = false; + } + } + } + + Ok((None, None, None)) + } + + pub fn match_pc(&self, probe: u64) -> bool { + if let Some(range_offset) = self.attrs.ranges_offset { + return self.match_range(probe, range_offset); + } + + match (self.attrs.low_pc, self.attrs.high_pc) { + (Some(low), Some(high)) => { + probe >= low + && match high { + HighPc::Addr(high) => probe < high, + HighPc::Offset(size) => probe < low + size, + } + } + _ => false, + } + } + + pub fn find_frames(&self, probe: u64) -> gimli::Result> { + let mut functions_frames = vec![]; + if let Some(offset) = self.find_subprogram(probe)? { + let _ = self.find_function(offset, probe, &mut functions_frames); + } + + Ok(functions_frames) + } + + pub fn get_address(&self, idx: DebugAddrIndex) -> u64 { + self.debug_addr + .get_address(self.head.encoding().address_size, self.attrs.addr_base, idx) + .unwrap() + } +} diff --git a/src/common/exception/src/elf/library_loader.rs b/src/common/exception/src/elf/library_loader.rs new file mode 100644 index 000000000000..e356cb71791a --- /dev/null +++ b/src/common/exception/src/elf/library_loader.rs @@ -0,0 +1,227 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ffi::CStr; +use std::ffi::OsStr; +use std::ffi::OsString; +use std::fmt::Write; +use std::os::fd::AsRawFd; +use std::os::unix::ffi::OsStrExt; +use std::path::PathBuf; +use std::ptr; +use std::sync::Arc; + +use libc::c_int; +use libc::c_void; +use libc::dl_iterate_phdr; +use libc::dl_phdr_info; +use libc::size_t; +use object::Object; +use object::ObjectSymbol; +use object::ObjectSymbolTable; + +use crate::elf::library_manager::Library; +use crate::elf::library_symbol::Symbol; +use crate::elf::ElfFile; + +#[derive(Debug)] +pub struct LibraryLoader { + symbols: Vec, + libraries: Vec, +} + +impl LibraryLoader { + pub fn load() -> LibraryLoader { + unsafe { + let mut loader = LibraryLoader { + symbols: vec![], + libraries: vec![], + }; + + dl_iterate_phdr(Some(callback), std::ptr::addr_of_mut!(loader).cast()); + + loader + } + } + + pub unsafe fn load_symbols(&mut self, library: &mut Library) { + let library_data = library.data(); + + let Ok(elf) = ElfFile::parse(library_data) else { + return; + }; + + let symbol_table = match elf.symbol_table() { + Some(symbol_table) => symbol_table, + None => match elf.dynamic_symbol_table() { + Some(dynamic_symbol_table) => dynamic_symbol_table, + None => { + return; + } + }, + }; + + for symbol in symbol_table.symbols() { + let Ok(sym_name) = symbol.name() else { + continue; + }; + + if sym_name.is_empty() || symbol.size() == 0 { + continue; + } + + self.symbols.push(Symbol { + #[allow(clippy::missing_transmute_annotations)] + name: unsafe { std::mem::transmute(sym_name) }, + address_begin: library.address_begin as u64 + symbol.address(), + address_end: library.address_begin as u64 + symbol.address() + symbol.size(), + }); + } + + library.elf = Some(Arc::new(elf)); + } + + unsafe fn mmap_library(&mut self, library_path: PathBuf) -> std::io::Result { + let name = format!("{}", library_path.display()); + let file = std::fs::File::open(library_path)?; + let file_len = file.metadata()?.len(); + let ptr = libc::mmap( + ptr::null_mut(), + file_len as libc::size_t, + libc::PROT_READ, + libc::MAP_PRIVATE, + file.as_raw_fd(), + 0, + ); + + match ptr == libc::MAP_FAILED { + true => Err(std::io::Error::other("Cannot mmap")), + false => Ok(Library::create(name, ptr as *const u8, file_len as usize)), + } + } + + unsafe fn load_library(&mut self, info: &dl_phdr_info) -> std::io::Result { + let library_name = match info.dlpi_name.is_null() || *info.dlpi_name == 0 { + true => match self.libraries.is_empty() { + true => OsString::from("/proc/self/exe"), + false => OsString::new(), + }, + false => { + let bytes = CStr::from_ptr(info.dlpi_name).to_bytes(); + OsStr::from_bytes(bytes).to_owned() + } + }; + + if library_name.is_empty() { + return Err(std::io::Error::other("empty library name")); + } + + let binary_path = std::fs::canonicalize(library_name)?.to_path_buf(); + let mut binary_library = self.mmap_library(binary_path.clone())?; + + let Some(binary_build_id) = binary_library.build_id() else { + let binary_data = binary_library.data(); + binary_library.address_begin = info.dlpi_addr as usize; + binary_library.address_end = info.dlpi_addr as usize + binary_data.len(); + return Ok(binary_library); + }; + + // Symbol binary in the same dir ./databend-query.debug + let mut binary_debug_path = binary_path.clone(); + binary_debug_path.set_extension("debug"); + + if std::fs::exists(&binary_debug_path)? { + let mut library = self.mmap_library(binary_debug_path)?; + if matches!(library.build_id(), Some(v) if v == binary_build_id) { + let library_data = library.data(); + library.address_begin = info.dlpi_addr as usize; + library.address_end = info.dlpi_addr as usize + library_data.len(); + return Ok(library); + } + } + + // Symbol binary in the system lib/debug/relative_path dir + // /usr/lib/debug/home/ubuntu/databend/databend-query.debug + let Ok(relative_path) = binary_path.strip_prefix("/") else { + return Err(std::io::Error::other("Cannot strip_prefix for path")); + }; + + let lib_debug = std::path::Path::new("/usr/lib/debug"); + let mut system_named_debug_path = lib_debug.join(relative_path); + system_named_debug_path.set_extension("debug"); + + if std::fs::exists(&system_named_debug_path)? { + let mut library = self.mmap_library(system_named_debug_path)?; + if matches!(library.build_id(), Some(v) if v == binary_build_id) { + let library_data = library.data(); + library.address_begin = info.dlpi_addr as usize; + library.address_end = info.dlpi_addr as usize + library_data.len(); + return Ok(library); + } + } + + if binary_build_id.len() >= 2 { + fn encode_hex(bytes: &[u8]) -> String { + let mut encoded_hex = String::with_capacity(bytes.len() * 2); + for &b in bytes { + write!(&mut encoded_hex, "{:02x}", b).unwrap(); + } + encoded_hex + } + + let mut system_id_debug_path = lib_debug + .join(encode_hex(&binary_build_id[..1])) + .join(encode_hex(&binary_build_id[1..])); + + system_id_debug_path.set_extension("debug"); + + if std::fs::exists(&system_id_debug_path)? { + let mut library = self.mmap_library(system_id_debug_path)?; + + if matches!(library.build_id(), Some(v) if v == binary_build_id) { + let library_data = library.data(); + library.address_begin = info.dlpi_addr as usize; + library.address_end = info.dlpi_addr as usize + library_data.len(); + return Ok(library); + } + } + } + + let binary_library_data = binary_library.data(); + binary_library.address_begin = info.dlpi_addr as usize; + binary_library.address_end = info.dlpi_addr as usize + binary_library_data.len(); + Ok(binary_library) + } + + pub unsafe fn load_libraries(&mut self, info: &dl_phdr_info) { + if let Ok(mut library) = self.load_library(info) { + self.load_symbols(&mut library); + self.libraries.push(library); + } + } + + pub fn finalize(mut self) -> (Vec, Vec) { + self.symbols.sort_by(Symbol::sort_begin_address); + self.libraries.sort_by(Library::sort_begin_address); + self.symbols.dedup_by(Symbol::same_address); + + (self.libraries, self.symbols) + } +} + +unsafe extern "C" fn callback(info: *mut dl_phdr_info, _size: size_t, v: *mut c_void) -> c_int { + let loader = &mut *v.cast::(); + loader.load_libraries(&*info); + 0 +} diff --git a/src/common/exception/src/elf/library_manager.rs b/src/common/exception/src/elf/library_manager.rs new file mode 100644 index 000000000000..daa4bffec7d0 --- /dev/null +++ b/src/common/exception/src/elf/library_manager.rs @@ -0,0 +1,197 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cmp::Ordering; +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Formatter; +use std::sync::Arc; + +use object::Object; +use once_cell::sync::OnceCell; + +use crate::elf::dwarf::Dwarf; +use crate::elf::library_loader::LibraryLoader; +use crate::elf::library_symbol::Symbol; +use crate::elf::ElfFile; +use crate::exception_backtrace::ResolvedStackFrame; +use crate::exception_backtrace::StackFrame; + +pub struct Library { + pub name: String, + pub address_begin: usize, + pub address_end: usize, + pub elf: Option>, + library_data: &'static [u8], +} + +impl Library { + pub fn create(name: String, data: *const u8, size: usize) -> Library { + Library { + name, + address_begin: 0, + address_end: 0, + elf: None, + // Leak memory + library_data: unsafe { std::slice::from_raw_parts(data, size) }, + } + } + pub fn sort_begin_address(&self, other: &Self) -> Ordering { + self.address_begin.cmp(&other.address_begin) + } + + pub fn data(&self) -> &'static [u8] { + self.library_data + } + + pub unsafe fn build_id(&self) -> Option<&'static [u8]> { + let elf_file = ElfFile::parse(self.data()).ok()?; + match elf_file.build_id() { + Ok(None) | Err(_) => None, + Ok(Some(build)) => Some(build), + } + } +} + +static INSTANCE: OnceCell> = OnceCell::new(); + +impl Debug for Library { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Library") + .field("name", &self.name) + .field("address_begin", &self.address_begin) + .field("address_end", &self.address_end) + .finish() + } +} + +// #[derive(Debug)] +pub struct LibraryManager { + symbols: Vec, + libraries: Vec, +} + +impl Debug for LibraryManager { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LibraryManager") + .field("libraries", &self.libraries) + .field("symbols", &self.symbols.len()) + .finish() + } +} + +impl LibraryManager { + fn find_library(&self, addr: usize) -> Option<&Library> { + self.libraries + .iter() + .find(|library| library.address_begin <= addr && addr <= library.address_end) + } + + pub fn resolve_frames Result<(), E>>( + &self, + frames: &[StackFrame], + only_address: bool, + mut f: F, + ) -> Result<(), E> { + let mut dwarf_cache = HashMap::with_capacity(self.libraries.len()); + + for frame in frames { + let StackFrame::Ip(addr) = frame; + + let mut resolved_frame = ResolvedStackFrame { + virtual_address: *addr, + physical_address: *addr, + symbol: String::from(""), + inlined: false, + file: None, + line: None, + column: None, + }; + + if let Some(library) = self.find_library(*addr) { + resolved_frame.physical_address = *addr - library.address_begin; + } + let Some(library) = self.find_library(*addr) else { + f(ResolvedStackFrame { + virtual_address: *addr, + physical_address: *addr, + symbol: String::from(""), + inlined: false, + file: None, + line: None, + column: None, + })?; + + continue; + }; + + let physical_address = *addr - library.address_begin; + + if !only_address { + let dwarf = match library.elf.as_ref() { + None => &None, + Some(elf) => match dwarf_cache.get(&library.name) { + Some(v) => v, + None => { + dwarf_cache.insert(library.name.clone(), Dwarf::create(elf.clone())); + dwarf_cache.get(&library.name).unwrap() + } + }, + }; + + if let Some(dwarf) = dwarf { + let adjusted_addr = (physical_address - 1) as u64; + + if let Ok(locations) = dwarf.find_frames(adjusted_addr) { + for location in locations { + f(ResolvedStackFrame { + virtual_address: 0, + physical_address, + symbol: location.symbol.unwrap_or("".to_string()), + inlined: location.is_inlined, + file: location.file, + line: location.line, + column: location.column, + })?; + } + + continue; + } + } + } + + f(ResolvedStackFrame { + physical_address, + virtual_address: *addr, + inlined: false, + symbol: String::from(""), + file: None, + line: None, + column: None, + })?; + } + + Ok(()) + } + + pub fn create() -> Arc { + let loader = LibraryLoader::load(); + let (libraries, symbols) = loader.finalize(); + Arc::new(LibraryManager { symbols, libraries }) + } + + pub fn instance() -> Arc { + INSTANCE.get_or_init(LibraryManager::create).clone() + } +} diff --git a/src/common/exception/src/elf/library_symbol.rs b/src/common/exception/src/elf/library_symbol.rs new file mode 100644 index 000000000000..84804fbbea2a --- /dev/null +++ b/src/common/exception/src/elf/library_symbol.rs @@ -0,0 +1,46 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cmp::Ordering; +use std::fmt::Debug; +use std::fmt::Formatter; + +pub struct Symbol { + pub name: &'static [u8], + pub address_end: u64, + pub address_begin: u64, +} + +impl Symbol { + pub fn sort_begin_address(&self, other: &Self) -> Ordering { + self.address_begin.cmp(&other.address_begin) + } + + pub fn same_address(&mut self, other: &mut Self) -> bool { + self.address_begin == other.address_begin && self.address_end == other.address_end + } +} + +impl Debug for Symbol { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Symbol") + .field( + "name", + &rustc_demangle::demangle(std::str::from_utf8(self.name).unwrap()), + ) + .field("address_begin", &self.address_begin) + .field("address_end", &self.address_end) + .finish() + } +} diff --git a/src/common/exception/src/elf/mod.rs b/src/common/exception/src/elf/mod.rs new file mode 100644 index 000000000000..c1696452ef16 --- /dev/null +++ b/src/common/exception/src/elf/mod.rs @@ -0,0 +1,29 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod dwarf; +mod dwarf_inline_functions; +mod dwarf_subprogram; +mod dwarf_unit; +mod library_loader; +mod library_manager; +mod library_symbol; + +#[cfg(target_pointer_width = "32")] +type ElfFile = object::read::elf::ElfFile32<'static, object::NativeEndian, &'static [u8]>; + +#[cfg(target_pointer_width = "64")] +type ElfFile = object::read::elf::ElfFile64<'static, object::NativeEndian, &'static [u8]>; + +pub use library_manager::LibraryManager; diff --git a/src/common/exception/src/exception.rs b/src/common/exception/src/exception.rs index 5f488f51058b..668aeba8c6b5 100644 --- a/src/common/exception/src/exception.rs +++ b/src/common/exception/src/exception.rs @@ -18,75 +18,14 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; use std::marker::PhantomData; -use std::sync::Arc; -use backtrace::Backtrace; use databend_common_ast::span::pretty_print_error; use databend_common_ast::Span; use thiserror::Error; use crate::exception_backtrace::capture; use crate::ErrorFrame; - -#[derive(Clone)] -pub enum ErrorCodeBacktrace { - Serialized(Arc), - Symbols(Arc), - Address(Arc), -} - -impl Display for ErrorCodeBacktrace { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - match self { - ErrorCodeBacktrace::Serialized(backtrace) => write!(f, "{}", backtrace), - ErrorCodeBacktrace::Symbols(backtrace) => write!(f, "{:?}", backtrace), - ErrorCodeBacktrace::Address(backtrace) => { - let frames_address = backtrace - .frames() - .iter() - .map(|f| (f.ip() as usize, f.symbol_address() as usize)) - .collect::>(); - write!(f, "{:?}", frames_address) - } - } - } -} - -impl From<&str> for ErrorCodeBacktrace { - fn from(s: &str) -> Self { - Self::Serialized(Arc::new(s.to_string())) - } -} - -impl From for ErrorCodeBacktrace { - fn from(s: String) -> Self { - Self::Serialized(Arc::new(s)) - } -} - -impl From> for ErrorCodeBacktrace { - fn from(s: Arc) -> Self { - Self::Serialized(s) - } -} - -impl From for ErrorCodeBacktrace { - fn from(bt: Backtrace) -> Self { - Self::Symbols(Arc::new(bt)) - } -} - -impl From<&Backtrace> for ErrorCodeBacktrace { - fn from(bt: &Backtrace) -> Self { - Self::Serialized(Arc::new(format!("{:?}", bt))) - } -} - -impl From> for ErrorCodeBacktrace { - fn from(bt: Arc) -> Self { - Self::Symbols(bt) - } -} +use crate::StackTrace; #[derive(Error)] pub struct ErrorCode { @@ -98,8 +37,8 @@ pub struct ErrorCode { // cause is only used to contain an `anyhow::Error`. // TODO: remove `cause` when we completely get rid of `anyhow::Error`. pub(crate) cause: Option>, - pub(crate) backtrace: Option, pub(crate) stacks: Vec, + pub(crate) backtrace: StackTrace, pub(crate) _phantom: PhantomData, } @@ -199,24 +138,12 @@ impl ErrorCode { self } - /// Set backtrace info for this error. - /// - /// Useful when trying to keep original backtrace - pub fn set_backtrace(mut self, bt: Option>) -> Self { - if let Some(b) = bt { - self.backtrace = Some(b.into()); - } - self - } - - pub fn backtrace(&self) -> Option { + pub fn backtrace(&self) -> StackTrace { self.backtrace.clone() } pub fn backtrace_str(&self) -> String { - self.backtrace - .as_ref() - .map_or("".to_string(), |x| x.to_string()) + format!("{:?}", &self.backtrace) } pub fn stacks(&self) -> &[ErrorFrame] { @@ -241,26 +168,12 @@ impl Debug for ErrorCode { self.message(), )?; - match self.backtrace.as_ref() { - None => write!( + match self.backtrace.frames.is_empty() { + true => write!( f, "\n\n " ), - Some(backtrace) => { - // TODO: Custom stack frame format for print - match backtrace { - ErrorCodeBacktrace::Symbols(backtrace) => write!(f, "\n\n{:?}", backtrace), - ErrorCodeBacktrace::Serialized(backtrace) => write!(f, "\n\n{}", backtrace), - ErrorCodeBacktrace::Address(backtrace) => { - let frames_address = backtrace - .frames() - .iter() - .map(|f| (f.ip() as usize, f.symbol_address() as usize)) - .collect::>(); - write!(f, "\n\n{:?}", frames_address) - } - } - } + false => write!(f, "\n\n{:?}", &self.backtrace), } } } @@ -318,8 +231,8 @@ impl ErrorCode { detail: String::new(), span: None, cause: None, - backtrace: None, stacks: vec![], + backtrace: StackTrace::no_capture(), _phantom: PhantomData::, } } @@ -330,7 +243,7 @@ impl ErrorCode { display_text: String, detail: String, cause: Option>, - backtrace: Option, + backtrace: StackTrace, ) -> Self { ErrorCode { code, diff --git a/src/common/exception/src/exception_backtrace.rs b/src/common/exception/src/exception_backtrace.rs index ef50db7f0c35..7dfdeaa5de3c 100644 --- a/src/common/exception/src/exception_backtrace.rs +++ b/src/common/exception/src/exception_backtrace.rs @@ -12,12 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -// use std::backtrace::Backtrace; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Formatter; +use std::fmt::Write; +use std::hash::Hash; +use std::hash::Hasher; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::sync::Arc; - -use crate::exception::ErrorCodeBacktrace; +use std::sync::LazyLock; +use std::sync::Mutex; +use std::sync::PoisonError; +use std::sync::RwLock; // 0: not specified 1: disable 2: enable pub static USER_SET_ENABLE_BACKTRACE: AtomicUsize = AtomicUsize::new(0); @@ -26,6 +34,8 @@ pub fn set_backtrace(switch: bool) { if switch { USER_SET_ENABLE_BACKTRACE.store(2, Ordering::Relaxed); } else { + let mut write_guard = STACK_CACHE.write().unwrap_or_else(PoisonError::into_inner); + write_guard.clear(); USER_SET_ENABLE_BACKTRACE.store(1, Ordering::Relaxed); } } @@ -49,42 +59,220 @@ fn enable_rust_backtrace() -> bool { enabled } -enum BacktraceStyle { - Symbols, - Address, +pub fn capture() -> StackTrace { + match enable_rust_backtrace() { + true => StackTrace::capture(), + false => StackTrace::no_capture(), + } +} + +#[cfg(target_os = "linux")] +pub struct ResolvedStackFrame { + pub virtual_address: usize, + pub physical_address: usize, + pub symbol: String, + pub inlined: bool, + pub file: Option, + pub line: Option, + pub column: Option, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] +pub enum StackFrame { + #[cfg(target_os = "linux")] + Ip(usize), + #[cfg(not(target_os = "linux"))] + Backtrace(backtrace::BacktraceFrame), } -fn backtrace_style() -> BacktraceStyle { - static ENABLED: AtomicUsize = AtomicUsize::new(0); - match ENABLED.load(Ordering::Relaxed) { - 1 => return BacktraceStyle::Address, - 2 => return BacktraceStyle::Symbols, - _ => {} +impl Eq for StackFrame {} + +impl PartialEq for StackFrame { + fn eq(&self, other: &Self) -> bool { + #[cfg(target_os = "linux")] + { + let StackFrame::Ip(addr) = &self; + let StackFrame::Ip(other_addr) = &other; + addr == other_addr + } + + #[cfg(not(target_os = "linux"))] + { + let StackFrame::Backtrace(addr) = &self; + let StackFrame::Backtrace(other_addr) = &other; + addr.ip() == other_addr.ip() + } } +} - let backtrace_style = match std::env::var("BACKTRACE_STYLE") { - Ok(style) if style.eq_ignore_ascii_case("ADDRESS") => 1, - _ => 2, - }; +impl Hash for StackFrame { + fn hash(&self, state: &mut H) { + #[cfg(target_os = "linux")] + { + let StackFrame::Ip(addr) = &self; + addr.hash(state); + } - ENABLED.store(backtrace_style, Ordering::Relaxed); - match backtrace_style { - 1 => BacktraceStyle::Address, - _ => BacktraceStyle::Symbols, + #[cfg(not(target_os = "linux"))] + { + let StackFrame::Backtrace(addr) = &self; + addr.ip().hash(state) + } } } -pub fn capture() -> Option { - match enable_rust_backtrace() { - false => None, - true => match backtrace_style() { - BacktraceStyle::Symbols => Some(ErrorCodeBacktrace::Symbols(Arc::new( - backtrace::Backtrace::new(), - ))), - // TODO: get offset address(https://github.com/rust-lang/backtrace-rs/issues/434) - BacktraceStyle::Address => Some(ErrorCodeBacktrace::Address(Arc::new( - backtrace::Backtrace::new_unresolved(), - ))), - }, +// Rewrite the backtrace on linux ELF using gimli-rs. +// +// Differences from backtrace-rs[https://github.com/rust-lang/backtrace-rs]: +// - Almost lock-free (backtrace-rs requires large-grained locks or frequent lock operations) +// - Symbol resolution is lazy, only resolved when outputting +// - Cache the all stack frames for the stack, not just a single stack frame +// - Output the physical addresses of the stack instead of virtual addresses, even in the absence of symbols (this will help us use backtraces to get cause in the case of split symbol tables) +// - Output inline functions and marked it +// +// What's different from gimli-addr2line[https://github.com/gimli-rs/addr2line](why not use gimli-addr2line): +// - Use aranges to optimize the lookup of DWARF units (if present) +// - gimli-addr2line caches and sorts the symbol tables to speed up symbol lookup, which would introduce locks and caching (but in reality, symbol lookup is a low-frequency operation in databend, and rapid reconstruction based on mmap is sufficient). +#[derive(Clone, serde::Serialize, serde::Deserialize)] +pub struct StackTrace { + pub(crate) frames: Vec, +} + +impl Eq for StackTrace {} + +impl PartialEq for StackTrace { + fn eq(&self, other: &Self) -> bool { + self.frames == other.frames + } +} + +impl StackTrace { + pub fn capture() -> StackTrace { + let mut frames = Vec::with_capacity(50); + Self::capture_frames(&mut frames); + StackTrace { frames } + } + + pub fn no_capture() -> StackTrace { + StackTrace { frames: vec![] } + } + + #[cfg(not(target_os = "linux"))] + fn capture_frames(frames: &mut Vec) { + unsafe { + backtrace::trace_unsynchronized(|frame| { + frames.push(StackFrame::Backtrace(backtrace::BacktraceFrame::from( + frame.clone(), + ))); + frames.len() != frames.capacity() + }); + } + } + + #[cfg(target_os = "linux")] + fn capture_frames(frames: &mut Vec) { + // Safety: + unsafe { + backtrace::trace_unsynchronized(|frame| { + frames.push(StackFrame::Ip(frame.ip() as usize)); + frames.len() != frames.capacity() + }); + } + } + + #[cfg(not(target_os = "linux"))] + fn fmt_frames(&self, display_text: &mut String, address: bool) -> std::fmt::Result { + let mut frames = std::vec::Vec::with_capacity(self.frames.len()); + for frame in &self.frames { + let StackFrame::Backtrace(frame) = frame; + frames.push(frame.clone()); + } + + let mut backtrace = backtrace::Backtrace::from(frames); + + if !address { + backtrace.resolve(); + } + + writeln!(display_text, "{:?}", backtrace) + } + + #[cfg(target_os = "linux")] + fn fmt_frames(&self, f: &mut String, address: bool) -> std::fmt::Result { + let mut idx = 0; + crate::elf::LibraryManager::instance().resolve_frames(&self.frames, address, |frame| { + write!(f, "{:4}: {}", idx, frame.symbol)?; + + if frame.inlined { + write!(f, "[inlined]")?; + } else if frame.physical_address != frame.virtual_address { + write!(f, "@{:x}", frame.physical_address)?; + } + + #[allow(clippy::writeln_empty_string)] + writeln!(f, "")?; + + if let Some(file) = frame.file { + write!(f, " at {}", file)?; + + if let Some(line) = frame.line { + write!(f, ":{}", line)?; + + if let Some(column) = frame.column { + write!(f, ":{}", column)?; + } + } + + #[allow(clippy::writeln_empty_string)] + writeln!(f, "")?; + } + + idx += 1; + Ok(()) + }) + } +} + +#[allow(clippy::type_complexity)] +static STACK_CACHE: LazyLock, Arc>>>>>> = + LazyLock::new(|| RwLock::new(HashMap::new())); + +impl Debug for StackTrace { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if self.frames.is_empty() { + return writeln!(f, ""); + } + + let mut display_text = { + let read_guard = STACK_CACHE.read().unwrap_or_else(PoisonError::into_inner); + read_guard.get(&self.frames).cloned() + }; + + if display_text.is_none() { + let mut guard = STACK_CACHE.write().unwrap_or_else(PoisonError::into_inner); + + display_text = Some(match guard.entry(self.frames.clone()) { + Entry::Occupied(v) => v.get().clone(), + Entry::Vacant(v) => v.insert(Arc::new(Mutex::new(None))).clone(), + }); + } + + let display_text_lock = display_text.as_ref().unwrap(); + let mut display_guard = display_text_lock + .lock() + .unwrap_or_else(PoisonError::into_inner); + + if display_guard.is_none() { + let mut display_text = String::new(); + + self.fmt_frames(&mut display_text, !enable_rust_backtrace())?; + *display_guard = Some(Arc::new(display_text)); + } + + let display_text = display_guard.as_ref().unwrap().clone(); + drop(display_guard); + + writeln!(f, "{}", display_text)?; + Ok(()) } } diff --git a/src/common/exception/src/exception_into.rs b/src/common/exception/src/exception_into.rs index cb18b0e451cc..a8f5a9e686e1 100644 --- a/src/common/exception/src/exception_into.rs +++ b/src/common/exception/src/exception_into.rs @@ -16,15 +16,14 @@ use std::error::Error; use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; -use std::sync::Arc; use databend_common_ast::Span; use geozero::error::GeozeroError; -use crate::exception::ErrorCodeBacktrace; use crate::exception_backtrace::capture; use crate::ErrorCode; use crate::ErrorFrame; +use crate::StackTrace; #[derive(thiserror::Error)] enum OtherErrors { @@ -279,7 +278,7 @@ pub struct SerializedError { pub name: String, pub message: String, pub span: Span, - pub backtrace: String, + pub backtrace: StackTrace, pub stacks: Vec, } @@ -296,7 +295,7 @@ impl From<&ErrorCode> for SerializedError { name: e.name(), message: e.message(), span: e.span(), - backtrace: e.backtrace_str(), + backtrace: e.backtrace.clone(), stacks: e.stacks().iter().map(|f| f.into()).collect(), } } @@ -304,19 +303,13 @@ impl From<&ErrorCode> for SerializedError { impl From<&SerializedError> for ErrorCode { fn from(se: &SerializedError) -> Self { - let backtrace = match se.backtrace.len() { - 0 => None, - _ => Some(ErrorCodeBacktrace::Serialized(Arc::new( - se.backtrace.clone(), - ))), - }; ErrorCode::create( se.code, se.name.clone(), se.message.clone(), String::new(), None, - backtrace, + se.backtrace.clone(), ) .set_span(se.span) .set_stacks(se.stacks.iter().map(|f| f.into()).collect()) @@ -383,28 +376,15 @@ impl From for ErrorCode { } match serde_json::from_slice::(details) { Err(error) => ErrorCode::from(error), - Ok(serialized_error) => match serialized_error.backtrace.len() { - 0 => ErrorCode::create( - serialized_error.code, - serialized_error.name, - serialized_error.message, - String::new(), - None, - None, - ) - .set_span(serialized_error.span), - _ => ErrorCode::create( - serialized_error.code, - serialized_error.name, - serialized_error.message, - String::new(), - None, - Some(ErrorCodeBacktrace::Serialized(Arc::new( - serialized_error.backtrace, - ))), - ) - .set_span(serialized_error.span), - }, + Ok(serialized_error) => ErrorCode::create( + serialized_error.code, + serialized_error.name, + serialized_error.message, + String::new(), + None, + serialized_error.backtrace, + ) + .set_span(serialized_error.span), } } _ => ErrorCode::Unimplemented(status.to_string()), @@ -414,18 +394,16 @@ impl From for ErrorCode { impl From for tonic::Status { fn from(err: ErrorCode) -> Self { - let error_json = serde_json::to_vec::(&SerializedError { + let serialized_error = SerializedError { code: err.code(), name: err.name(), message: err.message(), span: err.span(), - backtrace: { - let mut str = err.backtrace_str(); - str.truncate(2 * 1024); - str - }, stacks: err.stacks().iter().map(|f| f.into()).collect(), - }); + backtrace: err.backtrace, + }; + + let error_json = serde_json::to_vec::(&serialized_error); match error_json { Ok(serialized_error_json) => { @@ -433,7 +411,7 @@ impl From for tonic::Status { // To distinguish from that, we use Code::Unknown here tonic::Status::with_details( tonic::Code::Unknown, - err.message(), + serialized_error.message.clone(), serialized_error_json.into(), ) } diff --git a/src/common/exception/src/lib.rs b/src/common/exception/src/lib.rs index 548d1d02ae30..62332d2d4103 100644 --- a/src/common/exception/src/lib.rs +++ b/src/common/exception/src/lib.rs @@ -14,7 +14,11 @@ #![allow(clippy::uninlined_format_args)] +extern crate core; + mod context; +#[cfg(target_os = "linux")] +mod elf; pub mod exception; mod exception_backtrace; mod exception_code; @@ -28,5 +32,6 @@ pub use exception::ErrorCode; pub use exception::Result; pub use exception::ToErrorCode; pub use exception_backtrace::set_backtrace; +pub use exception_backtrace::StackTrace; pub use exception_backtrace::USER_SET_ENABLE_BACKTRACE; pub use exception_into::SerializedError; diff --git a/src/common/exception/tests/it/exception_flight.rs b/src/common/exception/tests/it/exception_flight.rs index cf360a397b83..59db53a0404e 100644 --- a/src/common/exception/tests/it/exception_flight.rs +++ b/src/common/exception/tests/it/exception_flight.rs @@ -12,13 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - use arrow_flight::FlightData; -use backtrace::Backtrace; -use databend_common_exception::exception::ErrorCodeBacktrace; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_exception::StackTrace; #[test] fn test_serialize() -> Result<()> { @@ -28,7 +25,7 @@ fn test_serialize() -> Result<()> { String::from("test_message"), String::new(), None, - Some(ErrorCodeBacktrace::Symbols(Arc::new(Backtrace::new()))), + StackTrace::capture(), ) .set_span(Some((0..1).into())); let backtrace_str = error_code.backtrace_str(); diff --git a/src/query/pipeline/core/src/processors/profile.rs b/src/query/pipeline/core/src/processors/profile.rs index 5a61e82ceb07..f3f776601fd6 100644 --- a/src/query/pipeline/core/src/processors/profile.rs +++ b/src/query/pipeline/core/src/processors/profile.rs @@ -26,6 +26,7 @@ use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileLabel; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_exception::ErrorCode; +use databend_common_exception::StackTrace; pub struct PlanScopeGuard { idx: usize, @@ -54,7 +55,7 @@ impl Drop for PlanScopeGuard { pub struct ErrorInfoDesc { message: String, detail: String, - backtrace: String, + backtrace: StackTrace, } impl ErrorInfoDesc { @@ -62,7 +63,7 @@ impl ErrorInfoDesc { ErrorInfoDesc { message: error.message(), detail: error.detail(), - backtrace: error.backtrace_str(), + backtrace: error.backtrace(), } } } diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0026_merge_into.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0026_merge_into.test index b56efaf7794d..00eb09c87786 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0026_merge_into.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0026_merge_into.test @@ -1,7 +1,1438 @@ statement ok set enable_distributed_merge_into = 1; -include ./09_0036_merge_into_without_distributed_enable.test +statement ok +set enable_experimental_merge_into = 1; + +statement ok +drop database if exists db; + +statement ok +create database db; + +statement ok +use db; + +statement ok +drop table if exists t1; + +statement ok +drop table if exists t2; + +statement ok +create table t1(a int,b string, c string) cluster by(a,b); + +statement ok +create table t2(a int,b string, c string) cluster by(a,b); + +statement ok +insert into t1 values(1,'b1','c1'),(2,'b2','c2'); + +statement ok +insert into t1 values(2,'b3','c3'),(3,'b4','c4'); + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c1 +2 b2 c2 +2 b3 c3 +3 b4 c4 + +statement ok +insert into t2 values(1,'b_5','c_5'),(3,'b_6','c_6'); + +statement ok +insert into t2 values(2,'b_7','c_7'); + +query TTT +select * from t2 order by a,b,c; +---- +1 b_5 c_5 +2 b_7 c_7 +3 b_6 c_6 + +## test source alias +statement error 1005 +merge into t1 using (select * from t2 ) on t1.a = t2.a when matched then update set t1.c = t2.c,t1.c = t2.c; + +# section I: basic test for match and unmatch + +statement error 1006 +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then update set t1.c = t2.c,t1.c = t2.c; + +query T +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then update set t1.c = t2.c; +---- +4 + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c_5 +2 b2 c_7 +2 b3 c_7 +3 b4 c_6 + +statement ok +insert into t2 values(4,'b_8','c_8'); + +query TTT +select * from t2 order by a,b,c; +---- +1 b_5 c_5 +2 b_7 c_7 +3 b_6 c_6 +4 b_8 c_8 + +query TT +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then update set t1.c = t2.c when not matched then insert (a,b,c) values(t2.a,t2.b,t2.c); +---- +1 4 + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c_5 +2 b2 c_7 +2 b3 c_7 +3 b4 c_6 +4 b_8 c_8 + +statement ok +insert into t2 values(1,'b_9','c_9'); + +statement error 4001 +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then update set t1.c = t2.c when not matched then insert (a,b,c) values(t2.a,t2.b,t2.c); + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c_5 +2 b2 c_7 +2 b3 c_7 +3 b4 c_6 +4 b_8 c_8 + +statement ok +delete from t2 where a = 1; + +query TTT +select * from t2 order by a,b,c; +---- +2 b_7 c_7 +3 b_6 c_6 +4 b_8 c_8 + +statement ok +insert into t2 values(5,'b_9','c_9'); + +query TTT +select * from t2 order by a,b,c; +---- +2 b_7 c_7 +3 b_6 c_6 +4 b_8 c_8 +5 b_9 c_9 + +query TT +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then delete; +---- +4 + +query ITT +select * from t1 order by a,b,c; +---- +1 b1 c_5 + +# section 2 multi clauses +statement ok +insert into t1 values(2,'b_1','c_1'),(3,'b_2','c_2'); + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c_5 +2 b_1 c_1 +3 b_2 c_2 + +statement error 1005 +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then delete when matched then update set t1.c = t2.c when not matched and t2.c = 'c_8' then insert (a,b,c) values(t2.a,t2.b,t2.c); + +query TTT +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched and t1.b = 'b_1' then delete when matched then update set t1.c = t2.c when not matched and t2.c = 'c_8' then insert (a,b,c) values(t2.a,t2.b,t2.c); +---- +1 1 1 + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c_5 +3 b_2 c_6 +4 b_8 c_8 + +query TT +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then delete when not matched and t2.c = 'c_9' then insert (a,b,c) values(t2.a,t2.b,t2.c); +---- +1 2 + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c_5 +5 b_9 c_9 + +query T +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when not matched and t2.c = 'c_8' then insert (a,b) values(t2.a,t2.b) when not matched and t2.c = 'c_7' then insert (a,c) values(t2.a,t2.c); +---- +2 + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c_5 +2 NULL c_7 +4 b_8 NULL +5 b_9 c_9 + +statement ok +insert into t2 values(5,'b_10','c_10'); + +query TTT +select * from t2 order by a,b,c; +---- +2 b_7 c_7 +3 b_6 c_6 +4 b_8 c_8 +5 b_10 c_10 +5 b_9 c_9 + +statement error 4001 +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched and t2.c = 'c_9' then update set t1.b = 'b_11' when matched and t2.c = 'c_10' then delete; + +## idempotent delete test +query T +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then delete; +---- +3 + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c_5 + +## test star for merge into +statement ok +truncate table t1; + +statement ok +truncate table t2; + +query I +select count(*) from t1; +---- +0 + +query I +select count(*) from t2; +---- +0 + +statement ok +insert into t1 values(1,'b1','c1'),(2,'b2','c2'); + +query TTT +select * from t1 order by a,b,c; +---- +1 b1 c1 +2 b2 c2 + +statement ok +insert into t2 values(1,'b3','c3'),(3,'b4','c4'); + +query TTT +select * from t2 order by a,b,c; +---- +1 b3 c3 +3 b4 c4 + +## test insert columns mismatch +statement error 1065 +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when not matched then insert values(t2.a,t2.c); + +query TT +merge into t1 using (select * from t2 ) as t2 on t1.a = t2.a when matched then update * when not matched then insert *; +---- +1 1 + + +query TTT +select * from t1 order by a,b,c; +---- +1 b3 c3 +2 b2 c2 +3 b4 c4 + +## test multi same name for star +statement error 1065 +merge into t1 using (select a,b,c,a from t2 ) as t2 on t1.a = t2.a when matched then update *; + +statement error 1065 +merge into t1 using (select a,b,c,a,b from t2 ) as t2 on t1.a = t2.a when not matched then insert *; + +## stage file test +statement ok +drop table if exists test_stage; + +statement ok +drop table if exists target_table; + +statement ok +create table target_table(a int,b string,c string) cluster by(a,b); + +statement ok +insert into target_table values(1,'a_1','b_1'),(2,'a_2','b_2'); + +query TTT +select * from target_table order by a,b,c; +---- +1 a_1 b_1 +2 a_2 b_2 + +statement ok +create table test_stage(a int,b string,c string) cluster by(a,b); + +statement ok +insert into test_stage values(1,'a1','b1'),(2,'a2','b2'),(3,'a3','b3'); + +query TTT +select * from test_stage order by a,b,c; +---- +1 a1 b1 +2 a2 b2 +3 a3 b3 + +statement ok +drop stage if exists s6_merge_into; + +statement ok +drop stage if exists s7_merge_into; + +statement ok +create stage s6_merge_into FILE_FORMAT = (TYPE = CSV); + +statement ok +remove @s6_merge_into; + +statement ok +copy into @s6_merge_into from (select a,b,c from test_stage order by a,b,c); + +query TTT +select $1,$2,$3 from @s6_merge_into order by $1,$2,$3; +---- +1 a1 b1 +2 a2 b2 +3 a3 b3 + +## test CSV +query TT +merge into target_table using (select $1,$2,$3 from @s6_merge_into) as cdc on cast(cdc.$1 as int) = target_table.a when matched then delete when not matched then insert values(cdc.$1,cdc.$2,cdc.$3); +---- +1 2 + +query TTT +select * from target_table order by a,b,c; +---- +3 a3 b3 + +## test parquet +statement ok +truncate table target_table; + +query I +select count(*) from target_table; +---- +0 + +statement ok +create stage s7_merge_into FILE_FORMAT = (TYPE = PARQUET); + +statement ok +remove @s7_merge_into; + +statement ok +copy into @s7_merge_into from (select a,b,c from test_stage order by a,b,c); + +query TTT +select $1,$2,$3 from @s7_merge_into order by $1,$2,$3; +---- +1 a1 b1 +2 a2 b2 +3 a3 b3 + +statement ok +insert into target_table values(1,'a_1','b_1'),(2,'a_2','b_2'); + +query TTT +select * from target_table order by a,b,c; +---- +1 a_1 b_1 +2 a_2 b_2 + +query TT +merge into target_table using (select $1,$2,$3 from @s7_merge_into) as cdc on cdc.$1 = target_table.a when matched then delete when not matched then insert values(cdc.$1,cdc.$2,cdc.$3); +---- +1 2 + +query TTT +select * from target_table order by a,b,c; +---- +3 a3 b3 + +## NULL test, for join, if join_expr result is +## NULL, it will be treated as not matched. +statement ok +truncate table t1; + +statement ok +truncate table t2; + +query I +select count(*) from t1; +---- +0 + +query I +select count(*) from t2; +---- +0 + +statement ok +insert into t1 values(NULL,'b_1','c_1'); + +query TTT +select * from t1 order by a,b,c; +---- +NULL b_1 c_1 + +statement ok +insert into t2 values(1,'b_4','c_4'),(2,'b_2','c_2'),(NULL,'b_3','c_3'); + +query TTT +select * from t2 order by a,b,c; +---- +1 b_4 c_4 +2 b_2 c_2 +NULL b_3 c_3 + +query TT +merge into t1 using (select * from t2) as t2 on t1.a = t2.a when matched then delete when not matched then insert *; +---- +3 0 + +query TTT +select * from t1 order by a,b,c; +---- +1 b_4 c_4 +2 b_2 c_2 +NULL b_1 c_1 +NULL b_3 c_3 + +query T +merge into t1 using (select * from t2) as t2 on t1.a = t2.a when matched then delete; +---- +2 + +query TTT +select * from t1 order by a,b,c; +---- +NULL b_1 c_1 +NULL b_3 c_3 + +statement ok +truncate table t1; + +statement ok +truncate table t2; + +query I +select count(*) from t1; +---- +0 + +query I +select count(*) from t2; +---- +0 + +## test target table alias +statement ok +insert into t2 values(1,'a1','b1'); + +query TT +merge into t1 as t3 using (select * from t2 ) as t2 on t3.a = t2.a when not matched then insert (a,b,c) values(t2.a,t2.b,t2.c); +---- +1 + +query TTT +select * from t1 order by a,b,c; +---- +1 a1 b1 + +statement ok +drop table if exists employees; + +statement ok +drop table if exists salaries; + +statement ok +CREATE TABLE employees (employee_id INT, employee_name VARCHAR(255),department VARCHAR(255)) cluster by(employee_id,employee_name); + +statement ok +drop table if exists salaries; + +statement ok +CREATE TABLE salaries (employee_id INT,salary DECIMAL(10, 2)) cluster by(employee_id,salary); + +statement ok +INSERT INTO employees VALUES(1, 'Alice', 'HR'),(2, 'Bob', 'IT'),(3, 'Charlie', 'Finance'),(4, 'David', 'HR'); + +statement ok +INSERT INTO salaries VALUES(1, 50000.00),(2, 60000.00); + +query TT +MERGE INTO salaries USING (SELECT * FROM employees) as employees ON salaries.employee_id = employees.employee_id WHEN MATCHED AND employees.department = 'HR' THEN UPDATE SET salaries.salary = salaries.salary + 1000.00 WHEN MATCHED THEN UPDATE SET salaries.salary = salaries.salary + 500.00 WHEN NOT MATCHED THEN INSERT (employee_id, salary) VALUES (employees.employee_id, 55000.00); +---- +2 2 + +query TTT +select * from salaries order by employee_id; +---- +1 51000.00 +2 60500.00 +3 55000.00 +4 55000.00 + +statement ok +drop table if exists t1_target; + +## null cast bug fix +statement ok +drop table if exists t1_target; + +statement ok +drop table if exists t2_source; + +statement ok +create table t1_target(a int not null) cluster by(a); + +statement ok +drop table if exists t2_source; + +statement ok +create table t2_source(a int not null) cluster by(a); + +statement ok +insert into t1_target values(1); + +statement ok +insert into t2_source values(1),(2); + +query TT +merge into t1_target using (select * from t2_source) as t2_source on t1_target.a = t2_source.a when matched then update * when not matched then insert *; +---- +1 1 + +query T +select * from t1_target order by a; +---- +1 +2 + +statement ok +drop table if exists cluster_target; + +## cluster table test +statement ok +drop table if exists cluster_target; + +statement ok +drop table if exists cluster_source; + +statement ok +create table cluster_target(a int,b string,c int) cluster by(a,b); + +statement ok +drop table if exists cluster_source; + +statement ok +create table cluster_source(a int,b string,c int); + +statement ok +insert into cluster_source values(12,'b',1),(1,'a',2),(2,'b',3),(2,'a',4),(3,'a',3); + +## test update indetify error +statement error 1006 +merge into cluster_target as t1 using (select * from cluster_source) as t2 on t1.a = t2.a when matched then update set cluster_target.a = t2.a; + +statement error 1006 +merge into cluster_target as t1 using (select * from cluster_source) as t2 on t1.a = t2.a when matched then update set t2.a = t2.a; + +query TT +merge into cluster_target as t1 using (select * from cluster_source) as t2 on t1.a = t2.a when not matched then insert *; +---- +5 + +# By default setting, all rows merged from `cluster_source` will be resident in a single block of `cluster_target`, +# as table `cluster_target` is clustered by `(a,b)`, the rows inside the one block are assumed to be sorted +# by `(a, b)`, consequently, the result of the following query should be ordered by `(a,b)` without an explicit +# `order by` clause. +query TTT +select * from cluster_target; +---- +1 a 2 +2 a 4 +2 b 3 +3 a 3 +12 b 1 + +## add more tests +statement ok +drop table if exists target_test; + +statement ok +drop table if exists source_test; + +statement ok +create table target_test(a int,b string) cluster by(a,b); + +statement ok +insert into target_test values(1,'a'),(2,'b'),(3,'c'); + +statement ok +create table source_test(a int,b string,delete_flag bool) cluster by(a,b); + +statement ok +insert into source_test values(1,'d',true),(2,'e',true),(3,'f',false),(4,'e',true),(5,'f',false); + +############################################################################### +# To avoid flakiness, using different stage names for http and mysql handlers # +# testing of these 2 handlers may be run concurrently, and conflict with each # +# other, leading to flaky tests. # +############################################################################### + +onlyif mysql +statement ok +drop stage if exists source_parquet2; + +onlyif mysql +statement ok +create stage source_parquet2 file_format = (type = parquet); + +onlyif mysql +statement ok +remove @source_parquet2; + +onlyif mysql +statement ok +copy into @source_parquet2 from (select * from source_test); + +onlyif mysql +query TTT +merge into `target_test` as tt using (select `a`,`b`,`delete_flag` from @source_parquet2 (pattern => '.*[.]parquet')) as ss on (ss.`a` = tt.`a`) +when matched and ss.`delete_flag` = true then delete when matched then update * when not matched and ss.`delete_flag` = false then insert *; +---- +1 1 2 + + +onlyif http +statement ok +drop stage if exists source_parquet_http2; + +onlyif http +statement ok +create stage source_parquet_http2 file_format = (type = parquet); + +onlyif http +statement ok +remove @source_parquet_http2; + +onlyif http +statement ok +copy into @source_parquet_http2 from (select * from source_test); + +onlyif http +query TTT +merge into `target_test` as tt using (select `a`,`b`,`delete_flag` from @source_parquet_http2 (pattern => '.*[.]parquet')) as ss on (ss.`a` = tt.`a`) +when matched and ss.`delete_flag` = true then delete when matched then update * when not matched and ss.`delete_flag` = false then insert *; +---- +1 1 2 + +query TT +select * from target_test order by a; +---- +3 f +5 f + +## test not match cast and predicate index +statement ok +drop table if exists test_order; + +statement ok +drop table if exists random_source; + +statement ok +create table test_order(id bigint, id1 bigint, id2 bigint, id3 bigint, id4 bigint, id5 bigint, id6 bigint, id7 bigint, s1 varchar, s2 varchar, s3 varchar, s4 varchar, s5 varchar, s6 varchar, s7 varchar, s8 varchar, s9 varchar, s10 varchar, s11 varchar, s12 varchar, s13 varchar, d1 DECIMAL(20, 8), d2 DECIMAL(20, 8), d3 DECIMAL(20, 8), d4 DECIMAL(20, 8), d5 DECIMAL(20, 8), d6 DECIMAL(30, 8), d7 DECIMAL(30, 8), d8 DECIMAL(30, 8), d9 DECIMAL(30, 8), d10 DECIMAL(30, 8),insert_time datetime, insert_time1 datetime, insert_time2 datetime, insert_time3 datetime,i int) CLUSTER BY(to_yyyymmdd(insert_time), id) bloom_index_columns='insert_time,id'; + +statement ok +create table random_source(id bigint not null, id1 bigint, id2 bigint, id3 bigint, id4 bigint, id5 bigint, id6 bigint, id7 bigint,s1 varchar, s2 varchar, s3 varchar, s4 varchar, s5 varchar, s6 varchar, s7 varchar, s8 varchar, s9 varchar, s10 varchar, s11 varchar, s12 varchar, s13 varchar,d1 DECIMAL(20, 8), d2 DECIMAL(20, 8), d3 DECIMAL(20, 8), d4 DECIMAL(20, 8), d5 DECIMAL(20, 8), d6 DECIMAL(30, 8), d7 DECIMAL(30, 8), d8 DECIMAL(30, 8), d9 DECIMAL(30, 8), d10 DECIMAL(30, 8),insert_time datetime not null, insert_time1 datetime, insert_time2 datetime, insert_time3 datetime,i int) Engine = Random; + +statement ok +merge into test_order as t using (select id,34 as id1,238 as id2, id3, id4, id5, id6, id7,s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,d1, d2, d3, d4, d5, d6, d7, d8, d9, d10,insert_time,insert_time1,insert_time2,insert_time3,i from random_source limit 1) as s on t.id = s.id and t.insert_time = s.insert_time when matched then update * when not matched then insert *; + +## test update list #13297 +statement ok +drop table if exists t11 + +statement ok +drop table if exists t12 + +statement ok +create table t11(a int,b string, c string) cluster by(a,b); + +statement ok +create table t12(a int,b string, c string) cluster by(a,b); + +statement ok +insert into t11 values(1,'b1','c1'),(2,'b2','c2'); + +statement ok +insert into t12 values(1,'b_5','c_5'),(3,'b_6','c_6'); + +statement error 1065 +merge into t11 using (select a, c from t12) as t12 on t11.a = t12.a when matched and max(t11.a) > 0 then update set c = t12.c; + +statement error 1065 +merge into t11 using (select a, c from t12) as t12 on t11.a = t12.a when matched then update set c = count(*); + +## test issue #13287 +statement ok +drop table if exists tt1 + +statement ok +create table tt1 (a int, b int) cluster by(a,b); + +statement error 1065 +merge into tt1 using(select 10, 20) as tt2 on tt1.a = 1 when not matched and tt1.b = 2 then insert values (10, 20); + +query TT +merge into tt1 using(select 10 as a, 20 as b) as tt2 on tt1.a = 1 when not matched and tt2.b = 2 then insert values (10, 20); +---- +0 + +query T +select count(*) from tt1; +---- +0 + +## test issue #13367 +statement ok +drop table if exists tt2 + +statement ok +create table tt2(a bool, b variant, c map(string, string)) cluster by(a); + +statement ok +insert into tt2 values (true, '10', {'k1':'v1'}), (false, '20', {'k2':'v2'}) + +query T +merge into tt2 using(select true as x) as t on (x and tt2.a) when matched and tt2.a then update set tt2.b = parse_json('30'); +---- +1 + +query TTT +select a, b, c from tt2 order by b; +---- +0 20 {'k2':'v2'} +1 30 {'k1':'v1'} + +## add test: source is table +statement ok +drop table if exists t1; + +statement ok +drop table if exists t2; + +statement ok +create table t1(a int) cluster by(a); + +statement ok +create table t2(a int) cluster by(a); + +statement ok +insert into t1 values(1); + +statement ok +insert into t2 values(1),(2); + +query TT +merge into t1 using t2 on t1.a = t2.a when matched then delete when not matched then insert *; +---- +1 1 + +query T +select * from t1; +---- +2 + +statement ok +drop table if exists t1; + +statement ok +drop table if exists t2; + +statement ok +create table t1(b int) cluster by(b); + +statement ok +create table t2(a int) cluster by(a); + +statement ok +insert into t1 values(1); + +statement ok +insert into t2 values(1),(2); + +statement error 1065 +merge into t1 using t2 on t1.a = t2.a when matched then delete when not matched then insert *; + +## add more multi matched statement test +statement ok +drop table if exists t1; + +statement ok +drop table if exists t2; + +statement ok +create table t1(a int,b string,c bool) cluster by(a,b); + +statement ok +create table t2(a int,b string,c bool) cluster by(a,b); + +statement ok +insert into t1 values(1,'a1',true),(2,'a2',false),(3,'a3',true); + +statement ok +insert into t2 values(1,'b1',true),(2,'b2',false),(3,'b3',true); + +query TTT +select * from t1; +---- +1 a1 1 +2 a2 0 +3 a3 1 + +query TTT +select * from t2; +---- +1 b1 1 +2 b2 0 +3 b3 1 + +query TT +merge into t1 using t2 on t1.a = t2.a when matched and t1.a = 1 then delete when matched and t1.a = 2 then update * when matched and t1.a = 3 then delete; +---- +1 2 + +query TTT +select * from t1; +---- +2 b2 0 + +query T +merge into t1 using t2 on t1.a = t2.a when matched then delete; +---- +1 + +query T +select count(*) from t1; +---- +0 + +statement ok +insert into t1 values(1,'a1',true),(2,'a2',false),(3,'a3',true); + +query TT +merge into t1 using t2 on t1.a = t2.a when matched and t1.a = 2 then update * when matched and t1.a = 1 then delete when matched and t1.a = 3 then update *; +---- +2 1 + +query TTT +select * from t1; +---- +2 b2 0 +3 b3 1 + +## issue 13454 +statement ok +drop table if exists tt1; + +statement ok +create table tt1(a bool, b int) cluster by(a,b); + +statement ok +insert into tt1 values (true, 1), (false, 2); + +query T +merge into tt1 using (select 1 as x) as tt2 on (2 > 1) when matched and a then delete; +---- +1 + +query TT +select * from tt1; +---- +0 2 + +## issue #13298 +statement ok +drop table if exists t11; + +statement ok +drop table if exists t12; + +statement ok +create table t12 (a int, b int) cluster by(a,b); + +statement ok +create table t11 (a int, b int) cluster by(a,b); + +statement ok +insert into t11 values (1, 10),(2, 20),(3, 30),(4, 40); + +statement ok +insert into t12 values (1, 10),(2, 20),(3, 30),(4, 40); + +query T +MERGE INTO t11 USING(SELECT NULL AS c0 FROM t12) AS t12 ON (t11.a OR TRUE) WHEN MATCHED AND TRUE THEN DELETE; +---- +4 + +query T +select count(*) from t11; +---- +0 + +## test issue #13732 +statement ok +CREATE TABLE orders CLUSTER BY (to_yyyymmddhh(created_at), user_id) AS SELECT + number % 5000 AS order_id, + number % 10000 AS user_id, + CASE WHEN (rand() * 10)::int % 2 = 0 THEN 'buy' + ELSE 'sell' + END AS order_type, + CASE WHEN (rand() * 10)::int % 3 = 0 THEN 'BTC' + WHEN (rand() * 10)::int % 3 = 1 THEN 'ETH' + ELSE 'XRP' + END AS asset_type, + (rand() * 100)::decimal(18, 8) AS quantity, + (rand() * 1000)::decimal(18, 8) AS price, + CASE WHEN (rand() * 10)::int % 3 = 0 THEN 'completed' + WHEN (rand() * 10)::int % 3 = 1 THEN 'pending' + ELSE 'cancelled' + END AS status, + date_add('day', floor(rand() * 10 % 365)::int, '2021-01-01') AS created_at, + date_add('day', floor(rand() * 10 % 365)::int, '2021-01-01') AS updated_at +FROM numbers(5000); + +### for now, we disable target_table_optimization for native. Native will +### spilt one block into multi pages. We should fix this one in the future. +statement ok +MERGE INTO orders USING +( + SELECT + number % 5000 AS order_id, + number % 100000 AS user_id, + CASE WHEN (rand() * 10)::int % 2 = 0 THEN 'buy' + ELSE 'sell' + END AS order_type, + CASE WHEN (rand() * 10)::int % 3 = 0 THEN 'BTC' + WHEN (rand() * 10)::int % 3 = 1 THEN 'ETH' + ELSE 'XRP' + END AS asset_type, + (rand() * 100)::decimal(18, 8) AS quantity, + (rand() * 1000)::decimal(18, 8) AS price, + CASE WHEN (rand() * 10)::int % 3 = 0 THEN 'completed' + WHEN (rand() * 10)::int % 3 = 1 THEN 'pending' + ELSE 'cancelled' + END AS status, + date_add('day', floor(rand() * 10 % 365)::int, '2021-01-01') AS created_at, + date_add('day', floor(rand() * 10 % 365)::int, '2021-01-01') AS updated_at + FROM numbers(5000) +) AS source +ON orders.order_id = source.order_id +WHEN MATCHED THEN + UPDATE SET + orders.user_id = source.user_id, + orders.order_type = source.order_type, + orders.asset_type = source.asset_type, + orders.quantity = source.quantity, + orders.price = source.price, + orders.status = source.status, + orders.created_at = source.created_at, + orders.updated_at = source.updated_at +WHEN NOT MATCHED THEN + INSERT (order_id, user_id, order_type, asset_type, quantity, price, status, created_at, updated_at) + VALUES (source.order_id, source.user_id, source.order_type, source.asset_type, source.quantity, source.price, source.status, source.created_at, source.updated_at); + +## test issue #13733 +statement ok +CREATE TABLE transactions CLUSTER BY (to_yyyymmddhh(transaction_time), user_id) AS SELECT + number % 1000000 AS transaction_id, + number % 100000 AS user_id, + CASE WHEN (rand() * 10)::int % 3 = 0 THEN 'deposit' + WHEN (rand() * 10)::int % 3 = 1 THEN 'withdrawal' + ELSE 'trade' +END AS transaction_type, + CASE WHEN (rand() * 10)::int % 3 = 0 THEN 'BTC' + WHEN (rand() * 10)::int % 3 = 1 THEN 'ETH' + ELSE 'XRP' +END AS asset_type, + (rand() * 100)::decimal(18, 8) AS quantity, + date_add('day', floor(rand() * 10 % 365)::int, '2021-01-01') AS transaction_time +FROM numbers(1000000); + +statement ok +MERGE INTO orders AS tt USING +( + SELECT + CASE + WHEN number % 2 = 0 THEN (number / 2) % 250000 + ELSE (SELECT MAX(order_id) FROM orders) + number + 1 + END AS order_id, + number % 100000 AS user_id, + CASE WHEN (rand() * 10)::int % 2 = 0 THEN 'buy' + ELSE 'sell' + END AS order_type, + CASE WHEN (rand() * 10)::int % 3 = 0 THEN 'BTC' + WHEN (rand() * 10)::int % 3 = 1 THEN 'ETH' + ELSE 'XRP' + END AS asset_type, + (rand() * 100)::decimal(18, 8) AS quantity, + (rand() * 1000)::decimal(18, 8) AS price, + CASE WHEN (rand() * 10)::int % 3 = 0 THEN 'completed' + WHEN (rand() * 10)::int % 3 = 1 THEN 'pending' + ELSE 'cancelled' + END AS status, + date_add('day', floor(rand() * 10 % 365)::int, '2021-01-01') AS created_at, + date_add('day', floor(rand() * 10 % 365)::int, '2021-01-01') AS updated_at, + CASE WHEN number % 2 = 0 THEN false ELSE true END AS is_delete + FROM numbers(5000) +) AS ss +ON (tt.user_id = ss.user_id AND tt.asset_type = ss.asset_type) +WHEN MATCHED AND ss.is_delete = true THEN + DELETE +WHEN MATCHED AND ss.is_delete = false THEN + UPDATE * WHEN NOT MATCHED THEN + INSERT *; + +## unsupport complex exprs for now. +## #13798 we need to support non-correlated-subquery for unmatched values exprs +statement error 1065 +MERGE INTO orders USING ( + SELECT t.user_id, t.asset_type, 'buy' AS synthetic_order_type, SUM(t.quantity) AS total_quantity, today() AS synthetic_date + FROM transactions t + WHERE t.transaction_type = 'deposit' + GROUP BY t.user_id, t.asset_type + HAVING SUM(t.quantity) > 100 +) AS synthetic_orders ON orders.user_id = synthetic_orders.user_id AND orders.asset_type = synthetic_orders.asset_type +WHEN NOT MATCHED THEN + INSERT (order_id, user_id, order_type, asset_type, quantity, price, status, created_at, updated_at) + VALUES ((SELECT MAX(order_id) FROM orders) + 1, synthetic_orders.user_id, synthetic_orders.synthetic_order_type, synthetic_orders.asset_type, synthetic_orders.total_quantity, 0, 'pending', synthetic_orders.synthetic_date, synthetic_orders.synthetic_date); + +## issue #13810: rewrite rule test +statement ok +DROP TABLE IF EXISTS orders; + +statement ok +CREATE TABLE orders ( + order_id INT NOT NULL, + user_id INT NOT NULL, + order_type VARCHAR NOT NULL, + asset_type VARCHAR NOT NULL, + quantity DECIMAL(18,8) NOT NULL, + price DECIMAL(18,8) NOT NULL, + status VARCHAR NOT NULL, + created_at DATE NOT NULL, + updated_at DATE NOT NULL +) row_per_block=5113; + +statement ok +insert into orders values(200007,7,'buy','BTC',4.81412194,48.14121943,'completed',to_date('2021-01-01'),to_date('2021-01-01')), +(200015,15,'buy','BTC',3.78463552,37.84635523,'completed',to_date('2021-01-01'),to_date('2021-01-01')), +(200019,19,'buy','BTC',1.61186913,16.11869132,'completed',to_date('2021-01-01'),to_date('2021-01-01')), +(200031,31,'buy','BTC',3.99013730,39.90137297,'completed',to_date('2021-01-01'),to_date('2021-01-01')), +(200047,47,'buy','BTC',0.98841829,9.88418289,'completed',to_date('2021-01-01'),to_date('2021-01-01')), +(200077,77,'buy','BTC',2.07360391,20.73603908,'completed',to_date('2021-01-01'),to_date('2021-01-01')), +(200087,87,'sell','ETH',9.64567442,96.45674419,'pending',to_date('2021-01-01'),to_date('2021-01-01')), +(200095,95,'buy','BTC',2.26686563,22.66865634,'completed',to_date('2021-01-01'),to_date('2021-01-01')), +(200098,98,'buy','BTC',1.37252960,13.72529599,'completed',to_date('2021-01-01'),to_date('2021-01-01')), +(200102,102,'buy','BTC',1.53596481,15.35964815,'completed',to_date('2021-01-01'),to_date('2021-01-01')); + +query T +MERGE INTO orders USING ( + SELECT o.order_id, o.user_id, o.order_type, o.asset_type, o.quantity + a.avg_quantity AS new_quantity, o.price, o.status, o.created_at, o.updated_at + FROM orders o + INNER JOIN ( + SELECT user_id, asset_type, sum(quantity) AS avg_quantity + FROM orders + GROUP BY user_id, asset_type + ) a ON o.user_id = a.user_id AND o.asset_type = a.asset_type +) AS joined_data ON orders.order_id = joined_data.order_id + WHEN MATCHED THEN + UPDATE SET orders.quantity = joined_data.new_quantity; +---- +10 + +query TTTT +SELECT SUM(quantity) AS total_quantity, + AVG(quantity) AS average_quantity, + MIN(quantity) AS min_quantity, + MAX(quantity) AS max_quantity +FROM orders; +---- +64.16764110 6.416764110000 1.97683658 19.29134884 + +statement ok +create table tb_01 (id int,c1 varchar,c2 datetime(0),c3 json) cluster by(c1,c2); + +statement ok +create table tmp_01 like tb_01; + +statement ok +insert into tmp_01 values(1,'abc',to_date('2023-11-29'),parse_json('{"a":1}')); + +query TT +merge into tb_01 as T using ( select * from tmp_01) as S on t.id = s.id when matched then update * when not matched then insert *; +---- +1 0 + +query TTT +select id,c1,to_date(c2),c3 from tb_01; +---- +1 abc 2023-11-29 {"a":1} + +## test #issue13932 +statement ok +create table null_target(a int not null,b text) cluster by(a,b); + +statement ok +create table null_source(a int not null,b text) cluster by(a,b); + +statement ok +insert into null_target values(1,'a1'); + +statement ok +insert into null_target values(2,'a2'); + +statement ok +insert into null_source values(1,'a3'); + +statement ok +insert into null_source values(3,'a4'); + +statement error 1006 +merge into null_target using null_source on null_target.a = null_source.a when matched then update * +when not matched then insert (b) values(null_source.b); + +statement ok +delete from null_source where a = 3; + +query TT +merge into null_target using null_source on null_target.a = null_source.a when matched then update * +when not matched then insert (b) values(null_source.b); +---- +0 1 + +query TT +select * from null_target order by a,b; +---- +1 a3 +2 a2 + +## issue#13972 +statement ok +create table tt1_(a bool not null, b int not null, c int not null); + +statement ok +insert into tt1_ values(true, 10, 11),(false, 20, 21); + +query TT +MERGE INTO tt1_ USING + (SELECT + 657 AS cc0, + 658 AS cc1 + ) AS tRIA7K(cc0, cc1) ON ( + cc0 < cc1) + WHEN MATCHED AND FALSE THEN UPDATE SET + a = FALSE, + b = 332366211 + WHEN MATCHED AND a THEN DELETE + WHEN NOT MATCHED AND TRUE THEN INSERT (b, c) VALUES(10, 20); +---- +0 0 1 + +query TTT +select * from tt1_; +---- +0 20 21 + +## issue#14474 +statement ok +create table target_tt1 (a bool not null default true, b int not null default 1); + +query T +merge into target_tt1 using(select false, 10) as tt2(a, b) on (target_tt1.b = 1) when not matched then insert (b) values (20); +---- +1 + +query TT +select * from target_tt1; +---- +1 20 + +## test multi insert clauses with specified default values +statement ok +create table target_default_values(a int default 12,b string default 'yes'); + +statement ok +create table source_default_values(a int default 12,b string default 'yes'); + +statement ok +insert into target_default_values values(1,'a'); + +statement ok +insert into target_default_values values(2,'b'); + +statement ok +insert into source_default_values values(1,'c'); + +statement ok +insert into source_default_values values(3,'d'); + +statement ok +insert into source_default_values values(2,'e'); + +statement ok +insert into source_default_values values(4,'f'); + +query TTT +merge into target_default_values as t1 using source_default_values as t2 on t1.a = t2.a when matched and t1.b = 'a' +then update set t1.b = t2.b when matched then delete when not matched and t2.b = 'd' then insert (a) values(t2.a) when not matched +then insert(b) values(t2.b); +---- +2 1 1 + +query TT +select * from target_default_values order by a,b; +---- +1 c +3 yes +12 f + + +## test update column only optimization +statement ok +drop table if exists column_only_optimization_target; + +statement ok +drop table if exists column_only_optimization_source; + +statement ok +create table column_only_optimization_target(a int,b string); + +statement ok +create table column_only_optimization_source(a int,b string); + +statement ok +insert into column_only_optimization_target values(1,'a1'),(2,'a2'); + +statement ok +insert into column_only_optimization_target values(3,'a3'),(4,'a4'); + +statement ok +insert into column_only_optimization_target values(5,'a5'),(6,'a6'); + +statement ok +insert into column_only_optimization_target values(7,'a7'),(8,'a8'); + +query TT +select * from column_only_optimization_target order by a,b; +---- +1 a1 +2 a2 +3 a3 +4 a4 +5 a5 +6 a6 +7 a7 +8 a8 + +statement ok +insert into column_only_optimization_source values(1,'b1'),(2,'b2'); + +statement ok +insert into column_only_optimization_source values(3,'b3'),(4,'b4'); + +query TT +select * from column_only_optimization_source order by a,b; +---- +1 b1 +2 b2 +3 b3 +4 b4 + +query TT +merge into column_only_optimization_target as t1 using column_only_optimization_source as t2 on +t1.a = t2.a when matched then update set t1.b = t2.b when not matched then insert *; +---- +0 4 + +query TT +select * from column_only_optimization_target order by a,b; +---- +1 b1 +2 b2 +3 b3 +4 b4 +5 a5 +6 a6 +7 a7 +8 a8 + +## add more tests cases for distributed modes. +statement ok +CREATE TABLE IF NOT EXISTS lineitem_target_origin_200_blocks1 ( + l_orderkey BIGINT not null, + l_partkey BIGINT not null, + l_suppkey BIGINT not null, + l_linenumber BIGINT not null, + l_quantity DECIMAL(15, 2) not null, + l_extendedprice DECIMAL(15, 2) not null, + l_discount DECIMAL(15, 2) not null, + l_tax DECIMAL(15, 2) not null, + l_returnflag STRING not null, + l_linestatus STRING not null, + l_shipdate DATE not null, + l_commitdate DATE not null, + l_receiptdate DATE not null, + l_shipinstruct STRING not null, + l_shipmode STRING not null, + l_comment STRING not null +) CLUSTER BY(l_shipdate, l_orderkey); + +statement ok +CREATE TABLE IF NOT EXISTS lineitem_target_origin_400_blocks1 ( + l_orderkey BIGINT not null, + l_partkey BIGINT not null, + l_suppkey BIGINT not null, + l_linenumber BIGINT not null, + l_quantity DECIMAL(15, 2) not null, + l_extendedprice DECIMAL(15, 2) not null, + l_discount DECIMAL(15, 2) not null, + l_tax DECIMAL(15, 2) not null, + l_returnflag STRING not null, + l_linestatus STRING not null, + l_shipdate DATE not null, + l_commitdate DATE not null, + l_receiptdate DATE not null, + l_shipinstruct STRING not null, + l_shipmode STRING not null, + l_comment STRING not null +) CLUSTER BY(l_shipdate, l_orderkey); + +statement ok +CREATE TABLE IF NOT EXISTS lineitem_random( + l_orderkey BIGINT not null, + l_partkey BIGINT not null, + l_suppkey BIGINT not null, + l_linenumber BIGINT not null, + l_quantity DECIMAL(15, 2) not null, + l_extendedprice DECIMAL(15, 2) not null, + l_discount DECIMAL(15, 2) not null, + l_tax DECIMAL(15, 2) not null, + l_returnflag STRING not null, + l_linestatus STRING not null, + l_shipdate DATE not null, + l_commitdate DATE not null, + l_receiptdate DATE not null, + l_shipinstruct STRING not null, + l_shipmode STRING not null, + l_comment STRING not null +) engine = random; + +## add 4w rows +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +query T +select count(*) from lineitem_target_origin_400_blocks1; +---- +40000 + +statement ok +insert into lineitem_target_origin_200_blocks1 select * from lineitem_target_origin_400_blocks1; + +query T +select count(*) from lineitem_target_origin_200_blocks1; +---- +40000 + +statement ok +insert into lineitem_target_origin_400_blocks1 select * from lineitem_random limit 5000; + +query T +select count(*) from lineitem_target_origin_400_blocks1; +---- +45000 + +## it maybe flaky test, but in most times, it's normal. +query TT +MERGE INTO lineitem_target_origin_400_blocks1 as t1 using lineitem_target_origin_200_blocks1 as t2 on +t1.l_orderkey = t2.l_orderkey and +t1.l_partkey = t2.l_partkey +and t1.l_suppkey = t2.l_suppkey and +t1.l_linenumber = t2.l_linenumber and +t1.l_quantity = t2.l_quantity and +t1.l_extendedprice = t2.l_extendedprice and +t1.l_discount = t2.l_discount +when matched then update * +when not matched then insert *; +---- +0 40000 + +statement ok +set enable_experimental_merge_into = 0; statement ok set enable_distributed_merge_into = 0; diff --git a/tests/suites/1_stateful/02_query/02_0000_kill_query.py b/tests/suites/1_stateful/02_query/02_0000_kill_query.py index f3dc069f6223..590a8821f1cc 100755 --- a/tests/suites/1_stateful/02_query/02_0000_kill_query.py +++ b/tests/suites/1_stateful/02_query/02_0000_kill_query.py @@ -37,7 +37,7 @@ res = mycursor.fetchone() kill_query = "kill query " + str(res[0]) + ";" mycursor.execute(kill_query) - time.sleep(0.5) + time.sleep(1) mycursor.execute( "SELECT * FROM system.processes WHERE extra_info LIKE '%SELECT max(number)%' AND extra_info NOT LIKE '%system.processes%';" ) From cdb90dcc90d57d9a2f577436d65fa3effeffef27 Mon Sep 17 00:00:00 2001 From: Bohu Date: Tue, 12 Nov 2024 09:31:25 +0800 Subject: [PATCH 17/92] docs: add stream to readme (#16810) README: add stream doc --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index dcf2eb65b1c8..32ea05cd7583 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ - **Data Simplification**: Streamlines data ingestion, no external ETL needed. 👉 [Data Loading](https://docs.databend.com/guides/load-data/). +- **Real-Time CDC**: Supports real-time incremental data updates to keep data current and accurate. 👉 [Stream](https://docs.databend.com/guides/load-data/continuous-data-pipelines/stream) + - **Format Flexibility**: Supports multiple data formats and types, including JSON, CSV, Parquet, GEO, and more. - **ACID Transactions**: Ensures data integrity with atomic, consistent, isolated, and durable operations. From 4728f446b811a8fc192bb6114f12f41bc654333d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Tue, 12 Nov 2024 09:56:16 +0800 Subject: [PATCH 18/92] refactor: when upgrading, do not import already purged garbage log entries (#16809) Otherwise non-consecutive log indexes, such as `[5..100], [200..]` will panic the upgrading sub-routine. --- .../raft-store/src/ondisk/upgrade_to_v004.rs | 67 +++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs b/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs index e036892b8554..6d978d119dc1 100644 --- a/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs +++ b/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs @@ -23,9 +23,12 @@ use databend_common_meta_sled_store::init_get_sled_db; use databend_common_meta_sled_store::SledTree; use databend_common_meta_stoerr::MetaStorageError; use fs_extra::dir::CopyOptions; +use log::debug; +use openraft::LogIdOptionExt; use raft_log::codeq::error_context_ext::ErrorContextExt; use tokio::io; +use crate::key_spaces::LogMeta; use crate::key_spaces::RaftStoreEntry; use crate::ondisk::DataVersion; use crate::ondisk::OnDisk; @@ -33,6 +36,7 @@ use crate::raft_log_v004::importer; use crate::raft_log_v004::RaftLogV004; use crate::sm_v003::SnapshotStoreV003; use crate::sm_v003::SnapshotStoreV004; +use crate::state_machine::LogMetaKey; impl OnDisk { /// Upgrade the on-disk data form [`DataVersion::V003`] to [`DataVersion::V004`]. @@ -57,14 +61,69 @@ impl OnDisk { let db = init_get_sled_db(self.config.raft_dir.clone(), self.config.sled_cache_size()); - let tree_names = ["raft_state", "raft_log"]; + // Read the purged index + let first_log_index = { + let tree = SledTree::open(&db, "raft_log", self.config.is_sync())?; + let ks = tree.key_space::(); + let purged = ks.get(&LogMetaKey::LastPurged).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!( + "{}; when(get last purged index from sled db for upgrading V003 to V004)", + e + ), + ) + })?; + + purged.map(|v| v.log_id()).next_index() + }; + + // import logs + { + let tree = SledTree::open(&db, "raft_log", self.config.is_sync())?; + let it = tree.tree.iter(); + + for (i, rkv) in it.enumerate() { + let (k, v) = rkv.map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!( + "{}; when(iterating raft log in sled db for upgrading V003 to V004)", + e + ), + ) + })?; + + let ent = RaftStoreEntry::deserialize(&k, &v)?; + let upgraded = ent.upgrade(); + if let RaftStoreEntry::LogEntry(entry) = &upgraded { + if entry.log_id.index < first_log_index { + debug!( + "skip already purged log: {} when:(import V003 log to V004)", + entry.log_id + ); + continue; + } + } + + debug!("import upgraded V003 entry: {:?}", upgraded); + importer.import_raft_store_entry(upgraded)?; + + if i % 5000 == 0 { + self.progress(format_args!(" Imported {} logs", i)); + } + } + } - for tree_name in tree_names.iter() { - let tree = SledTree::open(&db, tree_name, self.config.is_sync())?; + // import raft_state + { + let tree = SledTree::open(&db, "raft_state", self.config.is_sync())?; let kvs = tree.export()?; for kv in kvs { let ent = RaftStoreEntry::deserialize(&kv[0], &kv[1])?; - importer.import_raft_store_entry(ent.upgrade())?; + debug!("import V003 entry: {:?}", ent); + let upgraded = ent.upgrade(); + importer.import_raft_store_entry(upgraded)?; } } From 6175b1a08ccd1f1e4406f13af4ea1f33735b9949 Mon Sep 17 00:00:00 2001 From: TCeason <33082201+TCeason@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:33:24 +0800 Subject: [PATCH 19/92] feat(query): expand trim/ltrim/rtrim (#16802) * feat(query): expand trim/ltrim/rtrim * fmt code --- src/query/ast/src/parser/expr.rs | 18 ++- src/query/functions/src/scalars/string.rs | 30 +++++ .../it/scalars/testdata/function_list.txt | 4 + .../base/20+_others/20_0001_planner.test | 98 ---------------- .../02_0055_function_strings_trim.test | 108 ++++++++++++++++++ 5 files changed, 156 insertions(+), 102 deletions(-) diff --git a/src/query/ast/src/parser/expr.rs b/src/query/ast/src/parser/expr.rs index 599748fcd497..58fdb9cc7780 100644 --- a/src/query/ast/src/parser/expr.rs +++ b/src/query/ast/src/parser/expr.rs @@ -961,12 +961,22 @@ pub fn expr_element(i: Input) -> IResult> { rule! { TRIM ~ "(" - ~ #subexpr(0) + ~ #subexpr(0) ~ ("," ~ #subexpr(0))? ~ ^")" }, - |(_, _, expr, _)| ExprElement::Trim { - expr: Box::new(expr), - trim_where: None, + |(_, _, expr, trim_str, _)| { + if let Some(trim_str) = trim_str { + let trim_str = trim_str.1; + ExprElement::Trim { + expr: Box::new(expr), + trim_where: Some((TrimWhere::Both, Box::new(trim_str))), + } + } else { + ExprElement::Trim { + expr: Box::new(expr), + trim_where: None, + } + } }, ); let trim_from = map( diff --git a/src/query/functions/src/scalars/string.rs b/src/query/functions/src/scalars/string.rs index 273a1d9dc9ed..ecf7cb11ac6d 100644 --- a/src/query/functions/src/scalars/string.rs +++ b/src/query/functions/src/scalars/string.rs @@ -430,6 +430,21 @@ pub fn register(registry: &mut FunctionRegistry) { }), ); + registry.register_passthrough_nullable_2_arg::( + "ltrim", + |_, _, _| FunctionDomain::Full, + vectorize_with_builder_2_arg::( + |val, trim_str, output, _| { + if trim_str.is_empty() { + output.put_and_commit(val); + return; + } + + output.put_and_commit(val.trim_start_matches(trim_str)); + }, + ), + ); + registry.register_passthrough_nullable_2_arg::( "trim_leading", |_, _, _| FunctionDomain::Full, @@ -445,6 +460,21 @@ pub fn register(registry: &mut FunctionRegistry) { ), ); + registry.register_passthrough_nullable_2_arg::( + "rtrim", + |_, _, _| FunctionDomain::Full, + vectorize_with_builder_2_arg::( + |val, trim_str, output, _| { + if trim_str.is_empty() { + output.put_and_commit(val); + return; + } + + output.put_and_commit(val.trim_end_matches(trim_str)); + }, + ), + ); + registry.register_passthrough_nullable_2_arg::( "trim_trailing", |_, _, _| FunctionDomain::Full, diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index 7517b17c9233..ec521332ed50 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -2534,6 +2534,8 @@ Functions overloads: 35 lte FACTORY 0 ltrim(String) :: String 1 ltrim(String NULL) :: String NULL +2 ltrim(String, String) :: String +3 ltrim(String NULL, String NULL) :: String NULL 0 map(Array(Nothing), Array(Nothing)) :: Map(Nothing) 1 map(Array(Nothing) NULL, Array(Nothing) NULL) :: Map(Nothing) NULL 2 map(Array(T0), Array(T1)) :: Map(T0, T1) @@ -3532,6 +3534,8 @@ Functions overloads: 1 rpad(String NULL, UInt64 NULL, String NULL) :: String NULL 0 rtrim(String) :: String 1 rtrim(String NULL) :: String NULL +2 rtrim(String, String) :: String +3 rtrim(String NULL, String NULL) :: String NULL 0 running_difference(Int64) :: Int64 1 running_difference(Int64 NULL) :: Int64 NULL 2 running_difference(Date) :: Int32 diff --git a/tests/sqllogictests/suites/base/20+_others/20_0001_planner.test b/tests/sqllogictests/suites/base/20+_others/20_0001_planner.test index 35adfda51ee6..4b4244aeeb9c 100644 --- a/tests/sqllogictests/suites/base/20+_others/20_0001_planner.test +++ b/tests/sqllogictests/suites/base/20+_others/20_0001_planner.test @@ -779,104 +779,6 @@ SELECT DISTINCT count(number %3) as c FROM numbers(10) group by number % 3 ORDE 3 4 -query T -select trim(leading ' ' from ' abc') ----- -abc - - -statement ok -select trim(leading ' ' from '') - - -statement ok -select trim(leading 'ab' from 'abab') - - -query T -select trim(leading 'ab' from 'abc') ----- -c - - -query T -select trim(leading ' ' from NULL) ----- -NULL - - -query T -select trim(leading NULL from 'aaa') ----- -NULL - - -query T -select trim(trailing ' ' from 'abc ') ----- -abc - - -statement ok -select trim(trailing ' ' from '') - - -statement ok -select trim(trailing 'ab' from 'abab') - - -query T -select trim(trailing 'ab' from 'cab') ----- -c - - -query T -select trim(trailing ' ' from NULL) ----- -NULL - - -query T -select trim(trailing NULL from 'aaa') ----- -NULL - - -statement ok -select trim(both 'ab' from 'abab') - - -query T -select trim(both 'ab' from 'abcab') ----- -c - - -query T -select trim(both ' ' from NULL) ----- -NULL - - -query T -select trim(both NULL from 'aaa') ----- -NULL - - -query T -select trim(' abc ') ----- -abc - - -query T -select trim(NULL) ----- -NULL - - query I select [1, 2, 3] ---- diff --git a/tests/sqllogictests/suites/query/functions/02_0055_function_strings_trim.test b/tests/sqllogictests/suites/query/functions/02_0055_function_strings_trim.test index 9ffc2b20c32c..5efdde864a29 100644 --- a/tests/sqllogictests/suites/query/functions/02_0055_function_strings_trim.test +++ b/tests/sqllogictests/suites/query/functions/02_0055_function_strings_trim.test @@ -31,12 +31,120 @@ select trim_leading('aaabbaaa', '') ---- aaabbaaa +query T +select ltrim('aaabbaaa', 'aa') +---- +abbaaa + query T select trim_trailing('aaabbaaa', 'aa') ---- aaabba +query T +select rtrim('aaabbaaa', 'aa') +---- +aaabba + query T select trim_trailing('aaabbaaa', '') ---- aaabbaaa + +query T +select trim(leading ' ' from ' abc') +---- +abc + + +query T +select trim(leading ' ' from '') +---- +(empty) + +statement ok +select trim(leading 'ab' from 'abab') + + +query T +select trim(leading 'ab' from 'abc') +---- +c + + +query T +select trim(leading ' ' from NULL) +---- +NULL + + +query T +select trim(leading NULL from 'aaa') +---- +NULL + + +query T +select trim(trailing ' ' from 'abc ') +---- +abc + + +query T +select trim(trailing ' ' from '') +---- +(empty) + +query T +select trim(trailing 'ab' from 'abab') +---- +(empty) + +query T +select trim(trailing 'ab' from 'cab') +---- +c + + +query T +select trim(trailing ' ' from NULL) +---- +NULL + + +query T +select trim(trailing NULL from 'aaa') +---- +NULL + + +query T +select trim(both 'ab' from 'abab') +---- +(empty) + +query T +select trim(both 'ab' from 'abcab') +---- +c + +query T +select trim('abcab', 'ab') +---- +c + +query T +select trim(both ' ' from NULL) +---- +NULL + + +query T +select trim(both NULL from 'aaa') +---- +NULL + +query T +select trim(NULL) +---- +NULL From 9d3cd15bd76996513e1c8fcdfbef2f5d720cebf8 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Tue, 12 Nov 2024 14:56:18 +0800 Subject: [PATCH 20/92] chore(query): embedded public key of license (#16813) * chore(query): embedded public key of license key * chore(query): embedded public key of license key --- Cargo.lock | 2 + src/common/building/Cargo.toml | 1 + src/common/building/src/lib.rs | 7 +++ src/query/ee/Cargo.toml | 1 + src/query/ee/src/license/license_mgr.rs | 76 ++++++++++++++++++++----- 5 files changed, 73 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b737905655a..c601c0a1c107 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3171,6 +3171,7 @@ name = "databend-common-building" version = "0.1.0" dependencies = [ "anyhow", + "base64 0.22.1", "cargo-license", "cargo_metadata", "gix", @@ -4846,6 +4847,7 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-s3", + "base64 0.22.1", "chrono", "chrono-tz 0.8.6", "dashmap 6.1.0", diff --git a/src/common/building/Cargo.toml b/src/common/building/Cargo.toml index 0631dcd9c358..8b51da396998 100644 --- a/src/common/building/Cargo.toml +++ b/src/common/building/Cargo.toml @@ -12,6 +12,7 @@ test = true [dependencies] anyhow = { workspace = true } +base64 = { workspace = true } cargo-license = { workspace = true } cargo_metadata = { workspace = true } gix = { workspace = true } diff --git a/src/common/building/src/lib.rs b/src/common/building/src/lib.rs index 21a7459e8e98..070381a7873c 100644 --- a/src/common/building/src/lib.rs +++ b/src/common/building/src/lib.rs @@ -58,6 +58,7 @@ pub fn add_building_env_vars() { add_target_features(); add_env_version(); add_env_license(); + add_license_public_key(); } pub fn set_env_config() { @@ -88,6 +89,12 @@ fn discover_version() -> Result { } } +pub fn add_license_public_key() { + let v = env::var("DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY").unwrap_or_default(); + let v = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, v.as_bytes()); + println!("cargo:rustc-env=DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY={v}"); +} + pub fn add_env_license() { let v = env::var("DATABEND_ENTERPRISE_LICENSE_EMBEDDED").unwrap_or_default(); println!("cargo:rustc-env=DATABEND_ENTERPRISE_LICENSE_EMBEDDED={v}"); diff --git a/src/query/ee/Cargo.toml b/src/query/ee/Cargo.toml index 29824d20420f..def5aeb0bc62 100644 --- a/src/query/ee/Cargo.toml +++ b/src/query/ee/Cargo.toml @@ -14,6 +14,7 @@ test = true [dependencies] async-backtrace = { workspace = true } async-trait = { workspace = true } +base64 = { workspace = true } chrono = { workspace = true } chrono-tz = { workspace = true } dashmap = { workspace = true } diff --git a/src/query/ee/src/license/license_mgr.rs b/src/query/ee/src/license/license_mgr.rs index 6d4176687eed..c22defff1922 100644 --- a/src/query/ee/src/license/license_mgr.rs +++ b/src/query/ee/src/license/license_mgr.rs @@ -37,7 +37,7 @@ pWjW3wxSdeARerxs/BeoWK7FspDtfLaAT8iJe4YEmR0JpkRQ8foWs0ve3w== pub struct RealLicenseManager { tenant: String, - public_key: String, + public_keys: Vec, // cache available settings to get avoid of unneeded license parsing time. pub(crate) cache: DashMap>, @@ -45,10 +45,24 @@ pub struct RealLicenseManager { impl LicenseManager for RealLicenseManager { fn init(tenant: String) -> Result<()> { + let public_key_str = embedded_public_keys()?; + + let mut public_keys = Vec::new(); + let mut remain_str = public_key_str.as_str(); + + let len = "-----END PUBLIC KEY-----".len(); + while let Some(r_pos) = remain_str.find("-----END PUBLIC KEY-----") { + let key_str = &remain_str[..r_pos + len]; + public_keys.push(key_str.to_string()); + remain_str = remain_str[r_pos + len..].trim(); + } + + public_keys.push(LICENSE_PUBLIC_KEY.to_string()); + let rm = RealLicenseManager { tenant, + public_keys, cache: DashMap::new(), - public_key: LICENSE_PUBLIC_KEY.to_string(), }; GlobalInstance::set(Arc::new(LicenseManagerSwitch::create(Box::new(rm)))); @@ -85,17 +99,25 @@ impl LicenseManager for RealLicenseManager { } fn parse_license(&self, raw: &str) -> Result> { - let public_key = ES256PublicKey::from_pem(self.public_key.as_str()) - .map_err_to_code(ErrorCode::LicenseKeyParseError, || "public key load failed")?; - match public_key.verify_token::(raw, None) { - Ok(v) => Ok(v), - Err(cause) => match cause.downcast_ref::() { - Some(JWTError::TokenHasExpired) => { - Err(ErrorCode::LicenseKeyExpired("license key is expired.")) - } - _ => Err(ErrorCode::LicenseKeyParseError("jwt claim decode failed")), - }, + for public_key in &self.public_keys { + let public_key = ES256PublicKey::from_pem(public_key) + .map_err_to_code(ErrorCode::LicenseKeyParseError, || "public key load failed")?; + + return match public_key.verify_token::(raw, None) { + Ok(v) => Ok(v), + Err(cause) => match cause.downcast_ref::() { + Some(JWTError::TokenHasExpired) => { + Err(ErrorCode::LicenseKeyExpired("license key is expired.")) + } + Some(JWTError::InvalidSignature) => { + continue; + } + _ => Err(ErrorCode::LicenseKeyParseError("jwt claim decode failed")), + }, + }; } + + Err(ErrorCode::LicenseKeyParseError("wt claim decode failed")) } fn get_storage_quota(&self, license_key: String) -> Result { @@ -136,9 +158,8 @@ impl RealLicenseManager { pub fn new(tenant: String, public_key: String) -> Self { RealLicenseManager { tenant, - public_key, - cache: DashMap::new(), + public_keys: vec![public_key], } } @@ -184,3 +205,30 @@ impl RealLicenseManager { ) } } + +fn embedded_public_keys() -> Result { + let pub_key = env!("DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY").to_string(); + + if pub_key.is_empty() { + return Ok(pub_key); + } + + let decode_res = base64::Engine::decode( + &base64::engine::general_purpose::STANDARD, + pub_key.as_bytes(), + ); + + match decode_res { + Err(e) => Err(ErrorCode::Internal(format!( + "Cannot parse embedded public key {:?}", + e + ))), + Ok(bytes) => match String::from_utf8(bytes) { + Err(e) => Err(ErrorCode::Internal(format!( + "Cannot parse embedded public key {:?}", + e + ))), + Ok(keys) => Ok(keys), + }, + } +} From a34da6f32d9f4223bc9094390fbd4c8f4b2be0f0 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Tue, 12 Nov 2024 15:43:09 +0800 Subject: [PATCH 21/92] feat: support pushdown predicate into iceberg engine (#16650) * feat: support pushdown predicate into iceberg engine * feat: support pushdown predicate into iceberg engine * fix tests * fix tests --- src/query/expression/src/expression.rs | 2 +- src/query/expression/src/type_check.rs | 25 +- .../src/catalogs/default/immutable_catalog.rs | 5 + .../sql/src/planner/semantic/type_check.rs | 14 +- src/query/storages/iceberg/src/catalog.rs | 4 + src/query/storages/iceberg/src/lib.rs | 1 + src/query/storages/iceberg/src/predicate.rs | 238 ++++++++++++++++++ src/query/storages/iceberg/src/table.rs | 19 +- .../base/03_common/03_0005_select_filter.test | 13 + .../suites/tpch_iceberg/prune.test | 141 +++++++++++ 10 files changed, 441 insertions(+), 21 deletions(-) create mode 100644 src/query/storages/iceberg/src/predicate.rs create mode 100644 tests/sqllogictests/suites/tpch_iceberg/prune.test diff --git a/src/query/expression/src/expression.rs b/src/query/expression/src/expression.rs index 92c84c7ac540..ef32aeba4a17 100644 --- a/src/query/expression/src/expression.rs +++ b/src/query/expression/src/expression.rs @@ -290,7 +290,7 @@ impl PartialEq for Expr { /// /// The remote node will recover the `Arc` pointer within `FunctionCall` by looking /// up the function registry with the `FunctionID`. -#[derive(Debug, Clone, Educe, Serialize, Deserialize)] +#[derive(Debug, Clone, Educe, Serialize, Deserialize, EnumAsInner)] #[educe(PartialEq, Eq, Hash)] pub enum RemoteExpr { Constant { diff --git a/src/query/expression/src/type_check.rs b/src/query/expression/src/type_check.rs index 132189349ba6..a7c2fca9f80c 100755 --- a/src/query/expression/src/type_check.rs +++ b/src/query/expression/src/type_check.rs @@ -75,7 +75,7 @@ pub fn check( args, params, } => { - let args_expr: Vec<_> = args + let mut args_expr: Vec<_> = args .iter() .map(|arg| check(arg, fn_registry)) .try_collect()?; @@ -84,41 +84,32 @@ pub fn check( // c:int16 = 12456 will be resolve as `to_int32(c) == to_int32(12456)` // This may hurt the bloom filter, we should try cast to literal as the datatype of column if name == "eq" && args_expr.len() == 2 { - match args_expr.as_slice() { + match args_expr.as_mut_slice() { [ e, Expr::Constant { span, scalar, - data_type: src_ty, + data_type, }, ] | [ Expr::Constant { span, scalar, - data_type: src_ty, + data_type, }, e, ] => { - let src_ty = src_ty.remove_nullable(); + let src_ty = data_type.remove_nullable(); let dest_ty = e.data_type().remove_nullable(); if dest_ty.is_integer() && src_ty.is_integer() { - if let Ok(scalar) = + if let Ok(casted_scalar) = cast_scalar(*span, scalar.clone(), dest_ty, fn_registry) { - return check_function( - *span, - name, - params, - &[e.clone(), Expr::Constant { - span: *span, - data_type: scalar.as_ref().infer_data_type(), - scalar, - }], - fn_registry, - ); + *scalar = casted_scalar; + *data_type = scalar.as_ref().infer_data_type(); } } } diff --git a/src/query/service/src/catalogs/default/immutable_catalog.rs b/src/query/service/src/catalogs/default/immutable_catalog.rs index e2a37915e0cf..cfb88afa8ad8 100644 --- a/src/query/service/src/catalogs/default/immutable_catalog.rs +++ b/src/query/service/src/catalogs/default/immutable_catalog.rs @@ -153,6 +153,11 @@ impl Catalog for ImmutableCatalog { CatalogInfo::default().into() } + fn disable_table_info_refresh(self: Arc) -> Result> { + let me = self.as_ref().clone(); + Ok(Arc::new(me)) + } + #[async_backtrace::framed] async fn get_database(&self, _tenant: &Tenant, db_name: &str) -> Result> { match db_name { diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index fdb7355a241e..5edd35bd756d 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -2761,13 +2761,16 @@ impl<'a> TypeChecker<'a> { params: params.clone(), args: arguments, }; + let expr = type_check::check(&raw_expr, &BUILTIN_FUNCTIONS)?; // Run constant folding for arguments of the scalar function. // This will be helpful to simplify some constant expressions, especially // the implicitly casted literal values, e.g. `timestamp > '2001-01-01'` // will be folded from `timestamp > to_timestamp('2001-01-01')` to `timestamp > 978307200000000` - let folded_args = match &expr { + // Note: check function may reorder the args + + let mut folded_args = match &expr { databend_common_expression::Expr::FunctionCall { args: checked_args, .. } => { @@ -2795,6 +2798,15 @@ impl<'a> TypeChecker<'a> { return Ok(constant); } + // reorder + if func_name == "eq" + && folded_args.len() == 2 + && matches!(folded_args[0], ScalarExpr::ConstantExpr(_)) + && !matches!(folded_args[1], ScalarExpr::ConstantExpr(_)) + { + folded_args.swap(0, 1); + } + Ok(Box::new(( FunctionCall { span, diff --git a/src/query/storages/iceberg/src/catalog.rs b/src/query/storages/iceberg/src/catalog.rs index d627c95f4cc2..92ef717b8685 100644 --- a/src/query/storages/iceberg/src/catalog.rs +++ b/src/query/storages/iceberg/src/catalog.rs @@ -216,6 +216,10 @@ impl Catalog for IcebergCatalog { self.info.clone() } + fn disable_table_info_refresh(self: Arc) -> Result> { + Ok(self) + } + #[fastrace::trace] #[async_backtrace::framed] async fn get_database(&self, _tenant: &Tenant, db_name: &str) -> Result> { diff --git a/src/query/storages/iceberg/src/lib.rs b/src/query/storages/iceberg/src/lib.rs index c7b5a87908c6..8949eee3c52d 100644 --- a/src/query/storages/iceberg/src/lib.rs +++ b/src/query/storages/iceberg/src/lib.rs @@ -22,6 +22,7 @@ mod catalog; mod database; mod partition; +mod predicate; mod table; mod table_source; diff --git a/src/query/storages/iceberg/src/predicate.rs b/src/query/storages/iceberg/src/predicate.rs new file mode 100644 index 000000000000..11a69727710c --- /dev/null +++ b/src/query/storages/iceberg/src/predicate.rs @@ -0,0 +1,238 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::RemoteExpr; +use databend_common_expression::Scalar; +use iceberg::expr::Predicate; +use iceberg::expr::Reference; +use iceberg::spec::Datum; + +#[derive(Default, Copy, Clone, Debug)] +pub struct PredicateBuilder { + uncertain: bool, +} + +impl PredicateBuilder { + pub fn build(&mut self, expr: &RemoteExpr) -> Predicate { + match expr { + RemoteExpr::Constant { + span: _, + scalar, + data_type, + } if data_type.remove_nullable() == DataType::Boolean => { + let value = scalar.as_boolean(); + let is_true = value.copied().unwrap_or(false); + if is_true { + Predicate::AlwaysTrue + } else { + Predicate::AlwaysFalse + } + } + + // is_true + RemoteExpr::FunctionCall { + span: _, + id, + generics: _, + args, + return_type: _, + } if args.len() == 1 && id.name().as_ref() == "is_true" => { + let predicate = self.build(&args[0]); + if self.uncertain { + return Predicate::AlwaysTrue; + } + match predicate { + Predicate::AlwaysTrue => Predicate::AlwaysTrue, + Predicate::AlwaysFalse => Predicate::AlwaysFalse, + _ => predicate, + } + } + + // unary + RemoteExpr::FunctionCall { + span: _, + id, + generics: _, + args, + return_type: _, + } if args.len() == 1 && matches!(args[0], RemoteExpr::ColumnRef { .. }) => { + let (_, name, _, _) = args[0].as_column_ref().unwrap(); + let r = Reference::new(name); + if let Some(op) = build_unary(r, id.name().as_ref()) { + return op; + } + self.uncertain = true; + Predicate::AlwaysTrue + } + + // not + RemoteExpr::FunctionCall { + span: _, + id, + generics: _, + args, + return_type: _, + } if args.len() == 1 && id.name().as_ref() == "not" => { + let predicate = self.build(&args[0]); + if self.uncertain { + return Predicate::AlwaysTrue; + } + match predicate { + Predicate::AlwaysTrue => Predicate::AlwaysFalse, + Predicate::AlwaysFalse => Predicate::AlwaysTrue, + _ => predicate.negate(), + } + } + + // binary {a op datum} + RemoteExpr::FunctionCall { + span: _, + id, + generics: _, + args, + return_type: _, + } if args.len() == 2 && ["and", "and_filters", "or"].contains(&id.name().as_ref()) => { + let left = self.build(&args[0]); + let right = self.build(&args[1]); + if self.uncertain { + return Predicate::AlwaysTrue; + } + match id.name().as_ref() { + "and" | "and_filters" => left.and(right), + "or" => left.or(right), + _ => unreachable!(), + } + } + + // binary {a op datum} + RemoteExpr::FunctionCall { + span: _, + id, + generics: _, + args, + return_type: _, + } if args.len() == 2 + && matches!(args[0], RemoteExpr::ColumnRef { .. }) + && matches!(args[1], RemoteExpr::Constant { .. }) => + { + let val = args[1].as_constant().unwrap(); + let val = scalar_to_datatum(val.1); + if let Some(datum) = val { + let (_, name, _, _) = args[0].as_column_ref().unwrap(); + let r = Reference::new(name); + let p = build_binary(r, id.name().as_ref(), datum); + if let Some(op) = p { + return op; + } + } + self.uncertain = true; + Predicate::AlwaysTrue + } + + // binary {datum op a} + RemoteExpr::FunctionCall { + span: _, + id, + generics: _, + args, + return_type: _, + } if args.len() == 2 + && matches!(args[1], RemoteExpr::ColumnRef { .. }) + && matches!(args[0], RemoteExpr::Constant { .. }) => + { + let val = args[0].as_constant().unwrap(); + let val = scalar_to_datatum(val.1); + if let Some(datum) = val { + let (_, name, _, _) = args[1].as_column_ref().unwrap(); + let r = Reference::new(name); + let p = build_reverse_binary(r, id.name().as_ref(), datum); + if let Some(op) = p { + return op; + } + } + self.uncertain = true; + Predicate::AlwaysTrue + } + + _ => { + self.uncertain = true; + Predicate::AlwaysTrue + } + } + } +} + +fn build_unary(r: Reference, op: &str) -> Option { + let op = match op { + "is_null" => r.is_null(), + "is_not_null" => r.is_not_null(), + _ => return None, + }; + Some(op) +} + +// a op datum +fn build_binary(r: Reference, op: &str, datum: Datum) -> Option { + let op = match op { + "lt" | "<" => r.less_than(datum), + "le" | "<=" => r.less_than_or_equal_to(datum), + "gt" | ">" => r.greater_than(datum), + "ge" | ">=" => r.greater_than_or_equal_to(datum), + "eq" | "=" => r.equal_to(datum), + "ne" | "!=" => r.not_equal_to(datum), + _ => return None, + }; + Some(op) +} + +// datum op a to a op_v datum +fn build_reverse_binary(r: Reference, op: &str, datum: Datum) -> Option { + let op = match op { + "lt" | "<" => r.greater_than(datum), + "le" | "<=" => r.greater_than_or_equal_to(datum), + "gt" | ">" => r.less_than(datum), + "ge" | ">=" => r.less_than_or_equal_to(datum), + "eq" | "=" => r.equal_to(datum), + "ne" | "!=" => r.not_equal_to(datum), + _ => return None, + }; + Some(op) +} + +fn scalar_to_datatum(scalar: &Scalar) -> Option { + let val = match scalar { + Scalar::Number(n) => match n { + NumberScalar::Int8(i) => Datum::int(*i as i32), + NumberScalar::Int16(i) => Datum::int(*i as i32), + NumberScalar::Int32(i) => Datum::int(*i), + NumberScalar::Int64(i) => Datum::long(*i), + NumberScalar::UInt8(i) => Datum::int(*i as i32), + NumberScalar::UInt16(i) => Datum::int(*i as i32), + NumberScalar::UInt32(i) if *i <= i32::MAX as u32 => Datum::int(*i as i32), + NumberScalar::UInt64(i) if *i <= i64::MAX as u64 => Datum::long(*i as i64), /* Potential loss of precision */ + NumberScalar::Float32(f) => Datum::float(*f), + NumberScalar::Float64(f) => Datum::double(*f), + _ => return None, + }, + Scalar::Timestamp(ts) => Datum::timestamp_micros(*ts), + Scalar::Date(d) => Datum::date(*d), + Scalar::Boolean(b) => Datum::bool(*b), + Scalar::Binary(b) => Datum::binary(b.clone()), + Scalar::String(s) => Datum::string(s), + _ => return None, + }; + Some(val) +} diff --git a/src/query/storages/iceberg/src/table.rs b/src/query/storages/iceberg/src/table.rs index 12e512fcff15..eba5034fc5b7 100644 --- a/src/query/storages/iceberg/src/table.rs +++ b/src/query/storages/iceberg/src/table.rs @@ -27,6 +27,7 @@ use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PartitionsShuffleKind; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; +use databend_common_catalog::table::TableStatistics; use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; @@ -37,10 +38,12 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_pipeline_core::Pipeline; +use databend_storages_common_table_meta::table::ChangeType; use futures::TryStreamExt; use tokio::sync::OnceCell; use crate::partition::IcebergPartInfo; +use crate::predicate::PredicateBuilder; use crate::table_source::IcebergTableSource; use crate::IcebergCatalog; @@ -200,8 +203,10 @@ impl IcebergTable { .map(|v| v.name.clone()), ); } - // TODO: Implement filter based on iceberg-rust's scan builder. - // if let Some(filter) = &push_downs.filters {} + if let Some(filter) = &push_downs.filters { + let predicate = PredicateBuilder::default().build(&filter.filter); + scan = scan.with_filter(predicate) + } } let tasks: Vec<_> = scan @@ -251,6 +256,16 @@ impl Table for IcebergTable { &self.get_table_info().name } + // TODO load summary + async fn table_statistics( + &self, + _ctx: Arc, + _require_fresh: bool, + _change_type: Option, + ) -> Result> { + Ok(None) + } + #[async_backtrace::framed] async fn read_partitions( &self, diff --git a/tests/sqllogictests/suites/base/03_common/03_0005_select_filter.test b/tests/sqllogictests/suites/base/03_common/03_0005_select_filter.test index f5e640eea604..ecd3d84098b0 100644 --- a/tests/sqllogictests/suites/base/03_common/03_0005_select_filter.test +++ b/tests/sqllogictests/suites/base/03_common/03_0005_select_filter.test @@ -25,6 +25,19 @@ SELECT number as c1, (number+1) as c2 FROM numbers_mt (3) where number+1>1 1 2 2 3 +query II +SELECT 3 = number, number = 3 FROM numbers_mt (30) where 3 = number +---- +1 1 + +query II +SELECT 3 = number, number = 3 FROM numbers_mt (4) order by number +---- +0 0 +0 0 +0 0 +1 1 + query I SELECT count() FROM numbers_mt (10) where -1 ---- diff --git a/tests/sqllogictests/suites/tpch_iceberg/prune.test b/tests/sqllogictests/suites/tpch_iceberg/prune.test new file mode 100644 index 000000000000..f79f367d7145 --- /dev/null +++ b/tests/sqllogictests/suites/tpch_iceberg/prune.test @@ -0,0 +1,141 @@ +statement ok +DROP CATALOG IF EXISTS ctl; + +statement ok +CREATE CATALOG ctl +TYPE=ICEBERG +CONNECTION=( + TYPE='rest' + ADDRESS='http://127.0.0.1:8181' + WAREHOUSE='s3://iceberg-tpch' + "s3.region"='us-east-1' + "s3.endpoint"='http://127.0.0.1:9000' +); + +## note: the tests only cover standalone mode +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey < 1; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [is_true(lineitem.l_orderkey (#0) < 1)] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [is_true(lineitem.l_orderkey (#0) < 1)], limit: NONE] + └── estimated rows: 0.00 + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey < 1 or l_commitdate < '1992-01-31'; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [is_true((lineitem.l_orderkey (#0) < 1 OR lineitem.l_commitdate (#11) < '1992-01-31'))] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0), l_commitdate (#11)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [is_true((lineitem.l_orderkey (#0) < 1 OR lineitem.l_commitdate (#11) < '1992-01-31'))], limit: NONE] + └── estimated rows: 0.00 + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey < 1 and l_commitdate > '1992-01-31'; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [is_true(lineitem.l_orderkey (#0) < 1), is_true(lineitem.l_commitdate (#11) > '1992-01-31')] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0), l_commitdate (#11)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [and_filters(lineitem.l_orderkey (#0) < 1, lineitem.l_commitdate (#11) > '1992-01-31')], limit: NONE] + └── estimated rows: 0.00 + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey > 1 and l_commitdate = '1992-01-22'; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [is_true(lineitem.l_orderkey (#0) > 1), is_true(lineitem.l_commitdate (#11) = '1992-01-22')] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0), l_commitdate (#11)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [and_filters(lineitem.l_orderkey (#0) > 1, lineitem.l_commitdate (#11) = '1992-01-22')], limit: NONE] + └── estimated rows: 0.00 + + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey is null; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [NOT is_not_null(lineitem.l_orderkey (#0))] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [NOT is_not_null(lineitem.l_orderkey (#0))], limit: NONE] + └── estimated rows: 0.00 + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey is null or l_commitdate is not null; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [(NOT is_not_null(lineitem.l_orderkey (#0)) OR is_not_null(lineitem.l_commitdate (#11)))] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0), l_commitdate (#11)] + ├── read rows: 600572 + ├── read size: 14.41 MiB + ├── partitions total: 6 + ├── partitions scanned: 6 + ├── push downs: [filters: [(NOT is_not_null(lineitem.l_orderkey (#0)) OR is_not_null(lineitem.l_commitdate (#11)))], limit: NONE] + └── estimated rows: 0.00 From b9d4c60bb7d99bf021bc7f1ea914ae7127f8db80 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Tue, 12 Nov 2024 17:14:41 +0800 Subject: [PATCH 22/92] chore(query): refactor common/arrow removes arrow/io crates (#16808) * refactor native engine * update * update * [ci skip] update * [ci skip] update * remove ffi * remove ffi * remove ffi * fix init nested * fix init nested * fix stat --- Cargo.lock | 106 +- Cargo.toml | 10 - src/common/arrow/Cargo.toml | 63 +- .../arrow/src/arrow/array/binary/ffi.rs | 80 -- .../arrow/src/arrow/array/binary/mod.rs | 1 - .../arrow/src/arrow/array/binview/ffi.rs | 117 -- .../arrow/src/arrow/array/binview/mod.rs | 1 - .../arrow/src/arrow/array/boolean/ffi.rs | 70 - .../arrow/src/arrow/array/boolean/mod.rs | 2 +- .../arrow/src/arrow/array/dictionary/ffi.rs | 59 - .../arrow/src/arrow/array/dictionary/mod.rs | 2 +- src/common/arrow/src/arrow/array/ffi.rs | 103 -- .../src/arrow/array/fixed_size_binary/ffi.rs | 72 - .../src/arrow/array/fixed_size_binary/mod.rs | 2 +- .../src/arrow/array/fixed_size_list/ffi.rs | 55 - .../src/arrow/array/fixed_size_list/mod.rs | 2 +- src/common/arrow/src/arrow/array/list/ffi.rs | 84 -- src/common/arrow/src/arrow/array/list/mod.rs | 2 +- src/common/arrow/src/arrow/array/map/ffi.rs | 83 -- src/common/arrow/src/arrow/array/map/mod.rs | 2 +- src/common/arrow/src/arrow/array/mod.rs | 6 +- src/common/arrow/src/arrow/array/null.rs | 26 - .../arrow/src/arrow/array/primitive/ffi.rs | 72 - .../arrow/src/arrow/array/primitive/mod.rs | 2 +- .../arrow/src/arrow/array/struct_/ffi.rs | 88 -- .../arrow/src/arrow/array/struct_/mod.rs | 4 +- src/common/arrow/src/arrow/array/union/ffi.rs | 75 -- src/common/arrow/src/arrow/array/union/mod.rs | 2 +- src/common/arrow/src/arrow/array/utf8/ffi.rs | 79 -- src/common/arrow/src/arrow/array/utf8/mod.rs | 2 +- src/common/arrow/src/arrow/buffer/mod.rs | 4 - .../src/arrow/compute/aggregate/memory.rs | 141 -- .../arrow/src/arrow/compute/aggregate/mod.rs | 18 - src/common/arrow/src/arrow/compute/arity.rs | 296 ----- .../arrow/src/arrow/compute/cast/binary_to.rs | 178 --- .../src/arrow/compute/cast/boolean_to.rs | 63 - .../src/arrow/compute/cast/decimal_to.rs | 150 --- .../src/arrow/compute/cast/dictionary_to.rs | 198 --- .../arrow/src/arrow/compute/cast/mod.rs | 1030 --------------- .../src/arrow/compute/cast/primitive_to.rs | 607 --------- .../arrow/src/arrow/compute/cast/utf8_to.rs | 186 --- .../arrow/src/arrow/compute/merge_sort/mod.rs | 560 -------- src/common/arrow/src/arrow/compute/mod.rs | 16 - .../arrow/src/arrow/compute/sort/binary.rs | 31 - .../arrow/src/arrow/compute/sort/boolean.rs | 66 - .../arrow/src/arrow/compute/sort/common.rs | 182 --- .../arrow/src/arrow/compute/sort/lex_sort.rs | 250 ---- .../arrow/src/arrow/compute/sort/mod.rs | 389 ------ .../arrow/compute/sort/primitive/indices.rs | 233 ---- .../src/arrow/compute/sort/primitive/mod.rs | 20 - .../src/arrow/compute/sort/primitive/sort.rs | 237 ---- .../src/arrow/compute/sort/row/dictionary.rs | 118 -- .../arrow/src/arrow/compute/sort/row/fixed.rs | 194 --- .../src/arrow/compute/sort/row/interner.rs | 417 ------ .../arrow/src/arrow/compute/sort/row/mod.rs | 835 ------------ .../src/arrow/compute/sort/row/variable.rs | 116 -- .../arrow/src/arrow/compute/sort/utf8.rs | 59 - .../arrow/src/arrow/compute/take/binary.rs | 41 - .../arrow/src/arrow/compute/take/binview.rs | 39 - .../arrow/src/arrow/compute/take/boolean.rs | 156 --- .../arrow/src/arrow/compute/take/dict.rs | 41 - .../src/arrow/compute/take/fixed_size_list.rs | 63 - .../src/arrow/compute/take/generic_binary.rs | 174 --- .../arrow/src/arrow/compute/take/list.rs | 62 - .../arrow/src/arrow/compute/take/mod.rs | 155 --- .../arrow/src/arrow/compute/take/primitive.rs | 190 --- .../arrow/src/arrow/compute/take/structure.rs | 64 - .../arrow/src/arrow/compute/take/utf8.rs | 86 -- src/common/arrow/src/arrow/error.rs | 6 + src/common/arrow/src/arrow/ffi/array.rs | 618 --------- src/common/arrow/src/arrow/ffi/bridge.rs | 58 - src/common/arrow/src/arrow/ffi/generated.rs | 70 - src/common/arrow/src/arrow/ffi/mmap.rs | 182 --- src/common/arrow/src/arrow/ffi/mod.rs | 67 - src/common/arrow/src/arrow/ffi/schema.rs | 660 ---------- src/common/arrow/src/arrow/ffi/stream.rs | 245 ---- src/common/arrow/src/arrow/io/README.md | 24 - src/common/arrow/src/arrow/io/flight/mod.rs | 264 ---- .../arrow/src/arrow/io/ipc/append/mod.rs | 92 -- .../arrow/src/arrow/io/ipc/compression.rs | 106 -- .../arrow/src/arrow/io/ipc/endianness.rs | 26 - src/common/arrow/src/arrow/io/ipc/mod.rs | 122 -- .../src/arrow/io/ipc/read/array/binary.rs | 111 -- .../src/arrow/io/ipc/read/array/binview.rs | 102 -- .../src/arrow/io/ipc/read/array/boolean.rs | 92 -- .../src/arrow/io/ipc/read/array/dictionary.rs | 86 -- .../io/ipc/read/array/fixed_size_binary.rs | 96 -- .../io/ipc/read/array/fixed_size_list.rs | 107 -- .../arrow/src/arrow/io/ipc/read/array/list.rs | 133 -- .../arrow/src/arrow/io/ipc/read/array/map.rs | 128 -- .../arrow/src/arrow/io/ipc/read/array/mod.rs | 42 - .../arrow/src/arrow/io/ipc/read/array/null.rs | 45 - .../src/arrow/io/ipc/read/array/primitive.rs | 97 -- .../src/arrow/io/ipc/read/array/struct_.rs | 112 -- .../src/arrow/io/ipc/read/array/union.rs | 150 --- .../arrow/src/arrow/io/ipc/read/array/utf8.rs | 112 -- .../arrow/src/arrow/io/ipc/read/common.rs | 388 ------ .../src/arrow/io/ipc/read/deserialize.rs | 304 ----- .../arrow/src/arrow/io/ipc/read/error.rs | 127 -- .../arrow/src/arrow/io/ipc/read/file.rs | 330 ----- .../arrow/src/arrow/io/ipc/read/file_async.rs | 376 ------ src/common/arrow/src/arrow/io/ipc/read/mod.rs | 58 - .../arrow/src/arrow/io/ipc/read/read_basic.rs | 386 ------ .../arrow/src/arrow/io/ipc/read/reader.rs | 156 --- .../arrow/src/arrow/io/ipc/read/schema.rs | 446 ------- .../arrow/src/arrow/io/ipc/read/stream.rs | 335 ----- .../src/arrow/io/ipc/read/stream_async.rs | 259 ---- .../arrow/src/arrow/io/ipc/write/common.rs | 518 -------- .../src/arrow/io/ipc/write/common_async.rs | 83 -- .../src/arrow/io/ipc/write/common_sync.rs | 75 -- .../src/arrow/io/ipc/write/file_async.rs | 277 ---- .../arrow/src/arrow/io/ipc/write/mod.rs | 78 -- .../arrow/src/arrow/io/ipc/write/schema.rs | 357 ----- .../arrow/src/arrow/io/ipc/write/serialize.rs | 833 ------------ .../arrow/src/arrow/io/ipc/write/stream.rs | 134 -- .../src/arrow/io/ipc/write/stream_async.rs | 206 --- .../arrow/src/arrow/io/ipc/write/writer.rs | 232 ---- src/common/arrow/src/arrow/io/iterator.rs | 80 -- src/common/arrow/src/arrow/io/mod.rs | 30 - src/common/arrow/src/arrow/io/parquet/mod.rs | 46 - .../arrow/src/arrow/io/parquet/read/README.md | 34 - .../io/parquet/read/deserialize/README.md | 70 - .../parquet/read/deserialize/binary/basic.rs | 549 -------- .../read/deserialize/binary/dictionary.rs | 196 --- .../io/parquet/read/deserialize/binary/mod.rs | 25 - .../parquet/read/deserialize/binary/nested.rs | 217 --- .../parquet/read/deserialize/binary/utils.rs | 186 --- .../parquet/read/deserialize/binview/basic.rs | 292 ----- .../read/deserialize/binview/dictionary.rs | 181 --- .../parquet/read/deserialize/binview/mod.rs | 23 - .../read/deserialize/binview/nested.rs | 165 --- .../parquet/read/deserialize/boolean/basic.rs | 251 ---- .../parquet/read/deserialize/boolean/mod.rs | 21 - .../read/deserialize/boolean/nested.rs | 171 --- .../read/deserialize/dictionary/mod.rs | 337 ----- .../read/deserialize/dictionary/nested.rs | 234 ---- .../deserialize/fixed_size_binary/basic.rs | 347 ----- .../fixed_size_binary/dictionary.rs | 169 --- .../read/deserialize/fixed_size_binary/mod.rs | 24 - .../deserialize/fixed_size_binary/nested.rs | 213 --- .../deserialize/fixed_size_binary/utils.rs | 73 -- .../arrow/io/parquet/read/deserialize/mod.rs | 258 ---- .../io/parquet/read/deserialize/nested.rs | 620 --------- .../parquet/read/deserialize/nested_utils.rs | 586 --------- .../io/parquet/read/deserialize/null/mod.rs | 124 -- .../parquet/read/deserialize/null/nested.rs | 140 -- .../read/deserialize/primitive/basic.rs | 387 ------ .../read/deserialize/primitive/dictionary.rs | 208 --- .../read/deserialize/primitive/integer.rs | 280 ---- .../parquet/read/deserialize/primitive/mod.rs | 25 - .../read/deserialize/primitive/nested.rs | 259 ---- .../io/parquet/read/deserialize/simple.rs | 687 ---------- .../io/parquet/read/deserialize/struct_.rs | 76 -- .../io/parquet/read/deserialize/utils.rs | 584 --------- .../arrow/src/arrow/io/parquet/read/file.rs | 222 ---- .../arrow/io/parquet/read/indexes/binary.rs | 59 - .../arrow/io/parquet/read/indexes/boolean.rs | 36 - .../parquet/read/indexes/fixed_len_binary.rs | 88 -- .../src/arrow/io/parquet/read/indexes/mod.rs | 410 ------ .../io/parquet/read/indexes/primitive.rs | 243 ---- .../arrow/src/arrow/io/parquet/read/mod.rs | 133 -- .../src/arrow/io/parquet/read/row_group.rs | 374 ------ .../arrow/io/parquet/read/schema/convert.rs | 1129 ---------------- .../arrow/io/parquet/read/schema/metadata.rs | 72 - .../src/arrow/io/parquet/read/schema/mod.rs | 77 -- .../io/parquet/read/statistics/binary.rs | 41 - .../io/parquet/read/statistics/boolean.rs | 40 - .../io/parquet/read/statistics/dictionary.rs | 85 -- .../io/parquet/read/statistics/fixlen.rs | 164 --- .../arrow/io/parquet/read/statistics/list.rs | 100 -- .../arrow/io/parquet/read/statistics/map.rs | 82 -- .../arrow/io/parquet/read/statistics/mod.rs | 596 --------- .../arrow/io/parquet/read/statistics/null.rs | 26 - .../io/parquet/read/statistics/primitive.rs | 72 - .../io/parquet/read/statistics/struct_.rs | 81 -- .../arrow/io/parquet/read/statistics/utf8.rs | 48 - .../arrow/io/parquet/write/binary/basic.rs | 190 --- .../src/arrow/io/parquet/write/binary/mod.rs | 24 - .../arrow/io/parquet/write/binary/nested.rs | 67 - .../arrow/io/parquet/write/binview/basic.rs | 140 -- .../src/arrow/io/parquet/write/binview/mod.rs | 22 - .../arrow/io/parquet/write/binview/nested.rs | 60 - .../arrow/io/parquet/write/boolean/basic.rs | 109 -- .../src/arrow/io/parquet/write/boolean/mod.rs | 20 - .../arrow/io/parquet/write/boolean/nested.rs | 63 - .../src/arrow/io/parquet/write/dictionary.rs | 321 ----- .../arrow/src/arrow/io/parquet/write/file.rs | 115 -- .../arrow/io/parquet/write/fixed_len_bytes.rs | 197 --- .../arrow/src/arrow/io/parquet/write/mod.rs | 951 -------------- .../src/arrow/io/parquet/write/nested/def.rs | 584 --------- .../src/arrow/io/parquet/write/nested/mod.rs | 133 -- .../src/arrow/io/parquet/write/nested/rep.rs | 376 ------ .../arrow/src/arrow/io/parquet/write/pages.rs | 653 ---------- .../arrow/io/parquet/write/primitive/basic.rs | 210 --- .../arrow/io/parquet/write/primitive/mod.rs | 23 - .../io/parquet/write/primitive/nested.rs | 75 -- .../src/arrow/io/parquet/write/row_group.rs | 146 --- .../src/arrow/io/parquet/write/schema.rs | 407 ------ .../arrow/src/arrow/io/parquet/write/sink.rs | 256 ---- .../src/arrow/io/parquet/write/utf8/basic.rs | 139 -- .../src/arrow/io/parquet/write/utf8/mod.rs | 22 - .../src/arrow/io/parquet/write/utf8/nested.rs | 67 - .../arrow/src/arrow/io/parquet/write/utils.rs | 164 --- src/common/arrow/src/arrow/mod.rs | 2 - src/common/arrow/src/arrow/util/lexical.rs | 57 - src/common/arrow/src/arrow/util/mod.rs | 2 - src/common/arrow/src/lib.rs | 3 - src/common/arrow/src/native/mod.rs | 9 +- src/common/arrow/src/native/nested.rs | 403 ++++++ .../arrow/src/native/read/array/binary.rs | 168 +-- .../arrow/src/native/read/array/boolean.rs | 138 +- .../arrow/src/native/read/array/double.rs | 146 +-- .../arrow/src/native/read/array/integer.rs | 146 +-- .../arrow/src/native/read/array/list.rs | 4 +- src/common/arrow/src/native/read/array/map.rs | 4 +- src/common/arrow/src/native/read/array/mod.rs | 1 - .../arrow/src/native/read/array/struct_.rs | 30 +- .../arrow/src/native/read/array/view.rs | 126 +- .../arrow/src/native/read/batch_read.rs | 91 +- .../arrow/src/native/read/deserialize.rs | 80 +- src/common/arrow/src/native/read/mod.rs | 29 +- .../arrow/src/native/read/read_basic.rs | 196 +-- src/common/arrow/src/native/read/reader.rs | 14 +- src/common/arrow/src/native/stat.rs | 26 +- src/common/arrow/src/native/util/bit_util.rs | 41 + src/common/arrow/src/native/util/mod.rs | 39 + src/common/arrow/src/native/write/common.rs | 73 +- .../arrow/src/native/write/serialize.rs | 212 +-- src/common/arrow/src/native/write/writer.rs | 24 +- .../it/arrow/compute/aggregate/memory.rs | 46 - .../tests/it/arrow/compute/aggregate/mod.rs | 16 - .../arrow/tests/it/arrow/compute/cast.rs | 903 ------------- .../tests/it/arrow/compute/merge_sort.rs | 299 ----- .../arrow/tests/it/arrow/compute/mod.rs | 10 - .../tests/it/arrow/compute/sort/lex_sort.rs | 227 ---- .../arrow/tests/it/arrow/compute/sort/mod.rs | 613 --------- .../tests/it/arrow/compute/sort/row/mod.rs | 302 ----- .../arrow/tests/it/arrow/compute/take.rs | 347 ----- src/common/arrow/tests/it/arrow/ffi/data.rs | 383 ------ src/common/arrow/tests/it/arrow/ffi/mod.rs | 37 - src/common/arrow/tests/it/arrow/ffi/stream.rs | 60 - src/common/arrow/tests/it/arrow/mod.rs | 2 +- src/common/arrow/tests/it/native/io.rs | 25 +- src/common/exception/src/exception_into.rs | 6 - src/common/parquet2/.gitignore | 5 - src/common/parquet2/Cargo.toml | 44 - src/common/parquet2/LICENSE | 14 - src/common/parquet2/src/bloom_filter/hash.rs | 32 - src/common/parquet2/src/bloom_filter/mod.rs | 86 -- src/common/parquet2/src/bloom_filter/read.rs | 69 - .../parquet2/src/bloom_filter/split_block.rs | 97 -- src/common/parquet2/src/compression.rs | 402 ------ src/common/parquet2/src/deserialize/binary.rs | 87 -- .../parquet2/src/deserialize/boolean.rs | 56 - .../parquet2/src/deserialize/filtered_rle.rs | 290 ----- .../parquet2/src/deserialize/fixed_len.rs | 124 -- .../parquet2/src/deserialize/hybrid_rle.rs | 213 --- src/common/parquet2/src/deserialize/mod.rs | 33 - src/common/parquet2/src/deserialize/native.rs | 114 -- src/common/parquet2/src/deserialize/utils.rs | 192 --- .../parquet2/src/encoding/bitpacked/decode.rs | 228 ---- .../parquet2/src/encoding/bitpacked/encode.rs | 71 - .../parquet2/src/encoding/bitpacked/mod.rs | 236 ---- .../parquet2/src/encoding/bitpacked/pack.rs | 123 -- .../parquet2/src/encoding/bitpacked/unpack.rs | 135 -- .../src/encoding/delta_bitpacked/decoder.rs | 379 ------ .../src/encoding/delta_bitpacked/encoder.rs | 139 -- .../src/encoding/delta_bitpacked/mod.rs | 105 -- .../src/encoding/delta_byte_array/decoder.rs | 122 -- .../src/encoding/delta_byte_array/encoder.rs | 47 - .../src/encoding/delta_byte_array/mod.rs | 48 - .../delta_length_byte_array/decoder.rs | 95 -- .../delta_length_byte_array/encoder.rs | 34 - .../encoding/delta_length_byte_array/mod.rs | 65 - .../src/encoding/hybrid_rle/bitmap.rs | 117 -- .../src/encoding/hybrid_rle/decoder.rs | 158 --- .../src/encoding/hybrid_rle/encoder.rs | 185 --- .../parquet2/src/encoding/hybrid_rle/mod.rs | 280 ---- src/common/parquet2/src/encoding/mod.rs | 42 - .../parquet2/src/encoding/plain_byte_array.rs | 61 - src/common/parquet2/src/encoding/uleb128.rs | 112 -- .../parquet2/src/encoding/zigzag_leb128.rs | 84 -- src/common/parquet2/src/error.rs | 123 -- src/common/parquet2/src/indexes/index.rs | 338 ----- src/common/parquet2/src/indexes/intervals.rs | 150 --- src/common/parquet2/src/indexes/mod.rs | 246 ---- src/common/parquet2/src/lib.rs | 48 - .../src/metadata/column_chunk_metadata.rs | 229 ---- .../src/metadata/column_descriptor.rs | 68 - .../parquet2/src/metadata/column_order.rs | 47 - .../parquet2/src/metadata/file_metadata.rs | 144 -- src/common/parquet2/src/metadata/mod.rs | 34 - .../parquet2/src/metadata/row_metadata.rs | 125 -- .../src/metadata/schema_descriptor.rs | 161 --- src/common/parquet2/src/metadata/sort.rs | 112 -- src/common/parquet2/src/page/mod.rs | 440 ------- src/common/parquet2/src/parquet_bridge.rs | 728 ----------- src/common/parquet2/src/read/column/mod.rs | 225 ---- src/common/parquet2/src/read/column/stream.rs | 70 - src/common/parquet2/src/read/compression.rs | 305 ----- .../parquet2/src/read/indexes/deserialize.rs | 47 - src/common/parquet2/src/read/indexes/mod.rs | 19 - src/common/parquet2/src/read/indexes/read.rs | 151 --- src/common/parquet2/src/read/levels.rs | 42 - src/common/parquet2/src/read/metadata.rs | 122 -- src/common/parquet2/src/read/mod.rs | 106 -- .../parquet2/src/read/page/indexed_reader.rs | 228 ---- src/common/parquet2/src/read/page/mod.rs | 38 - src/common/parquet2/src/read/page/reader.rs | 325 ----- src/common/parquet2/src/read/page/stream.rs | 163 --- src/common/parquet2/src/read/stream.rs | 92 -- .../src/schema/io_message/from_message.rs | 1160 ----------------- .../parquet2/src/schema/io_message/mod.rs | 18 - .../src/schema/io_thrift/from_thrift.rs | 150 --- .../parquet2/src/schema/io_thrift/mod.rs | 102 -- .../src/schema/io_thrift/to_thrift.rs | 98 -- src/common/parquet2/src/schema/mod.rs | 22 - .../parquet2/src/schema/types/basic_type.rs | 33 - .../src/schema/types/converted_type.rs | 252 ---- src/common/parquet2/src/schema/types/mod.rs | 33 - .../parquet2/src/schema/types/parquet_type.rs | 227 ---- .../src/schema/types/physical_type.rs | 75 -- src/common/parquet2/src/schema/types/spec.rs | 201 --- src/common/parquet2/src/statistics/binary.rs | 67 - src/common/parquet2/src/statistics/boolean.rs | 88 -- .../src/statistics/fixed_len_binary.rs | 93 -- src/common/parquet2/src/statistics/mod.rs | 150 --- .../parquet2/src/statistics/primitive.rs | 87 -- src/common/parquet2/src/types.rs | 156 --- src/common/parquet2/src/write/column_chunk.rs | 228 ---- src/common/parquet2/src/write/compression.rs | 180 --- src/common/parquet2/src/write/dyn_iter.rs | 76 -- src/common/parquet2/src/write/file.rs | 261 ---- src/common/parquet2/src/write/indexes/mod.rs | 19 - .../parquet2/src/write/indexes/serialize.rs | 99 -- .../parquet2/src/write/indexes/write.rs | 63 - src/common/parquet2/src/write/mod.rs | 75 -- src/common/parquet2/src/write/page.rs | 260 ---- src/common/parquet2/src/write/row_group.rs | 220 ---- src/common/parquet2/src/write/statistics.rs | 324 ----- src/common/parquet2/src/write/stream.rs | 214 --- src/common/storage/src/column_node.rs | 5 +- src/meta/client/Cargo.toml | 2 +- src/meta/client/src/grpc_client.rs | 2 +- src/meta/service/Cargo.toml | 1 + src/meta/service/src/api/grpc/grpc_service.rs | 2 +- .../expression/src/kernels/take_chunks.rs | 3 +- src/query/expression/tests/it/common.rs | 3 +- src/query/expression/tests/it/serde.rs | 25 - .../tests/it/servers/flight/flight_service.rs | 4 +- .../table_meta/src/table/table_compression.rs | 15 - .../block/block_reader_native_deserialize.rs | 13 +- .../fuse/src/io/read/meta/meta_readers.rs | 6 +- .../09_0027_func_fuse_encoding.test | 2 +- .../09_http_handler/09_0007_token.py | 6 +- 355 files changed, 806 insertions(+), 56032 deletions(-) delete mode 100644 src/common/arrow/src/arrow/array/binary/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/binview/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/boolean/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/dictionary/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_binary/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_list/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/list/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/map/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/primitive/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/struct_/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/union/ffi.rs delete mode 100644 src/common/arrow/src/arrow/array/utf8/ffi.rs delete mode 100644 src/common/arrow/src/arrow/compute/aggregate/memory.rs delete mode 100644 src/common/arrow/src/arrow/compute/aggregate/mod.rs delete mode 100644 src/common/arrow/src/arrow/compute/arity.rs delete mode 100644 src/common/arrow/src/arrow/compute/cast/binary_to.rs delete mode 100644 src/common/arrow/src/arrow/compute/cast/boolean_to.rs delete mode 100644 src/common/arrow/src/arrow/compute/cast/decimal_to.rs delete mode 100644 src/common/arrow/src/arrow/compute/cast/dictionary_to.rs delete mode 100644 src/common/arrow/src/arrow/compute/cast/mod.rs delete mode 100644 src/common/arrow/src/arrow/compute/cast/primitive_to.rs delete mode 100644 src/common/arrow/src/arrow/compute/cast/utf8_to.rs delete mode 100644 src/common/arrow/src/arrow/compute/merge_sort/mod.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/binary.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/boolean.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/common.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/lex_sort.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/mod.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/primitive/indices.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/primitive/mod.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/primitive/sort.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/row/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/row/fixed.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/row/interner.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/row/mod.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/row/variable.rs delete mode 100644 src/common/arrow/src/arrow/compute/sort/utf8.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/binary.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/binview.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/boolean.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/dict.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/fixed_size_list.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/generic_binary.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/list.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/mod.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/primitive.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/structure.rs delete mode 100644 src/common/arrow/src/arrow/compute/take/utf8.rs delete mode 100644 src/common/arrow/src/arrow/ffi/array.rs delete mode 100644 src/common/arrow/src/arrow/ffi/bridge.rs delete mode 100644 src/common/arrow/src/arrow/ffi/generated.rs delete mode 100644 src/common/arrow/src/arrow/ffi/mmap.rs delete mode 100644 src/common/arrow/src/arrow/ffi/mod.rs delete mode 100644 src/common/arrow/src/arrow/ffi/schema.rs delete mode 100644 src/common/arrow/src/arrow/ffi/stream.rs delete mode 100644 src/common/arrow/src/arrow/io/README.md delete mode 100644 src/common/arrow/src/arrow/io/flight/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/append/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/compression.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/endianness.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/binary.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/binview.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/boolean.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/fixed_size_binary.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/fixed_size_list.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/list.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/map.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/null.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/primitive.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/struct_.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/union.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/array/utf8.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/common.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/deserialize.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/error.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/file.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/file_async.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/read_basic.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/reader.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/schema.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/stream.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/read/stream_async.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/common.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/common_async.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/common_sync.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/file_async.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/schema.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/serialize.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/stream.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/stream_async.rs delete mode 100644 src/common/arrow/src/arrow/io/ipc/write/writer.rs delete mode 100644 src/common/arrow/src/arrow/io/iterator.rs delete mode 100644 src/common/arrow/src/arrow/io/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/README.md delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/README.md delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/utils.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/dictionary/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/dictionary/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/utils.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/nested_utils.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/null/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/null/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/integer.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/simple.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/struct_.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/deserialize/utils.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/file.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/indexes/binary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/indexes/boolean.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/indexes/fixed_len_binary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/indexes/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/indexes/primitive.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/row_group.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/schema/convert.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/schema/metadata.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/schema/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/binary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/boolean.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/fixlen.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/list.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/map.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/null.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/primitive.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/struct_.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/read/statistics/utf8.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/binary/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/binary/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/binary/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/binview/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/binview/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/binview/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/boolean/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/boolean/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/boolean/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/file.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/fixed_len_bytes.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/nested/def.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/nested/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/nested/rep.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/pages.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/primitive/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/primitive/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/primitive/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/row_group.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/schema.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/sink.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/utf8/basic.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/utf8/mod.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/utf8/nested.rs delete mode 100644 src/common/arrow/src/arrow/io/parquet/write/utils.rs delete mode 100644 src/common/arrow/src/arrow/util/lexical.rs create mode 100644 src/common/arrow/src/native/nested.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/aggregate/memory.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/aggregate/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/cast.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/merge_sort.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/sort/lex_sort.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/sort/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/sort/row/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/take.rs delete mode 100644 src/common/arrow/tests/it/arrow/ffi/data.rs delete mode 100644 src/common/arrow/tests/it/arrow/ffi/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/ffi/stream.rs delete mode 100644 src/common/parquet2/.gitignore delete mode 100644 src/common/parquet2/Cargo.toml delete mode 100644 src/common/parquet2/LICENSE delete mode 100644 src/common/parquet2/src/bloom_filter/hash.rs delete mode 100644 src/common/parquet2/src/bloom_filter/mod.rs delete mode 100644 src/common/parquet2/src/bloom_filter/read.rs delete mode 100644 src/common/parquet2/src/bloom_filter/split_block.rs delete mode 100644 src/common/parquet2/src/compression.rs delete mode 100644 src/common/parquet2/src/deserialize/binary.rs delete mode 100644 src/common/parquet2/src/deserialize/boolean.rs delete mode 100644 src/common/parquet2/src/deserialize/filtered_rle.rs delete mode 100644 src/common/parquet2/src/deserialize/fixed_len.rs delete mode 100644 src/common/parquet2/src/deserialize/hybrid_rle.rs delete mode 100644 src/common/parquet2/src/deserialize/mod.rs delete mode 100644 src/common/parquet2/src/deserialize/native.rs delete mode 100644 src/common/parquet2/src/deserialize/utils.rs delete mode 100644 src/common/parquet2/src/encoding/bitpacked/decode.rs delete mode 100644 src/common/parquet2/src/encoding/bitpacked/encode.rs delete mode 100644 src/common/parquet2/src/encoding/bitpacked/mod.rs delete mode 100644 src/common/parquet2/src/encoding/bitpacked/pack.rs delete mode 100644 src/common/parquet2/src/encoding/bitpacked/unpack.rs delete mode 100644 src/common/parquet2/src/encoding/delta_bitpacked/decoder.rs delete mode 100644 src/common/parquet2/src/encoding/delta_bitpacked/encoder.rs delete mode 100644 src/common/parquet2/src/encoding/delta_bitpacked/mod.rs delete mode 100644 src/common/parquet2/src/encoding/delta_byte_array/decoder.rs delete mode 100644 src/common/parquet2/src/encoding/delta_byte_array/encoder.rs delete mode 100644 src/common/parquet2/src/encoding/delta_byte_array/mod.rs delete mode 100644 src/common/parquet2/src/encoding/delta_length_byte_array/decoder.rs delete mode 100644 src/common/parquet2/src/encoding/delta_length_byte_array/encoder.rs delete mode 100644 src/common/parquet2/src/encoding/delta_length_byte_array/mod.rs delete mode 100644 src/common/parquet2/src/encoding/hybrid_rle/bitmap.rs delete mode 100644 src/common/parquet2/src/encoding/hybrid_rle/decoder.rs delete mode 100644 src/common/parquet2/src/encoding/hybrid_rle/encoder.rs delete mode 100644 src/common/parquet2/src/encoding/hybrid_rle/mod.rs delete mode 100644 src/common/parquet2/src/encoding/mod.rs delete mode 100644 src/common/parquet2/src/encoding/plain_byte_array.rs delete mode 100644 src/common/parquet2/src/encoding/uleb128.rs delete mode 100644 src/common/parquet2/src/encoding/zigzag_leb128.rs delete mode 100644 src/common/parquet2/src/error.rs delete mode 100644 src/common/parquet2/src/indexes/index.rs delete mode 100644 src/common/parquet2/src/indexes/intervals.rs delete mode 100644 src/common/parquet2/src/indexes/mod.rs delete mode 100644 src/common/parquet2/src/lib.rs delete mode 100644 src/common/parquet2/src/metadata/column_chunk_metadata.rs delete mode 100644 src/common/parquet2/src/metadata/column_descriptor.rs delete mode 100644 src/common/parquet2/src/metadata/column_order.rs delete mode 100644 src/common/parquet2/src/metadata/file_metadata.rs delete mode 100644 src/common/parquet2/src/metadata/mod.rs delete mode 100644 src/common/parquet2/src/metadata/row_metadata.rs delete mode 100644 src/common/parquet2/src/metadata/schema_descriptor.rs delete mode 100644 src/common/parquet2/src/metadata/sort.rs delete mode 100644 src/common/parquet2/src/page/mod.rs delete mode 100644 src/common/parquet2/src/parquet_bridge.rs delete mode 100644 src/common/parquet2/src/read/column/mod.rs delete mode 100644 src/common/parquet2/src/read/column/stream.rs delete mode 100644 src/common/parquet2/src/read/compression.rs delete mode 100644 src/common/parquet2/src/read/indexes/deserialize.rs delete mode 100644 src/common/parquet2/src/read/indexes/mod.rs delete mode 100644 src/common/parquet2/src/read/indexes/read.rs delete mode 100644 src/common/parquet2/src/read/levels.rs delete mode 100644 src/common/parquet2/src/read/metadata.rs delete mode 100644 src/common/parquet2/src/read/mod.rs delete mode 100644 src/common/parquet2/src/read/page/indexed_reader.rs delete mode 100644 src/common/parquet2/src/read/page/mod.rs delete mode 100644 src/common/parquet2/src/read/page/reader.rs delete mode 100644 src/common/parquet2/src/read/page/stream.rs delete mode 100644 src/common/parquet2/src/read/stream.rs delete mode 100644 src/common/parquet2/src/schema/io_message/from_message.rs delete mode 100644 src/common/parquet2/src/schema/io_message/mod.rs delete mode 100644 src/common/parquet2/src/schema/io_thrift/from_thrift.rs delete mode 100644 src/common/parquet2/src/schema/io_thrift/mod.rs delete mode 100644 src/common/parquet2/src/schema/io_thrift/to_thrift.rs delete mode 100644 src/common/parquet2/src/schema/mod.rs delete mode 100644 src/common/parquet2/src/schema/types/basic_type.rs delete mode 100644 src/common/parquet2/src/schema/types/converted_type.rs delete mode 100644 src/common/parquet2/src/schema/types/mod.rs delete mode 100644 src/common/parquet2/src/schema/types/parquet_type.rs delete mode 100644 src/common/parquet2/src/schema/types/physical_type.rs delete mode 100644 src/common/parquet2/src/schema/types/spec.rs delete mode 100644 src/common/parquet2/src/statistics/binary.rs delete mode 100644 src/common/parquet2/src/statistics/boolean.rs delete mode 100644 src/common/parquet2/src/statistics/fixed_len_binary.rs delete mode 100644 src/common/parquet2/src/statistics/mod.rs delete mode 100644 src/common/parquet2/src/statistics/primitive.rs delete mode 100644 src/common/parquet2/src/types.rs delete mode 100644 src/common/parquet2/src/write/column_chunk.rs delete mode 100644 src/common/parquet2/src/write/compression.rs delete mode 100644 src/common/parquet2/src/write/dyn_iter.rs delete mode 100644 src/common/parquet2/src/write/file.rs delete mode 100644 src/common/parquet2/src/write/indexes/mod.rs delete mode 100644 src/common/parquet2/src/write/indexes/serialize.rs delete mode 100644 src/common/parquet2/src/write/indexes/write.rs delete mode 100644 src/common/parquet2/src/write/mod.rs delete mode 100644 src/common/parquet2/src/write/page.rs delete mode 100644 src/common/parquet2/src/write/row_group.rs delete mode 100644 src/common/parquet2/src/write/statistics.rs delete mode 100644 src/common/parquet2/src/write/stream.rs diff --git a/Cargo.lock b/Cargo.lock index c601c0a1c107..44a3a3fa546a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,12 +287,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" -[[package]] -name = "array-init-cursor" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" - [[package]] name = "arrayref" version = "0.3.8" @@ -455,18 +449,6 @@ dependencies = [ "tonic", ] -[[package]] -name = "arrow-format" -version = "0.8.1" -source = "git+https://github.com/Xuanwo/arrow-format?rev=5502823a#5502823aba94e0e3256691210432864b57006578" -dependencies = [ - "planus", - "prost", - "prost-derive", - "serde", - "tonic", -] - [[package]] name = "arrow-ipc" version = "53.2.0" @@ -3030,29 +3012,21 @@ dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", - "arrow-format", "arrow-schema", - "async-stream", - "base64 0.22.1", "bitpacking 0.8.4", "bytemuck", "byteorder", "bytes", "chrono", "chrono-tz 0.8.6", - "databend-common-parquet2", "dyn-clone", "either", "env_logger 0.11.5", "ethnum", - "fallible-streaming-iterator", "flate2", "foreign_vec", - "futures", "hashbrown 0.14.5", "indexmap 2.6.0", - "itertools 0.13.0", - "lexical-core", "log", "lz4", "num", @@ -3066,9 +3040,9 @@ dependencies = [ "roaring", "serde", "serde_derive", + "serde_json", "simdutf8", "snap", - "streaming-iterator", "tokio", "tokio-util", "zstd 0.12.4", @@ -3674,8 +3648,8 @@ version = "0.1.0" dependencies = [ "anyerror", "anyhow", + "arrow-flight", "async-backtrace", - "databend-common-arrow", "databend-common-base", "databend-common-building", "databend-common-exception", @@ -3903,29 +3877,6 @@ dependencies = [ "openai_api_rust", ] -[[package]] -name = "databend-common-parquet2" -version = "0.1.0" -dependencies = [ - "async-stream", - "brotli 3.5.0", - "bytes", - "criterion", - "flate2", - "futures", - "lz4", - "opendal", - "parquet-format-safe", - "rand", - "seq-macro", - "serde", - "snap", - "streaming-decompression", - "tokio", - "xxhash-rust", - "zstd 0.12.4", -] - [[package]] name = "databend-common-pipeline-core" version = "0.1.0" @@ -4969,6 +4920,7 @@ version = "0.1.0" dependencies = [ "anyerror", "anyhow", + "arrow-flight", "async-trait", "backon", "clap", @@ -6440,7 +6392,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" dependencies = [ "crc32fast", - "libz-ng-sys", "libz-sys", "miniz_oxide", ] @@ -9393,16 +9344,6 @@ dependencies = [ "threadpool", ] -[[package]] -name = "libz-ng-sys" -version = "1.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6409efc61b12687963e602df8ecf70e8ddacf95bc6576bcf16e3ac6328083c5" -dependencies = [ - "cmake", - "libc", -] - [[package]] name = "libz-sys" version = "1.1.19" @@ -10974,16 +10915,6 @@ dependencies = [ "zstd-sys", ] -[[package]] -name = "parquet-format-safe" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" -dependencies = [ - "async-trait", - "futures", -] - [[package]] name = "parse-zoneinfo" version = "0.3.1" @@ -11213,16 +11144,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" -[[package]] -name = "planus" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ed29d35a76575bb7aeff0222dc0fa7435a990a73810e405574ea81bd848af4" -dependencies = [ - "array-init-cursor", - "hashbrown 0.13.2", -] - [[package]] name = "plotters" version = "0.3.6" @@ -13974,21 +13895,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "streaming-decompression" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" -dependencies = [ - "fallible-streaming-iterator", -] - -[[package]] -name = "streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" - [[package]] name = "strength_reduce" version = "0.2.4" @@ -16568,12 +16474,6 @@ dependencies = [ "cbordata", ] -[[package]] -name = "xxhash-rust" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" - [[package]] name = "xz2" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index 7d68a2405e59..113ad08c77ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ members = [ "src/common/storage", "src/common/vector", "src/common/license", - "src/common/parquet2", "src/query/ast", "src/query/codegen", "src/query/config", @@ -140,7 +139,6 @@ databend-common-meta-store = { path = "src/meta/store" } databend-common-meta-types = { path = "src/meta/types" } databend-common-metrics = { path = "src/common/metrics" } databend-common-openai = { path = "src/common/openai" } -databend-common-parquet2 = { path = "src/common/parquet2" } databend-common-pipeline-core = { path = "src/query/pipeline/core" } databend-common-pipeline-sinks = { path = "src/query/pipeline/sinks" } databend-common-pipeline-sources = { path = "src/query/pipeline/sources" } @@ -196,12 +194,6 @@ databend-storages-common-session = { path = "src/query/storages/common/session" databend-storages-common-stage = { path = "src/query/storages/common/stage" } databend-storages-common-table-meta = { path = "src/query/storages/common/table_meta" } -# Specific dependencies -parquet2 = { package = "databend-common-parquet2", path = "src/common/parquet2", default-features = false, features = [ - "serde_types", - "async", -] } - # Crates.io dependencies ahash = "0.8" aho-corasick = { version = "1.0.1" } # @@ -214,7 +206,6 @@ arrow-buffer = { version = "53" } arrow-cast = { version = "53", features = ["prettyprint"] } arrow-data = { version = "53" } arrow-flight = { version = "53", features = ["flight-sql-experimental", "tls"] } -arrow-format = { version = "0.8.1", features = ["flight-data", "flight-service", "ipc"] } arrow-ipc = { version = "53" } arrow-ord = { version = "53" } arrow-schema = { version = "53", features = ["serde"] } @@ -612,7 +603,6 @@ overflow-checks = true rpath = false [patch.crates-io] -arrow-format = { git = "https://github.com/Xuanwo/arrow-format", rev = "5502823a" } async-backtrace = { git = "https://github.com/datafuse-extras/async-backtrace.git", rev = "dea4553" } async-recursion = { git = "https://github.com/datafuse-extras/async-recursion.git", rev = "a353334" } backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "72265be", features = [ diff --git a/src/common/arrow/Cargo.toml b/src/common/arrow/Cargo.toml index 3108780557f0..0380e70f9f9f 100644 --- a/src/common/arrow/Cargo.toml +++ b/src/common/arrow/Cargo.toml @@ -11,87 +11,33 @@ doctest = false test = true [features] -default = ["arrow-default", "parquet-default"] +default = ["arrow-default"] arrow = ["arrow-buffer", "arrow-schema", "arrow-data", "arrow-array"] -io_flight = ["io_ipc", "arrow-format/flight-data"] -io_ipc = [] -io_ipc_compression = [] - -# base64 + io_ipc because arrow schemas are stored as base64-encoded ipc format. -io_parquet = ["io_ipc", "base64", "streaming-iterator", "fallible-streaming-iterator"] -io_parquet_async = ["futures", "io_parquet", "parquet2/async"] - -io_parquet_compression = [ - "io_parquet_zstd", - "io_parquet_gzip", - "io_parquet_snappy", - "io_parquet_lz4", - "io_parquet_brotli", -] # sample testing of generated arrow data -io_parquet_sample_test = ["io_parquet_async"] - -# compression backends -io_parquet_brotli = ["parquet2/brotli"] -io_parquet_gzip = ["parquet2/gzip"] -io_parquet_lz4 = ["parquet2/lz4"] -io_parquet_snappy = ["parquet2/snappy"] -io_parquet_zstd = ["parquet2/zstd"] - -# parquet bloom filter functions -io_parquet_bloom_filter = ["parquet2/bloom_filter"] compute = [ - "compute_aggregate", - "compute_cast", "compute_concatenate", - "compute_merge_sort", - "compute_sort", - "compute_take", ] -compute_aggregate = [] -compute_cast = ["lexical-core", "compute_take"] compute_concatenate = [] -compute_merge_sort = ["itertools", "compute_sort"] -compute_sort = ["compute_take"] -compute_take = [] serde_types = ["serde", "serde_derive"] simd = [] arrow-default = [ "arrow", - "io_ipc", - "io_ipc_compression", - "io_flight", - "io_parquet_async", - "io_parquet_compression", - "io_parquet", "compute", "serde_types", "simd", ] -parquet-default = [ - "parquet2/lz4", - "parquet2/zstd", - "parquet2/snappy", - # this feature can't be built in musl - # "parquet2/gzip_zlib_ng", - "parquet2/brotli", -] - [dependencies] ahash = { workspace = true } arrow-array = { workspace = true, optional = true } arrow-buffer = { workspace = true, optional = true } arrow-data = { workspace = true, optional = true } -arrow-format = { workspace = true } arrow-schema = { workspace = true, optional = true } -async-stream = { workspace = true, optional = true } -base64 = { workspace = true, optional = true } bitpacking = { workspace = true } bytemuck = { workspace = true } byteorder = { workspace = true } @@ -101,28 +47,23 @@ chrono-tz = { workspace = true, optional = true } dyn-clone = { workspace = true } either = { workspace = true } ethnum = { workspace = true } -fallible-streaming-iterator = { workspace = true, optional = true } foreign_vec = { workspace = true } -futures = { workspace = true, optional = true } hashbrown_v0_14 = { workspace = true } indexmap = { workspace = true } -itertools = { workspace = true, optional = true } -lexical-core = { workspace = true, optional = true } log = { workspace = true } lz4 = { workspace = true } num = { workspace = true, features = ["std"] } num-traits = { workspace = true } opendal = { workspace = true } ordered-float = { workspace = true } -parquet2 = { workspace = true } rand = { workspace = true } ringbuffer = { workspace = true } roaring = { workspace = true } serde = { workspace = true, features = ["rc"], optional = true } serde_derive = { workspace = true, optional = true } +serde_json = { workspace = true } simdutf8 = { workspace = true } snap = { workspace = true } -streaming-iterator = { workspace = true, optional = true } zstd = { workspace = true } [dev-dependencies] diff --git a/src/common/arrow/src/arrow/array/binary/ffi.rs b/src/common/arrow/src/arrow/array/binary/ffi.rs deleted file mode 100644 index c57f211acd03..000000000000 --- a/src/common/arrow/src/arrow/array/binary/ffi.rs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::BinaryArray; -use crate::arrow::array::FromFfi; -use crate::arrow::array::ToFfi; -use crate::arrow::bitmap::align; -use crate::arrow::error::Result; -use crate::arrow::ffi; -use crate::arrow::offset::Offset; -use crate::arrow::offset::OffsetsBuffer; - -unsafe impl ToFfi for BinaryArray { - fn buffers(&self) -> Vec> { - vec![ - self.validity.as_ref().map(|x| x.as_ptr()), - Some(self.offsets.buffer().data_ptr().cast::()), - Some(self.values.data_ptr().cast::()), - ] - } - - fn offset(&self) -> Option { - let offset = self.offsets.buffer().offset(); - if let Some(bitmap) = self.validity.as_ref() { - if bitmap.offset() == offset { - Some(offset) - } else { - None - } - } else { - Some(offset) - } - } - - fn to_ffi_aligned(&self) -> Self { - let offset = self.offsets.buffer().offset(); - - let validity = self.validity.as_ref().map(|bitmap| { - if bitmap.offset() == offset { - bitmap.clone() - } else { - align(bitmap, offset) - } - }); - - Self { - data_type: self.data_type.clone(), - validity, - offsets: self.offsets.clone(), - values: self.values.clone(), - } - } -} - -impl FromFfi for BinaryArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - - let validity = unsafe { array.validity() }?; - let offsets = unsafe { array.buffer::(1) }?; - let values = unsafe { array.buffer::(2) }?; - - // assumption that data from FFI is well constructed - let offsets = unsafe { OffsetsBuffer::new_unchecked(offsets) }; - - Ok(Self::new(data_type, offsets, values, validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/binary/mod.rs b/src/common/arrow/src/arrow/array/binary/mod.rs index 98d92fcb37c6..c19f4fc217ee 100644 --- a/src/common/arrow/src/arrow/array/binary/mod.rs +++ b/src/common/arrow/src/arrow/array/binary/mod.rs @@ -29,7 +29,6 @@ use crate::arrow::offset::Offsets; use crate::arrow::offset::OffsetsBuffer; use crate::arrow::trusted_len::TrustedLen; -mod ffi; pub(super) mod fmt; mod iterator; pub use iterator::*; diff --git a/src/common/arrow/src/arrow/array/binview/ffi.rs b/src/common/arrow/src/arrow/array/binview/ffi.rs deleted file mode 100644 index 8d98daa62a95..000000000000 --- a/src/common/arrow/src/arrow/array/binview/ffi.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::atomic::AtomicU64; -use std::sync::atomic::Ordering; -use std::sync::Arc; - -use crate::arrow::array::binview::BinaryViewArrayGeneric; -use crate::arrow::array::binview::View; -use crate::arrow::array::binview::ViewType; -use crate::arrow::array::FromFfi; -use crate::arrow::array::ToFfi; -use crate::arrow::bitmap::align; -use crate::arrow::error::Result; -use crate::arrow::ffi; - -unsafe impl ToFfi for BinaryViewArrayGeneric { - fn buffers(&self) -> Vec> { - let mut buffers = Vec::with_capacity(self.buffers.len() + 2); - buffers.push(self.validity.as_ref().map(|x| x.as_ptr())); - buffers.push(Some(self.views.data_ptr().cast::())); - buffers.extend(self.buffers.iter().map(|b| Some(b.data_ptr()))); - buffers - } - - fn offset(&self) -> Option { - let offset = self.views.offset(); - if let Some(bitmap) = self.validity.as_ref() { - if bitmap.offset() == offset { - Some(offset) - } else { - None - } - } else { - Some(offset) - } - } - - fn to_ffi_aligned(&self) -> Self { - let offset = self.views.offset(); - - let validity = self.validity.as_ref().map(|bitmap| { - if bitmap.offset() == offset { - bitmap.clone() - } else { - align(bitmap, offset) - } - }); - - Self { - data_type: self.data_type.clone(), - validity, - views: self.views.clone(), - buffers: self.buffers.clone(), - phantom: Default::default(), - total_bytes_len: AtomicU64::new(self.total_bytes_len.load(Ordering::Relaxed)), - total_buffer_len: self.total_buffer_len, - } - } -} - -impl FromFfi for BinaryViewArrayGeneric { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - - let validity = unsafe { array.validity() }?; - let views = unsafe { array.buffer::(1) }?; - - // n_buffers - 2, 2 means validity + views - let n_buffers = array.n_buffers(); - let mut remaining_buffers = n_buffers - 2; - if remaining_buffers <= 1 { - return Ok(Self::new_unchecked_unknown_md( - data_type, - views, - Arc::from([]), - validity, - None, - )); - } - - let n_variadic_buffers = remaining_buffers - 1; - let variadic_buffer_offset = n_buffers - 1; - - let variadic_buffer_sizes = - array.buffer_known_len::(variadic_buffer_offset, n_variadic_buffers)?; - remaining_buffers -= 1; - - let mut variadic_buffers = Vec::with_capacity(remaining_buffers); - - let offset = 2; - for (i, &size) in (offset..remaining_buffers + offset).zip(variadic_buffer_sizes.iter()) { - let values = unsafe { array.buffer_known_len::(i, size as usize) }?; - variadic_buffers.push(values); - } - - Ok(Self::new_unchecked_unknown_md( - data_type, - views, - Arc::from(variadic_buffers), - validity, - None, - )) - } -} diff --git a/src/common/arrow/src/arrow/array/binview/mod.rs b/src/common/arrow/src/arrow/array/binview/mod.rs index 35850b10e0ab..69bda66794b5 100644 --- a/src/common/arrow/src/arrow/array/binview/mod.rs +++ b/src/common/arrow/src/arrow/array/binview/mod.rs @@ -13,7 +13,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod ffi; pub(crate) mod fmt; mod from; mod iterator; diff --git a/src/common/arrow/src/arrow/array/boolean/ffi.rs b/src/common/arrow/src/arrow/array/boolean/ffi.rs deleted file mode 100644 index 082fb6c39f64..000000000000 --- a/src/common/arrow/src/arrow/array/boolean/ffi.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::BooleanArray; -use crate::arrow::array::FromFfi; -use crate::arrow::array::ToFfi; -use crate::arrow::bitmap::align; -use crate::arrow::error::Result; -use crate::arrow::ffi; - -unsafe impl ToFfi for BooleanArray { - fn buffers(&self) -> Vec> { - vec![ - self.validity.as_ref().map(|x| x.as_ptr()), - Some(self.values.as_ptr()), - ] - } - - fn offset(&self) -> Option { - let offset = self.values.offset(); - if let Some(bitmap) = self.validity.as_ref() { - if bitmap.offset() == offset { - Some(offset) - } else { - None - } - } else { - Some(offset) - } - } - - fn to_ffi_aligned(&self) -> Self { - let offset = self.values.offset(); - - let validity = self.validity.as_ref().map(|bitmap| { - if bitmap.offset() == offset { - bitmap.clone() - } else { - align(bitmap, offset) - } - }); - - Self { - data_type: self.data_type.clone(), - validity, - values: self.values.clone(), - } - } -} - -impl FromFfi for BooleanArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let validity = unsafe { array.validity() }?; - let values = unsafe { array.bitmap(1) }?; - Self::try_new(data_type, values, validity) - } -} diff --git a/src/common/arrow/src/arrow/array/boolean/mod.rs b/src/common/arrow/src/arrow/array/boolean/mod.rs index e8bf0b462f8c..82cbd63bab32 100644 --- a/src/common/arrow/src/arrow/array/boolean/mod.rs +++ b/src/common/arrow/src/arrow/array/boolean/mod.rs @@ -27,7 +27,7 @@ use crate::arrow::trusted_len::TrustedLen; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod from; mod iterator; diff --git a/src/common/arrow/src/arrow/array/dictionary/ffi.rs b/src/common/arrow/src/arrow/array/dictionary/ffi.rs deleted file mode 100644 index 64d3800e8b7f..000000000000 --- a/src/common/arrow/src/arrow/array/dictionary/ffi.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::DictionaryArray; -use super::DictionaryKey; -use crate::arrow::array::FromFfi; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::ToFfi; -use crate::arrow::error::Error; -use crate::arrow::ffi; - -unsafe impl ToFfi for DictionaryArray { - fn buffers(&self) -> Vec> { - self.keys.buffers() - } - - fn offset(&self) -> Option { - self.keys.offset() - } - - fn to_ffi_aligned(&self) -> Self { - Self { - data_type: self.data_type.clone(), - keys: self.keys.to_ffi_aligned(), - values: self.values.clone(), - } - } -} - -impl FromFfi for DictionaryArray { - unsafe fn try_from_ffi(array: A) -> Result { - // keys: similar to PrimitiveArray, but the datatype is the inner one - let validity = unsafe { array.validity() }?; - let values = unsafe { array.buffer::(1) }?; - - let data_type = array.data_type().clone(); - - let keys = PrimitiveArray::::try_new(K::PRIMITIVE.into(), values, validity)?; - let values = array - .dictionary()? - .ok_or_else(|| Error::oos("Dictionary Array must contain a dictionary in ffi"))?; - let values = ffi::try_from(values)?; - - // the assumption of this trait - DictionaryArray::::try_new_unchecked(data_type, keys, values) - } -} diff --git a/src/common/arrow/src/arrow/array/dictionary/mod.rs b/src/common/arrow/src/arrow/array/dictionary/mod.rs index de7ec681d648..4ccc27135824 100644 --- a/src/common/arrow/src/arrow/array/dictionary/mod.rs +++ b/src/common/arrow/src/arrow/array/dictionary/mod.rs @@ -29,7 +29,7 @@ use crate::arrow::types::NativeType; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod iterator; mod mutable; diff --git a/src/common/arrow/src/arrow/array/ffi.rs b/src/common/arrow/src/arrow/array/ffi.rs deleted file mode 100644 index 35756afc8b15..000000000000 --- a/src/common/arrow/src/arrow/array/ffi.rs +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::*; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::arrow::ffi; - -/// Trait describing how a struct presents itself to the -/// [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) (FFI). -/// # Safety -/// Implementing this trait incorrect will lead to UB -pub(crate) unsafe trait ToFfi { - /// The pointers to the buffers. - fn buffers(&self) -> Vec>; - - /// The children - fn children(&self) -> Vec> { - vec![] - } - - /// The offset - fn offset(&self) -> Option; - - /// return a partial clone of self with an offset. - fn to_ffi_aligned(&self) -> Self; -} - -/// Trait describing how a struct imports into itself from the -/// [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) (FFI). -pub(crate) trait FromFfi: Sized { - /// Convert itself from FFI. - /// # Safety - /// This function is intrinsically `unsafe` as it requires the FFI to be made according - /// to the [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) - unsafe fn try_from_ffi(array: T) -> Result; -} - -macro_rules! ffi_dyn { - ($array:expr, $ty:ty) => {{ - let array = $array.as_any().downcast_ref::<$ty>().unwrap(); - ( - array.offset().unwrap(), - array.buffers(), - array.children(), - None, - ) - }}; -} - -type BuffersChildren = ( - usize, - Vec>, - Vec>, - Option>, -); - -pub fn offset_buffers_children_dictionary(array: &dyn Array) -> BuffersChildren { - use PhysicalType::*; - match array.data_type().to_physical_type() { - Null => ffi_dyn!(array, NullArray), - Boolean => ffi_dyn!(array, BooleanArray), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - ffi_dyn!(array, PrimitiveArray<$T>) - }), - Binary => ffi_dyn!(array, BinaryArray), - LargeBinary => ffi_dyn!(array, BinaryArray), - FixedSizeBinary => ffi_dyn!(array, FixedSizeBinaryArray), - Utf8 => ffi_dyn!(array, Utf8Array::), - LargeUtf8 => ffi_dyn!(array, Utf8Array::), - List => ffi_dyn!(array, ListArray::), - LargeList => ffi_dyn!(array, ListArray::), - FixedSizeList => ffi_dyn!(array, FixedSizeListArray), - Struct => ffi_dyn!(array, StructArray), - Union => ffi_dyn!(array, UnionArray), - Map => ffi_dyn!(array, MapArray), - BinaryView => ffi_dyn!(array, BinaryViewArray), - Utf8View => ffi_dyn!(array, Utf8ViewArray), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - let array = array.as_any().downcast_ref::>().unwrap(); - ( - array.offset().unwrap(), - array.buffers(), - array.children(), - Some(array.values().clone()), - ) - }) - } - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_binary/ffi.rs b/src/common/arrow/src/arrow/array/fixed_size_binary/ffi.rs deleted file mode 100644 index 88700d81d912..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_binary/ffi.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::FixedSizeBinaryArray; -use crate::arrow::array::FromFfi; -use crate::arrow::array::ToFfi; -use crate::arrow::bitmap::align; -use crate::arrow::error::Result; -use crate::arrow::ffi; - -unsafe impl ToFfi for FixedSizeBinaryArray { - fn buffers(&self) -> Vec> { - vec![ - self.validity.as_ref().map(|x| x.as_ptr()), - Some(self.values.data_ptr().cast::()), - ] - } - - fn offset(&self) -> Option { - let offset = self.values.offset() / self.size; - if let Some(bitmap) = self.validity.as_ref() { - if bitmap.offset() == offset { - Some(offset) - } else { - None - } - } else { - Some(offset) - } - } - - fn to_ffi_aligned(&self) -> Self { - let offset = self.values.offset() / self.size; - - let validity = self.validity.as_ref().map(|bitmap| { - if bitmap.offset() == offset { - bitmap.clone() - } else { - align(bitmap, offset) - } - }); - - Self { - size: self.size, - data_type: self.data_type.clone(), - validity, - values: self.values.clone(), - } - } -} - -impl FromFfi for FixedSizeBinaryArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let validity = unsafe { array.validity() }?; - let values = unsafe { array.buffer::(1) }?; - - Self::try_new(data_type, values, validity) - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_binary/mod.rs b/src/common/arrow/src/arrow/array/fixed_size_binary/mod.rs index 4122f7915015..7787790e4ffd 100644 --- a/src/common/arrow/src/arrow/array/fixed_size_binary/mod.rs +++ b/src/common/arrow/src/arrow/array/fixed_size_binary/mod.rs @@ -21,7 +21,7 @@ use crate::arrow::error::Error; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod iterator; mod mutable; diff --git a/src/common/arrow/src/arrow/array/fixed_size_list/ffi.rs b/src/common/arrow/src/arrow/array/fixed_size_list/ffi.rs deleted file mode 100644 index 1b6314a3c2cf..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_list/ffi.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::FixedSizeListArray; -use crate::arrow::array::ffi::FromFfi; -use crate::arrow::array::ffi::ToFfi; -use crate::arrow::array::Array; -use crate::arrow::error::Result; -use crate::arrow::ffi; - -unsafe impl ToFfi for FixedSizeListArray { - fn buffers(&self) -> Vec> { - vec![self.validity.as_ref().map(|x| x.as_ptr())] - } - - fn children(&self) -> Vec> { - vec![self.values.clone()] - } - - fn offset(&self) -> Option { - Some( - self.validity - .as_ref() - .map(|bitmap| bitmap.offset()) - .unwrap_or_default(), - ) - } - - fn to_ffi_aligned(&self) -> Self { - self.clone() - } -} - -impl FromFfi for FixedSizeListArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let validity = unsafe { array.validity() }?; - let child = unsafe { array.child(0)? }; - let values = ffi::try_from(child)?; - - Self::try_new(data_type, values, validity) - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_list/mod.rs b/src/common/arrow/src/arrow/array/fixed_size_list/mod.rs index 7a212b879b62..2fbb4c63405f 100644 --- a/src/common/arrow/src/arrow/array/fixed_size_list/mod.rs +++ b/src/common/arrow/src/arrow/array/fixed_size_list/mod.rs @@ -23,7 +23,7 @@ use crate::arrow::error::Error; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod iterator; mod mutable; diff --git a/src/common/arrow/src/arrow/array/list/ffi.rs b/src/common/arrow/src/arrow/array/list/ffi.rs deleted file mode 100644 index fae1a6d4ea1c..000000000000 --- a/src/common/arrow/src/arrow/array/list/ffi.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::ffi::ToFfi; -use super::super::Array; -use super::ListArray; -use crate::arrow::array::FromFfi; -use crate::arrow::bitmap::align; -use crate::arrow::error::Result; -use crate::arrow::ffi; -use crate::arrow::offset::Offset; -use crate::arrow::offset::OffsetsBuffer; - -unsafe impl ToFfi for ListArray { - fn buffers(&self) -> Vec> { - vec![ - self.validity.as_ref().map(|x| x.as_ptr()), - Some(self.offsets.buffer().data_ptr().cast::()), - ] - } - - fn children(&self) -> Vec> { - vec![self.values.clone()] - } - - fn offset(&self) -> Option { - let offset = self.offsets.buffer().offset(); - if let Some(bitmap) = self.validity.as_ref() { - if bitmap.offset() == offset { - Some(offset) - } else { - None - } - } else { - Some(offset) - } - } - - fn to_ffi_aligned(&self) -> Self { - let offset = self.offsets.buffer().offset(); - - let validity = self.validity.as_ref().map(|bitmap| { - if bitmap.offset() == offset { - bitmap.clone() - } else { - align(bitmap, offset) - } - }); - - Self { - data_type: self.data_type.clone(), - validity, - offsets: self.offsets.clone(), - values: self.values.clone(), - } - } -} - -impl FromFfi for ListArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let validity = unsafe { array.validity() }?; - let offsets = unsafe { array.buffer::(1) }?; - let child = unsafe { array.child(0)? }; - let values = ffi::try_from(child)?; - - // assumption that data from FFI is well constructed - let offsets = unsafe { OffsetsBuffer::new_unchecked(offsets) }; - - Ok(Self::new(data_type, offsets, values, validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/list/mod.rs b/src/common/arrow/src/arrow/array/list/mod.rs index 4ed711a92116..db68a7d03033 100644 --- a/src/common/arrow/src/arrow/array/list/mod.rs +++ b/src/common/arrow/src/arrow/array/list/mod.rs @@ -26,7 +26,7 @@ use crate::arrow::offset::OffsetsBuffer; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod iterator; pub use iterator::*; diff --git a/src/common/arrow/src/arrow/array/map/ffi.rs b/src/common/arrow/src/arrow/array/map/ffi.rs deleted file mode 100644 index 315d47cf12d1..000000000000 --- a/src/common/arrow/src/arrow/array/map/ffi.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::ffi::ToFfi; -use super::super::Array; -use super::MapArray; -use crate::arrow::array::FromFfi; -use crate::arrow::bitmap::align; -use crate::arrow::error::Result; -use crate::arrow::ffi; -use crate::arrow::offset::OffsetsBuffer; - -unsafe impl ToFfi for MapArray { - fn buffers(&self) -> Vec> { - vec![ - self.validity.as_ref().map(|x| x.as_ptr()), - Some(self.offsets.buffer().data_ptr().cast::()), - ] - } - - fn children(&self) -> Vec> { - vec![self.field.clone()] - } - - fn offset(&self) -> Option { - let offset = self.offsets.buffer().offset(); - if let Some(bitmap) = self.validity.as_ref() { - if bitmap.offset() == offset { - Some(offset) - } else { - None - } - } else { - Some(offset) - } - } - - fn to_ffi_aligned(&self) -> Self { - let offset = self.offsets.buffer().offset(); - - let validity = self.validity.as_ref().map(|bitmap| { - if bitmap.offset() == offset { - bitmap.clone() - } else { - align(bitmap, offset) - } - }); - - Self { - data_type: self.data_type.clone(), - validity, - offsets: self.offsets.clone(), - field: self.field.clone(), - } - } -} - -impl FromFfi for MapArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let validity = unsafe { array.validity() }?; - let offsets = unsafe { array.buffer::(1) }?; - let child = array.child(0)?; - let values = ffi::try_from(child)?; - - // assumption that data from FFI is well constructed - let offsets = unsafe { OffsetsBuffer::new_unchecked(offsets) }; - - Self::try_new(data_type, offsets, values, validity) - } -} diff --git a/src/common/arrow/src/arrow/array/map/mod.rs b/src/common/arrow/src/arrow/array/map/mod.rs index d329f8208be3..193084293f4b 100644 --- a/src/common/arrow/src/arrow/array/map/mod.rs +++ b/src/common/arrow/src/arrow/array/map/mod.rs @@ -24,7 +24,7 @@ use crate::arrow::offset::OffsetsBuffer; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod iterator; diff --git a/src/common/arrow/src/arrow/array/mod.rs b/src/common/arrow/src/arrow/array/mod.rs index 349a3345f6db..fc72f7e21518 100644 --- a/src/common/arrow/src/arrow/array/mod.rs +++ b/src/common/arrow/src/arrow/array/mod.rs @@ -750,7 +750,7 @@ mod union; mod utf8; mod equal; -mod ffi; + mod fmt; #[doc(hidden)] pub mod indexable; @@ -799,10 +799,6 @@ pub use utf8::MutableUtf8ValuesArray; pub use utf8::Utf8Array; pub use utf8::Utf8ValuesIter; -pub(crate) use self::ffi::offset_buffers_children_dictionary; -pub(crate) use self::ffi::FromFfi; -pub(crate) use self::ffi::ToFfi; - /// A trait describing the ability of a struct to create itself from a iterator. /// This is similar to [`Extend`], but accepted the creation to error. pub trait TryExtend { diff --git a/src/common/arrow/src/arrow/array/null.rs b/src/common/arrow/src/arrow/array/null.rs index 8bead030fade..22617e47fa3d 100644 --- a/src/common/arrow/src/arrow/array/null.rs +++ b/src/common/arrow/src/arrow/array/null.rs @@ -16,15 +16,12 @@ use std::any::Any; use crate::arrow::array::Array; -use crate::arrow::array::FromFfi; use crate::arrow::array::MutableArray; -use crate::arrow::array::ToFfi; use crate::arrow::bitmap::Bitmap; use crate::arrow::bitmap::MutableBitmap; use crate::arrow::datatypes::DataType; use crate::arrow::datatypes::PhysicalType; use crate::arrow::error::Error; -use crate::arrow::ffi; /// The concrete [`Array`] of [`DataType::Null`]. #[derive(Clone)] @@ -175,29 +172,6 @@ impl std::fmt::Debug for NullArray { } } -unsafe impl ToFfi for NullArray { - fn buffers(&self) -> Vec> { - // `None` is technically not required by the specification, but older C++ implementations require it, so leaving - // it here for backward compatibility - vec![None] - } - - fn offset(&self) -> Option { - Some(0) - } - - fn to_ffi_aligned(&self) -> Self { - self.clone() - } -} - -impl FromFfi for NullArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - Self::try_new(data_type, array.array().len()) - } -} - #[cfg(feature = "arrow")] mod arrow { use arrow_data::ArrayData; diff --git a/src/common/arrow/src/arrow/array/primitive/ffi.rs b/src/common/arrow/src/arrow/array/primitive/ffi.rs deleted file mode 100644 index cdcba358e20f..000000000000 --- a/src/common/arrow/src/arrow/array/primitive/ffi.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::PrimitiveArray; -use crate::arrow::array::FromFfi; -use crate::arrow::array::ToFfi; -use crate::arrow::bitmap::align; -use crate::arrow::error::Result; -use crate::arrow::ffi; -use crate::arrow::types::NativeType; - -unsafe impl ToFfi for PrimitiveArray { - fn buffers(&self) -> Vec> { - vec![ - self.validity.as_ref().map(|x| x.as_ptr()), - Some(self.values.data_ptr().cast::()), - ] - } - - fn offset(&self) -> Option { - let offset = self.values.offset(); - if let Some(bitmap) = self.validity.as_ref() { - if bitmap.offset() == offset { - Some(offset) - } else { - None - } - } else { - Some(offset) - } - } - - fn to_ffi_aligned(&self) -> Self { - let offset = self.values.offset(); - - let validity = self.validity.as_ref().map(|bitmap| { - if bitmap.offset() == offset { - bitmap.clone() - } else { - align(bitmap, offset) - } - }); - - Self { - data_type: self.data_type.clone(), - validity, - values: self.values.clone(), - } - } -} - -impl FromFfi for PrimitiveArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let validity = unsafe { array.validity() }?; - let values = unsafe { array.buffer::(1) }?; - - Self::try_new(data_type, values, validity) - } -} diff --git a/src/common/arrow/src/arrow/array/primitive/mod.rs b/src/common/arrow/src/arrow/array/primitive/mod.rs index f1d000652941..f9fdc0176e71 100644 --- a/src/common/arrow/src/arrow/array/primitive/mod.rs +++ b/src/common/arrow/src/arrow/array/primitive/mod.rs @@ -31,7 +31,7 @@ use crate::arrow::types::NativeType; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod from_natural; mod iterator; diff --git a/src/common/arrow/src/arrow/array/struct_/ffi.rs b/src/common/arrow/src/arrow/array/struct_/ffi.rs deleted file mode 100644 index c9d991239af1..000000000000 --- a/src/common/arrow/src/arrow/array/struct_/ffi.rs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::ffi::ToFfi; -use super::super::Array; -use super::super::FromFfi; -use super::StructArray; -use crate::arrow::error::Result; -use crate::arrow::ffi; - -unsafe impl ToFfi for StructArray { - fn buffers(&self) -> Vec> { - vec![self.validity.as_ref().map(|x| x.as_ptr())] - } - - fn children(&self) -> Vec> { - self.values.clone() - } - - fn offset(&self) -> Option { - Some( - self.validity - .as_ref() - .map(|bitmap| bitmap.offset()) - .unwrap_or_default(), - ) - } - - fn to_ffi_aligned(&self) -> Self { - self.clone() - } -} - -impl FromFfi for StructArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let fields = Self::get_fields(&data_type); - - let arrow_array = array.array(); - let validity = unsafe { array.validity() }?; - let len = arrow_array.len(); - let offset = arrow_array.offset(); - let values = (0..fields.len()) - .map(|index| { - let child = array.child(index)?; - ffi::try_from(child).map(|arr| { - // there is a discrepancy with how arrow2 exports sliced - // struct array and how pyarrow does it. - // # Pyarrow - // ## struct array len 3 - // * slice 1 by with len 2 - // offset on struct array: 1 - // length on struct array: 2 - // offset on value array: 0 - // length on value array: 3 - // # Arrow2 - // ## struct array len 3 - // * slice 1 by with len 2 - // offset on struct array: 0 - // length on struct array: 3 - // offset on value array: 1 - // length on value array: 2 - // - // this branch will ensure both can round trip - if arr.len() >= (len + offset) { - arr.sliced(offset, len) - } else { - arr - } - }) - }) - .collect::>>>()?; - - Self::try_new(data_type, values, validity) - } -} diff --git a/src/common/arrow/src/arrow/array/struct_/mod.rs b/src/common/arrow/src/arrow/array/struct_/mod.rs index e6e8d258b37f..b213f401bfd7 100644 --- a/src/common/arrow/src/arrow/array/struct_/mod.rs +++ b/src/common/arrow/src/arrow/array/struct_/mod.rs @@ -23,7 +23,7 @@ use crate::arrow::error::Error; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod iterator; mod mutable; @@ -84,7 +84,7 @@ impl StructArray { .try_for_each(|(index, (data_type, child))| { if data_type != child { Err(Error::oos(format!( - "The children DataTypes of a StructArray must equal the children data types. + "The children DataTypes of a StructArray must equal the children data types. However, the field {index} has data type {data_type:?} but the value has data type {child:?}" ))) } else { diff --git a/src/common/arrow/src/arrow/array/union/ffi.rs b/src/common/arrow/src/arrow/array/union/ffi.rs deleted file mode 100644 index db697f4c620c..000000000000 --- a/src/common/arrow/src/arrow/array/union/ffi.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::ffi::ToFfi; -use super::super::Array; -use super::UnionArray; -use crate::arrow::array::FromFfi; -use crate::arrow::error::Result; -use crate::arrow::ffi; - -unsafe impl ToFfi for UnionArray { - fn buffers(&self) -> Vec> { - if let Some(offsets) = &self.offsets { - vec![ - Some(self.types.data_ptr().cast::()), - Some(offsets.data_ptr().cast::()), - ] - } else { - vec![Some(self.types.data_ptr().cast::())] - } - } - - fn children(&self) -> Vec> { - self.fields.clone() - } - - fn offset(&self) -> Option { - Some(self.types.offset()) - } - - fn to_ffi_aligned(&self) -> Self { - self.clone() - } -} - -impl FromFfi for UnionArray { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let fields = Self::get_fields(&data_type); - - let mut types = unsafe { array.buffer::(0) }?; - let offsets = if Self::is_sparse(&data_type) { - None - } else { - Some(unsafe { array.buffer::(1) }?) - }; - - let length = array.array().len(); - let offset = array.array().offset(); - let fields = (0..fields.len()) - .map(|index| { - let child = array.child(index)?; - ffi::try_from(child) - }) - .collect::>>>()?; - - if offset > 0 { - types.slice(offset, length); - }; - - Self::try_new(data_type, types, fields, offsets) - } -} diff --git a/src/common/arrow/src/arrow/array/union/mod.rs b/src/common/arrow/src/arrow/array/union/mod.rs index 67381659e6c5..2bf2947d5ece 100644 --- a/src/common/arrow/src/arrow/array/union/mod.rs +++ b/src/common/arrow/src/arrow/array/union/mod.rs @@ -27,7 +27,7 @@ use crate::arrow::scalar::Scalar; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod iterator; diff --git a/src/common/arrow/src/arrow/array/utf8/ffi.rs b/src/common/arrow/src/arrow/array/utf8/ffi.rs deleted file mode 100644 index f6c479630408..000000000000 --- a/src/common/arrow/src/arrow/array/utf8/ffi.rs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Utf8Array; -use crate::arrow::array::FromFfi; -use crate::arrow::array::ToFfi; -use crate::arrow::bitmap::align; -use crate::arrow::error::Result; -use crate::arrow::ffi; -use crate::arrow::offset::Offset; -use crate::arrow::offset::OffsetsBuffer; - -unsafe impl ToFfi for Utf8Array { - fn buffers(&self) -> Vec> { - vec![ - self.validity.as_ref().map(|x| x.as_ptr()), - Some(self.offsets.buffer().data_ptr().cast::()), - Some(self.values.data_ptr().cast::()), - ] - } - - fn offset(&self) -> Option { - let offset = self.offsets.buffer().offset(); - if let Some(bitmap) = self.validity.as_ref() { - if bitmap.offset() == offset { - Some(offset) - } else { - None - } - } else { - Some(offset) - } - } - - fn to_ffi_aligned(&self) -> Self { - let offset = self.offsets.buffer().offset(); - - let validity = self.validity.as_ref().map(|bitmap| { - if bitmap.offset() == offset { - bitmap.clone() - } else { - align(bitmap, offset) - } - }); - - Self { - data_type: self.data_type.clone(), - validity, - offsets: self.offsets.clone(), - values: self.values.clone(), - } - } -} - -impl FromFfi for Utf8Array { - unsafe fn try_from_ffi(array: A) -> Result { - let data_type = array.data_type().clone(); - let validity = unsafe { array.validity() }?; - let offsets = unsafe { array.buffer::(1) }?; - let values = unsafe { array.buffer::(2)? }; - - // assumption that data from FFI is well constructed - let offsets = unsafe { OffsetsBuffer::new_unchecked(offsets) }; - - Ok(Self::new_unchecked(data_type, offsets, values, validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/utf8/mod.rs b/src/common/arrow/src/arrow/array/utf8/mod.rs index 53768a2dc590..8010b5118189 100644 --- a/src/common/arrow/src/arrow/array/utf8/mod.rs +++ b/src/common/arrow/src/arrow/array/utf8/mod.rs @@ -33,7 +33,7 @@ use crate::arrow::trusted_len::TrustedLen; #[cfg(feature = "arrow")] mod data; -mod ffi; + pub(super) mod fmt; mod from; mod iterator; diff --git a/src/common/arrow/src/arrow/buffer/mod.rs b/src/common/arrow/src/arrow/buffer/mod.rs index d3e82ce25bcc..873655a813da 100644 --- a/src/common/arrow/src/arrow/buffer/mod.rs +++ b/src/common/arrow/src/arrow/buffer/mod.rs @@ -20,12 +20,8 @@ mod iterator; use std::ops::Deref; -use crate::arrow::ffi::InternalArrowArray; - #[allow(dead_code)] pub(crate) enum BytesAllocator { - InternalArrowArray(InternalArrowArray), - #[cfg(feature = "arrow")] Arrow(arrow_buffer::Buffer), } diff --git a/src/common/arrow/src/arrow/compute/aggregate/memory.rs b/src/common/arrow/src/arrow/compute/aggregate/memory.rs deleted file mode 100644 index b8a78f31b021..000000000000 --- a/src/common/arrow/src/arrow/compute/aggregate/memory.rs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::*; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::PhysicalType; - -fn validity_size(validity: Option<&Bitmap>) -> usize { - validity.as_ref().map(|b| b.as_slice().0.len()).unwrap_or(0) -} - -macro_rules! dyn_binary { - ($array:expr, $ty:ty, $o:ty) => {{ - let array = $array.as_any().downcast_ref::<$ty>().unwrap(); - let offsets = array.offsets().buffer(); - - // in case of Binary/Utf8/List the offsets are sliced, - // not the values buffer - let values_start = offsets[0] as usize; - let values_end = offsets[offsets.len() - 1] as usize; - - values_end - values_start - + offsets.len() * std::mem::size_of::<$o>() - + validity_size(array.validity()) - }}; -} - -fn binview_size(array: &BinaryViewArrayGeneric) -> usize { - array.views().len() * std::mem::size_of::() - + array.data_buffers().iter().map(|b| b.len()).sum::() - + validity_size(array.validity()) -} - -/// Returns the total (heap) allocated size of the array in bytes. -/// # Implementation -/// This estimation is the sum of the size of its buffers, validity, including nested arrays. -/// Multiple arrays may share buffers and bitmaps. Therefore, the size of 2 arrays is not the -/// sum of the sizes computed from this function. In particular, [`StructArray`]'s size is an upper bound. -/// -/// When an array is sliced, its allocated size remains constant because the buffer unchanged. -/// However, this function will yield a smaller number. This is because this function returns -/// the visible size of the buffer, not its total capacity. -/// -/// FFI buffers are included in this estimation. -pub fn estimated_bytes_size(array: &dyn Array) -> usize { - use PhysicalType::*; - match array.data_type().to_physical_type() { - Null => 0, - Boolean => { - let array = array.as_any().downcast_ref::().unwrap(); - array.values().as_slice().0.len() + validity_size(array.validity()) - } - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - - array.values().len() * std::mem::size_of::<$T>() + validity_size(array.validity()) - }), - Binary => dyn_binary!(array, BinaryArray, i32), - FixedSizeBinary => { - let array = array - .as_any() - .downcast_ref::() - .unwrap(); - array.values().len() + validity_size(array.validity()) - } - LargeBinary => dyn_binary!(array, BinaryArray, i64), - Utf8 => dyn_binary!(array, Utf8Array, i32), - LargeUtf8 => dyn_binary!(array, Utf8Array, i64), - List => { - let array = array.as_any().downcast_ref::>().unwrap(); - estimated_bytes_size(array.values().as_ref()) - + array.offsets().len_proxy() * std::mem::size_of::() - + validity_size(array.validity()) - } - FixedSizeList => { - let array = array.as_any().downcast_ref::().unwrap(); - estimated_bytes_size(array.values().as_ref()) + validity_size(array.validity()) - } - LargeList => { - let array = array.as_any().downcast_ref::>().unwrap(); - estimated_bytes_size(array.values().as_ref()) - + array.offsets().len_proxy() * std::mem::size_of::() - + validity_size(array.validity()) - } - Struct => { - let array = array.as_any().downcast_ref::().unwrap(); - array - .values() - .iter() - .map(|x| x.as_ref()) - .map(estimated_bytes_size) - .sum::() - + validity_size(array.validity()) - } - Union => { - let array = array.as_any().downcast_ref::().unwrap(); - let types = array.types().len() * std::mem::size_of::(); - let offsets = array - .offsets() - .as_ref() - .map(|x| x.len() * std::mem::size_of::()) - .unwrap_or_default(); - let fields = array - .fields() - .iter() - .map(|x| x.as_ref()) - .map(estimated_bytes_size) - .sum::(); - types + offsets + fields - } - Dictionary(key_type) => match_integer_type!(key_type, |$T| { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - estimated_bytes_size(array.keys()) + estimated_bytes_size(array.values().as_ref()) - }), - Map => { - let array = array.as_any().downcast_ref::().unwrap(); - let offsets = array.offsets().len_proxy() * std::mem::size_of::(); - offsets + estimated_bytes_size(array.field().as_ref()) + validity_size(array.validity()) - } - Utf8View => binview_size::(array.as_any().downcast_ref().unwrap()), - BinaryView => binview_size::<[u8]>(array.as_any().downcast_ref().unwrap()), - } -} diff --git a/src/common/arrow/src/arrow/compute/aggregate/mod.rs b/src/common/arrow/src/arrow/compute/aggregate/mod.rs deleted file mode 100644 index 4367822a7d1c..000000000000 --- a/src/common/arrow/src/arrow/compute/aggregate/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains different aggregation functions -mod memory; -pub use memory::*; diff --git a/src/common/arrow/src/arrow/compute/arity.rs b/src/common/arrow/src/arrow/compute/arity.rs deleted file mode 100644 index c70630026423..000000000000 --- a/src/common/arrow/src/arrow/compute/arity.rs +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Defines kernels suitable to perform operations to primitive arrays. - -use super::utils::check_same_len; -use super::utils::combine_validities; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; - -/// Applies an unary and infallible function to a [`PrimitiveArray`]. This is the -/// fastest way to perform an operation on a [`PrimitiveArray`] when the benefits -/// of a vectorized operation outweighs the cost of branching nulls and -/// non-nulls. -/// -/// # Implementation -/// This will apply the function for all values, including those on null slots. -/// This implies that the operation must be infallible for any value of the -/// corresponding type or this function may panic. -#[inline] -pub fn unary(array: &PrimitiveArray, op: F, data_type: DataType) -> PrimitiveArray -where - I: NativeType, - O: NativeType, - F: Fn(I) -> O, -{ - let values = array.values().iter().map(|v| op(*v)).collect::>(); - - PrimitiveArray::::new(data_type, values.into(), array.validity().cloned()) -} - -/// Version of unary that checks for errors in the closure used to create the -/// buffer -pub fn try_unary( - array: &PrimitiveArray, - op: F, - data_type: DataType, -) -> Result> -where - I: NativeType, - O: NativeType, - F: Fn(I) -> Result, -{ - let values = array - .values() - .iter() - .map(|v| op(*v)) - .collect::>>()? - .into(); - - Ok(PrimitiveArray::::new( - data_type, - values, - array.validity().cloned(), - )) -} - -/// Version of unary that returns an array and bitmap. Used when working with -/// overflowing operations -pub fn unary_with_bitmap( - array: &PrimitiveArray, - op: F, - data_type: DataType, -) -> (PrimitiveArray, Bitmap) -where - I: NativeType, - O: NativeType, - F: Fn(I) -> (O, bool), -{ - let mut mut_bitmap = MutableBitmap::with_capacity(array.len()); - - let values = array - .values() - .iter() - .map(|v| { - let (res, over) = op(*v); - mut_bitmap.push(over); - res - }) - .collect::>() - .into(); - - ( - PrimitiveArray::::new(data_type, values, array.validity().cloned()), - mut_bitmap.into(), - ) -} - -/// Version of unary that creates a mutable bitmap that is used to keep track -/// of checked operations. The resulting bitmap is compared with the array -/// bitmap to create the final validity array. -pub fn unary_checked( - array: &PrimitiveArray, - op: F, - data_type: DataType, -) -> PrimitiveArray -where - I: NativeType, - O: NativeType, - F: Fn(I) -> Option, -{ - let mut mut_bitmap = MutableBitmap::with_capacity(array.len()); - - let values = array - .values() - .iter() - .map(|v| match op(*v) { - Some(val) => { - mut_bitmap.push(true); - val - } - None => { - mut_bitmap.push(false); - O::default() - } - }) - .collect::>() - .into(); - - // The validity has to be checked against the bitmap created during the - // creation of the values with the iterator. If an error was found during - // the iteration, then the validity is changed to None to mark the value - // as Null - let bitmap: Bitmap = mut_bitmap.into(); - let validity = combine_validities(array.validity(), Some(&bitmap)); - - PrimitiveArray::::new(data_type, values, validity) -} - -/// Applies a binary operations to two primitive arrays. This is the fastest -/// way to perform an operation on two primitive array when the benefits of a -/// vectorized operation outweighs the cost of branching nulls and non-nulls. -/// # Errors -/// This function errors iff the arrays have a different length. -/// # Implementation -/// This will apply the function for all values, including those on null slots. -/// This implies that the operation must be infallible for any value of the -/// corresponding type. -/// The types of the arrays are not checked with this operation. The closure -/// "op" needs to handle the different types in the arrays. The datatype for the -/// resulting array has to be selected by the implementer of the function as -/// an argument for the function. -#[inline] -pub fn binary( - lhs: &PrimitiveArray, - rhs: &PrimitiveArray, - data_type: DataType, - op: F, -) -> PrimitiveArray -where - T: NativeType, - D: NativeType, - F: Fn(T, D) -> T, -{ - check_same_len(lhs, rhs).unwrap(); - - let validity = combine_validities(lhs.validity(), rhs.validity()); - - let values = lhs - .values() - .iter() - .zip(rhs.values().iter()) - .map(|(l, r)| op(*l, *r)) - .collect::>() - .into(); - - PrimitiveArray::::new(data_type, values, validity) -} - -/// Version of binary that checks for errors in the closure used to create the -/// buffer -pub fn try_binary( - lhs: &PrimitiveArray, - rhs: &PrimitiveArray, - data_type: DataType, - op: F, -) -> Result> -where - T: NativeType, - D: NativeType, - F: Fn(T, D) -> Result, -{ - check_same_len(lhs, rhs)?; - - let validity = combine_validities(lhs.validity(), rhs.validity()); - - let values = lhs - .values() - .iter() - .zip(rhs.values().iter()) - .map(|(l, r)| op(*l, *r)) - .collect::>>()? - .into(); - - Ok(PrimitiveArray::::new(data_type, values, validity)) -} - -/// Version of binary that returns an array and bitmap. Used when working with -/// overflowing operations -pub fn binary_with_bitmap( - lhs: &PrimitiveArray, - rhs: &PrimitiveArray, - data_type: DataType, - op: F, -) -> (PrimitiveArray, Bitmap) -where - T: NativeType, - D: NativeType, - F: Fn(T, D) -> (T, bool), -{ - check_same_len(lhs, rhs).unwrap(); - - let validity = combine_validities(lhs.validity(), rhs.validity()); - - let mut mut_bitmap = MutableBitmap::with_capacity(lhs.len()); - - let values = lhs - .values() - .iter() - .zip(rhs.values().iter()) - .map(|(l, r)| { - let (res, over) = op(*l, *r); - mut_bitmap.push(over); - res - }) - .collect::>() - .into(); - - ( - PrimitiveArray::::new(data_type, values, validity), - mut_bitmap.into(), - ) -} - -/// Version of binary that creates a mutable bitmap that is used to keep track -/// of checked operations. The resulting bitmap is compared with the array -/// bitmap to create the final validity array. -pub fn binary_checked( - lhs: &PrimitiveArray, - rhs: &PrimitiveArray, - data_type: DataType, - op: F, -) -> PrimitiveArray -where - T: NativeType, - D: NativeType, - F: Fn(T, D) -> Option, -{ - check_same_len(lhs, rhs).unwrap(); - - let mut mut_bitmap = MutableBitmap::with_capacity(lhs.len()); - - let values = lhs - .values() - .iter() - .zip(rhs.values().iter()) - .map(|(l, r)| match op(*l, *r) { - Some(val) => { - mut_bitmap.push(true); - val - } - None => { - mut_bitmap.push(false); - T::default() - } - }) - .collect::>() - .into(); - - let bitmap: Bitmap = mut_bitmap.into(); - let validity = combine_validities(lhs.validity(), rhs.validity()); - - // The validity has to be checked against the bitmap created during the - // creation of the values with the iterator. If an error was found during - // the iteration, then the validity is changed to None to mark the value - // as Null - let validity = combine_validities(validity.as_ref(), Some(&bitmap)); - - PrimitiveArray::::new(data_type, values, validity) -} diff --git a/src/common/arrow/src/arrow/compute/cast/binary_to.rs b/src/common/arrow/src/arrow/compute/cast/binary_to.rs deleted file mode 100644 index 4f5df7f1a01c..000000000000 --- a/src/common/arrow/src/arrow/compute/cast/binary_to.rs +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::CastOptions; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::types::NativeType; - -/// Conversion of binary -pub fn binary_to_large_binary(from: &BinaryArray, to_data_type: DataType) -> BinaryArray { - let values = from.values().clone(); - BinaryArray::::new( - to_data_type, - from.offsets().into(), - values, - from.validity().cloned(), - ) -} - -/// Conversion of binary -pub fn binary_large_to_binary( - from: &BinaryArray, - to_data_type: DataType, -) -> Result> { - let values = from.values().clone(); - let offsets = from.offsets().try_into()?; - Ok(BinaryArray::::new( - to_data_type, - offsets, - values, - from.validity().cloned(), - )) -} - -/// Conversion to utf8 -pub fn binary_to_utf8( - from: &BinaryArray, - to_data_type: DataType, -) -> Result> { - Utf8Array::::try_new( - to_data_type, - from.offsets().clone(), - from.values().clone(), - from.validity().cloned(), - ) -} - -/// Conversion to utf8 -/// # Errors -/// This function errors if the values are not valid utf8 -pub fn binary_to_large_utf8( - from: &BinaryArray, - to_data_type: DataType, -) -> Result> { - let values = from.values().clone(); - let offsets = from.offsets().into(); - - Utf8Array::::try_new(to_data_type, offsets, values, from.validity().cloned()) -} - -/// Casts a [`BinaryArray`] to a [`PrimitiveArray`] at best-effort using `lexical_core::parse_partial`, making any uncastable value as zero. -pub fn partial_binary_to_primitive( - from: &BinaryArray, - to: &DataType, -) -> PrimitiveArray -where - T: NativeType + lexical_core::FromLexical, -{ - let iter = from - .iter() - .map(|x| x.and_then::(|x| lexical_core::parse_partial(x).ok().map(|x| x.0))); - - PrimitiveArray::::from_trusted_len_iter(iter).to(to.clone()) -} - -/// Casts a [`BinaryArray`] to a [`PrimitiveArray`], making any uncastable value a Null. -pub fn binary_to_primitive( - from: &BinaryArray, - to: &DataType, -) -> PrimitiveArray -where - T: NativeType + lexical_core::FromLexical, -{ - let iter = from - .iter() - .map(|x| x.and_then::(|x| lexical_core::parse(x).ok())); - - PrimitiveArray::::from_trusted_len_iter(iter).to(to.clone()) -} - -pub(super) fn binary_to_primitive_dyn( - from: &dyn Array, - to: &DataType, - options: CastOptions, -) -> Result> -where - T: NativeType + lexical_core::FromLexical, -{ - let from = from.as_any().downcast_ref().unwrap(); - if options.partial { - Ok(Box::new(partial_binary_to_primitive::(from, to))) - } else { - Ok(Box::new(binary_to_primitive::(from, to))) - } -} - -/// Cast [`BinaryArray`] to [`DictionaryArray`], also known as packing. -/// # Errors -/// This function errors if the maximum key is smaller than the number of distinct elements -/// in the array. -pub fn binary_to_dictionary( - from: &BinaryArray, -) -> Result> { - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(from.iter())?; - - Ok(array.into()) -} - -pub(super) fn binary_to_dictionary_dyn( - from: &dyn Array, -) -> Result> { - let values = from.as_any().downcast_ref().unwrap(); - binary_to_dictionary::(values).map(|x| Box::new(x) as Box) -} - -fn fixed_size_to_offsets(values_len: usize, fixed_size: usize) -> Offsets { - let offsets = (0..(values_len + 1)) - .step_by(fixed_size) - .map(|v| O::from_usize(v).unwrap()) - .collect(); - // Safety - // * every element is `>= 0` - // * element at position `i` is >= than element at position `i-1`. - unsafe { Offsets::new_unchecked(offsets) } -} - -/// Conversion of `FixedSizeBinary` to `Binary`. -pub fn fixed_size_binary_binary( - from: &FixedSizeBinaryArray, - to_data_type: DataType, -) -> BinaryArray { - let values = from.values().clone(); - let offsets = fixed_size_to_offsets(values.len(), from.size()); - BinaryArray::::new( - to_data_type, - offsets.into(), - values, - from.validity().cloned(), - ) -} - -/// Conversion of binary -pub fn binary_to_list(from: &BinaryArray, to_data_type: DataType) -> ListArray { - let values = from.values().clone(); - let values = PrimitiveArray::new(DataType::UInt8, values, None); - ListArray::::new( - to_data_type, - from.offsets().clone(), - values.boxed(), - from.validity().cloned(), - ) -} diff --git a/src/common/arrow/src/arrow/compute/cast/boolean_to.rs b/src/common/arrow/src/arrow/compute/cast/boolean_to.rs deleted file mode 100644 index 35ac806ba914..000000000000 --- a/src/common/arrow/src/arrow/compute/cast/boolean_to.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::array::BooleanArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::types::NativeType; - -pub(super) fn boolean_to_primitive_dyn(array: &dyn Array) -> Result> -where T: NativeType + num_traits::One { - let array = array.as_any().downcast_ref().unwrap(); - Ok(Box::new(boolean_to_primitive::(array))) -} - -/// Casts the [`BooleanArray`] to a [`PrimitiveArray`]. -pub fn boolean_to_primitive(from: &BooleanArray) -> PrimitiveArray -where T: NativeType + num_traits::One { - let values = from - .values() - .iter() - .map(|x| if x { T::one() } else { T::default() }) - .collect::>(); - - PrimitiveArray::::new(T::PRIMITIVE.into(), values.into(), from.validity().cloned()) -} - -/// Casts the [`BooleanArray`] to a [`Utf8Array`], casting trues to `"1"` and falses to `"0"` -pub fn boolean_to_utf8(from: &BooleanArray) -> Utf8Array { - let iter = from.values().iter().map(|x| if x { "1" } else { "0" }); - Utf8Array::from_trusted_len_values_iter(iter) -} - -pub(super) fn boolean_to_utf8_dyn(array: &dyn Array) -> Result> { - let array = array.as_any().downcast_ref().unwrap(); - Ok(Box::new(boolean_to_utf8::(array))) -} - -/// Casts the [`BooleanArray`] to a [`BinaryArray`], casting trues to `"1"` and falses to `"0"` -pub fn boolean_to_binary(from: &BooleanArray) -> BinaryArray { - let iter = from.values().iter().map(|x| if x { b"1" } else { b"0" }); - BinaryArray::from_trusted_len_values_iter(iter) -} - -pub(super) fn boolean_to_binary_dyn(array: &dyn Array) -> Result> { - let array = array.as_any().downcast_ref().unwrap(); - Ok(Box::new(boolean_to_binary::(array))) -} diff --git a/src/common/arrow/src/arrow/compute/cast/decimal_to.rs b/src/common/arrow/src/arrow/compute/cast/decimal_to.rs deleted file mode 100644 index 1f1a069aa803..000000000000 --- a/src/common/arrow/src/arrow/compute/cast/decimal_to.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use num_traits::AsPrimitive; -use num_traits::Float; -use num_traits::NumCast; - -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; - -#[inline] -fn decimal_to_decimal_impl Option>( - from: &PrimitiveArray, - op: F, - to_precision: usize, - to_scale: usize, -) -> PrimitiveArray { - let min_for_precision = 9_i128 - .saturating_pow(1 + to_precision as u32) - .saturating_neg(); - let max_for_precision = 9_i128.saturating_pow(1 + to_precision as u32); - - let values = from.iter().map(|x| { - x.and_then(|x| { - op(*x).and_then(|x| { - if x > max_for_precision || x < min_for_precision { - None - } else { - Some(x) - } - }) - }) - }); - PrimitiveArray::::from_trusted_len_iter(values) - .to(DataType::Decimal(to_precision, to_scale)) -} - -/// Returns a [`PrimitiveArray`] with the casted values. Values are `None` on overflow -pub fn decimal_to_decimal( - from: &PrimitiveArray, - to_precision: usize, - to_scale: usize, -) -> PrimitiveArray { - let (from_precision, from_scale) = - if let DataType::Decimal(p, s) = from.data_type().to_logical_type() { - (*p, *s) - } else { - panic!("internal error: i128 is always a decimal") - }; - - if to_scale == from_scale && to_precision >= from_precision { - // fast path - return from.clone().to(DataType::Decimal(to_precision, to_scale)); - } - // todo: other fast paths include increasing scale and precision by so that - // a number will never overflow (validity is preserved) - - if from_scale > to_scale { - let factor = 10_i128.pow((from_scale - to_scale) as u32); - decimal_to_decimal_impl( - from, - |x: i128| x.checked_div(factor), - to_precision, - to_scale, - ) - } else { - let factor = 10_i128.pow((to_scale - from_scale) as u32); - decimal_to_decimal_impl( - from, - |x: i128| x.checked_mul(factor), - to_precision, - to_scale, - ) - } -} - -pub(super) fn decimal_to_decimal_dyn( - from: &dyn Array, - to_precision: usize, - to_scale: usize, -) -> Result> { - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(decimal_to_decimal(from, to_precision, to_scale))) -} - -/// Returns a [`PrimitiveArray`] with the casted values. Values are `None` on overflow -pub fn decimal_to_float(from: &PrimitiveArray) -> PrimitiveArray -where - T: NativeType + Float, - f64: AsPrimitive, -{ - let (_, from_scale) = if let DataType::Decimal(p, s) = from.data_type().to_logical_type() { - (*p, *s) - } else { - panic!("internal error: i128 is always a decimal") - }; - - let div = 10_f64.powi(from_scale as i32); - let values = from - .values() - .iter() - .map(|x| (*x as f64 / div).as_()) - .collect(); - - PrimitiveArray::::new(T::PRIMITIVE.into(), values, from.validity().cloned()) -} - -pub(super) fn decimal_to_float_dyn(from: &dyn Array) -> Result> -where - T: NativeType + Float, - f64: AsPrimitive, -{ - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(decimal_to_float::(from))) -} - -/// Returns a [`PrimitiveArray`] with the casted values. Values are `None` on overflow -pub fn decimal_to_integer(from: &PrimitiveArray) -> PrimitiveArray -where T: NativeType + NumCast { - let (_, from_scale) = if let DataType::Decimal(p, s) = from.data_type().to_logical_type() { - (*p, *s) - } else { - panic!("internal error: i128 is always a decimal") - }; - - let factor = 10_i128.pow(from_scale as u32); - let values = from.iter().map(|x| x.and_then(|x| T::from(*x / factor))); - - PrimitiveArray::from_trusted_len_iter(values) -} - -pub(super) fn decimal_to_integer_dyn(from: &dyn Array) -> Result> -where T: NativeType + NumCast { - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(decimal_to_integer::(from))) -} diff --git a/src/common/arrow/src/arrow/compute/cast/dictionary_to.rs b/src/common/arrow/src/arrow/compute/cast/dictionary_to.rs deleted file mode 100644 index bd810f8055e2..000000000000 --- a/src/common/arrow/src/arrow/compute/cast/dictionary_to.rs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::primitive_as_primitive; -use super::primitive_to_primitive; -use super::CastOptions; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::compute::cast::cast; -use crate::arrow::compute::take::take; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -macro_rules! key_cast { - ($keys:expr, $values:expr, $array:expr, $to_keys_type:expr, $to_type:ty, $to_datatype:expr) => {{ - let cast_keys = primitive_to_primitive::<_, $to_type>($keys, $to_keys_type); - - // Failure to cast keys (because they don't fit in the - // target type) results in NULL values; - if cast_keys.null_count() > $keys.null_count() { - return Err(Error::Overflow); - } - // Safety: this is safe because given a type `T` that fits in a `usize`, casting it to type `P` either overflows or also fits in a `usize` - unsafe { - DictionaryArray::try_new_unchecked($to_datatype, cast_keys, $values.clone()) - } - .map(|x| x.boxed()) - }}; -} - -/// Casts a [`DictionaryArray`] to a new [`DictionaryArray`] by keeping the -/// keys and casting the values to `values_type`. -/// # Errors -/// This function errors if the values are not castable to `values_type` -pub fn dictionary_to_dictionary_values( - from: &DictionaryArray, - values_type: &DataType, -) -> Result> { - let keys = from.keys(); - let values = from.values(); - let length = values.len(); - - let values = cast(values.as_ref(), values_type, CastOptions::default())?; - - assert_eq!(values.len(), length); // this is guaranteed by `cast` - unsafe { - DictionaryArray::try_new_unchecked(from.data_type().clone(), keys.clone(), values.clone()) - } -} - -/// Similar to dictionary_to_dictionary_values, but overflowing cast is wrapped -pub fn wrapping_dictionary_to_dictionary_values( - from: &DictionaryArray, - values_type: &DataType, -) -> Result> { - let keys = from.keys(); - let values = from.values(); - let length = values.len(); - - let values = cast(values.as_ref(), values_type, CastOptions { - wrapped: true, - partial: false, - })?; - assert_eq!(values.len(), length); // this is guaranteed by `cast` - unsafe { - DictionaryArray::try_new_unchecked(from.data_type().clone(), keys.clone(), values.clone()) - } -} - -/// Casts a [`DictionaryArray`] to a new [`DictionaryArray`] backed by a -/// different physical type of the keys, while keeping the values equal. -/// # Errors -/// Errors if any of the old keys' values is larger than the maximum value -/// supported by the new physical type. -pub fn dictionary_to_dictionary_keys( - from: &DictionaryArray, -) -> Result> -where - K1: DictionaryKey + num_traits::NumCast, - K2: DictionaryKey + num_traits::NumCast, -{ - let keys = from.keys(); - let values = from.values(); - let is_ordered = from.is_ordered(); - - let casted_keys = primitive_to_primitive::(keys, &K2::PRIMITIVE.into()); - - if casted_keys.null_count() > keys.null_count() { - Err(Error::Overflow) - } else { - let data_type = DataType::Dictionary( - K2::KEY_TYPE, - Box::new(values.data_type().clone()), - is_ordered, - ); - // Safety: this is safe because given a type `T` that fits in a `usize`, casting it to type `P` either overflows or also fits in a `usize` - unsafe { DictionaryArray::try_new_unchecked(data_type, casted_keys, values.clone()) } - } -} - -/// Similar to dictionary_to_dictionary_keys, but overflowing cast is wrapped -pub fn wrapping_dictionary_to_dictionary_keys( - from: &DictionaryArray, -) -> Result> -where - K1: DictionaryKey + num_traits::AsPrimitive, - K2: DictionaryKey, -{ - let keys = from.keys(); - let values = from.values(); - let is_ordered = from.is_ordered(); - - let casted_keys = primitive_as_primitive::(keys, &K2::PRIMITIVE.into()); - - if casted_keys.null_count() > keys.null_count() { - Err(Error::Overflow) - } else { - let data_type = DataType::Dictionary( - K2::KEY_TYPE, - Box::new(values.data_type().clone()), - is_ordered, - ); - // some of the values may not fit in `usize` and thus this needs to be checked - DictionaryArray::try_new(data_type, casted_keys, values.clone()) - } -} - -pub(super) fn dictionary_cast_dyn( - array: &dyn Array, - to_type: &DataType, - options: CastOptions, -) -> Result> { - let array = array.as_any().downcast_ref::>().unwrap(); - let keys = array.keys(); - let values = array.values(); - - match to_type { - DataType::Dictionary(to_keys_type, to_values_type, _) => { - let values = cast(values.as_ref(), to_values_type, options)?; - - // create the appropriate array type - let to_key_type = (*to_keys_type).into(); - - // Safety: - // we return an error on overflow so the integers remain within bounds - match_integer_type!(to_keys_type, |$T| { - key_cast!(keys, values, array, &to_key_type, $T, to_type.clone()) - }) - } - _ => unpack_dictionary::(keys, values.as_ref(), to_type, options), - } -} - -// Unpack the dictionary -fn unpack_dictionary( - keys: &PrimitiveArray, - values: &dyn Array, - to_type: &DataType, - options: CastOptions, -) -> Result> -where - K: DictionaryKey + num_traits::NumCast, -{ - // attempt to cast the dict values to the target type - // use the take kernel to expand out the dictionary - let values = cast(values, to_type, options)?; - - // take requires first casting i32 - let indices = primitive_to_primitive::<_, i32>(keys, &DataType::Int32); - - take(values.as_ref(), &indices) -} - -/// Casts a [`DictionaryArray`] to its values' [`DataType`], also known as unpacking. -/// The resulting array has the same length. -pub fn dictionary_to_values(from: &DictionaryArray) -> Box -where K: DictionaryKey + num_traits::NumCast { - // take requires first casting i64 - let indices = primitive_to_primitive::<_, i64>(from.keys(), &DataType::Int64); - - // unwrap: The dictionary guarantees that the keys are not out-of-bounds. - take(from.values().as_ref(), &indices).unwrap() -} diff --git a/src/common/arrow/src/arrow/compute/cast/mod.rs b/src/common/arrow/src/arrow/compute/cast/mod.rs deleted file mode 100644 index 9de683146bd4..000000000000 --- a/src/common/arrow/src/arrow/compute/cast/mod.rs +++ /dev/null @@ -1,1030 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Defines different casting operators such as [`cast`] or [`primitive_to_binary`]. - -mod binary_to; -mod boolean_to; -mod decimal_to; -mod dictionary_to; -mod primitive_to; -mod utf8_to; - -pub use binary_to::*; -pub use boolean_to::*; -pub use decimal_to::*; -pub use dictionary_to::*; -pub use primitive_to::*; -pub use utf8_to::*; - -use crate::arrow::array::*; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; - -/// options defining how Cast kernels behave -#[derive(Clone, Copy, Debug, Default)] -pub struct CastOptions { - /// default to false - /// whether an overflowing cast should be converted to `None` (default), or be wrapped (i.e. `256i16 as u8 = 0` vectorized). - /// Settings this to `true` is 5-6x faster for numeric types. - pub wrapped: bool, - /// default to false - /// whether to cast to an integer at the best-effort - pub partial: bool, -} - -impl CastOptions { - fn with_wrapped(&self, v: bool) -> Self { - let mut option = *self; - option.wrapped = v; - option - } -} - -/// Returns true if this type is numeric: (UInt*, Unit*, or Float*). -fn is_numeric(t: &DataType) -> bool { - use DataType::*; - matches!( - t, - UInt8 | UInt16 | UInt32 | UInt64 | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 - ) -} - -macro_rules! primitive_dyn { - ($from:expr, $expr:tt) => {{ - let from = $from.as_any().downcast_ref().unwrap(); - Ok(Box::new($expr(from))) - }}; - ($from:expr, $expr:tt, $to:expr) => {{ - let from = $from.as_any().downcast_ref().unwrap(); - Ok(Box::new($expr(from, $to))) - }}; - ($from:expr, $expr:tt, $from_t:expr, $to:expr) => {{ - let from = $from.as_any().downcast_ref().unwrap(); - Ok(Box::new($expr(from, $from_t, $to))) - }}; - ($from:expr, $expr:tt, $arg1:expr, $arg2:expr, $arg3:expr) => {{ - let from = $from.as_any().downcast_ref().unwrap(); - Ok(Box::new($expr(from, $arg1, $arg2, $arg3))) - }}; -} - -/// Return true if a value of type `from_type` can be cast into a -/// value of `to_type`. Note that such as cast may be lossy. -/// -/// If this function returns true to stay consistent with the `cast` kernel below. -pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { - use self::DataType::*; - if from_type == to_type { - return true; - } - - match (from_type, to_type) { - (Null, _) | (_, Null) => true, - (Struct(_), _) => false, - (_, Struct(_)) => false, - (FixedSizeList(list_from, _), List(list_to)) => { - can_cast_types(&list_from.data_type, &list_to.data_type) - } - (FixedSizeList(list_from, _), LargeList(list_to)) => { - can_cast_types(&list_from.data_type, &list_to.data_type) - } - (List(list_from), FixedSizeList(list_to, _)) => { - can_cast_types(&list_from.data_type, &list_to.data_type) - } - (LargeList(list_from), FixedSizeList(list_to, _)) => { - can_cast_types(&list_from.data_type, &list_to.data_type) - } - (List(list_from), List(list_to)) => { - can_cast_types(&list_from.data_type, &list_to.data_type) - } - (LargeList(list_from), LargeList(list_to)) => { - can_cast_types(&list_from.data_type, &list_to.data_type) - } - (List(list_from), LargeList(list_to)) if list_from == list_to => true, - (LargeList(list_from), List(list_to)) if list_from == list_to => true, - (_, List(list_to)) => can_cast_types(from_type, &list_to.data_type), - (_, LargeList(list_to)) if from_type != &LargeBinary => { - can_cast_types(from_type, &list_to.data_type) - } - (Dictionary(_, from_value_type, _), Dictionary(_, to_value_type, _)) => { - can_cast_types(from_value_type, to_value_type) - } - (Dictionary(_, value_type, _), _) => can_cast_types(value_type, to_type), - (_, Dictionary(_, value_type, _)) => can_cast_types(from_type, value_type), - - (_, Boolean) => is_numeric(from_type), - (Boolean, _) => { - is_numeric(to_type) - || to_type == &Utf8 - || to_type == &LargeUtf8 - || to_type == &Binary - || to_type == &LargeBinary - } - - (Utf8, to_type) => { - is_numeric(to_type) - || matches!( - to_type, - LargeUtf8 | Binary | Date32 | Date64 | Timestamp(TimeUnit::Nanosecond, _) - ) - } - (LargeUtf8, to_type) => { - is_numeric(to_type) - || matches!( - to_type, - Utf8 | LargeBinary | Date32 | Date64 | Timestamp(TimeUnit::Nanosecond, _) - ) - } - - (Binary, to_type) => { - is_numeric(to_type) || matches!(to_type, LargeBinary | Utf8 | LargeUtf8) - } - (LargeBinary, to_type) => { - is_numeric(to_type) - || match to_type { - Binary | LargeUtf8 => true, - LargeList(field) => matches!(field.data_type, UInt8), - _ => false, - } - } - (FixedSizeBinary(_), to_type) => matches!(to_type, Binary | LargeBinary), - (Timestamp(_, _), Utf8) => true, - (Timestamp(_, _), LargeUtf8) => true, - (_, Utf8) => is_numeric(from_type) || from_type == &Binary, - (_, LargeUtf8) => is_numeric(from_type) || from_type == &LargeBinary, - - (_, Binary) => is_numeric(from_type), - (_, LargeBinary) => is_numeric(from_type), - - // start numeric casts - (UInt8, UInt16) => true, - (UInt8, UInt32) => true, - (UInt8, UInt64) => true, - (UInt8, Int8) => true, - (UInt8, Int16) => true, - (UInt8, Int32) => true, - (UInt8, Int64) => true, - (UInt8, Float32) => true, - (UInt8, Float64) => true, - (UInt8, Decimal(_, _)) => true, - - (UInt16, UInt8) => true, - (UInt16, UInt32) => true, - (UInt16, UInt64) => true, - (UInt16, Int8) => true, - (UInt16, Int16) => true, - (UInt16, Int32) => true, - (UInt16, Int64) => true, - (UInt16, Float32) => true, - (UInt16, Float64) => true, - (UInt16, Decimal(_, _)) => true, - - (UInt32, UInt8) => true, - (UInt32, UInt16) => true, - (UInt32, UInt64) => true, - (UInt32, Int8) => true, - (UInt32, Int16) => true, - (UInt32, Int32) => true, - (UInt32, Int64) => true, - (UInt32, Float32) => true, - (UInt32, Float64) => true, - (UInt32, Decimal(_, _)) => true, - - (UInt64, UInt8) => true, - (UInt64, UInt16) => true, - (UInt64, UInt32) => true, - (UInt64, Int8) => true, - (UInt64, Int16) => true, - (UInt64, Int32) => true, - (UInt64, Int64) => true, - (UInt64, Float32) => true, - (UInt64, Float64) => true, - (UInt64, Decimal(_, _)) => true, - - (Int8, UInt8) => true, - (Int8, UInt16) => true, - (Int8, UInt32) => true, - (Int8, UInt64) => true, - (Int8, Int16) => true, - (Int8, Int32) => true, - (Int8, Int64) => true, - (Int8, Float32) => true, - (Int8, Float64) => true, - (Int8, Decimal(_, _)) => true, - - (Int16, UInt8) => true, - (Int16, UInt16) => true, - (Int16, UInt32) => true, - (Int16, UInt64) => true, - (Int16, Int8) => true, - (Int16, Int32) => true, - (Int16, Int64) => true, - (Int16, Float32) => true, - (Int16, Float64) => true, - (Int16, Decimal(_, _)) => true, - - (Int32, UInt8) => true, - (Int32, UInt16) => true, - (Int32, UInt32) => true, - (Int32, UInt64) => true, - (Int32, Int8) => true, - (Int32, Int16) => true, - (Int32, Int64) => true, - (Int32, Float32) => true, - (Int32, Float64) => true, - (Int32, Decimal(_, _)) => true, - - (Int64, UInt8) => true, - (Int64, UInt16) => true, - (Int64, UInt32) => true, - (Int64, UInt64) => true, - (Int64, Int8) => true, - (Int64, Int16) => true, - (Int64, Int32) => true, - (Int64, Float32) => true, - (Int64, Float64) => true, - (Int64, Decimal(_, _)) => true, - - (Float16, Float32) => true, - - (Float32, UInt8) => true, - (Float32, UInt16) => true, - (Float32, UInt32) => true, - (Float32, UInt64) => true, - (Float32, Int8) => true, - (Float32, Int16) => true, - (Float32, Int32) => true, - (Float32, Int64) => true, - (Float32, Float64) => true, - (Float32, Decimal(_, _)) => true, - - (Float64, UInt8) => true, - (Float64, UInt16) => true, - (Float64, UInt32) => true, - (Float64, UInt64) => true, - (Float64, Int8) => true, - (Float64, Int16) => true, - (Float64, Int32) => true, - (Float64, Int64) => true, - (Float64, Float32) => true, - (Float64, Decimal(_, _)) => true, - - ( - Decimal(_, _), - UInt8 - | UInt16 - | UInt32 - | UInt64 - | Int8 - | Int16 - | Int32 - | Int64 - | Float32 - | Float64 - | Decimal(_, _), - ) => true, - // end numeric casts - - // temporal casts - (Int32, Date32) => true, - (Int32, Time32(_)) => true, - (Date32, Int32) => true, - (Date32, Int64) => true, - (Time32(_), Int32) => true, - (Int64, Date64) => true, - (Int64, Time64(_)) => true, - (Date64, Int32) => true, - (Date64, Int64) => true, - (Time64(_), Int64) => true, - (Date32, Date64) => true, - (Date64, Date32) => true, - (Time32(TimeUnit::Second), Time32(TimeUnit::Millisecond)) => true, - (Time32(TimeUnit::Millisecond), Time32(TimeUnit::Second)) => true, - (Time32(_), Time64(_)) => true, - (Time64(TimeUnit::Microsecond), Time64(TimeUnit::Nanosecond)) => true, - (Time64(TimeUnit::Nanosecond), Time64(TimeUnit::Microsecond)) => true, - (Time64(_), Time32(to_unit)) => { - matches!(to_unit, TimeUnit::Second | TimeUnit::Millisecond) - } - (Timestamp(_, _), Int64) => true, - (Int64, Timestamp(_, _)) => true, - (Timestamp(_, _), Timestamp(_, _)) => true, - (Timestamp(_, _), Date32) => true, - (Timestamp(_, _), Date64) => true, - (Int64, Duration(_)) => true, - (Duration(_), Int64) => true, - (Interval(_), Interval(IntervalUnit::MonthDayNano)) => true, - (_, _) => false, - } -} - -fn cast_list( - array: &ListArray, - to_type: &DataType, - options: CastOptions, -) -> Result> { - let values = array.values(); - let new_values = cast( - values.as_ref(), - ListArray::::get_child_type(to_type), - options, - )?; - - Ok(ListArray::::new( - to_type.clone(), - array.offsets().clone(), - new_values, - array.validity().cloned(), - )) -} - -fn cast_list_to_large_list(array: &ListArray, to_type: &DataType) -> ListArray { - let offsets = array.offsets().into(); - - ListArray::::new( - to_type.clone(), - offsets, - array.values().clone(), - array.validity().cloned(), - ) -} - -fn cast_large_to_list(array: &ListArray, to_type: &DataType) -> ListArray { - let offsets = array.offsets().try_into().expect("Convert me to error"); - - ListArray::::new( - to_type.clone(), - offsets, - array.values().clone(), - array.validity().cloned(), - ) -} - -fn cast_fixed_size_list_to_list( - fixed: &FixedSizeListArray, - to_type: &DataType, - options: CastOptions, -) -> Result> { - let new_values = cast( - fixed.values().as_ref(), - ListArray::::get_child_type(to_type), - options, - )?; - - let offsets = (0..=fixed.len()) - .map(|ix| O::from_as_usize(ix * fixed.size())) - .collect::>(); - // Safety: offsets _are_ monotonically increasing - let offsets = unsafe { Offsets::new_unchecked(offsets) }; - - Ok(ListArray::::new( - to_type.clone(), - offsets.into(), - new_values, - fixed.validity().cloned(), - )) -} - -fn cast_list_to_fixed_size_list( - list: &ListArray, - inner: &Field, - size: usize, - options: CastOptions, -) -> Result { - let offsets = list.offsets().buffer().iter(); - let expected = (0..list.len()).map(|ix| O::from_as_usize(ix * size)); - - match offsets - .zip(expected) - .find(|(actual, expected)| *actual != expected) - { - Some(_) => Err(Error::InvalidArgumentError( - "incompatible offsets in source list".to_string(), - )), - None => { - let sliced_values = list.values().sliced( - list.offsets().first().to_usize(), - list.offsets().range().to_usize(), - ); - let new_values = cast(sliced_values.as_ref(), inner.data_type(), options)?; - Ok(FixedSizeListArray::new( - DataType::FixedSizeList(Box::new(inner.clone()), size), - new_values, - list.validity().cloned(), - )) - } - } -} - -/// Cast `array` to the provided data type and return a new [`Array`] with -/// type `to_type`, if possible. -/// -/// Behavior: -/// * PrimitiveArray to PrimitiveArray: overflowing cast will be None -/// * Boolean to Utf8: `true` => '1', `false` => `0` -/// * Utf8 to numeric: strings that can't be parsed to numbers return null, float strings -/// in integer casts return null -/// * Numeric to boolean: 0 returns `false`, any other value returns `true` -/// * List to List: the underlying data type is cast -/// * Fixed Size List to List: the underlying data type is cast -/// * List to Fixed Size List: the offsets are checked for valid order, then the -/// underlying type is cast. -/// * PrimitiveArray to List: a list array with 1 value per slot is created -/// * Date32 and Date64: precision lost when going to higher interval -/// * Time32 and Time64: precision lost when going to higher interval -/// * Timestamp and Date{32|64}: precision lost when going to higher interval -/// * Temporal to/from backing primitive: zero-copy with data type change -/// -/// Unsupported Casts: -/// * To or from `StructArray` -/// * List to primitive -/// * Utf8 to boolean -/// * Interval and duration -pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Result> { - use DataType::*; - let from_type = array.data_type(); - - // clone array if types are the same - if from_type == to_type { - return Ok(clone(array)); - } - - let as_options = options.with_wrapped(true); - match (from_type, to_type) { - (Null, _) | (_, Null) => Ok(new_null_array(to_type.clone(), array.len())), - (Struct(_), _) => Err(Error::NotYetImplemented( - "Cannot cast from struct to other types".to_string(), - )), - (_, Struct(_)) => Err(Error::NotYetImplemented( - "Cannot cast to struct from other types".to_string(), - )), - (List(_), FixedSizeList(inner, size)) => cast_list_to_fixed_size_list::( - array.as_any().downcast_ref().unwrap(), - inner.as_ref(), - *size, - options, - ) - .map(|x| x.boxed()), - (LargeList(_), FixedSizeList(inner, size)) => cast_list_to_fixed_size_list::( - array.as_any().downcast_ref().unwrap(), - inner.as_ref(), - *size, - options, - ) - .map(|x| x.boxed()), - (FixedSizeList(_, _), List(_)) => cast_fixed_size_list_to_list::( - array.as_any().downcast_ref().unwrap(), - to_type, - options, - ) - .map(|x| x.boxed()), - (FixedSizeList(_, _), LargeList(_)) => cast_fixed_size_list_to_list::( - array.as_any().downcast_ref().unwrap(), - to_type, - options, - ) - .map(|x| x.boxed()), - (List(_), List(_)) => { - cast_list::(array.as_any().downcast_ref().unwrap(), to_type, options) - .map(|x| x.boxed()) - } - (LargeList(_), LargeList(_)) => { - cast_list::(array.as_any().downcast_ref().unwrap(), to_type, options) - .map(|x| x.boxed()) - } - (List(lhs), LargeList(rhs)) if lhs == rhs => { - Ok(cast_list_to_large_list(array.as_any().downcast_ref().unwrap(), to_type).boxed()) - } - (LargeList(lhs), List(rhs)) if lhs == rhs => { - Ok(cast_large_to_list(array.as_any().downcast_ref().unwrap(), to_type).boxed()) - } - - (_, List(to)) => { - // cast primitive to list's primitive - let values = cast(array, &to.data_type, options)?; - // create offsets, where if array.len() = 2, we have [0,1,2] - let offsets = (0..=array.len() as i32).collect::>(); - // Safety: offsets _are_ monotonically increasing - let offsets = unsafe { Offsets::new_unchecked(offsets) }; - - let list_array = ListArray::::new(to_type.clone(), offsets.into(), values, None); - - Ok(Box::new(list_array)) - } - - (_, LargeList(to)) if from_type != &LargeBinary => { - // cast primitive to list's primitive - let values = cast(array, &to.data_type, options)?; - // create offsets, where if array.len() = 2, we have [0,1,2] - let offsets = (0..=array.len() as i64).collect::>(); - // Safety: offsets _are_ monotonically increasing - let offsets = unsafe { Offsets::new_unchecked(offsets) }; - - let list_array = ListArray::::new(to_type.clone(), offsets.into(), values, None); - - Ok(Box::new(list_array)) - } - - (Dictionary(index_type, ..), _) => match_integer_type!(index_type, |$T| { - dictionary_cast_dyn::<$T>(array, to_type, options) - }), - (_, Dictionary(index_type, value_type, _)) => match_integer_type!(index_type, |$T| { - cast_to_dictionary::<$T>(array, value_type, options) - }), - (_, Boolean) => match from_type { - UInt8 => primitive_to_boolean_dyn::(array, to_type.clone()), - UInt16 => primitive_to_boolean_dyn::(array, to_type.clone()), - UInt32 => primitive_to_boolean_dyn::(array, to_type.clone()), - UInt64 => primitive_to_boolean_dyn::(array, to_type.clone()), - Int8 => primitive_to_boolean_dyn::(array, to_type.clone()), - Int16 => primitive_to_boolean_dyn::(array, to_type.clone()), - Int32 => primitive_to_boolean_dyn::(array, to_type.clone()), - Int64 => primitive_to_boolean_dyn::(array, to_type.clone()), - Float32 => primitive_to_boolean_dyn::(array, to_type.clone()), - Float64 => primitive_to_boolean_dyn::(array, to_type.clone()), - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - (Boolean, _) => match to_type { - UInt8 => boolean_to_primitive_dyn::(array), - UInt16 => boolean_to_primitive_dyn::(array), - UInt32 => boolean_to_primitive_dyn::(array), - UInt64 => boolean_to_primitive_dyn::(array), - Int8 => boolean_to_primitive_dyn::(array), - Int16 => boolean_to_primitive_dyn::(array), - Int32 => boolean_to_primitive_dyn::(array), - Int64 => boolean_to_primitive_dyn::(array), - Float32 => boolean_to_primitive_dyn::(array), - Float64 => boolean_to_primitive_dyn::(array), - Utf8 => boolean_to_utf8_dyn::(array), - LargeUtf8 => boolean_to_utf8_dyn::(array), - Binary => boolean_to_binary_dyn::(array), - LargeBinary => boolean_to_binary_dyn::(array), - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - - (Utf8, _) => match to_type { - UInt8 => utf8_to_primitive_dyn::(array, to_type, options), - UInt16 => utf8_to_primitive_dyn::(array, to_type, options), - UInt32 => utf8_to_primitive_dyn::(array, to_type, options), - UInt64 => utf8_to_primitive_dyn::(array, to_type, options), - Int8 => utf8_to_primitive_dyn::(array, to_type, options), - Int16 => utf8_to_primitive_dyn::(array, to_type, options), - Int32 => utf8_to_primitive_dyn::(array, to_type, options), - Int64 => utf8_to_primitive_dyn::(array, to_type, options), - Float32 => utf8_to_primitive_dyn::(array, to_type, options), - Float64 => utf8_to_primitive_dyn::(array, to_type, options), - Date32 => utf8_to_date32_dyn::(array), - Date64 => utf8_to_date64_dyn::(array), - LargeUtf8 => Ok(Box::new(utf8_to_large_utf8( - array.as_any().downcast_ref().unwrap(), - ))), - Binary => Ok(utf8_to_binary::( - array.as_any().downcast_ref().unwrap(), - to_type.clone(), - ) - .boxed()), - Timestamp(TimeUnit::Nanosecond, None) => utf8_to_naive_timestamp_ns_dyn::(array), - Timestamp(TimeUnit::Nanosecond, Some(tz)) => { - utf8_to_timestamp_ns_dyn::(array, tz.clone()) - } - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - (LargeUtf8, _) => match to_type { - UInt8 => utf8_to_primitive_dyn::(array, to_type, options), - UInt16 => utf8_to_primitive_dyn::(array, to_type, options), - UInt32 => utf8_to_primitive_dyn::(array, to_type, options), - UInt64 => utf8_to_primitive_dyn::(array, to_type, options), - Int8 => utf8_to_primitive_dyn::(array, to_type, options), - Int16 => utf8_to_primitive_dyn::(array, to_type, options), - Int32 => utf8_to_primitive_dyn::(array, to_type, options), - Int64 => utf8_to_primitive_dyn::(array, to_type, options), - Float32 => utf8_to_primitive_dyn::(array, to_type, options), - Float64 => utf8_to_primitive_dyn::(array, to_type, options), - Date32 => utf8_to_date32_dyn::(array), - Date64 => utf8_to_date64_dyn::(array), - Utf8 => utf8_large_to_utf8(array.as_any().downcast_ref().unwrap()).map(|x| x.boxed()), - LargeBinary => Ok(utf8_to_binary::( - array.as_any().downcast_ref().unwrap(), - to_type.clone(), - ) - .boxed()), - Timestamp(TimeUnit::Nanosecond, None) => utf8_to_naive_timestamp_ns_dyn::(array), - Timestamp(TimeUnit::Nanosecond, Some(tz)) => { - utf8_to_timestamp_ns_dyn::(array, tz.clone()) - } - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - - (_, Utf8) => match from_type { - UInt8 => primitive_to_utf8_dyn::(array), - UInt16 => primitive_to_utf8_dyn::(array), - UInt32 => primitive_to_utf8_dyn::(array), - UInt64 => primitive_to_utf8_dyn::(array), - Int8 => primitive_to_utf8_dyn::(array), - Int16 => primitive_to_utf8_dyn::(array), - Int32 => primitive_to_utf8_dyn::(array), - Int64 => primitive_to_utf8_dyn::(array), - Float32 => primitive_to_utf8_dyn::(array), - Float64 => primitive_to_utf8_dyn::(array), - Binary => { - let array = array.as_any().downcast_ref::>().unwrap(); - - // perf todo: the offsets are equal; we can speed-up this - let iter = array - .iter() - .map(|x| x.and_then(|x| simdutf8::basic::from_utf8(x).ok())); - - let array = Utf8Array::::from_trusted_len_iter(iter); - Ok(Box::new(array)) - } - Timestamp(from_unit, Some(tz)) => { - let from = array.as_any().downcast_ref().unwrap(); - Ok(Box::new(timestamp_to_utf8::(from, *from_unit, tz)?)) - } - Timestamp(from_unit, None) => { - let from = array.as_any().downcast_ref().unwrap(); - Ok(Box::new(naive_timestamp_to_utf8::(from, *from_unit))) - } - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - - (_, LargeUtf8) => match from_type { - UInt8 => primitive_to_utf8_dyn::(array), - UInt16 => primitive_to_utf8_dyn::(array), - UInt32 => primitive_to_utf8_dyn::(array), - UInt64 => primitive_to_utf8_dyn::(array), - Int8 => primitive_to_utf8_dyn::(array), - Int16 => primitive_to_utf8_dyn::(array), - Int32 => primitive_to_utf8_dyn::(array), - Int64 => primitive_to_utf8_dyn::(array), - Float32 => primitive_to_utf8_dyn::(array), - Float64 => primitive_to_utf8_dyn::(array), - Binary => binary_to_large_utf8(array.as_any().downcast_ref().unwrap(), to_type.clone()) - .map(|x| x.boxed()), - LargeBinary => { - binary_to_utf8::(array.as_any().downcast_ref().unwrap(), to_type.clone()) - .map(|x| x.boxed()) - } - Timestamp(from_unit, Some(tz)) => { - let from = array.as_any().downcast_ref().unwrap(); - Ok(Box::new(timestamp_to_utf8::(from, *from_unit, tz)?)) - } - Timestamp(from_unit, None) => { - let from = array.as_any().downcast_ref().unwrap(); - Ok(Box::new(naive_timestamp_to_utf8::(from, *from_unit))) - } - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - - (Binary, _) => match to_type { - UInt8 => binary_to_primitive_dyn::(array, to_type, options), - UInt16 => binary_to_primitive_dyn::(array, to_type, options), - UInt32 => binary_to_primitive_dyn::(array, to_type, options), - UInt64 => binary_to_primitive_dyn::(array, to_type, options), - Int8 => binary_to_primitive_dyn::(array, to_type, options), - Int16 => binary_to_primitive_dyn::(array, to_type, options), - Int32 => binary_to_primitive_dyn::(array, to_type, options), - Int64 => binary_to_primitive_dyn::(array, to_type, options), - Float32 => binary_to_primitive_dyn::(array, to_type, options), - Float64 => binary_to_primitive_dyn::(array, to_type, options), - LargeBinary => Ok(Box::new(binary_to_large_binary( - array.as_any().downcast_ref().unwrap(), - to_type.clone(), - ))), - Utf8 => binary_to_utf8::(array.as_any().downcast_ref().unwrap(), to_type.clone()) - .map(|x| x.boxed()), - - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - - (LargeBinary, _) => { - match to_type { - UInt8 => binary_to_primitive_dyn::(array, to_type, options), - UInt16 => binary_to_primitive_dyn::(array, to_type, options), - UInt32 => binary_to_primitive_dyn::(array, to_type, options), - UInt64 => binary_to_primitive_dyn::(array, to_type, options), - Int8 => binary_to_primitive_dyn::(array, to_type, options), - Int16 => binary_to_primitive_dyn::(array, to_type, options), - Int32 => binary_to_primitive_dyn::(array, to_type, options), - Int64 => binary_to_primitive_dyn::(array, to_type, options), - Float32 => binary_to_primitive_dyn::(array, to_type, options), - Float64 => binary_to_primitive_dyn::(array, to_type, options), - Binary => { - binary_large_to_binary(array.as_any().downcast_ref().unwrap(), to_type.clone()) - .map(|x| x.boxed()) - } - LargeUtf8 => { - binary_to_utf8::(array.as_any().downcast_ref().unwrap(), to_type.clone()) - .map(|x| x.boxed()) - } - LargeList(inner) if matches!(inner.data_type, DataType::UInt8) => Ok( - binary_to_list::(array.as_any().downcast_ref().unwrap(), to_type.clone()) - .boxed(), - ), - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - } - } - (FixedSizeBinary(_), _) => match to_type { - Binary => Ok(fixed_size_binary_binary::( - array.as_any().downcast_ref().unwrap(), - to_type.clone(), - ) - .boxed()), - LargeBinary => Ok(fixed_size_binary_binary::( - array.as_any().downcast_ref().unwrap(), - to_type.clone(), - ) - .boxed()), - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - - (_, Binary) => match from_type { - UInt8 => primitive_to_binary_dyn::(array), - UInt16 => primitive_to_binary_dyn::(array), - UInt32 => primitive_to_binary_dyn::(array), - UInt64 => primitive_to_binary_dyn::(array), - Int8 => primitive_to_binary_dyn::(array), - Int16 => primitive_to_binary_dyn::(array), - Int32 => primitive_to_binary_dyn::(array), - Int64 => primitive_to_binary_dyn::(array), - Float32 => primitive_to_binary_dyn::(array), - Float64 => primitive_to_binary_dyn::(array), - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - - (_, LargeBinary) => match from_type { - UInt8 => primitive_to_binary_dyn::(array), - UInt16 => primitive_to_binary_dyn::(array), - UInt32 => primitive_to_binary_dyn::(array), - UInt64 => primitive_to_binary_dyn::(array), - Int8 => primitive_to_binary_dyn::(array), - Int16 => primitive_to_binary_dyn::(array), - Int32 => primitive_to_binary_dyn::(array), - Int64 => primitive_to_binary_dyn::(array), - Float32 => primitive_to_binary_dyn::(array), - Float64 => primitive_to_binary_dyn::(array), - _ => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - }, - - // start numeric casts - (UInt8, UInt16) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt8, UInt32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt8, UInt64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt8, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt8, Int16) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt8, Int32) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt8, Int64) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt8, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt8, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt8, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), - - (UInt16, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt16, UInt32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt16, UInt64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt16, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt16, Int16) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt16, Int32) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt16, Int64) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt16, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt16, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt16, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), - - (UInt32, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt32, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt32, UInt64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt32, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt32, Int16) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt32, Int32) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt32, Int64) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt32, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt32, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt32, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), - - (UInt64, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt64, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt64, UInt32) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt64, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt64, Int16) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt64, Int32) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt64, Int64) => primitive_to_primitive_dyn::(array, to_type, options), - (UInt64, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt64, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt64, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), - - (Int8, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (Int8, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), - (Int8, UInt32) => primitive_to_primitive_dyn::(array, to_type, options), - (Int8, UInt64) => primitive_to_primitive_dyn::(array, to_type, options), - (Int8, Int16) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int8, Int32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int8, Int64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int8, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int8, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int8, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), - - (Int16, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (Int16, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), - (Int16, UInt32) => primitive_to_primitive_dyn::(array, to_type, options), - (Int16, UInt64) => primitive_to_primitive_dyn::(array, to_type, options), - (Int16, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (Int16, Int32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int16, Int64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int16, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int16, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int16, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), - - (Int32, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (Int32, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), - (Int32, UInt32) => primitive_to_primitive_dyn::(array, to_type, options), - (Int32, UInt64) => primitive_to_primitive_dyn::(array, to_type, options), - (Int32, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (Int32, Int16) => primitive_to_primitive_dyn::(array, to_type, options), - (Int32, Int64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int32, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int32, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int32, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), - - (Int64, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (Int64, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), - (Int64, UInt32) => primitive_to_primitive_dyn::(array, to_type, options), - (Int64, UInt64) => primitive_to_primitive_dyn::(array, to_type, options), - (Int64, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (Int64, Int16) => primitive_to_primitive_dyn::(array, to_type, options), - (Int64, Int32) => primitive_to_primitive_dyn::(array, to_type, options), - (Int64, Float32) => primitive_to_primitive_dyn::(array, to_type, options), - (Int64, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int64, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), - - (Float16, Float32) => { - let from = array.as_any().downcast_ref().unwrap(); - Ok(f16_to_f32(from).boxed()) - } - - (Float32, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (Float32, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), - (Float32, UInt32) => primitive_to_primitive_dyn::(array, to_type, options), - (Float32, UInt64) => primitive_to_primitive_dyn::(array, to_type, options), - (Float32, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (Float32, Int16) => primitive_to_primitive_dyn::(array, to_type, options), - (Float32, Int32) => primitive_to_primitive_dyn::(array, to_type, options), - (Float32, Int64) => primitive_to_primitive_dyn::(array, to_type, options), - (Float32, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Float32, Decimal(p, s)) => float_to_decimal_dyn::(array, *p, *s), - - (Float64, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, UInt32) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, UInt64) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, Int8) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, Int16) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, Int32) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, Int64) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, Float32) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, Decimal(p, s)) => float_to_decimal_dyn::(array, *p, *s), - - (Decimal(_, _), UInt8) => decimal_to_integer_dyn::(array), - (Decimal(_, _), UInt16) => decimal_to_integer_dyn::(array), - (Decimal(_, _), UInt32) => decimal_to_integer_dyn::(array), - (Decimal(_, _), UInt64) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Int8) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Int16) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Int32) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Int64) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Float32) => decimal_to_float_dyn::(array), - (Decimal(_, _), Float64) => decimal_to_float_dyn::(array), - (Decimal(_, _), Decimal(to_p, to_s)) => decimal_to_decimal_dyn(array, *to_p, *to_s), - // end numeric casts - - // temporal casts - (Int32, Date32) => primitive_to_same_primitive_dyn::(array, to_type), - (Int32, Time32(TimeUnit::Second)) => primitive_to_same_primitive_dyn::(array, to_type), - (Int32, Time32(TimeUnit::Millisecond)) => { - primitive_to_same_primitive_dyn::(array, to_type) - } - // No support for microsecond/nanosecond with i32 - (Date32, Int32) => primitive_to_same_primitive_dyn::(array, to_type), - (Date32, Int64) => primitive_to_primitive_dyn::(array, to_type, options), - (Time32(_), Int32) => primitive_to_same_primitive_dyn::(array, to_type), - (Int64, Date64) => primitive_to_same_primitive_dyn::(array, to_type), - // No support for second/milliseconds with i64 - (Int64, Time64(TimeUnit::Microsecond)) => { - primitive_to_same_primitive_dyn::(array, to_type) - } - (Int64, Time64(TimeUnit::Nanosecond)) => { - primitive_to_same_primitive_dyn::(array, to_type) - } - - (Date64, Int32) => primitive_to_primitive_dyn::(array, to_type, options), - (Date64, Int64) => primitive_to_same_primitive_dyn::(array, to_type), - (Time64(_), Int64) => primitive_to_same_primitive_dyn::(array, to_type), - (Date32, Date64) => primitive_dyn!(array, date32_to_date64), - (Date64, Date32) => primitive_dyn!(array, date64_to_date32), - (Time32(TimeUnit::Second), Time32(TimeUnit::Millisecond)) => { - primitive_dyn!(array, time32s_to_time32ms) - } - (Time32(TimeUnit::Millisecond), Time32(TimeUnit::Second)) => { - primitive_dyn!(array, time32ms_to_time32s) - } - (Time32(from_unit), Time64(to_unit)) => { - primitive_dyn!(array, time32_to_time64, *from_unit, *to_unit) - } - (Time64(TimeUnit::Microsecond), Time64(TimeUnit::Nanosecond)) => { - primitive_dyn!(array, time64us_to_time64ns) - } - (Time64(TimeUnit::Nanosecond), Time64(TimeUnit::Microsecond)) => { - primitive_dyn!(array, time64ns_to_time64us) - } - (Time64(from_unit), Time32(to_unit)) => { - primitive_dyn!(array, time64_to_time32, *from_unit, *to_unit) - } - (Timestamp(_, _), Int64) => primitive_to_same_primitive_dyn::(array, to_type), - (Int64, Timestamp(_, _)) => primitive_to_same_primitive_dyn::(array, to_type), - (Timestamp(from_unit, _), Timestamp(to_unit, tz)) => { - primitive_dyn!(array, timestamp_to_timestamp, *from_unit, *to_unit, tz) - } - (Timestamp(from_unit, _), Date32) => primitive_dyn!(array, timestamp_to_date32, *from_unit), - (Timestamp(from_unit, _), Date64) => primitive_dyn!(array, timestamp_to_date64, *from_unit), - - (Int64, Duration(_)) => primitive_to_same_primitive_dyn::(array, to_type), - (Duration(_), Int64) => primitive_to_same_primitive_dyn::(array, to_type), - - (Interval(IntervalUnit::DayTime), Interval(IntervalUnit::MonthDayNano)) => { - primitive_dyn!(array, days_ms_to_months_days_ns) - } - (Interval(IntervalUnit::YearMonth), Interval(IntervalUnit::MonthDayNano)) => { - primitive_dyn!(array, months_to_months_days_ns) - } - - (_, _) => Err(Error::NotYetImplemented(format!( - "Casting from {from_type:?} to {to_type:?} not supported", - ))), - } -} - -/// Attempts to encode an array into an `ArrayDictionary` with index -/// type K and value (dictionary) type value_type -/// -/// K is the key type -fn cast_to_dictionary( - array: &dyn Array, - dict_value_type: &DataType, - options: CastOptions, -) -> Result> { - let array = cast(array, dict_value_type, options)?; - let array = array.as_ref(); - match *dict_value_type { - DataType::Int8 => primitive_to_dictionary_dyn::(array), - DataType::Int16 => primitive_to_dictionary_dyn::(array), - DataType::Int32 => primitive_to_dictionary_dyn::(array), - DataType::Int64 => primitive_to_dictionary_dyn::(array), - DataType::UInt8 => primitive_to_dictionary_dyn::(array), - DataType::UInt16 => primitive_to_dictionary_dyn::(array), - DataType::UInt32 => primitive_to_dictionary_dyn::(array), - DataType::UInt64 => primitive_to_dictionary_dyn::(array), - DataType::Utf8 => utf8_to_dictionary_dyn::(array), - DataType::LargeUtf8 => utf8_to_dictionary_dyn::(array), - DataType::Binary => binary_to_dictionary_dyn::(array), - DataType::LargeBinary => binary_to_dictionary_dyn::(array), - _ => Err(Error::NotYetImplemented(format!( - "Unsupported output type for dictionary packing: {dict_value_type:?}" - ))), - } -} diff --git a/src/common/arrow/src/arrow/compute/cast/primitive_to.rs b/src/common/arrow/src/arrow/compute/cast/primitive_to.rs deleted file mode 100644 index d9d6fab20399..000000000000 --- a/src/common/arrow/src/arrow/compute/cast/primitive_to.rs +++ /dev/null @@ -1,607 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::hash::Hash; - -use num_traits::AsPrimitive; -use num_traits::Float; -use num_traits::ToPrimitive; - -use super::CastOptions; -use crate::arrow::array::*; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::compute::arity::unary; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::temporal_conversions::*; -use crate::arrow::types::days_ms; -use crate::arrow::types::f16; -use crate::arrow::types::months_days_ns; -use crate::arrow::types::NativeType; - -/// Returns a [`BinaryArray`] where every element is the binary representation of the number. -pub fn primitive_to_binary( - from: &PrimitiveArray, -) -> BinaryArray { - let mut values: Vec = Vec::with_capacity(from.len()); - let mut offsets: Vec = Vec::with_capacity(from.len() + 1); - offsets.push(O::default()); - - let mut offset: usize = 0; - - unsafe { - for x in from.values().iter() { - values.reserve(offset + T::FORMATTED_SIZE_DECIMAL); - - let bytes = std::slice::from_raw_parts_mut( - values.as_mut_ptr().add(offset), - values.capacity() - offset, - ); - let len = lexical_core::write(*x, bytes).len(); - - offset += len; - offsets.push(O::from_usize(offset).unwrap()); - } - values.set_len(offset); - values.shrink_to_fit(); - // Safety: offsets _are_ monotonically increasing - let offsets = Offsets::new_unchecked(offsets); - BinaryArray::::new( - BinaryArray::::default_data_type(), - offsets.into(), - values.into(), - from.validity().cloned(), - ) - } -} - -pub(super) fn primitive_to_binary_dyn(from: &dyn Array) -> Result> -where - O: Offset, - T: NativeType + lexical_core::ToLexical, -{ - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(primitive_to_binary::(from))) -} - -/// Returns a [`BooleanArray`] where every element is different from zero. -/// Validity is preserved. -pub fn primitive_to_boolean( - from: &PrimitiveArray, - to_type: DataType, -) -> BooleanArray { - let iter = from.values().iter().map(|v| *v != T::default()); - let values = Bitmap::from_trusted_len_iter(iter); - - BooleanArray::new(to_type, values, from.validity().cloned()) -} - -pub(super) fn primitive_to_boolean_dyn( - from: &dyn Array, - to_type: DataType, -) -> Result> -where - T: NativeType, -{ - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(primitive_to_boolean::(from, to_type))) -} - -/// Returns a [`Utf8Array`] where every element is the utf8 representation of the number. -pub fn primitive_to_utf8( - from: &PrimitiveArray, -) -> Utf8Array { - let mut values: Vec = Vec::with_capacity(from.len()); - let mut offsets: Vec = Vec::with_capacity(from.len() + 1); - offsets.push(O::default()); - - let mut offset: usize = 0; - - unsafe { - for x in from.values().iter() { - values.reserve(offset + T::FORMATTED_SIZE_DECIMAL); - - let bytes = std::slice::from_raw_parts_mut( - values.as_mut_ptr().add(offset), - values.capacity() - offset, - ); - let len = lexical_core::write(*x, bytes).len(); - - offset += len; - offsets.push(O::from_usize(offset).unwrap()); - } - values.set_len(offset); - values.shrink_to_fit(); - // Safety: offsets _are_ monotonically increasing - let offsets = Offsets::new_unchecked(offsets); - Utf8Array::::new_unchecked( - Utf8Array::::default_data_type(), - offsets.into(), - values.into(), - from.validity().cloned(), - ) - } -} - -pub(super) fn primitive_to_utf8_dyn(from: &dyn Array) -> Result> -where - O: Offset, - T: NativeType + lexical_core::ToLexical, -{ - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(primitive_to_utf8::(from))) -} - -pub(super) fn primitive_to_primitive_dyn( - from: &dyn Array, - to_type: &DataType, - options: CastOptions, -) -> Result> -where - I: NativeType + num_traits::NumCast + num_traits::AsPrimitive, - O: NativeType + num_traits::NumCast, -{ - let from = from.as_any().downcast_ref::>().unwrap(); - if options.wrapped { - Ok(Box::new(primitive_as_primitive::(from, to_type))) - } else { - Ok(Box::new(primitive_to_primitive::(from, to_type))) - } -} - -/// Cast [`PrimitiveArray`] to a [`PrimitiveArray`] of another physical type via numeric conversion. -pub fn primitive_to_primitive( - from: &PrimitiveArray, - to_type: &DataType, -) -> PrimitiveArray -where - I: NativeType + num_traits::NumCast, - O: NativeType + num_traits::NumCast, -{ - let iter = from - .iter() - .map(|v| v.and_then(|x| num_traits::cast::cast::(*x))); - PrimitiveArray::::from_trusted_len_iter(iter).to(to_type.clone()) -} - -/// Returns a [`PrimitiveArray`] with the casted values. Values are `None` on overflow -pub fn integer_to_decimal>( - from: &PrimitiveArray, - to_precision: usize, - to_scale: usize, -) -> PrimitiveArray { - let multiplier = 10_i128.pow(to_scale as u32); - - let min_for_precision = 9_i128 - .saturating_pow(1 + to_precision as u32) - .saturating_neg(); - let max_for_precision = 9_i128.saturating_pow(1 + to_precision as u32); - - let values = from.iter().map(|x| { - x.and_then(|x| { - x.as_().checked_mul(multiplier).and_then(|x| { - if x > max_for_precision || x < min_for_precision { - None - } else { - Some(x) - } - }) - }) - }); - - PrimitiveArray::::from_trusted_len_iter(values) - .to(DataType::Decimal(to_precision, to_scale)) -} - -pub(super) fn integer_to_decimal_dyn( - from: &dyn Array, - precision: usize, - scale: usize, -) -> Result> -where - T: NativeType + AsPrimitive, -{ - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(integer_to_decimal::(from, precision, scale))) -} - -/// Returns a [`PrimitiveArray`] with the casted values. Values are `None` on overflow -pub fn float_to_decimal( - from: &PrimitiveArray, - to_precision: usize, - to_scale: usize, -) -> PrimitiveArray -where - T: NativeType + Float + ToPrimitive, - f64: AsPrimitive, -{ - // 1.2 => 12 - let multiplier: T = (10_f64).powi(to_scale as i32).as_(); - - let min_for_precision = 9_i128 - .saturating_pow(1 + to_precision as u32) - .saturating_neg(); - let max_for_precision = 9_i128.saturating_pow(1 + to_precision as u32); - - let values = from.iter().map(|x| { - x.and_then(|x| { - let x = (*x * multiplier).to_i128().unwrap(); - if x > max_for_precision || x < min_for_precision { - None - } else { - Some(x) - } - }) - }); - - PrimitiveArray::::from_trusted_len_iter(values) - .to(DataType::Decimal(to_precision, to_scale)) -} - -pub(super) fn float_to_decimal_dyn( - from: &dyn Array, - precision: usize, - scale: usize, -) -> Result> -where - T: NativeType + Float + ToPrimitive, - f64: AsPrimitive, -{ - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(float_to_decimal::(from, precision, scale))) -} - -/// Cast [`PrimitiveArray`] as a [`PrimitiveArray`] -/// Same as `number as to_number_type` in rust -pub fn primitive_as_primitive( - from: &PrimitiveArray, - to_type: &DataType, -) -> PrimitiveArray -where - I: NativeType + num_traits::AsPrimitive, - O: NativeType, -{ - unary(from, num_traits::AsPrimitive::::as_, to_type.clone()) -} - -/// Cast [`PrimitiveArray`] to a [`PrimitiveArray`] of the same physical type. -/// This is O(1). -pub fn primitive_to_same_primitive( - from: &PrimitiveArray, - to_type: &DataType, -) -> PrimitiveArray -where - T: NativeType, -{ - PrimitiveArray::::new( - to_type.clone(), - from.values().clone(), - from.validity().cloned(), - ) -} - -/// Cast [`PrimitiveArray`] to a [`PrimitiveArray`] of the same physical type. -/// This is O(1). -pub(super) fn primitive_to_same_primitive_dyn( - from: &dyn Array, - to_type: &DataType, -) -> Result> -where - T: NativeType, -{ - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(primitive_to_same_primitive::(from, to_type))) -} - -pub(super) fn primitive_to_dictionary_dyn( - from: &dyn Array, -) -> Result> { - let from = from.as_any().downcast_ref().unwrap(); - primitive_to_dictionary::(from).map(|x| Box::new(x) as Box) -} - -/// Cast [`PrimitiveArray`] to [`DictionaryArray`]. Also known as packing. -/// # Errors -/// This function errors if the maximum key is smaller than the number of distinct elements -/// in the array. -pub fn primitive_to_dictionary( - from: &PrimitiveArray, -) -> Result> { - let iter = from.iter().map(|x| x.copied()); - let mut array = MutableDictionaryArray::::try_empty(MutablePrimitiveArray::::from( - from.data_type().clone(), - ))?; - array.try_extend(iter)?; - - Ok(array.into()) -} - -/// Get the time unit as a multiple of a second -const fn time_unit_multiple(unit: TimeUnit) -> i64 { - match unit { - TimeUnit::Second => 1, - TimeUnit::Millisecond => MILLISECONDS, - TimeUnit::Microsecond => MICROSECONDS, - TimeUnit::Nanosecond => NANOSECONDS, - } -} - -/// Conversion of dates -pub fn date32_to_date64(from: &PrimitiveArray) -> PrimitiveArray { - unary(from, |x| x as i64 * MILLISECONDS_IN_DAY, DataType::Date64) -} - -/// Conversion of dates -pub fn date64_to_date32(from: &PrimitiveArray) -> PrimitiveArray { - unary(from, |x| (x / MILLISECONDS_IN_DAY) as i32, DataType::Date32) -} - -/// Conversion of times -pub fn time32s_to_time32ms(from: &PrimitiveArray) -> PrimitiveArray { - unary(from, |x| x * 1000, DataType::Time32(TimeUnit::Millisecond)) -} - -/// Conversion of times -pub fn time32ms_to_time32s(from: &PrimitiveArray) -> PrimitiveArray { - unary(from, |x| x / 1000, DataType::Time32(TimeUnit::Second)) -} - -/// Conversion of times -pub fn time64us_to_time64ns(from: &PrimitiveArray) -> PrimitiveArray { - unary(from, |x| x * 1000, DataType::Time64(TimeUnit::Nanosecond)) -} - -/// Conversion of times -pub fn time64ns_to_time64us(from: &PrimitiveArray) -> PrimitiveArray { - unary(from, |x| x / 1000, DataType::Time64(TimeUnit::Microsecond)) -} - -/// Conversion of timestamp -pub fn timestamp_to_date64(from: &PrimitiveArray, from_unit: TimeUnit) -> PrimitiveArray { - let from_size = time_unit_multiple(from_unit); - let to_size = MILLISECONDS; - let to_type = DataType::Date64; - - // Scale time_array by (to_size / from_size) using a - // single integer operation, but need to avoid integer - // math rounding down to zero - - match to_size.cmp(&from_size) { - std::cmp::Ordering::Less => unary(from, |x| (x / (from_size / to_size)), to_type), - std::cmp::Ordering::Equal => primitive_to_same_primitive(from, &to_type), - std::cmp::Ordering::Greater => unary(from, |x| (x * (to_size / from_size)), to_type), - } -} - -/// Conversion of timestamp -pub fn timestamp_to_date32(from: &PrimitiveArray, from_unit: TimeUnit) -> PrimitiveArray { - let from_size = time_unit_multiple(from_unit) * SECONDS_IN_DAY; - unary(from, |x| (x / from_size) as i32, DataType::Date32) -} - -/// Conversion of time -pub fn time32_to_time64( - from: &PrimitiveArray, - from_unit: TimeUnit, - to_unit: TimeUnit, -) -> PrimitiveArray { - let from_size = time_unit_multiple(from_unit); - let to_size = time_unit_multiple(to_unit); - let divisor = to_size / from_size; - unary(from, |x| (x as i64 * divisor), DataType::Time64(to_unit)) -} - -/// Conversion of time -pub fn time64_to_time32( - from: &PrimitiveArray, - from_unit: TimeUnit, - to_unit: TimeUnit, -) -> PrimitiveArray { - let from_size = time_unit_multiple(from_unit); - let to_size = time_unit_multiple(to_unit); - let divisor = from_size / to_size; - unary(from, |x| (x / divisor) as i32, DataType::Time32(to_unit)) -} - -/// Conversion of timestamp -pub fn timestamp_to_timestamp( - from: &PrimitiveArray, - from_unit: TimeUnit, - to_unit: TimeUnit, - tz: &Option, -) -> PrimitiveArray { - let from_size = time_unit_multiple(from_unit); - let to_size = time_unit_multiple(to_unit); - let to_type = DataType::Timestamp(to_unit, tz.clone()); - // we either divide or multiply, depending on size of each unit - if from_size >= to_size { - unary(from, |x| (x / (from_size / to_size)), to_type) - } else { - unary(from, |x| (x * (to_size / from_size)), to_type) - } -} - -fn timestamp_to_utf8_impl( - from: &PrimitiveArray, - time_unit: TimeUnit, - timezone: T, -) -> Utf8Array -where - T::Offset: std::fmt::Display, -{ - match time_unit { - TimeUnit::Nanosecond => { - let iter = from.iter().map(|x| { - x.map(|x| { - let datetime = timestamp_ns_to_datetime(*x); - let offset = timezone.offset_from_utc_datetime(&datetime); - chrono::DateTime::::from_naive_utc_and_offset(datetime, offset).to_rfc3339() - }) - }); - Utf8Array::from_trusted_len_iter(iter) - } - TimeUnit::Microsecond => { - let iter = from.iter().map(|x| { - x.map(|x| { - let datetime = timestamp_us_to_datetime(*x); - let offset = timezone.offset_from_utc_datetime(&datetime); - chrono::DateTime::::from_naive_utc_and_offset(datetime, offset).to_rfc3339() - }) - }); - Utf8Array::from_trusted_len_iter(iter) - } - TimeUnit::Millisecond => { - let iter = from.iter().map(|x| { - x.map(|x| { - let datetime = timestamp_ms_to_datetime(*x); - let offset = timezone.offset_from_utc_datetime(&datetime); - chrono::DateTime::::from_naive_utc_and_offset(datetime, offset).to_rfc3339() - }) - }); - Utf8Array::from_trusted_len_iter(iter) - } - TimeUnit::Second => { - let iter = from.iter().map(|x| { - x.map(|x| { - let datetime = timestamp_s_to_datetime(*x); - let offset = timezone.offset_from_utc_datetime(&datetime); - chrono::DateTime::::from_naive_utc_and_offset(datetime, offset).to_rfc3339() - }) - }); - Utf8Array::from_trusted_len_iter(iter) - } - } -} - -#[cfg(feature = "chrono-tz")] -#[cfg_attr(docsrs, doc(cfg(feature = "chrono-tz")))] -fn chrono_tz_timestamp_to_utf8( - from: &PrimitiveArray, - time_unit: TimeUnit, - timezone_str: &str, -) -> Result> { - let timezone = parse_offset_tz(timezone_str)?; - Ok(timestamp_to_utf8_impl::( - from, time_unit, timezone, - )) -} - -#[cfg(not(feature = "chrono-tz"))] -fn chrono_tz_timestamp_to_utf8( - _: &PrimitiveArray, - _: TimeUnit, - timezone_str: &str, -) -> Result> { - use crate::arrow::error::Error; - Err(Error::InvalidArgumentError(format!( - "timezone \"{}\" cannot be parsed (feature chrono-tz is not active)", - timezone_str - ))) -} - -/// Returns a [`Utf8Array`] where every element is the utf8 representation of the timestamp in the rfc3339 format. -pub fn timestamp_to_utf8( - from: &PrimitiveArray, - time_unit: TimeUnit, - timezone_str: &str, -) -> Result> { - let timezone = parse_offset(timezone_str); - - if let Ok(timezone) = timezone { - Ok(timestamp_to_utf8_impl::( - from, time_unit, timezone, - )) - } else { - chrono_tz_timestamp_to_utf8(from, time_unit, timezone_str) - } -} - -/// Returns a [`Utf8Array`] where every element is the utf8 representation of the timestamp in the rfc3339 format. -pub fn naive_timestamp_to_utf8( - from: &PrimitiveArray, - time_unit: TimeUnit, -) -> Utf8Array { - match time_unit { - TimeUnit::Nanosecond => { - let iter = from.iter().map(|x| { - x.copied() - .map(timestamp_ns_to_datetime) - .map(|x| x.to_string()) - }); - Utf8Array::from_trusted_len_iter(iter) - } - TimeUnit::Microsecond => { - let iter = from.iter().map(|x| { - x.copied() - .map(timestamp_us_to_datetime) - .map(|x| x.to_string()) - }); - Utf8Array::from_trusted_len_iter(iter) - } - TimeUnit::Millisecond => { - let iter = from.iter().map(|x| { - x.copied() - .map(timestamp_ms_to_datetime) - .map(|x| x.to_string()) - }); - Utf8Array::from_trusted_len_iter(iter) - } - TimeUnit::Second => { - let iter = from.iter().map(|x| { - x.copied() - .map(timestamp_s_to_datetime) - .map(|x| x.to_string()) - }); - Utf8Array::from_trusted_len_iter(iter) - } - } -} - -#[inline] -fn days_ms_to_months_days_ns_scalar(from: days_ms) -> months_days_ns { - months_days_ns::new(0, from.days(), from.milliseconds() as i64 * 1000) -} - -/// Casts [`days_ms`]s to [`months_days_ns`]. This operation is infalible and lossless. -pub fn days_ms_to_months_days_ns(from: &PrimitiveArray) -> PrimitiveArray { - unary( - from, - days_ms_to_months_days_ns_scalar, - DataType::Interval(IntervalUnit::MonthDayNano), - ) -} - -#[inline] -fn months_to_months_days_ns_scalar(from: i32) -> months_days_ns { - months_days_ns::new(from, 0, 0) -} - -/// Casts months represented as [`i32`]s to [`months_days_ns`]. This operation is infalible and lossless. -pub fn months_to_months_days_ns(from: &PrimitiveArray) -> PrimitiveArray { - unary( - from, - months_to_months_days_ns_scalar, - DataType::Interval(IntervalUnit::MonthDayNano), - ) -} - -/// Casts f16 into f32 -pub fn f16_to_f32(from: &PrimitiveArray) -> PrimitiveArray { - unary(from, |x| x.to_f32(), DataType::Float32) -} diff --git a/src/common/arrow/src/arrow/compute/cast/utf8_to.rs b/src/common/arrow/src/arrow/compute/cast/utf8_to.rs deleted file mode 100644 index 8703cf2cd78a..000000000000 --- a/src/common/arrow/src/arrow/compute/cast/utf8_to.rs +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use chrono::Datelike; - -use super::CastOptions; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::temporal_conversions::utf8_to_naive_timestamp_ns as utf8_to_naive_timestamp_ns_; -use crate::arrow::temporal_conversions::utf8_to_timestamp_ns as utf8_to_timestamp_ns_; -use crate::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; -use crate::arrow::types::NativeType; - -const RFC3339: &str = "%Y-%m-%dT%H:%M:%S%.f%:z"; - -/// Casts a [`Utf8Array`] to a [`PrimitiveArray`], making any uncastable value a Null. -pub fn utf8_to_primitive(from: &Utf8Array, to: &DataType) -> PrimitiveArray -where T: NativeType + lexical_core::FromLexical { - let iter = from - .iter() - .map(|x| x.and_then::(|x| lexical_core::parse(x.as_bytes()).ok())); - - PrimitiveArray::::from_trusted_len_iter(iter).to(to.clone()) -} - -/// Casts a [`Utf8Array`] to a [`PrimitiveArray`] at best-effort using `lexical_core::parse_partial`, making any uncastable value as zero. -pub fn partial_utf8_to_primitive( - from: &Utf8Array, - to: &DataType, -) -> PrimitiveArray -where - T: NativeType + lexical_core::FromLexical, -{ - let iter = from.iter().map(|x| { - x.and_then::(|x| lexical_core::parse_partial(x.as_bytes()).ok().map(|x| x.0)) - }); - - PrimitiveArray::::from_trusted_len_iter(iter).to(to.clone()) -} - -pub(super) fn utf8_to_primitive_dyn( - from: &dyn Array, - to: &DataType, - options: CastOptions, -) -> Result> -where - T: NativeType + lexical_core::FromLexical, -{ - let from = from.as_any().downcast_ref().unwrap(); - if options.partial { - Ok(Box::new(partial_utf8_to_primitive::(from, to))) - } else { - Ok(Box::new(utf8_to_primitive::(from, to))) - } -} - -/// Casts a [`Utf8Array`] to a Date32 primitive, making any uncastable value a Null. -pub fn utf8_to_date32(from: &Utf8Array) -> PrimitiveArray { - let iter = from.iter().map(|x| { - x.and_then(|x| { - x.parse::() - .ok() - .map(|x| x.num_days_from_ce() - EPOCH_DAYS_FROM_CE) - }) - }); - PrimitiveArray::::from_trusted_len_iter(iter).to(DataType::Date32) -} - -pub(super) fn utf8_to_date32_dyn(from: &dyn Array) -> Result> { - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(utf8_to_date32::(from))) -} - -/// Casts a [`Utf8Array`] to a Date64 primitive, making any uncastable value a Null. -pub fn utf8_to_date64(from: &Utf8Array) -> PrimitiveArray { - let iter = from.iter().map(|x| { - x.and_then(|x| { - x.parse::() - .ok() - .map(|x| (x.num_days_from_ce() - EPOCH_DAYS_FROM_CE) as i64 * 86400000) - }) - }); - PrimitiveArray::from_trusted_len_iter(iter).to(DataType::Date64) -} - -pub(super) fn utf8_to_date64_dyn(from: &dyn Array) -> Result> { - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(utf8_to_date64::(from))) -} - -pub(super) fn utf8_to_dictionary_dyn( - from: &dyn Array, -) -> Result> { - let values = from.as_any().downcast_ref().unwrap(); - utf8_to_dictionary::(values).map(|x| Box::new(x) as Box) -} - -/// Cast [`Utf8Array`] to [`DictionaryArray`], also known as packing. -/// # Errors -/// This function errors if the maximum key is smaller than the number of distinct elements -/// in the array. -pub fn utf8_to_dictionary( - from: &Utf8Array, -) -> Result> { - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(from.iter())?; - - Ok(array.into()) -} - -pub(super) fn utf8_to_naive_timestamp_ns_dyn( - from: &dyn Array, -) -> Result> { - let from = from.as_any().downcast_ref().unwrap(); - Ok(Box::new(utf8_to_naive_timestamp_ns::(from))) -} - -/// [`crate::arrow::temporal_conversions::utf8_to_timestamp_ns`] applied for RFC3339 formatting -pub fn utf8_to_naive_timestamp_ns(from: &Utf8Array) -> PrimitiveArray { - utf8_to_naive_timestamp_ns_(from, RFC3339) -} - -pub(super) fn utf8_to_timestamp_ns_dyn( - from: &dyn Array, - timezone: String, -) -> Result> { - let from = from.as_any().downcast_ref().unwrap(); - utf8_to_timestamp_ns::(from, timezone) - .map(Box::new) - .map(|x| x as Box) -} - -/// [`crate::arrow::temporal_conversions::utf8_to_timestamp_ns`] applied for RFC3339 formatting -pub fn utf8_to_timestamp_ns( - from: &Utf8Array, - timezone: String, -) -> Result> { - utf8_to_timestamp_ns_(from, RFC3339, timezone) -} - -/// Conversion of utf8 -pub fn utf8_to_large_utf8(from: &Utf8Array) -> Utf8Array { - let data_type = Utf8Array::::default_data_type(); - let validity = from.validity().cloned(); - let values = from.values().clone(); - - let offsets = from.offsets().into(); - // Safety: sound because `values` fulfills the same invariants as `from.values()` - unsafe { Utf8Array::::new_unchecked(data_type, offsets, values, validity) } -} - -/// Conversion of utf8 -pub fn utf8_large_to_utf8(from: &Utf8Array) -> Result> { - let data_type = Utf8Array::::default_data_type(); - let validity = from.validity().cloned(); - let values = from.values().clone(); - let offsets = from.offsets().try_into()?; - - // Safety: sound because `values` fulfills the same invariants as `from.values()` - Ok(unsafe { Utf8Array::::new_unchecked(data_type, offsets, values, validity) }) -} - -/// Conversion to binary -pub fn utf8_to_binary(from: &Utf8Array, to_data_type: DataType) -> BinaryArray { - // Safety: erasure of an invariant is always safe - BinaryArray::::new( - to_data_type, - from.offsets().clone(), - from.values().clone(), - from.validity().cloned(), - ) -} diff --git a/src/common/arrow/src/arrow/compute/merge_sort/mod.rs b/src/common/arrow/src/arrow/compute/merge_sort/mod.rs deleted file mode 100644 index fac1505dea26..000000000000 --- a/src/common/arrow/src/arrow/compute/merge_sort/mod.rs +++ /dev/null @@ -1,560 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Functions to perform merge-sorts. -//! -//! The goal of merge-sort is to merge two sorted arrays, `[a0, a1]`, `merge_sort(a0, a1)`, -//! so that the resulting array is sorted, i.e. the following invariant upholds: -//! `sort(merge_sort(a0, a1)) == merge_sort(a0, a1)` for any two sorted arrays `a0` and `a1`. -//! -//! Given that two sorted arrays are more likely to be partially sorted within each other, -//! and that the resulting array is built by taking elements from each array, it is -//! advantageous to `take` slices of items, not items, from each array. -//! As such, this module's main data representation is `(i: usize, start: usize, len: usize)`, -//! which represents a slice of array `i`. -//! -//! In this context, `merge_sort` is composed by two main operations: -//! -//! 1. compute the array of slices `v` that construct a new sorted array from `a0` and `a1`. -//! 2. `take_arrays` from `a0` and `a1`, creating the sorted array. -//! -//! In the extreme case where the two arrays are already sorted between then (e.g. `[0, 2]`, `[3, 4]`), -//! we need two slices, `v = vec![(0, 0, a0.len()), (1, 0, a1.len())]`. The higher the -//! inter-leave between the two arrays, the more slices will be needed, and -//! generally the more expensive the `take` operation will be. -//! -//! ## Merge-sort multiple arrays -//! -//! The main advantage of merge-sort over `sort` is that it can be parallelized. -//! For example, given a set of arrays `[a0, a1, a2, a3]` representing the same field, -//! e.g. over 4 batches of arrays, they can be sorted in parallel as follows (pseudo-code): -//! -//! ```rust,ignore -//! // in parallel -//! let a0 = sort(a0); -//! let a1 = sort(a1); -//! let a2 = sort(a2); -//! let a3 = sort(a3); -//! -//! // in parallel and recursively -//! let slices1 = merge_sort_slices(a0, a1); -//! let slices2 = merge_sort_slices(a2, a3); -//! let slices = merge_sort_slices(slices1, slices2); -//! -//! let array = take_arrays(&[a0, a1, a2, a3], slices, None); -//! ``` -//! -//! A common operation in query engines is to merge multiple fields based on the -//! same sorting field (e.g. merge-sort multiple batches of arrays). -//! To perform this, use the same idea as above, but use `take_arrays` over -//! each independent field (which can again be parallelized): -//! -//! ```rust,ignore -//! // `slices` computed before-hand -//! // in parallel -//! let array1 = take_arrays(&[a0, a1, a2, a3], slices, None); -//! let array2 = take_arrays(&[b0, b1, b2, b3], slices, None); -//! ``` -//! -//! To serialize slices, e.g. for checkpointing or transfer via Arrow's IPC, you can store -//! them as 3 non-null primitive arrays (e.g. `PrimitiveArray`). - -use std::cmp::Ordering; -use std::iter::once; - -use ahash::AHashMap; -use itertools::Itertools; - -use crate::arrow::array::growable::make_growable; -use crate::arrow::array::ord::build_compare; -use crate::arrow::array::ord::DynComparator; -use crate::arrow::array::Array; -pub use crate::arrow::compute::sort::SortOptions; -use crate::arrow::error::Result; - -/// A slice denoting `(array_index, start, len)` representing a slice from one of N arrays. -/// This is used to keep track of contiguous blocks of slots. -/// An array of MergeSlice, `[MergeSlice]`, represents inter-leaved array slices. -/// For example, `[(0, 0, 2), (1, 0, 1), (0, 2, 3)]` represents 2 arrays (a0 and a1) arranged as follows: -/// `[a0[0..2], a1[0..1], a0[2..5]]` -/// This representation is useful when building arrays in memory as it allows to memcopy slices of arrays. -/// This is particularly useful in merge-sort because sorted arrays (passed to the merge-sort) are more likely -/// to have contiguous blocks of sorted elements (than by random). -pub type MergeSlice = (usize, usize, usize); - -/// Takes N arrays together through `slices` under the assumption that the slices have -/// a total coverage of the arrays. -/// I.e. they are such that all elements on all arrays are picked (which is the case in sorting). -/// # Panic -/// This function panics if: -/// * `max(slices[i].0) >= arrays.len()`, as it indicates that the slices point to an array out of bounds from `arrays`. -/// * the arrays do not have the same [`crate::arrow::datatypes::DataType`] (as it makes no sense to take together from them) -pub fn take_arrays>( - arrays: &[&dyn Array], - slices: I, - limit: Option, -) -> Box { - let slices = slices.into_iter(); - let len = arrays.iter().map(|array| array.len()).sum(); - - let limit = limit.unwrap_or(len); - let limit = limit.min(len); - let mut growable = make_growable(arrays, false, limit); - - if limit != len { - let mut current_len = 0; - for (index, start, len) in slices { - if len + current_len >= limit { - growable.extend(index, start, limit - current_len); - break; - } else { - growable.extend(index, start, len); - current_len += len; - } - } - } else { - for (index, start, len) in slices { - growable.extend(index, start, len); - } - } - - growable.as_box() -} - -/// Combines two sorted [Array]s of the same [`crate::arrow::datatypes::DataType`] into a single sorted array. -/// If the arrays are not sorted (which this function does not check), the result is wrong. -/// # Error -/// This function errors when: -/// * the arrays have a different [`crate::arrow::datatypes::DataType`] -/// * the arrays have a [`crate::arrow::datatypes::DataType`] that has no order relationship -/// # Example -/// ```rust -/// use arrow2::array::Int32Array; -/// use arrow2::compute::merge_sort::merge_sort; -/// use arrow2::compute::merge_sort::SortOptions; -/// # use arrow2::error::Result; -/// # fn main() -> Result<()> { -/// let a = Int32Array::from_slice(&[2, 4, 6]); -/// let b = Int32Array::from_slice(&[0, 1, 3]); -/// let sorted = merge_sort(&a, &b, &SortOptions::default(), None)?; -/// let expected = Int32Array::from_slice(&[0, 1, 2, 3, 4, 6]); -/// assert_eq!(expected, sorted.as_ref()); -/// # Ok(()) -/// # } -/// ``` -pub fn merge_sort( - lhs: &dyn Array, - rhs: &dyn Array, - options: &SortOptions, - limit: Option, -) -> Result> { - let arrays = &[lhs, rhs]; - - let pairs: &[(&[&dyn Array], &SortOptions)] = &[(arrays, options)]; - let comparator = build_comparator(pairs)?; - - let lhs = (0, 0, lhs.len()); - let rhs = (1, 0, rhs.len()); - let slices = merge_sort_slices(once(&lhs), once(&rhs), &comparator); - Ok(take_arrays(arrays, slices, limit)) -} - -/// Returns a vector of slices from different sorted arrays that can be used to create sorted arrays. -/// `pairs` is an array representing multiple sorted array sets. The expected format is -/// -/// pairs: [([a00, a01], o1), ([a10, a11], o2), ...] -/// where aj0.len() == aj0.len() -/// aj1.len() == aj1.len() -/// ... -/// In other words, `pairs.i.0[j]` must be an array coming from a batch of equal len arrays. -/// # Example -/// ```rust -/// use arrow2::array::Int32Array; -/// use arrow2::compute::merge_sort::slices; -/// use arrow2::compute::merge_sort::SortOptions; -/// # use arrow2::error::Result; -/// # fn main() -> Result<()> { -/// let a = Int32Array::from_slice(&[2, 4, 6]); -/// let b = Int32Array::from_slice(&[0, 1, 3]); -/// let slices = slices(&[(&[&a, &b], &SortOptions::default())])?; -/// assert_eq!(slices, vec![(1, 0, 2), (0, 0, 1), (1, 2, 1), (0, 1, 2)]); -/// -/// # Ok(()) -/// # } -/// ``` -/// # Error -/// This function errors if the arrays `a0i` are not pairwise sortable. This happens when either -/// they have not the same [`crate::arrow::datatypes::DataType`] or when their [`crate::arrow::datatypes::DataType`] -/// does not correspond to a sortable type. -/// # Panic -/// This function panics if: -/// * `pairs` has no elements -/// * the length condition above is not fulfilled -pub fn slices(pairs: &[(&[&dyn Array], &SortOptions)]) -> Result> { - assert!(!pairs.is_empty()); - let comparator = build_comparator(pairs)?; - - // pairs: [([a00, a01], o1), ([a10, a11], o2), ...] - // slices: [(0, 0, len), (1, 0, len)] - - let slices = pairs[0] - .0 - .iter() - .enumerate() - .map(|(index, array)| vec![(index, 0, array.len())]) - .collect::>(); - - let slices = slices - .iter() - .map(|slice| slice.as_ref()) - .collect::>(); - Ok(recursive_merge_sort(&slices, &comparator)) -} - -/// recursively sort-merges multiple `slices` representing slices of sorted arrays according -/// to a comparison function between those arrays. -/// Note that `slices` is an array of arrays, `slices[i][j]`. The index `i` represents -/// the set of arrays `i`, while the index `j` represents -/// the array `j` within that set. -/// Note that this does not split to the smallest element as arrays: the smallest unit is a `slice` -fn recursive_merge_sort(slices: &[&[MergeSlice]], comparator: &Comparator) -> Vec { - let n = slices.len(); - let m = n / 2; - - if n == 1 { - // slices are assumed sort arrays - return slices[0].to_vec(); - } - if n == 2 { - return merge_sort_slices(slices[0].iter(), slices[1].iter(), comparator) - .collect::>(); - } - - // split in 2 and sort - let lhs = recursive_merge_sort(&slices[0..m], comparator); - let rhs = recursive_merge_sort(&slices[m..n], comparator); - - // merge-sort the splits - merge_sort_slices(lhs.iter(), rhs.iter(), comparator).collect::>() -} - -/// An iterator adapter that merge-sorts two iterators of `MergeSlice` into a single `MergeSlice` -/// such that the resulting `MergeSlice`s are ordered according to `comparator`. -pub struct MergeSortSlices<'a, L, R> -where - L: Iterator, - R: Iterator, -{ - lhs: L, - rhs: R, - comparator: &'a Comparator<'a>, - - left: Option<(MergeSlice, usize)>, // current left pile and index - right: Option<(MergeSlice, usize)>, // current right pile and index - - // track the current slice being constructed (from left or right) - has_started: bool, - current_start: usize, - current_len: usize, - current_is_left: bool, -} - -impl<'a, L, R> MergeSortSlices<'a, L, R> -where - L: Iterator, - R: Iterator, -{ - fn new(lhs: L, rhs: R, comparator: &'a Comparator<'a>) -> Self { - Self { - lhs, - rhs, - comparator, - left: None, - right: None, - has_started: false, - current_start: 0, - current_len: 0, - current_is_left: true, - } - } - - fn next_left(&mut self) { - match self.lhs.next() { - Some(slice) => { - self.left = Some((*slice, slice.1)); - self.current_start = slice.1; - } - None => self.left = None, - } - } - - fn next_right(&mut self) { - match self.rhs.next() { - Some(slice) => { - self.right = Some((*slice, slice.1)); - self.current_start = slice.1; - } - None => self.right = None, - } - } - - /// Collect the MergeSortSlices to be a vec for reusing - #[warn(dead_code)] - pub fn to_vec(self, limit: Option) -> Vec { - match limit { - Some(limit) => { - let mut v = Vec::with_capacity(limit); - let mut current_len = 0; - for (index, start, len) in self { - if len + current_len >= limit { - v.push((index, start, limit - current_len)); - break; - } else { - v.push((index, start, len)); - } - current_len += len; - } - - v - } - None => self.into_iter().collect(), - } - } -} - -impl<'a, L, R> Iterator for MergeSortSlices<'a, L, R> -where - L: Iterator, - R: Iterator, -{ - type Item = MergeSlice; - - fn next(&mut self) -> Option { - if !self.has_started { - // first call of `next` - self.next_left(); - self.next_right(); - } - - match (self.left, self.right) { - (None, None) => { - // both ended - None - } - (Some((left_slice, left_index)), None) => { - // right ended => push left - self.next_left(); - // pushing from left - if left_index != left_slice.1 { - // we are in the middle of some slice: push the - // remaining of that slice - Some(( - left_slice.0, - left_index, - left_slice.2 - (left_index - left_slice.1), - )) - } else { - Some(left_slice) - } - } - (None, Some((right_slice, right_index))) => { - // left ended => push right - self.next_right(); - if right_index != right_slice.1 { - // we are in the middle of some slice: push the - // remaining of that slice - Some(( - right_slice.0, - right_index, - right_slice.2 - (right_index - right_slice.1), - )) - } else { - Some(right_slice) - } - } - // both sides have elements - (Some((left_slice, mut left_index)), Some((right_slice, mut right_index))) => { - if !self.has_started { - let ordering = - (self.comparator)(left_slice.0, left_index, right_slice.0, right_index); - if ordering == Ordering::Greater { - self.current_is_left = false; - self.current_start = right_index; - } else { - self.current_is_left = true; - self.current_start = left_index; - } - self.has_started = true; - } - - // advance left_index or right_index until the next split - while (left_index < left_slice.1 + left_slice.2) - && (right_index < right_slice.1 + right_slice.2) - { - match ( - (self.comparator)(left_slice.0, left_index, right_slice.0, right_index), - self.current_is_left, - ) { - (Ordering::Less, true) | (Ordering::Equal, true) => { - // on the left and take from the left - self.current_len += 1; - left_index += 1; - } - (Ordering::Greater, false) | (Ordering::Equal, false) => { - // on the right and take from the right - self.current_len += 1; - right_index += 1; - } - (Ordering::Less, false) => { - // switch from right side to left side => push new slice from the right - let start = self.current_start; - let len = self.current_len; - self.current_is_left = true; - self.current_len = 0; - self.current_start = left_index; - if len > 0 { - self.left = Some((left_slice, left_index)); - self.right = Some((right_slice, right_index)); - return Some((right_slice.0, start, len)); - } - } - (Ordering::Greater, true) => { - // switch from left side to right side => push slice from the left - let start = self.current_start; - let len = self.current_len; - self.current_is_left = false; - self.current_len = 0; - self.current_start = right_index; - if len > 0 { - self.left = Some((left_slice, left_index)); - self.right = Some((right_slice, right_index)); - return Some((left_slice.0, start, len)); - } - } - } - } - let start = self.current_start; - let len = self.current_len; - if left_index == left_slice.1 + left_slice.2 { - // reached end of left slice => push it - self.current_len = 0; - self.next_left(); - Some((left_slice.0, start, len)) - } else { - debug_assert_eq!(right_index, right_slice.1 + right_slice.2); - // reached end of right slice => push it - self.current_len = 0; - self.next_right(); - Some((right_slice.0, start, len)) - } - } - } - } -} - -/// Given two iterators of slices representing two sets of sorted [`Array`]s, and a `comparator` bound to those [`Array`]s, -/// returns a new iterator of slices denoting how to `take` slices from each of the arrays such that the resulting -/// array is sorted according to `comparator` -pub fn merge_sort_slices< - 'a, - L: Iterator, - R: Iterator, ->( - lhs: L, - rhs: R, - comparator: &'a Comparator, -) -> MergeSortSlices<'a, L, R> { - MergeSortSlices::new(lhs, rhs, comparator) -} - -// (left index, left row), (right index, right row) -type Comparator<'a> = Box Ordering + 'a>; -type IsValid<'a> = Box bool + 'a>; - -/// returns a comparison function between any two arrays of each pair of arrays, according to `SortOptions`. -pub fn build_comparator<'a>( - pairs: &'a [(&'a [&'a dyn Array], &SortOptions)], -) -> Result> { - build_comparator_impl(pairs, &build_compare) -} - -/// returns a comparison function between any two arrays of each pair of arrays, according to `SortOptions`. -/// Implementing custom `build_compare_fn` for unsupported data types. -pub fn build_comparator_impl<'a>( - pairs: &'a [(&'a [&'a dyn Array], &SortOptions)], - build_compare_fn: &dyn Fn(&dyn Array, &dyn Array) -> Result, -) -> Result> { - // prepare the comparison function of _values_ between all pairs of arrays - let indices_pairs = (0..pairs[0].0.len()) - .combinations(2) - .map(|indices| (indices[0], indices[1])); - - let data = indices_pairs - .map(|(lhs_index, rhs_index)| { - let multi_column_comparator = pairs - .iter() - .map(move |(arrays, _)| { - Ok(( - Box::new(move |row| arrays[lhs_index].is_valid(row)) as IsValid<'a>, - Box::new(move |row| arrays[rhs_index].is_valid(row)) as IsValid<'a>, - build_compare_fn(arrays[lhs_index], arrays[rhs_index])?, - )) - }) - .collect::>>()?; - Ok(((lhs_index, rhs_index), multi_column_comparator)) - }) - .collect::>>>()?; - - // prepare a comparison function taking into account _nulls_ and sort options - let cmp = move |left_index, left_row, right_index, right_row| { - let data = data.get(&(left_index, right_index)).unwrap(); - // data.iter().zip(pairs.iter()).for_each() - for c in 0..pairs.len() { - let descending = pairs[c].1.descending; - let null_first = pairs[c].1.nulls_first; - let (l_is_valid, r_is_valid, value_comparator) = &data[c]; - let result = match ((l_is_valid)(left_row), (r_is_valid)(right_row)) { - (true, true) => { - let result = (value_comparator)(left_row, right_row); - match descending { - true => result.reverse(), - false => result, - } - } - (false, true) => { - if null_first { - Ordering::Less - } else { - Ordering::Greater - } - } - (true, false) => { - if null_first { - Ordering::Greater - } else { - Ordering::Less - } - } - (false, false) => Ordering::Equal, - }; - if result != Ordering::Equal { - // we found a relevant comparison => short-circuit and return it - return result; - } - } - Ordering::Equal - }; - Ok(Box::new(cmp)) -} diff --git a/src/common/arrow/src/arrow/compute/mod.rs b/src/common/arrow/src/arrow/compute/mod.rs index c4e4a4c90ce3..2b2108fb8985 100644 --- a/src/common/arrow/src/arrow/compute/mod.rs +++ b/src/common/arrow/src/arrow/compute/mod.rs @@ -26,23 +26,7 @@ //! Some dynamically-typed operators have an auxiliary function, `can_*`, that returns //! true if the operator can be applied to the particular `DataType`. -#[cfg(any(feature = "compute_aggregate", feature = "io_parquet"))] -#[cfg_attr(docsrs, doc(cfg(feature = "compute_aggregate")))] -pub mod aggregate; -pub mod arity; -#[cfg(feature = "compute_cast")] -#[cfg_attr(docsrs, doc(cfg(feature = "compute_cast")))] -pub mod cast; #[cfg(feature = "compute_concatenate")] #[cfg_attr(docsrs, doc(cfg(feature = "compute_concatenate")))] pub mod concatenate; -#[cfg(feature = "compute_merge_sort")] -#[cfg_attr(docsrs, doc(cfg(feature = "compute_merge_sort")))] -pub mod merge_sort; -#[cfg(feature = "compute_sort")] -#[cfg_attr(docsrs, doc(cfg(feature = "compute_sort")))] -pub mod sort; -#[cfg(feature = "compute_take")] -#[cfg_attr(docsrs, doc(cfg(feature = "compute_take")))] -pub mod take; mod utils; diff --git a/src/common/arrow/src/arrow/compute/sort/binary.rs b/src/common/arrow/src/arrow/compute/sort/binary.rs deleted file mode 100644 index 5ecf8ffcceaf..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/binary.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::common; -use super::SortOptions; -use crate::arrow::array::BinaryArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::offset::Offset; -use crate::arrow::types::Index; - -pub(super) fn indices_sorted_unstable_by( - array: &BinaryArray, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray { - let get = |idx| unsafe { array.value_unchecked(idx) }; - let cmp = |lhs: &&[u8], rhs: &&[u8]| lhs.cmp(rhs); - common::indices_sorted_unstable_by(array.validity(), get, cmp, array.len(), options, limit) -} diff --git a/src/common/arrow/src/arrow/compute/sort/boolean.rs b/src/common/arrow/src/arrow/compute/sort/boolean.rs deleted file mode 100644 index d2baa37ea322..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/boolean.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::SortOptions; -use crate::arrow::array::BooleanArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::types::Index; - -/// Returns the indices that would sort a [`BooleanArray`]. -pub fn sort_boolean( - values: &BooleanArray, - value_indices: Vec, - null_indices: Vec, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray { - let descending = options.descending; - - // create tuples that are used for sorting - let mut valids = value_indices - .into_iter() - .map(|index| (index, values.value(index.to_usize()))) - .collect::>(); - - let mut nulls = null_indices; - - if !descending { - valids.sort_by(|a, b| a.1.cmp(&b.1)); - } else { - valids.sort_by(|a, b| b.1.cmp(&a.1)); - // reverse to keep a stable ordering - nulls.reverse(); - } - - let mut values = Vec::::with_capacity(values.len()); - - if options.nulls_first { - values.extend_from_slice(nulls.as_slice()); - values.extend(valids.iter().map(|x| x.0)); - } else { - // nulls last - values.extend(valids.iter().map(|x| x.0)); - values.extend_from_slice(nulls.as_slice()); - } - - // un-efficient; there are much more performant ways of sorting nulls above, anyways. - if let Some(limit) = limit { - values.truncate(limit); - values.shrink_to_fit(); - } - - let data_type = I::PRIMITIVE.into(); - PrimitiveArray::::new(data_type, values.into(), None) -} diff --git a/src/common/arrow/src/arrow/compute/sort/common.rs b/src/common/arrow/src/arrow/compute/sort/common.rs deleted file mode 100644 index 5caa812b8f79..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/common.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::SortOptions; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::types::Index; - -/// # Safety -/// This function guarantees that: -/// * `get` is only called for `0 <= i < limit` -/// * `cmp` is only called from the co-domain of `get`. -#[inline] -fn k_element_sort_inner( - indices: &mut [I], - get: G, - descending: bool, - limit: usize, - mut cmp: F, -) where - G: Fn(usize) -> T, - F: FnMut(&T, &T) -> std::cmp::Ordering, -{ - if descending { - let mut compare = |lhs: &I, rhs: &I| { - let lhs = get(lhs.to_usize()); - let rhs = get(rhs.to_usize()); - cmp(&rhs, &lhs) - }; - let (before, _, _) = indices.select_nth_unstable_by(limit, &mut compare); - before.sort_unstable_by(&mut compare); - } else { - let mut compare = |lhs: &I, rhs: &I| { - let lhs = get(lhs.to_usize()); - let rhs = get(rhs.to_usize()); - cmp(&lhs, &rhs) - }; - let (before, _, _) = indices.select_nth_unstable_by(limit, &mut compare); - before.sort_unstable_by(&mut compare); - } -} - -/// # Safety -/// This function guarantees that: -/// * `get` is only called for `0 <= i < limit` -/// * `cmp` is only called from the co-domain of `get`. -#[inline] -fn sort_unstable_by( - indices: &mut [I], - get: G, - mut cmp: F, - descending: bool, - limit: usize, -) where - I: Index, - G: Fn(usize) -> T, - F: FnMut(&T, &T) -> std::cmp::Ordering, -{ - if limit != indices.len() { - return k_element_sort_inner(indices, get, descending, limit, cmp); - } - - if descending { - indices.sort_unstable_by(|lhs, rhs| { - let lhs = get(lhs.to_usize()); - let rhs = get(rhs.to_usize()); - cmp(&rhs, &lhs) - }) - } else { - indices.sort_unstable_by(|lhs, rhs| { - let lhs = get(lhs.to_usize()); - let rhs = get(rhs.to_usize()); - cmp(&lhs, &rhs) - }) - } -} - -/// # Safety -/// This function guarantees that: -/// * `get` is only called for `0 <= i < length` -/// * `cmp` is only called from the co-domain of `get`. -#[inline] -pub(super) fn indices_sorted_unstable_by( - validity: Option<&Bitmap>, - get: G, - cmp: F, - length: usize, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray -where - I: Index, - G: Fn(usize) -> T, - F: Fn(&T, &T) -> std::cmp::Ordering, -{ - let descending = options.descending; - - let limit = limit.unwrap_or(length); - // Safety: without this, we go out of bounds when limit >= length. - let limit = limit.min(length); - - let indices = if let Some(validity) = validity { - let mut indices = vec![I::default(); length]; - if options.nulls_first { - let mut nulls = 0; - let mut valids = 0; - validity - .iter() - .zip(I::range(0, length).unwrap()) - .for_each(|(is_valid, index)| { - if is_valid { - indices[validity.unset_bits() + valids] = index; - valids += 1; - } else { - indices[nulls] = index; - nulls += 1; - } - }); - - if limit > validity.unset_bits() { - // when limit is larger, we must sort values: - - // Soundness: - // all indices in `indices` are by construction `< array.len() == values.len()` - // limit is by construction < indices.len() - let limit = limit.saturating_sub(validity.unset_bits()); - let indices = &mut indices.as_mut_slice()[validity.unset_bits()..]; - sort_unstable_by(indices, get, cmp, options.descending, limit) - } - } else { - let last_valid_index = length.saturating_sub(validity.unset_bits()); - let mut nulls = 0; - let mut valids = 0; - validity - .iter() - .zip(I::range(0, length).unwrap()) - .for_each(|(x, index)| { - if x { - indices[valids] = index; - valids += 1; - } else { - indices[last_valid_index + nulls] = index; - nulls += 1; - } - }); - - // Soundness: - // all indices in `indices` are by construction `< array.len() == values.len()` - // limit is by construction <= values.len() - let limit = limit.min(last_valid_index); - let indices = &mut indices.as_mut_slice()[..last_valid_index]; - sort_unstable_by(indices, get, cmp, options.descending, limit); - } - - indices.truncate(limit); - indices.shrink_to_fit(); - - indices - } else { - let mut indices = I::range(0, length).unwrap().collect::>(); - - sort_unstable_by(&mut indices, get, cmp, descending, limit); - indices.truncate(limit); - indices.shrink_to_fit(); - indices - }; - - let data_type = I::PRIMITIVE.into(); - PrimitiveArray::::new(data_type, indices.into(), None) -} diff --git a/src/common/arrow/src/arrow/compute/sort/lex_sort.rs b/src/common/arrow/src/arrow/compute/sort/lex_sort.rs deleted file mode 100644 index 9d15373e0cb5..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/lex_sort.rs +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::cmp::Ordering; - -use super::sort_to_indices; -use super::SortOptions; -use crate::arrow::array::ord; -use crate::arrow::array::ord::DynComparator; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::compute::take; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::Index; - -type IsValid = Box bool + Send + Sync>; - -/// One column to be used in lexicographical sort -#[derive(Clone, Debug)] -pub struct SortColumn<'a> { - /// The array to sort - pub values: &'a dyn Array, - /// The options to apply to the sort - pub options: Option, -} - -/// Sort a list of [`Array`] using [`SortOptions`] provided for each array. -/// # Implementaqtion -/// The sort is stable and lexicographical on values. -/// -/// Returns an [`Error`] if any of the array type is either unsupported by -/// `lexsort_to_indices` or `take`. -/// -/// Example: -/// -/// ``` -/// use std::convert::From; -/// -/// use arrow2::array::Array; -/// use arrow2::array::Int64Array; -/// use arrow2::array::Utf8Array; -/// use arrow2::compute::sort::lexsort; -/// use arrow2::compute::sort::SortColumn; -/// use arrow2::compute::sort::SortOptions; -/// use arrow2::datatypes::DataType; -/// -/// let int64 = Int64Array::from(&[None, Some(-2), Some(89), Some(-64), Some(101)]); -/// let utf8 = Utf8Array::::from(&vec![ -/// Some("hello"), -/// Some("world"), -/// Some(","), -/// Some("foobar"), -/// Some("!"), -/// ]); -/// -/// let sorted_chunk = lexsort::( -/// &vec![ -/// SortColumn { -/// values: &int64, -/// options: None, -/// }, -/// SortColumn { -/// values: &utf8, -/// options: Some(SortOptions { -/// descending: true, -/// nulls_first: false, -/// }), -/// }, -/// ], -/// None, -/// ) -/// .unwrap(); -/// -/// let sorted = sorted_chunk[0] -/// .as_any() -/// .downcast_ref::() -/// .unwrap(); -/// assert_eq!(sorted.value(1), -64); -/// assert!(sorted.is_null(0)); -/// ``` -pub fn lexsort( - columns: &[SortColumn], - limit: Option, -) -> Result>> { - let indices = lexsort_to_indices::(columns, limit)?; - columns - .iter() - .map(|c| take::take(c.values, &indices)) - .collect() -} - -#[inline] -fn build_is_valid(array: &dyn Array) -> IsValid { - if let Some(validity) = array.validity() { - let validity = validity.clone(); - Box::new(move |x| unsafe { validity.get_bit_unchecked(x) }) - } else { - Box::new(move |_| true) - } -} - -pub fn build_compare(array: &dyn Array, sort_option: SortOptions) -> Result { - build_compare_impl(array, sort_option, &ord::build_compare) -} - -pub(crate) fn build_compare_impl( - array: &dyn Array, - sort_option: SortOptions, - build_compare_fn: &dyn Fn(&dyn Array, &dyn Array) -> Result, -) -> Result { - let is_valid = build_is_valid(array); - let comparator = build_compare_fn(array, array)?; - - Ok(match (sort_option.descending, sort_option.nulls_first) { - (true, true) => Box::new(move |i: usize, j: usize| match (is_valid(i), is_valid(j)) { - (true, true) => match (comparator)(i, j) { - Ordering::Equal => Ordering::Equal, - other => other.reverse(), - }, - (false, true) => Ordering::Less, - (true, false) => Ordering::Greater, - (false, false) => Ordering::Equal, - }), - (false, true) => Box::new(move |i: usize, j: usize| match (is_valid(i), is_valid(j)) { - (true, true) => match (comparator)(i, j) { - Ordering::Equal => Ordering::Equal, - other => other, - }, - (false, true) => Ordering::Less, - (true, false) => Ordering::Greater, - (false, false) => Ordering::Equal, - }), - (false, false) => Box::new(move |i: usize, j: usize| match (is_valid(i), is_valid(j)) { - (true, true) => match (comparator)(i, j) { - Ordering::Equal => Ordering::Equal, - other => other, - }, - (false, true) => Ordering::Greater, - (true, false) => Ordering::Less, - (false, false) => Ordering::Equal, - }), - (true, false) => Box::new(move |i: usize, j: usize| match (is_valid(i), is_valid(j)) { - (true, true) => match (comparator)(i, j) { - Ordering::Equal => Ordering::Equal, - other => other.reverse(), - }, - (false, true) => Ordering::Greater, - (true, false) => Ordering::Less, - (false, false) => Ordering::Equal, - }), - }) -} - -/// Sorts a list of [`SortColumn`] into a non-nullable [`PrimitiveArray`] -/// representing the indices that would sort the columns. -pub fn lexsort_to_indices( - columns: &[SortColumn], - limit: Option, -) -> Result> { - lexsort_to_indices_impl(columns, limit, &ord::build_compare) -} - -/// Sorts a list of [`SortColumn`] into a non-nullable [`PrimitiveArray`] -/// representing the indices that would sort the columns. -/// Implementing custom `build_compare_fn` for unsupported data types. -pub fn lexsort_to_indices_impl( - columns: &[SortColumn], - limit: Option, - build_compare_fn: &dyn Fn(&dyn Array, &dyn Array) -> Result, -) -> Result> { - if columns.is_empty() { - return Err(Error::InvalidArgumentError( - "Sort requires at least one column".to_string(), - )); - } - if columns.len() == 1 { - // fallback to non-lexical sort - let column = &columns[0]; - if let Ok(indices) = - sort_to_indices(column.values, &column.options.unwrap_or_default(), limit) - { - return Ok(indices); - } - } - - let row_count = columns[0].values.len(); - if columns.iter().any(|item| item.values.len() != row_count) { - return Err(Error::InvalidArgumentError( - "lexical sort columns have different row counts".to_string(), - )); - }; - - // map arrays to comparators - let comparators = columns - .iter() - .map(|column| -> Result { - build_compare_impl( - column.values, - column.options.unwrap_or_default(), - build_compare_fn, - ) - }) - .collect::>>()?; - - let lex_comparator = |a_idx: &I, b_idx: &I| -> Ordering { - let a_idx = a_idx.to_usize(); - let b_idx = b_idx.to_usize(); - for comparator in comparators.iter() { - match comparator(a_idx, b_idx) { - Ordering::Equal => continue, - other => return other, - } - } - - Ordering::Equal - }; - - let mut values = I::range(0, row_count).unwrap().collect::>(); - - if let Some(limit) = limit { - let limit = limit.min(row_count); - let before = if limit < row_count { - let (before, _, _) = values.select_nth_unstable_by(limit, lex_comparator); - before - } else { - &mut values[..] - }; - before.sort_unstable_by(lex_comparator); - values.truncate(limit); - values.shrink_to_fit(); - } else { - values.sort_unstable_by(lex_comparator); - } - - let data_type = I::PRIMITIVE.into(); - Ok(PrimitiveArray::::new(data_type, values.into(), None)) -} diff --git a/src/common/arrow/src/arrow/compute/sort/mod.rs b/src/common/arrow/src/arrow/compute/sort/mod.rs deleted file mode 100644 index 967c771bc111..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/mod.rs +++ /dev/null @@ -1,389 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains operators to sort individual and slices of [`Array`]s. -use std::cmp::Ordering; - -use crate::arrow::array::ord; -use crate::arrow::array::*; -use crate::arrow::compute::take; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::types::Index; - -mod binary; -mod boolean; -mod common; -mod lex_sort; -mod primitive; -mod utf8; - -pub mod row; -pub use lex_sort::build_compare; -pub use lex_sort::lexsort; -pub use lex_sort::lexsort_to_indices; -pub use lex_sort::lexsort_to_indices_impl; -pub use lex_sort::SortColumn; - -macro_rules! dyn_sort { - ($ty:ty, $array:expr, $cmp:expr, $options:expr, $limit:expr) => {{ - let array = $array - .as_any() - .downcast_ref::>() - .unwrap(); - Ok(Box::new(primitive::sort_by::<$ty, _>( - &array, $cmp, $options, $limit, - ))) - }}; -} - -/// Sort the [`Array`] using [`SortOptions`]. -/// -/// Performs an unstable sort on values and indices. Nulls are ordered according to the `nulls_first` flag in `options`. -/// Floats are sorted using IEEE 754 totalOrder -/// # Errors -/// Errors if the [`DataType`] is not supported. -pub fn sort( - values: &dyn Array, - options: &SortOptions, - limit: Option, -) -> Result> { - match values.data_type() { - DataType::Int8 => dyn_sort!(i8, values, ord::total_cmp, options, limit), - DataType::Int16 => dyn_sort!(i16, values, ord::total_cmp, options, limit), - DataType::Int32 - | DataType::Date32 - | DataType::Time32(_) - | DataType::Interval(IntervalUnit::YearMonth) => { - dyn_sort!(i32, values, ord::total_cmp, options, limit) - } - DataType::Int64 - | DataType::Date64 - | DataType::Time64(_) - | DataType::Timestamp(_, None) - | DataType::Duration(_) => dyn_sort!(i64, values, ord::total_cmp, options, limit), - DataType::UInt8 => dyn_sort!(u8, values, ord::total_cmp, options, limit), - DataType::UInt16 => dyn_sort!(u16, values, ord::total_cmp, options, limit), - DataType::UInt32 => dyn_sort!(u32, values, ord::total_cmp, options, limit), - DataType::UInt64 => dyn_sort!(u64, values, ord::total_cmp, options, limit), - DataType::Float32 => dyn_sort!(f32, values, ord::total_cmp_f32, options, limit), - DataType::Float64 => dyn_sort!(f64, values, ord::total_cmp_f64, options, limit), - _ => { - let indices = sort_to_indices::(values, options, limit)?; - take::take(values, &indices) - } - } -} - -// partition indices into valid and null indices -fn partition_validity(array: &dyn Array) -> (Vec, Vec) { - let length = array.len(); - let indices = (0..length).map(|x| I::from_usize(x).unwrap()); - if let Some(validity) = array.validity() { - indices.partition(|index| validity.get_bit(index.to_usize())) - } else { - (indices.collect(), vec![]) - } -} - -macro_rules! dyn_sort_indices { - ($index:ty, $ty:ty, $array:expr, $cmp:expr, $options:expr, $limit:expr) => {{ - let array = $array - .as_any() - .downcast_ref::>() - .unwrap(); - Ok(primitive::indices_sorted_unstable_by::<$index, $ty, _>( - &array, $cmp, $options, $limit, - )) - }}; -} - -/// Sort elements from `values` into a non-nullable [`PrimitiveArray`] of indices that sort `values`. -pub fn sort_to_indices( - values: &dyn Array, - options: &SortOptions, - limit: Option, -) -> Result> { - match values.data_type() { - DataType::Boolean => { - let (v, n) = partition_validity(values); - Ok(boolean::sort_boolean( - values.as_any().downcast_ref().unwrap(), - v, - n, - options, - limit, - )) - } - DataType::Int8 => dyn_sort_indices!(I, i8, values, ord::total_cmp, options, limit), - DataType::Int16 => dyn_sort_indices!(I, i16, values, ord::total_cmp, options, limit), - DataType::Int32 - | DataType::Date32 - | DataType::Time32(_) - | DataType::Interval(IntervalUnit::YearMonth) => { - dyn_sort_indices!(I, i32, values, ord::total_cmp, options, limit) - } - DataType::Int64 - | DataType::Date64 - | DataType::Time64(_) - | DataType::Timestamp(_, None) - | DataType::Duration(_) => { - dyn_sort_indices!(I, i64, values, ord::total_cmp, options, limit) - } - DataType::UInt8 => dyn_sort_indices!(I, u8, values, ord::total_cmp, options, limit), - DataType::UInt16 => dyn_sort_indices!(I, u16, values, ord::total_cmp, options, limit), - DataType::UInt32 => dyn_sort_indices!(I, u32, values, ord::total_cmp, options, limit), - DataType::UInt64 => dyn_sort_indices!(I, u64, values, ord::total_cmp, options, limit), - DataType::Float32 => dyn_sort_indices!(I, f32, values, ord::total_cmp_f32, options, limit), - DataType::Float64 => dyn_sort_indices!(I, f64, values, ord::total_cmp_f64, options, limit), - DataType::Utf8 => Ok(utf8::indices_sorted_unstable_by::( - values.as_any().downcast_ref().unwrap(), - options, - limit, - )), - DataType::LargeUtf8 => Ok(utf8::indices_sorted_unstable_by::( - values.as_any().downcast_ref().unwrap(), - options, - limit, - )), - DataType::Binary => Ok(binary::indices_sorted_unstable_by::( - values.as_any().downcast_ref().unwrap(), - options, - limit, - )), - DataType::LargeBinary => Ok(binary::indices_sorted_unstable_by::( - values.as_any().downcast_ref().unwrap(), - options, - limit, - )), - DataType::List(field) => { - let (v, n) = partition_validity(values); - match &field.data_type { - DataType::Int8 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int16 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int32 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int64 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt8 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt16 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt32 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt64 => Ok(sort_list::(values, v, n, options, limit)), - t => Err(Error::NotYetImplemented(format!( - "Sort not supported for list type {t:?}" - ))), - } - } - DataType::LargeList(field) => { - let (v, n) = partition_validity(values); - match field.data_type() { - DataType::Int8 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int16 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int32 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int64 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt8 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt16 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt32 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt64 => Ok(sort_list::(values, v, n, options, limit)), - t => Err(Error::NotYetImplemented(format!( - "Sort not supported for list type {t:?}" - ))), - } - } - DataType::FixedSizeList(field, _) => { - let (v, n) = partition_validity(values); - match field.data_type() { - DataType::Int8 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int16 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int32 => Ok(sort_list::(values, v, n, options, limit)), - DataType::Int64 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt8 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt16 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt32 => Ok(sort_list::(values, v, n, options, limit)), - DataType::UInt64 => Ok(sort_list::(values, v, n, options, limit)), - t => Err(Error::NotYetImplemented(format!( - "Sort not supported for list type {t:?}" - ))), - } - } - DataType::Dictionary(key_type, value_type, _) => match value_type.as_ref() { - DataType::Utf8 => Ok(sort_dict::(values, key_type, options, limit)), - DataType::LargeUtf8 => Ok(sort_dict::(values, key_type, options, limit)), - t => Err(Error::NotYetImplemented(format!( - "Sort not supported for dictionary type with keys {t:?}" - ))), - }, - t => Err(Error::NotYetImplemented(format!( - "Sort not supported for data type {t:?}" - ))), - } -} - -fn sort_dict( - values: &dyn Array, - key_type: &IntegerType, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray { - match_integer_type!(key_type, |$T| { - utf8::indices_sorted_unstable_by_dictionary::( - values.as_any().downcast_ref().unwrap(), - options, - limit, - ) - }) -} - -/// Checks if an array of type `datatype` can be sorted -/// -/// # Examples -/// ``` -/// use arrow2::compute::sort::can_sort; -/// use arrow2::datatypes::DataType; -/// -/// let data_type = DataType::Int8; -/// assert_eq!(can_sort(&data_type), true); -/// -/// let data_type = DataType::LargeBinary; -/// assert_eq!(can_sort(&data_type), true) -/// ``` -pub fn can_sort(data_type: &DataType) -> bool { - match data_type { - DataType::Boolean - | DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::Date32 - | DataType::Time32(_) - | DataType::Interval(_) - | DataType::Int64 - | DataType::Date64 - | DataType::Time64(_) - | DataType::Timestamp(_, None) - | DataType::Duration(_) - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::Float32 - | DataType::Float64 - | DataType::Utf8 - | DataType::LargeUtf8 - | DataType::Binary - | DataType::LargeBinary => true, - DataType::List(field) | DataType::LargeList(field) | DataType::FixedSizeList(field, _) => { - matches!( - field.data_type(), - DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::Int64 - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - ) - } - DataType::Dictionary(_, value_type, _) => { - matches!(*value_type.as_ref(), DataType::Utf8 | DataType::LargeUtf8) - } - _ => false, - } -} - -/// Options that define how sort kernels should behave -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct SortOptions { - /// Whether to sort in descending order - pub descending: bool, - /// Whether to sort nulls first - pub nulls_first: bool, -} - -impl Default for SortOptions { - fn default() -> Self { - Self { - descending: false, - // default to nulls first to match spark's behavior - nulls_first: true, - } - } -} - -fn sort_list( - values: &dyn Array, - value_indices: Vec, - null_indices: Vec, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray -where - I: Index, - O: Offset, -{ - let mut valids: Vec<(I, Box)> = values - .as_any() - .downcast_ref::() - .map_or_else( - || { - let values = values.as_any().downcast_ref::>().unwrap(); - value_indices - .iter() - .copied() - .map(|index| (index, values.value(index.to_usize()))) - .collect() - }, - |values| { - value_indices - .iter() - .copied() - .map(|index| (index, values.value(index.to_usize()))) - .collect() - }, - ); - - if !options.descending { - valids.sort_by(|a, b| cmp_array(a.1.as_ref(), b.1.as_ref())) - } else { - valids.sort_by(|a, b| cmp_array(b.1.as_ref(), a.1.as_ref())) - } - - let values = valids.iter().map(|tuple| tuple.0); - - let mut values = if options.nulls_first { - null_indices.into_iter().chain(values).collect::>() - } else { - values.chain(null_indices).collect::>() - }; - - values.truncate(limit.unwrap_or(values.len())); - - let data_type = I::PRIMITIVE.into(); - PrimitiveArray::::new(data_type, values.into(), None) -} - -/// Compare two `Array`s based on the ordering defined in [ord](crate::arrow::array::ord). -fn cmp_array(a: &dyn Array, b: &dyn Array) -> Ordering { - let cmp_op = ord::build_compare(a, b).unwrap(); - let length = a.len().min(b.len()); - - for i in 0..length { - let result = cmp_op(i, i); - if result != Ordering::Equal { - return result; - } - } - a.len().cmp(&b.len()) -} diff --git a/src/common/arrow/src/arrow/compute/sort/primitive/indices.rs b/src/common/arrow/src/arrow/compute/sort/primitive/indices.rs deleted file mode 100644 index 016b26aa0540..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/primitive/indices.rs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::common; -use super::super::SortOptions; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::types::Index; -use crate::arrow::types::NativeType; - -/// Unstable sort of indices. -pub fn indices_sorted_unstable_by( - array: &PrimitiveArray, - cmp: F, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray -where - I: Index, - T: NativeType, - F: Fn(&T, &T) -> std::cmp::Ordering, -{ - let values = array.values().as_slice(); - unsafe { - common::indices_sorted_unstable_by( - array.validity(), - |x: usize| *values.get_unchecked(x), - cmp, - array.len(), - options, - limit, - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrow::array::ord; - use crate::arrow::array::*; - use crate::arrow::datatypes::DataType; - - fn test( - data: &[Option], - data_type: DataType, - options: SortOptions, - limit: Option, - expected_data: &[i32], - ) where - T: NativeType + std::cmp::Ord, - { - let input = PrimitiveArray::::from(data).to(data_type); - let expected = Int32Array::from_slice(expected_data); - let output = - indices_sorted_unstable_by::(&input, ord::total_cmp, &options, limit); - assert_eq!(output, expected) - } - - #[test] - fn ascending_nulls_first() { - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: false, - nulls_first: true, - }, - None, - &[0, 5, 3, 1, 4, 2], - ); - } - - #[test] - fn ascending_nulls_last() { - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: false, - nulls_first: false, - }, - None, - &[3, 1, 4, 2, 0, 5], - ); - } - - #[test] - fn descending_nulls_first() { - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: true, - nulls_first: true, - }, - None, - &[0, 5, 2, 1, 4, 3], - ); - } - - #[test] - fn descending_nulls_last() { - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: true, - nulls_first: false, - }, - None, - &[2, 1, 4, 3, 0, 5], - ); - } - - #[test] - fn limit_ascending_nulls_first() { - // nulls sorted - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: false, - nulls_first: true, - }, - Some(2), - &[0, 5], - ); - - // nulls and values sorted - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: false, - nulls_first: true, - }, - Some(4), - &[0, 5, 3, 1], - ); - } - - #[test] - fn limit_ascending_nulls_last() { - // values - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: false, - nulls_first: false, - }, - Some(2), - &[3, 1], - ); - - // values and nulls - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: false, - nulls_first: false, - }, - Some(5), - &[3, 1, 4, 2, 0], - ); - } - - #[test] - fn limit_descending_nulls_first() { - // nulls - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: true, - nulls_first: true, - }, - Some(2), - &[0, 5], - ); - - // nulls and values - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: true, - nulls_first: true, - }, - Some(4), - &[0, 5, 2, 1], - ); - } - - #[test] - fn limit_descending_nulls_last() { - // values - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: true, - nulls_first: false, - }, - Some(2), - &[2, 1], - ); - - // values and nulls - test::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: true, - nulls_first: false, - }, - Some(5), - &[2, 1, 4, 3, 0], - ); - } -} diff --git a/src/common/arrow/src/arrow/compute/sort/primitive/mod.rs b/src/common/arrow/src/arrow/compute/sort/primitive/mod.rs deleted file mode 100644 index b1d3aa6382c3..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/primitive/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod indices; -mod sort; - -pub use indices::indices_sorted_unstable_by; -pub use sort::sort_by; diff --git a/src/common/arrow/src/arrow/compute/sort/primitive/sort.rs b/src/common/arrow/src/arrow/compute/sort/primitive/sort.rs deleted file mode 100644 index bc9256852f89..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/primitive/sort.rs +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::SortOptions; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::utils::SlicesIterator; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::types::NativeType; - -/// # Safety -/// `indices[i] < values.len()` for all i -#[inline] -fn k_element_sort_inner(values: &mut [T], descending: bool, limit: usize, mut cmp: F) -where - T: NativeType, - F: FnMut(&T, &T) -> std::cmp::Ordering, -{ - if descending { - let (before, _, _) = values.select_nth_unstable_by(limit, |x, y| cmp(y, x)); - before.sort_unstable_by(|x, y| cmp(x, y)); - } else { - let (before, _, _) = values.select_nth_unstable_by(limit, |x, y| cmp(x, y)); - before.sort_unstable_by(|x, y| cmp(x, y)); - } -} - -fn sort_values(values: &mut [T], mut cmp: F, descending: bool, limit: usize) -where - T: NativeType, - F: FnMut(&T, &T) -> std::cmp::Ordering, -{ - if limit != values.len() { - return k_element_sort_inner(values, descending, limit, cmp); - } - - if descending { - values.sort_unstable_by(|x, y| cmp(y, x)); - } else { - values.sort_unstable_by(cmp); - }; -} - -fn sort_nullable( - values: &[T], - validity: &Bitmap, - cmp: F, - options: &SortOptions, - limit: usize, -) -> (Buffer, Option) -where - T: NativeType, - F: FnMut(&T, &T) -> std::cmp::Ordering, -{ - assert!(limit <= values.len()); - if options.nulls_first && limit < validity.unset_bits() { - let buffer = vec![T::default(); limit]; - let bitmap = MutableBitmap::from_trusted_len_iter(std::iter::repeat(false).take(limit)); - return (buffer.into(), bitmap.into()); - } - - let nulls = std::iter::repeat(false).take(validity.unset_bits()); - let valids = std::iter::repeat(true).take(values.len() - validity.unset_bits()); - - let mut buffer = Vec::::with_capacity(values.len()); - let mut new_validity = MutableBitmap::with_capacity(values.len()); - let slices = SlicesIterator::new(validity); - - if options.nulls_first { - // validity is [0,0,0,...,1,1,1,1] - new_validity.extend_from_trusted_len_iter(nulls.chain(valids).take(limit)); - - // extend buffer with constants followed by non-null values - buffer.resize(validity.unset_bits(), T::default()); - for (start, len) in slices { - buffer.extend_from_slice(&values[start..start + len]) - } - - // sort values - sort_values( - &mut buffer.as_mut_slice()[validity.unset_bits()..], - cmp, - options.descending, - limit - validity.unset_bits(), - ); - } else { - // validity is [1,1,1,...,0,0,0,0] - new_validity.extend_from_trusted_len_iter(valids.chain(nulls).take(limit)); - - // extend buffer with non-null values - for (start, len) in slices { - buffer.extend_from_slice(&values[start..start + len]) - } - - // sort all non-null values - sort_values( - buffer.as_mut_slice(), - cmp, - options.descending, - limit - validity.unset_bits(), - ); - - if limit > values.len() - validity.unset_bits() { - // extend remaining with nulls - buffer.resize(buffer.len() + validity.unset_bits(), T::default()); - } - }; - // values are sorted, we can now truncate the remaining. - buffer.truncate(limit); - buffer.shrink_to_fit(); - - (buffer.into(), new_validity.into()) -} - -/// Sorts a [`PrimitiveArray`] according to `cmp` comparator and [`SortOptions`]. -pub fn sort_by( - array: &PrimitiveArray, - cmp: F, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray -where - T: NativeType, - F: FnMut(&T, &T) -> std::cmp::Ordering, -{ - let limit = limit.unwrap_or_else(|| array.len()); - let limit = limit.min(array.len()); - - let values = array.values(); - let validity = array.validity(); - - let (buffer, validity) = if let Some(validity) = validity { - sort_nullable(values, validity, cmp, options, limit) - } else { - let mut buffer = Vec::::new(); - buffer.extend_from_slice(values); - - sort_values(buffer.as_mut_slice(), cmp, options.descending, limit); - buffer.truncate(limit); - buffer.shrink_to_fit(); - - (buffer.into(), None) - }; - PrimitiveArray::::new(array.data_type().clone(), buffer, validity) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrow::array::ord; - use crate::arrow::array::PrimitiveArray; - use crate::arrow::datatypes::DataType; - - fn test_sort_primitive_arrays( - data: &[Option], - data_type: DataType, - options: SortOptions, - expected_data: &[Option], - ) where - T: NativeType + std::cmp::Ord, - { - let input = PrimitiveArray::::from(data).to(data_type.clone()); - let expected = PrimitiveArray::::from(expected_data).to(data_type.clone()); - let output = sort_by(&input, ord::total_cmp, &options, None); - assert_eq!(expected, output); - - // with limit - let expected = PrimitiveArray::::from(&expected_data[..3]).to(data_type); - let output = sort_by(&input, ord::total_cmp, &options, Some(3)); - assert_eq!(expected, output) - } - - #[test] - fn ascending_nulls_first() { - test_sort_primitive_arrays::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: false, - nulls_first: true, - }, - &[None, None, Some(2), Some(3), Some(3), Some(5)], - ); - } - - #[test] - fn ascending_nulls_last() { - test_sort_primitive_arrays::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: false, - nulls_first: false, - }, - &[Some(2), Some(3), Some(3), Some(5), None, None], - ); - } - - #[test] - fn descending_nulls_first() { - test_sort_primitive_arrays::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: true, - nulls_first: true, - }, - &[None, None, Some(5), Some(3), Some(3), Some(2)], - ); - } - - #[test] - fn descending_nulls_last() { - test_sort_primitive_arrays::( - &[None, Some(3), Some(5), Some(2), Some(3), None], - DataType::Int8, - SortOptions { - descending: true, - nulls_first: false, - }, - &[Some(5), Some(3), Some(3), Some(2), None, None], - ); - } -} diff --git a/src/common/arrow/src/arrow/compute/sort/row/dictionary.rs b/src/common/arrow/src/arrow/compute/sort/row/dictionary.rs deleted file mode 100644 index 1ccd3f30be67..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/row/dictionary.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::fixed::FixedLengthEncoding; -use super::interner::Interned; -use super::interner::OrderPreservingInterner; -use super::null_sentinel; -use super::Rows; -use crate::arrow::{ - array::{Array, BinaryArray, DictionaryArray, DictionaryKey, PrimitiveArray, Utf8Array}, - compute::sort::SortOptions, - datatypes::PhysicalType, - error::*, - // with_match_primitive_without_interval_type, -}; -use crate::with_match_primitive_without_interval_type; - -/// Computes the dictionary mapping for the given dictionary values -#[allow(clippy::borrowed_box)] -pub fn compute_dictionary_mapping( - interner: &mut OrderPreservingInterner, - values: &Box, -) -> Result>> { - Ok(match values.data_type().to_physical_type() { - PhysicalType::Primitive(primitive) => { - with_match_primitive_without_interval_type!(primitive, |$T| { - let values = values - .as_any() - .downcast_ref::>() - .unwrap(); - interner.intern(values.iter().map(|x| x.map(|x| x.encode()))) - }) - } - PhysicalType::Binary => { - let iter = values - .as_any() - .downcast_ref::>() - .unwrap() - .iter(); - interner.intern(iter) - } - PhysicalType::LargeBinary => { - let iter = values - .as_any() - .downcast_ref::>() - .unwrap() - .iter(); - interner.intern(iter) - } - PhysicalType::Utf8 => { - let iter = values - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .map(|x| x.map(|x| x.as_bytes())); - interner.intern(iter) - } - PhysicalType::LargeUtf8 => { - let iter = values - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .map(|x| x.map(|x| x.as_bytes())); - interner.intern(iter) - } - t => { - return Err(Error::NotYetImplemented(format!( - "dictionary value {t:?} is not supported" - ))); - } - }) -} - -/// Dictionary types are encoded as -/// -/// - single `0_u8` if null -/// - the bytes of the corresponding normalized key including the null terminator -pub fn encode_dictionary( - out: &mut Rows, - column: &DictionaryArray, - normalized_keys: &[Option<&[u8]>], - opts: SortOptions, -) { - for (offset, k) in out.offsets.iter_mut().skip(1).zip(column.keys()) { - match k.and_then(|k| normalized_keys[unsafe { k.as_usize() }]) { - Some(normalized_key) => { - let end_offset = *offset + 1 + normalized_key.len(); - out.buffer[*offset] = 1; - out.buffer[*offset + 1..end_offset].copy_from_slice(normalized_key); - // Negate if descending - if opts.descending { - out.buffer[*offset..end_offset] - .iter_mut() - .for_each(|v| *v = !*v) - } - *offset = end_offset; - } - None => { - out.buffer[*offset] = null_sentinel(opts); - *offset += 1; - } - } - } -} diff --git a/src/common/arrow/src/arrow/compute/sort/row/fixed.rs b/src/common/arrow/src/arrow/compute/sort/row/fixed.rs deleted file mode 100644 index a004dd6319f3..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/row/fixed.rs +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::null_sentinel; -use super::Rows; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::compute::sort::SortOptions; -use crate::arrow::types::f16; -use crate::arrow::types::i256; -use crate::arrow::types::NativeType; - -pub trait FromSlice { - fn from_slice(slice: &[u8], invert: bool) -> Self; -} - -impl FromSlice for [u8; N] { - #[inline] - fn from_slice(slice: &[u8], invert: bool) -> Self { - let mut t: Self = slice.try_into().unwrap(); - if invert { - t.iter_mut().for_each(|o| *o = !*o); - } - t - } -} - -/// Encodes a value of a particular fixed width type into bytes according to the rules -/// described on [`super::RowConverter`] -pub trait FixedLengthEncoding: Copy { - const ENCODED_LEN: usize = 1 + std::mem::size_of::(); - - type Encoded: Sized + Copy + FromSlice + AsRef<[u8]> + AsMut<[u8]>; - - fn encode(self) -> Self::Encoded; - - fn decode(encoded: Self::Encoded) -> Self; -} - -impl FixedLengthEncoding for bool { - type Encoded = [u8; 1]; - - fn encode(self) -> [u8; 1] { - [self as u8] - } - - fn decode(encoded: Self::Encoded) -> Self { - encoded[0] != 0 - } -} - -macro_rules! encode_signed { - ($n:expr, $t:ty) => { - impl FixedLengthEncoding for $t { - type Encoded = [u8; $n]; - - fn encode(self) -> [u8; $n] { - let mut b = self.to_be_bytes(); - // Toggle top "sign" bit to ensure consistent sort order - b[0] ^= 0x80; - b - } - - fn decode(mut encoded: Self::Encoded) -> Self { - // Toggle top "sign" bit - encoded[0] ^= 0x80; - Self::from_be_bytes(encoded) - } - } - }; -} - -encode_signed!(1, i8); -encode_signed!(2, i16); -encode_signed!(4, i32); -encode_signed!(8, i64); -encode_signed!(16, i128); -encode_signed!(32, i256); - -macro_rules! encode_unsigned { - ($n:expr, $t:ty) => { - impl FixedLengthEncoding for $t { - type Encoded = [u8; $n]; - - fn encode(self) -> [u8; $n] { - self.to_be_bytes() - } - - fn decode(encoded: Self::Encoded) -> Self { - Self::from_be_bytes(encoded) - } - } - }; -} - -encode_unsigned!(1, u8); -encode_unsigned!(2, u16); -encode_unsigned!(4, u32); -encode_unsigned!(8, u64); - -impl FixedLengthEncoding for f16 { - type Encoded = [u8; 2]; - - fn encode(self) -> [u8; 2] { - // https://github.com/rust-lang/rust/blob/9c20b2a8cc7588decb6de25ac6a7912dcef24d65/library/core/src/num/f32.rs#L1176-L1260 - let s = self.to_bits() as i16; - let val = s ^ (((s >> 15) as u16) >> 1) as i16; - val.encode() - } - - fn decode(encoded: Self::Encoded) -> Self { - let bits = i16::decode(encoded); - let val = bits ^ (((bits >> 15) as u16) >> 1) as i16; - Self::from_bits(val as u16) - } -} - -impl FixedLengthEncoding for f32 { - type Encoded = [u8; 4]; - - fn encode(self) -> [u8; 4] { - // https://github.com/rust-lang/rust/blob/9c20b2a8cc7588decb6de25ac6a7912dcef24d65/library/core/src/num/f32.rs#L1176-L1260 - let s = self.to_bits() as i32; - let val = s ^ (((s >> 31) as u32) >> 1) as i32; - val.encode() - } - - fn decode(encoded: Self::Encoded) -> Self { - let bits = i32::decode(encoded); - let val = bits ^ (((bits >> 31) as u32) >> 1) as i32; - Self::from_bits(val as u32) - } -} - -impl FixedLengthEncoding for f64 { - type Encoded = [u8; 8]; - - fn encode(self) -> [u8; 8] { - // https://github.com/rust-lang/rust/blob/9c20b2a8cc7588decb6de25ac6a7912dcef24d65/library/core/src/num/f32.rs#L1176-L1260 - let s = self.to_bits() as i64; - let val = s ^ (((s >> 63) as u64) >> 1) as i64; - val.encode() - } - - fn decode(encoded: Self::Encoded) -> Self { - let bits = i64::decode(encoded); - let val = bits ^ (((bits >> 63) as u64) >> 1) as i64; - Self::from_bits(val as u64) - } -} - -/// Returns the total encoded length (including null byte) for a value of type `T::Native` -pub const fn encoded_len(_col: &PrimitiveArray) -> usize -where T: NativeType + FixedLengthEncoding { - T::ENCODED_LEN -} - -/// Fixed width types are encoded as -/// -/// - 1 byte `0` if null or `1` if valid -/// - bytes of [`FixedLengthEncoding`] -pub fn encode>>( - out: &mut Rows, - i: I, - opts: SortOptions, -) { - for (offset, maybe_val) in out.offsets.iter_mut().skip(1).zip(i) { - let end_offset = *offset + T::ENCODED_LEN; - if let Some(val) = maybe_val { - let to_write = &mut out.buffer[*offset..end_offset]; - to_write[0] = 1; - let mut encoded = val.encode(); - if opts.descending { - // Flip bits to reverse order - encoded.as_mut().iter_mut().for_each(|v| *v = !*v) - } - to_write[1..].copy_from_slice(encoded.as_ref()) - } else { - out.buffer[*offset] = null_sentinel(opts); - } - *offset = end_offset; - } -} diff --git a/src/common/arrow/src/arrow/compute/sort/row/interner.rs b/src/common/arrow/src/arrow/compute/sort/row/interner.rs deleted file mode 100644 index 943f7b93ef56..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/row/interner.rs +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::hash::BuildHasher; -use std::hash::Hash; -use std::num::NonZeroU32; -use std::ops::Index; - -use hashbrown_v0_14::hash_map::RawEntryMut; -use hashbrown_v0_14::HashMap; - -/// An interned value of 32 bits. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Interned(NonZeroU32); // We use NonZeroU32 so that `Option` is 32 bits. - -/// A byte array interner that generates normalized keys that are sorted with respect -/// to the interned values, e.g. `intern(a) < intern(b) => a < b` -#[derive(Debug, Default, Clone)] -pub struct OrderPreservingInterner { - /// Provides a lookup from [`Interned`] to the normalized key - keys: InternBuffer, - /// Provides a lookup from [`Interned`] to the normalized value - values: InternBuffer, - /// Key allocation data structure - bucket: Box, - - // A hash table used to perform faster re-keying, and detect duplicates - hasher: ahash::RandomState, - // A SwissTable hashmap. - lookup: HashMap, -} - -trait HashSingle: BuildHasher { - /// Calculates the hash of a single value. - #[inline] - fn hash_single(&self, x: T) -> u64 - where Self: Sized { - self.hash_one(&x) - } -} - -impl HashSingle for ahash::RandomState {} - -impl OrderPreservingInterner { - /// Interns an iterator of values returning a list of [`Interned`] which can be - /// used with [`Self::normalized_key`] to retrieve the normalized keys with a - /// lifetime not tied to the mutable borrow passed to this method - pub fn intern(&mut self, input: I) -> Vec> - where - I: IntoIterator>, - V: AsRef<[u8]>, - { - let iter = input.into_iter(); - let capacity = iter.size_hint().0; - let mut out = Vec::with_capacity(capacity); - - // (index in output, hash value, value) - let mut to_intern: Vec<(usize, u64, V)> = Vec::with_capacity(capacity); - let mut to_intern_len = 0; - - for (idx, item) in iter.enumerate() { - let value: V = match item { - Some(value) => value, - None => { - out.push(None); - continue; - } - }; - - let v = value.as_ref(); - let hash = self.hasher.hash_single(v); - let entry = self - .lookup - .raw_entry_mut() - .from_hash(hash, |a| &self.values[*a] == v); - - match entry { - RawEntryMut::Occupied(o) => out.push(Some(*o.key())), - RawEntryMut::Vacant(_) => { - // Push placeholder - out.push(None); - to_intern_len += v.len(); - to_intern.push((idx, hash, value)); - } - }; - } - - to_intern.sort_unstable_by(|(_, _, a), (_, _, b)| a.as_ref().cmp(b.as_ref())); - - self.keys.offsets.reserve(to_intern.len()); - self.keys.values.reserve(to_intern.len()); // Approximation - self.values.offsets.reserve(to_intern.len()); - self.values.values.reserve(to_intern_len); - - for (idx, hash, value) in to_intern { - let val = value.as_ref(); - - let entry = self - .lookup - .raw_entry_mut() - .from_hash(hash, |a| &self.values[*a] == val); - - match entry { - RawEntryMut::Occupied(o) => { - out[idx] = Some(*o.key()); - } - RawEntryMut::Vacant(v) => { - let val = value.as_ref(); - self.bucket - .insert(&mut self.values, val, &mut self.keys.values); - self.keys.values.push(0); - let interned = self.keys.append(); - - let values = &self.values; - v.insert_with_hasher(hash, interned, (), |key| { - self.hasher.hash_single(&values[*key]) - }); - out[idx] = Some(interned); - } - } - } - - out - } - - /// Returns a null-terminated byte array that can be compared against other normalized_key - /// returned by this instance, to establish ordering of the interned values - pub fn normalized_key(&self, key: Interned) -> &[u8] { - self.keys.index(key) - } - - #[cfg(test)] - /// Converts a normalized key returned by [`Self::normalized_key`] to [`Interned`] - /// returning `None` if it cannot be found - pub fn lookup(&self, normalized_key: &[u8]) -> Option { - let len = normalized_key.len(); - if len <= 1 { - // `normalized_key` should terminate with a 0. - return None; - } - - let mut bucket = self.bucket.as_ref(); - if len > 2 { - for v in normalized_key.iter().take(len - 2) { - if *v == 255 { - bucket = bucket.next.as_ref()?; - } else { - let bucket_idx = v.checked_sub(1)?; - bucket = bucket.slots.get(bucket_idx as usize)?.child.as_ref()?; - } - } - } - - let slot_idx = normalized_key[len - 2].checked_sub(2)?; - Some(bucket.slots.get(slot_idx as usize)?.value) - } - - #[cfg(test)] - /// Returns the interned value for a given [`Interned`] - pub fn value(&self, key: Interned) -> &[u8] { - self.values.index(key) - } -} - -/// A buffer of `[u8]` indexed by `[Interned]` -#[derive(Debug, Clone)] -struct InternBuffer { - /// Raw values - values: Vec, - /// The ith value is `&values[offsets[i]..offsets[i+1]]` - offsets: Vec, -} - -impl Default for InternBuffer { - fn default() -> Self { - Self { - values: Default::default(), - offsets: vec![0], - } - } -} - -impl InternBuffer { - /// Insert `data` returning the corresponding [`Interned`] - fn insert(&mut self, data: &[u8]) -> Interned { - self.values.extend_from_slice(data); - self.append() - } - - /// Appends the next value based on data written to `self.values` - /// returning the corresponding [`Interned`] - fn append(&mut self) -> Interned { - let idx: u32 = self.offsets.len().try_into().unwrap(); - let key = Interned(NonZeroU32::new(idx).unwrap()); - self.offsets.push(self.values.len()); - key - } -} - -impl Index for InternBuffer { - type Output = [u8]; - - fn index(&self, key: Interned) -> &Self::Output { - let index = key.0.get() as usize; - let end = self.offsets[index]; - let start = self.offsets[index - 1]; - // SAFETY: - // self.values is never reduced in size and values appended - // to self.offsets are always less than self.values at the time - unsafe { self.values.get_unchecked(start..end) } - } -} - -/// A slot corresponds to a single byte-value in the generated normalized key -/// -/// It may contain a value, if not the first slot, and may contain a child [`Bucket`] representing -/// the next byte in the generated normalized key -#[derive(Debug, Clone)] -struct Slot { - value: Interned, - /// Child values less than `self.value` if any - child: Option>, -} - -/// Bucket is the root of the data-structure used to allocate normalized keys -/// -/// In particular it needs to generate keys that -/// -/// * Contain no `0` bytes other than the null terminator -/// * Compare lexicographically in the same manner as the encoded `data` -/// -/// The data structure consists of 254 slots, each of which can store a value. -/// Additionally each slot may contain a child bucket, containing values smaller -/// than the value within the slot. -/// -/// Each bucket also may contain a child bucket, containing values greater than -/// all values in the current bucket -/// -/// # Allocation Strategy -/// -/// The contiguous slice of slots containing values is searched to find the insertion -/// point for the new value, according to the sort order. -/// -/// If the insertion position exceeds 254, the number of slots, the value is inserted -/// into the child bucket of the current bucket. -/// -/// If the insertion position already contains a value, the value is inserted into the -/// child bucket of that slot. -/// -/// If the slot is not occupied, the value is inserted into that slot. -/// -/// The final key consists of the slot indexes visited incremented by 1, -/// with the final value incremented by 2, followed by a null terminator. -/// -/// Consider the case of the integers `[8, 6, 5, 7]` inserted in that order -/// -/// ```ignore -/// 8: &[2, 0] -/// 6: &[1, 2, 0] -/// 5: &[1, 1, 2, 0] -/// 7: &[1, 3, 0] -/// ``` -/// -/// Note: this allocation strategy is optimised for interning values in sorted order -#[derive(Debug, Clone)] -struct Bucket { - slots: Vec, - /// Bucket containing values larger than all of `slots` - next: Option>, -} - -impl Default for Bucket { - fn default() -> Self { - Self { - slots: Vec::with_capacity(254), - next: None, - } - } -} - -impl Bucket { - /// Insert `data` into this bucket or one of its children, appending the - /// normalized key to `out` as it is constructed - /// - /// # Panics - /// - /// Panics if the value already exists - fn insert(&mut self, values_buf: &mut InternBuffer, data: &[u8], out: &mut Vec) { - let slots_len = self.slots.len() as u8; - // We optimize the case of inserting a value directly after those already inserted - // as [`OrderPreservingInterner::intern`] sorts values prior to interning them - match self.slots.last() { - Some(slot) => { - if &values_buf[slot.value] < data { - if slots_len == 254 { - out.push(255); - self.next - .get_or_insert_with(Default::default) - .insert(values_buf, data, out) - } else { - out.push(slots_len + 2); - let value = values_buf.insert(data); - self.slots.push(Slot { value, child: None }); - } - } else { - // Find insertion point - match self - .slots - .binary_search_by(|slot| values_buf[slot.value].cmp(data)) - { - Ok(_) => unreachable!("value already exists"), - Err(idx) => { - out.push(idx as u8 + 1); - self.slots[idx] - .child - .get_or_insert_with(Default::default) - .insert(values_buf, data, out) - } - } - } - } - None => { - out.push(2); - let value = values_buf.insert(data); - self.slots.push(Slot { value, child: None }) - } - } - } -} - -#[cfg(test)] -mod tests { - use rand::seq::SliceRandom; - use rand::thread_rng; - - use super::*; - - // Clippy isn't smart enough to understand dropping mutability - #[allow(clippy::needless_collect)] - fn test_intern_values(values: &[u64]) { - let mut interner = OrderPreservingInterner::default(); - - // Intern a single value at a time to check ordering - let interned: Vec<_> = values - .iter() - .flat_map(|v| interner.intern([Some(&v.to_be_bytes())])) - .map(Option::unwrap) - .collect(); - - for (value, interned) in values.iter().zip(&interned) { - assert_eq!(interner.value(*interned), &value.to_be_bytes()); - } - - let normalized_keys: Vec<_> = interned - .iter() - .map(|x| interner.normalized_key(*x)) - .collect(); - - for (interned, normalized) in interned.iter().zip(&normalized_keys) { - assert_eq!(*interned, interner.lookup(normalized).unwrap()); - } - - for (i, a) in normalized_keys.iter().enumerate() { - for (j, b) in normalized_keys.iter().enumerate() { - let interned_cmp = a.cmp(b); - let values_cmp = values[i].cmp(&values[j]); - assert_eq!( - interned_cmp, values_cmp, - "({:?} vs {:?}) vs ({} vs {})", - a, b, values[i], values[j] - ) - } - } - } - - #[test] - #[cfg_attr(miri, ignore)] - fn test_interner() { - test_intern_values(&[8, 6, 5, 7]); - - let mut values: Vec<_> = (0_u64..2000).collect(); - test_intern_values(&values); - - let mut rng = thread_rng(); - values.shuffle(&mut rng); - test_intern_values(&values); - } - - #[test] - fn test_intern_duplicates() { - // Unsorted with duplicates - let values = [0_u8, 1, 8, 4, 1, 0]; - let mut interner = OrderPreservingInterner::default(); - - let interned = interner.intern(values.iter().map(std::slice::from_ref).map(Some)); - let interned: Vec<_> = interned.into_iter().map(Option::unwrap).collect(); - - assert_eq!(interned[0], interned[5]); - assert_eq!(interned[1], interned[4]); - assert!(interner.normalized_key(interned[0]) < interner.normalized_key(interned[1])); - assert!(interner.normalized_key(interned[1]) < interner.normalized_key(interned[2])); - assert!(interner.normalized_key(interned[1]) < interner.normalized_key(interned[3])); - assert!(interner.normalized_key(interned[3]) < interner.normalized_key(interned[2])); - } -} diff --git a/src/common/arrow/src/arrow/compute/sort/row/mod.rs b/src/common/arrow/src/arrow/compute/sort/row/mod.rs deleted file mode 100644 index c1d36b5d386c..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/row/mod.rs +++ /dev/null @@ -1,835 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! A comparable row-oriented representation of a collection of [`Array`]. -//! -//! **This module is an arrow2 version of [arrow::row]:[https://docs.rs/arrow/latest/arrow/row/index.html]** -//! -//! As [`Row`] are [normalized for sorting], they can be very efficiently [compared](PartialOrd), -//! using [`memcmp`] under the hood, or used in [non-comparison sorts] such as [radix sort]. This -//! makes the row format ideal for implementing efficient multi-column sorting, -//! grouping, aggregation, windowing and more. -//! -//! _Comparing [`Rows`] generated by different [`RowConverter`] is not guaranteed to -//! yield a meaningful ordering_ -//! -//! [non-comparison sorts]:[https://en.wikipedia.org/wiki/Sorting_algorithm#Non-comparison_sorts] -//! [radix sort]:[https://en.wikipedia.org/wiki/Radix_sort] -//! [normalized for sorting]:[https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.83.1080&rep=rep1&type=pdf] -//! [`memcmp`]:[https://www.man7.org/linux/man-pages/man3/memcmp.3.html] -use std::cmp::Ordering; -use std::hash::Hash; -use std::hash::Hasher; -use std::sync::Arc; - -use self::dictionary::compute_dictionary_mapping; -use self::dictionary::encode_dictionary; -use self::interner::OrderPreservingInterner; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::array::BooleanArray; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::compute::sort::SortOptions; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::*; - -mod dictionary; -mod fixed; -mod interner; -mod variable; - -/// Converts `Box` columns into a row-oriented format. -/// -/// # Format -/// -/// The encoding of the row format should not be considered stable, but is documented here -/// for reference. -/// -/// ## Unsigned Integer Encoding -/// -/// A null integer is encoded as a `0_u8`, followed by a zero-ed number of bytes corresponding -/// to the integer's length -/// -/// A valid integer is encoded as `1_u8`, followed by the big-endian representation of the -/// integer -/// -/// ## Signed Integer Encoding -/// -/// Signed integers have their most significant sign bit flipped, and are then encoded in the -/// same manner as an unsigned integer -/// -/// ## Float Encoding -/// -/// Floats are converted from IEEE 754 representation to a signed integer representation -/// by flipping all bar the sign bit if they are negative. -/// -/// They are then encoded in the same manner as a signed integer -/// -/// ## Variable Length Bytes Encoding -/// -/// A null is encoded as a `0_u8` -/// -/// An empty byte array is encoded as `1_u8` -/// -/// A non-null, non-empty byte array is encoded as `2_u8` followed by the byte array -/// encoded using a block based scheme described below. -/// -/// The byte array is broken up into 32-byte blocks, each block is written in turn -/// to the output, followed by `0xFF_u8`. The final block is padded to 32-bytes -/// with `0_u8` and written to the output, followed by the un-padded length in bytes -/// of this final block as a `u8` -/// -/// This is loosely inspired by [COBS] encoding, and chosen over more traditional -/// [byte stuffing] as it is more amenable to vectorisation, in particular AVX-256. -/// -/// ## Dictionary Encoding -/// -/// [`RowConverter`] needs to support converting dictionary encoded arrays with unsorted, and -/// potentially distinct dictionaries. One simple mechanism to avoid this would be to reverse -/// the dictionary encoding, and encode the array values directly, however, this would lose -/// the benefits of dictionary encoding to reduce memory and CPU consumption. -/// -/// As such the [`RowConverter`] maintains an order-preserving dictionary encoding for each -/// dictionary encoded column. As this is a variable-length encoding, new dictionary values -/// can be added whilst preserving the sort order. -/// -/// A null dictionary value is encoded as `0_u8`. -/// -/// A non-null dictionary value is encoded as `1_u8` followed by a null-terminated byte array -/// key determined by the order-preserving dictionary encoding -/// -/// # Ordering -/// -/// ## Float Ordering -/// -/// Floats are totally ordered in accordance to the `totalOrder` predicate as defined -/// in the IEEE 754 (2008 revision) floating point standard. -/// -/// The ordering established by this does not always agree with the -/// [`PartialOrd`] and [`PartialEq`] implementations of `f32`. For example, -/// they consider negative and positive zero equal, while this does not -/// -/// ## Null Ordering -/// -/// The encoding described above will order nulls first, this can be inverted by representing -/// nulls as `0xFF_u8` instead of `0_u8` -/// -/// ## Reverse Column Ordering -/// -/// The order of a given column can be reversed by negating the encoded bytes of non-null values -/// -/// [COBS]:[https://en.wikipedia.org/wiki/Consistent_Overhead_Byte_Stuffing] -/// [byte stuffing]:[https://en.wikipedia.org/wiki/High-Level_Data_Link_Control#Asynchronous_framing] -#[derive(Debug)] -pub struct RowConverter { - /// Sort fields - fields: Arc<[SortField]>, - /// interning state for column `i`, if column`i` is a dictionary - interners: Vec>>, -} - -/// Configure the data type and sort order for a given column -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SortField { - /// Sort options - options: SortOptions, - /// Data type - data_type: DataType, -} - -impl SortField { - /// Create a new column with the given data type - pub fn new(data_type: DataType) -> Self { - Self::new_with_options(data_type, SortOptions::default()) - } - - /// Create a new column with the given data type and [`SortOptions`] - pub fn new_with_options(data_type: DataType, options: SortOptions) -> Self { - Self { options, data_type } - } -} - -impl RowConverter { - /// Create a new [`RowConverter`] with the provided schema - pub fn new(fields: Vec) -> Self { - let interners = vec![None; fields.len()]; - Self { - fields: fields.into(), - interners, - } - } - - /// Convert a slice of [`Box`] columns into [`Rows`] - /// - /// See [`Row`] for information on when [`Row`] can be compared - /// - /// # Panics - /// - /// Panics if the schema of `columns` does not match that provided to [`RowConverter::new`] - pub fn convert_columns(&mut self, columns: &[Box]) -> Result { - if columns.len() != self.fields.len() { - return Err(Error::InvalidArgumentError(format!( - "Incorrect number of arrays provided to RowConverter, expected {} got {}", - self.fields.len(), - columns.len() - ))); - } - - let dictionaries = columns - .iter() - .zip(&mut self.interners) - .zip(self.fields.iter()) - .map(|((column, interner), field)| { - if column.data_type() != &field.data_type { - return Err(Error::InvalidArgumentError(format!( - "RowConverter column schema mismatch, expected {:?} got {:?}", - field.data_type, - column.data_type() - ))); - } - - let values = match column.data_type().to_logical_type() { - DataType::Dictionary(k, _, _) => match_integer_type!(k, |$T| { - let column = column - .as_any() - .downcast_ref::>() - .unwrap(); - column.values() - }), - _ => return Ok(None), - }; - - let interner = interner.get_or_insert_with(Default::default); - - let mapping = compute_dictionary_mapping(interner, values)? - .into_iter() - .map(|maybe_interned| { - maybe_interned.map(|interned| interner.normalized_key(interned)) - }) - .collect::>(); - - Ok(Some(mapping)) - }) - .collect::>>()?; - - let mut rows = new_empty_rows(columns, &dictionaries)?; - - // jorgecarleitao's comments in PR#1287: - // This seems to be embarassibly parallel. - // Given that this is a transpose of O(N x C) where N is length and C number of columns, I wonder if we could split this so users can parallelize. - // This is almost parallelizable - it is changing rows. - // However, there is still an optimization since modifying rows is O(1) but encoding is O(C). - // Will continue to think about this. - for ((column, field), dictionary) in - columns.iter().zip(self.fields.iter()).zip(dictionaries) - { - // We encode a column at a time to minimise dispatch overheads - encode_column(&mut rows, column, field.options, dictionary.as_deref()) - } - - Ok(rows) - } -} - -/// A row-oriented representation of arrow data, that is normalized for comparison -/// -/// See [`RowConverter`] -#[derive(Debug)] -pub struct Rows { - /// Underlying row bytes - buffer: Box<[u8]>, - /// Row `i` has data `&buffer[offsets[i]..offsets[i+1]]` - offsets: Box<[usize]>, -} - -impl Rows { - /// Get a reference to a certain row. - pub fn row(&self, row: usize) -> Row<'_> { - let end = self.offsets[row + 1]; - let start = self.offsets[row]; - Row { - data: unsafe { self.buffer.get_unchecked(start..end) }, - } - } - - /// Get a reference to a certain row but not check the bounds. - pub fn row_unchecked(&self, row: usize) -> Row<'_> { - let data = unsafe { - let end = *self.offsets.get_unchecked(row + 1); - let start = *self.offsets.get_unchecked(row); - self.buffer.get_unchecked(start..end) - }; - Row { data } - } - - /// Returns the number of rows - #[inline] - pub fn len(&self) -> usize { - self.offsets.len() - 1 - } - - /// Returns `true` if the number of rows is 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - #[inline] - /// Returns the iterator - pub fn iter(&self) -> RowsIter<'_> { - self.into_iter() - } -} - -impl<'a> IntoIterator for &'a Rows { - type Item = Row<'a>; - type IntoIter = RowsIter<'a>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - RowsIter { - rows: self, - start: 0, - end: self.len(), - } - } -} - -/// An iterator over [`Rows`] -#[derive(Debug)] -pub struct RowsIter<'a> { - rows: &'a Rows, - start: usize, - end: usize, -} - -impl<'a> Iterator for RowsIter<'a> { - type Item = Row<'a>; - - #[inline] - fn next(&mut self) -> Option { - if self.start < self.end { - let row = self.rows.row_unchecked(self.start); - self.start += 1; - Some(row) - } else { - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = self.len(); - (len, Some(len)) - } -} - -impl<'a> ExactSizeIterator for RowsIter<'a> { - #[inline] - fn len(&self) -> usize { - self.end - self.start - } -} - -impl<'a> DoubleEndedIterator for RowsIter<'a> { - fn next_back(&mut self) -> Option { - if self.end == self.start { - return None; - } - let row = self.rows.row(self.end); - self.end -= 1; - Some(row) - } -} - -unsafe impl<'a> crate::arrow::trusted_len::TrustedLen for RowsIter<'a> {} - -/// A comparable representation of a row -/// -/// Two [`Row`] can be compared if they both belong to [`Rows`] returned by calls to -/// [`RowConverter::convert_columns`] on the same [`RowConverter`] -/// -/// Otherwise any ordering established by comparing the [`Row`] is arbitrary -#[derive(Debug, Copy, Clone)] -pub struct Row<'a> { - data: &'a [u8], -} - -// Manually derive these as don't wish to include `fields` - -impl<'a> PartialEq for Row<'a> { - #[inline] - fn eq(&self, other: &Self) -> bool { - self.data.eq(other.data) - } -} - -impl<'a> Eq for Row<'a> {} - -impl<'a> PartialOrd for Row<'a> { - #[inline] - fn partial_cmp(&self, other: &Self) -> Option { - self.data.partial_cmp(other.data) - } -} - -impl<'a> Ord for Row<'a> { - #[inline] - fn cmp(&self, other: &Self) -> Ordering { - self.data.cmp(other.data) - } -} - -impl<'a> Hash for Row<'a> { - #[inline] - fn hash(&self, state: &mut H) { - self.data.hash(state) - } -} - -impl<'a> AsRef<[u8]> for Row<'a> { - #[inline] - fn as_ref(&self) -> &[u8] { - self.data - } -} - -/// Returns the null sentinel, negated if `invert` is true -#[inline] -fn null_sentinel(options: SortOptions) -> u8 { - match options.nulls_first { - true => 0, - false => 0xFF, - } -} - -/// Match `PrimitiveType` to standard Rust types -#[macro_export] -macro_rules! with_match_primitive_without_interval_type {( - $key_type:expr, | $_:tt $T:ident | $($body:tt)* -) => ({ - macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )} - use $crate::arrow::datatypes::PrimitiveType::*; - use $crate::arrow::types::{f16, i256}; - match $key_type { - Int8 => __with_ty__! { i8 }, - Int16 => __with_ty__! { i16 }, - Int32 => __with_ty__! { i32 }, - Int64 => __with_ty__! { i64 }, - Int128 => __with_ty__! { i128 }, - Int256 => __with_ty__! { i256 }, - UInt8 => __with_ty__! { u8 }, - UInt16 => __with_ty__! { u16 }, - UInt32 => __with_ty__! { u32 }, - UInt64 => __with_ty__! { u64 }, - Float16 => __with_ty__! { f16 }, - Float32 => __with_ty__! { f32 }, - Float64 => __with_ty__! { f64 }, - _ => unimplemented!("Unsupported type: {:?}", $key_type), - } -})} - -/// Computes the length of each encoded [`Rows`] and returns an empty [`Rows`] -fn new_empty_rows( - cols: &[Box], - dictionaries: &[Option>>], -) -> Result { - use fixed::FixedLengthEncoding; - - let num_rows = cols.first().map(|x| x.len()).unwrap_or(0); - let mut lengths = vec![0; num_rows]; - - for (array, dict) in cols.iter().zip(dictionaries) { - match array.data_type().to_physical_type() { - PhysicalType::Primitive(primitive) => { - with_match_primitive_without_interval_type!(primitive, |$T| { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - lengths.iter_mut().for_each(|x| *x += fixed::encoded_len(array)) - }) - } - PhysicalType::Null => {} - PhysicalType::Boolean => lengths.iter_mut().for_each(|x| *x += bool::ENCODED_LEN), - PhysicalType::Binary => array - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .zip(lengths.iter_mut()) - .for_each(|(slice, length)| *length += variable::encoded_len(slice)), - PhysicalType::LargeBinary => array - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .zip(lengths.iter_mut()) - .for_each(|(slice, length)| *length += variable::encoded_len(slice)), - PhysicalType::Utf8 => array - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .zip(lengths.iter_mut()) - .for_each(|(slice, length)| { - *length += variable::encoded_len(slice.map(|x| x.as_bytes())) - }), - PhysicalType::LargeUtf8 => array - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .zip(lengths.iter_mut()) - .for_each(|(slice, length)| { - *length += variable::encoded_len(slice.map(|x| x.as_bytes())) - }), - PhysicalType::Dictionary(k) => match_integer_type!(k, |$T| { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let dict = dict.as_ref().unwrap(); - for (v, length) in array.keys().iter().zip(lengths.iter_mut()) { - match v.and_then(|v| dict[*v as usize]) { - Some(k) => *length += k.len() + 1, - None => *length += 1, - } - } - }), - t => { - return Err(Error::NotYetImplemented(format!( - "not yet implemented: {t:?}" - ))); - } - } - } - - let mut offsets = Vec::with_capacity(num_rows + 1); - offsets.push(0); - - // We initialize the offsets shifted down by one row index. - // - // As the rows are appended to the offsets will be incremented to match - // - // For example, consider the case of 3 rows of length 3, 4, and 6 respectively. - // The offsets would be initialized to `0, 0, 3, 7` - // - // Writing the first row entirely would yield `0, 3, 3, 7` - // The second, `0, 3, 7, 7` - // The third, `0, 3, 7, 13` - // - // This would be the final offsets for reading - // - // In this way offsets tracks the position during writing whilst eventually serving - // as identifying the offsets of the written rows - let mut cur_offset = 0_usize; - for l in lengths { - offsets.push(cur_offset); - cur_offset = cur_offset.checked_add(l).expect("overflow"); - } - - let buffer = vec![0_u8; cur_offset]; - - Ok(Rows { - buffer: buffer.into(), - offsets: offsets.into(), - }) -} - -/// Encodes a column to the provided [`Rows`] incrementing the offsets as it progresses -#[allow(clippy::borrowed_box)] -fn encode_column( - out: &mut Rows, - column: &Box, - opts: SortOptions, - dictionary: Option<&[Option<&[u8]>]>, -) { - match column.data_type().to_physical_type() { - PhysicalType::Primitive(primitive) => { - with_match_primitive_without_interval_type!(primitive, |$T| { - let column = column - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .map(|v| v.map(|v| *v)); - fixed::encode(out, column, opts); - }) - } - PhysicalType::Null => {} - PhysicalType::Boolean => fixed::encode( - out, - column.as_any().downcast_ref::().unwrap(), - opts, - ), - PhysicalType::Binary => { - variable::encode( - out, - column - .as_any() - .downcast_ref::>() - .unwrap() - .iter(), - opts, - ); - } - PhysicalType::LargeBinary => { - variable::encode( - out, - column - .as_any() - .downcast_ref::>() - .unwrap() - .iter(), - opts, - ); - } - PhysicalType::Utf8 => variable::encode( - out, - column - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .map(|x| x.map(|x| x.as_bytes())), - opts, - ), - PhysicalType::LargeUtf8 => variable::encode( - out, - column - .as_any() - .downcast_ref::>() - .unwrap() - .iter() - .map(|x| x.map(|x| x.as_bytes())), - opts, - ), - PhysicalType::Dictionary(k) => match_integer_type!(k, |$T| { - let column = column - .as_any() - .downcast_ref::>() - .unwrap(); - encode_dictionary(out, column, dictionary.unwrap(), opts); - }), - t => unimplemented!("not yet implemented: {:?}", t), - } -} - -#[cfg(test)] -mod tests { - use std::fmt::Debug; - - use rand::distributions::uniform::SampleUniform; - use rand::distributions::Distribution; - use rand::distributions::Standard; - use rand::thread_rng; - use rand::Rng; - - use super::*; - use crate::arrow::array::Array; - use crate::arrow::array::DictionaryKey; - use crate::arrow::array::Float32Array; - use crate::arrow::array::Int16Array; - use crate::arrow::array::NullArray; - use crate::arrow::compute::sort::build_compare; - use crate::arrow::datatypes::DataType; - use crate::arrow::offset::Offset; - use crate::arrow::types::NativeType; - - #[test] - fn test_fixed_width() { - let cols = [ - Int16Array::from([Some(1), Some(2), None, Some(-5), Some(2), Some(2), Some(0)]) - .to_boxed(), - Float32Array::from([ - Some(1.3), - Some(2.5), - None, - Some(4.), - Some(0.1), - Some(-4.), - Some(-0.), - ]) - .to_boxed(), - ]; - - let mut converter = RowConverter::new(vec![ - SortField::new(DataType::Int16), - SortField::new(DataType::Float32), - ]); - let rows = converter.convert_columns(&cols).unwrap(); - - assert_eq!(rows.offsets.as_ref(), &[0, 8, 16, 24, 32, 40, 48, 56]); - assert_eq!(rows.buffer.as_ref(), &[ - 1, 128, 1, // - 1, 191, 166, 102, 102, // - 1, 128, 2, // - 1, 192, 32, 0, 0, // - 0, 0, 0, // - 0, 0, 0, 0, 0, // - 1, 127, 251, // - 1, 192, 128, 0, 0, // - 1, 128, 2, // - 1, 189, 204, 204, 205, // - 1, 128, 2, // - 1, 63, 127, 255, 255, // - 1, 128, 0, // - 1, 127, 255, 255, 255 // - ]); - - assert!(rows.row(3) < rows.row(6)); - assert!(rows.row(0) < rows.row(1)); - assert!(rows.row(3) < rows.row(0)); - assert!(rows.row(4) < rows.row(1)); - assert!(rows.row(5) < rows.row(4)); - } - - #[test] - fn test_null_encoding() { - let col = NullArray::new(DataType::Null, 10).to_boxed(); - let mut converter = RowConverter::new(vec![SortField::new(DataType::Null)]); - let rows = converter.convert_columns(&[col]).unwrap(); - assert_eq!(rows.len(), 10); - assert_eq!(rows.row(1).data.len(), 0); - } - - fn generate_primitive_array(len: usize, valid_percent: f64) -> PrimitiveArray - where - K: NativeType, - Standard: Distribution, - { - let mut rng = thread_rng(); - (0..len) - .map(|_| rng.gen_bool(valid_percent).then(|| rng.gen())) - .collect() - } - - fn generate_strings(len: usize, valid_percent: f64) -> Utf8Array { - let mut rng = thread_rng(); - (0..len) - .map(|_| { - rng.gen_bool(valid_percent).then(|| { - let len = rng.gen_range(0..100); - let bytes = (0..len).map(|_| rng.gen_range(0..128)).collect(); - String::from_utf8(bytes).unwrap() - }) - }) - .collect() - } - - fn generate_dictionary( - values: Box, - len: usize, - valid_percent: f64, - ) -> DictionaryArray - where - K: DictionaryKey + Ord + SampleUniform, - >::Error: Debug, - { - let mut rng = thread_rng(); - let min_key = 0_usize.try_into().unwrap(); - let max_key = values.len().try_into().unwrap(); - let keys: PrimitiveArray = (0..len) - .map(|_| { - rng.gen_bool(valid_percent) - .then(|| rng.gen_range(min_key..max_key)) - }) - .collect(); - - DictionaryArray::try_from_keys(keys, values).unwrap() - } - - fn generate_column(len: usize) -> Box { - let mut rng = thread_rng(); - match rng.gen_range(0..9) { - 0 => Box::new(generate_primitive_array::(len, 0.8)), - 1 => Box::new(generate_primitive_array::(len, 0.8)), - 2 => Box::new(generate_primitive_array::(len, 0.8)), - 3 => Box::new(generate_primitive_array::(len, 0.8)), - 4 => Box::new(generate_primitive_array::(len, 0.8)), - 5 => Box::new(generate_primitive_array::(len, 0.8)), - 6 => Box::new(generate_strings::(len, 0.8)), - 7 => Box::new(generate_dictionary::( - // Cannot test dictionaries containing null values because of #2687 - Box::new(generate_strings::(rng.gen_range(1..len), 1.0)), - len, - 0.8, - )), - 8 => Box::new(generate_dictionary::( - // Cannot test dictionaries containing null values because of #2687 - Box::new(generate_primitive_array::(rng.gen_range(1..len), 1.0)), - len, - 0.8, - )), - _ => unreachable!(), - } - } - - #[test] - #[cfg_attr(miri, ignore)] - fn fuzz_test() { - for _ in 0..100 { - let mut rng = thread_rng(); - let num_columns = rng.gen_range(1..5); - let len = rng.gen_range(5..100); - let arrays: Vec<_> = (0..num_columns).map(|_| generate_column(len)).collect(); - - let options: Vec<_> = (0..num_columns) - .map(|_| SortOptions { - descending: rng.gen_bool(0.5), - nulls_first: rng.gen_bool(0.5), - }) - .collect(); - - let comparators = arrays - .iter() - .zip(options.iter()) - .map(|(a, o)| build_compare(&**a, *o).unwrap()) - .collect::>(); - - let columns = options - .into_iter() - .zip(&arrays) - .map(|(o, a)| SortField::new_with_options(a.data_type().clone(), o)) - .collect(); - - let mut converter = RowConverter::new(columns); - let rows = converter.convert_columns(&arrays).unwrap(); - let cmp = |i, j| { - for cmp in comparators.iter() { - let cmp = cmp(i, j); - if cmp != Ordering::Equal { - return cmp; - } - } - Ordering::Equal - }; - - for i in 0..len { - for j in 0..len { - let row_i = rows.row(i); - let row_j = rows.row(j); - let row_cmp = row_i.cmp(&row_j); - let lex_cmp = cmp(i, j); - assert_eq!(row_cmp, lex_cmp); - } - } - } - } -} diff --git a/src/common/arrow/src/arrow/compute/sort/row/variable.rs b/src/common/arrow/src/arrow/compute/sort/row/variable.rs deleted file mode 100644 index cf0a06f09f02..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/row/variable.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::null_sentinel; -use super::Rows; -use crate::arrow::compute::sort::SortOptions; - -/// The block size of the variable length encoding -pub const BLOCK_SIZE: usize = 32; - -/// The continuation token -pub const BLOCK_CONTINUATION: u8 = 0xFF; - -/// Indicates an empty string -pub const EMPTY_SENTINEL: u8 = 1; - -/// Indicates a non-empty string -pub const NON_EMPTY_SENTINEL: u8 = 2; - -/// Returns the ceil of `value`/`divisor` -#[inline] -fn div_ceil(value: usize, divisor: usize) -> usize { - // Rewrite as `value.div_ceil(&divisor)` after - // https://github.com/rust-lang/rust/issues/88581 is merged. - value / divisor + (0 != value % divisor) as usize -} - -/// Returns the length of the encoded representation of a byte array, including the null byte -pub fn encoded_len(a: Option<&[u8]>) -> usize { - match a { - Some(a) => 1 + div_ceil(a.len(), BLOCK_SIZE) * (BLOCK_SIZE + 1), - None => 1, - } -} - -/// Variable length values are encoded as -/// -/// - single `0_u8` if null -/// - single `1_u8` if empty array -/// - `2_u8` if not empty, followed by one or more blocks -/// -/// where a block is encoded as -/// -/// - [`BLOCK_SIZE`] bytes of string data, padded with 0s -/// - `0xFF_u8` if this is not the last block for this string -/// - otherwise the length of the block as a `u8` -pub fn encode<'a, I: Iterator>>(out: &mut Rows, i: I, opts: SortOptions) { - for (offset, maybe_val) in out.offsets.iter_mut().skip(1).zip(i) { - match maybe_val { - Some([]) => { - out.buffer[*offset] = match opts.descending { - true => !EMPTY_SENTINEL, - false => EMPTY_SENTINEL, - }; - *offset += 1; - } - Some(val) => { - let block_count = div_ceil(val.len(), BLOCK_SIZE); - let end_offset = *offset + 1 + block_count * (BLOCK_SIZE + 1); - let to_write = &mut out.buffer[*offset..end_offset]; - - // Write `2_u8` to demarcate as non-empty, non-null string - to_write[0] = NON_EMPTY_SENTINEL; - - let mut chunks = val.chunks_exact(BLOCK_SIZE); - for (input, output) in chunks - .by_ref() - .zip(to_write[1..].chunks_exact_mut(BLOCK_SIZE + 1)) - { - let input: &[u8; BLOCK_SIZE] = input.try_into().unwrap(); - let out_block: &mut [u8; BLOCK_SIZE] = - (&mut output[..BLOCK_SIZE]).try_into().unwrap(); - - *out_block = *input; - - // Indicate that there are further blocks to follow - output[BLOCK_SIZE] = BLOCK_CONTINUATION; - } - - let remainder = chunks.remainder(); - if !remainder.is_empty() { - let start_offset = 1 + (block_count - 1) * (BLOCK_SIZE + 1); - to_write[start_offset..start_offset + remainder.len()] - .copy_from_slice(remainder); - *to_write.last_mut().unwrap() = remainder.len() as u8; - } else { - // We must overwrite the continuation marker written by the loop above - *to_write.last_mut().unwrap() = BLOCK_SIZE as u8; - } - - *offset = end_offset; - - if opts.descending { - // Invert bits - to_write.iter_mut().for_each(|v| *v = !*v) - } - } - None => { - out.buffer[*offset] = null_sentinel(opts); - *offset += 1; - } - } - } -} diff --git a/src/common/arrow/src/arrow/compute/sort/utf8.rs b/src/common/arrow/src/arrow/compute/sort/utf8.rs deleted file mode 100644 index 30767204069f..000000000000 --- a/src/common/arrow/src/arrow/compute/sort/utf8.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::common; -use super::SortOptions; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::offset::Offset; -use crate::arrow::types::Index; - -pub(super) fn indices_sorted_unstable_by( - array: &Utf8Array, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray { - let get = |idx| unsafe { array.value_unchecked(idx) }; - let cmp = |lhs: &&str, rhs: &&str| lhs.cmp(rhs); - common::indices_sorted_unstable_by(array.validity(), get, cmp, array.len(), options, limit) -} - -pub(super) fn indices_sorted_unstable_by_dictionary( - array: &DictionaryArray, - options: &SortOptions, - limit: Option, -) -> PrimitiveArray { - let keys = array.keys(); - - let dict = array - .values() - .as_any() - .downcast_ref::>() - .unwrap(); - - let get = |index| unsafe { - // safety: indices_sorted_unstable_by is guaranteed to get items in bounds - let index = keys.value_unchecked(index); - // safety: dictionaries are guaranteed to have valid usize keys - let index = index.as_usize(); - // safety: dictionaries are guaranteed to have keys in bounds - dict.value_unchecked(index) - }; - - let cmp = |lhs: &&str, rhs: &&str| lhs.cmp(rhs); - common::indices_sorted_unstable_by(array.validity(), get, cmp, array.len(), options, limit) -} diff --git a/src/common/arrow/src/arrow/compute/take/binary.rs b/src/common/arrow/src/arrow/compute/take/binary.rs deleted file mode 100644 index 6ebd57024dfd..000000000000 --- a/src/common/arrow/src/arrow/compute/take/binary.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::generic_binary::*; -use super::Index; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::offset::Offset; - -/// `take` implementation for utf8 arrays -pub fn take( - values: &BinaryArray, - indices: &PrimitiveArray, -) -> BinaryArray { - let data_type = values.data_type().clone(); - let indices_has_validity = indices.null_count() > 0; - let values_has_validity = values.null_count() > 0; - - let (offsets, values, validity) = match (values_has_validity, indices_has_validity) { - (false, false) => { - take_no_validity::(values.offsets(), values.values(), indices.values()) - } - (true, false) => take_values_validity(values, indices.values()), - (false, true) => take_indices_validity(values.offsets(), values.values(), indices), - (true, true) => take_values_indices_validity(values, indices), - }; - BinaryArray::::new(data_type, offsets, values, validity) -} diff --git a/src/common/arrow/src/arrow/compute/take/binview.rs b/src/common/arrow/src/arrow/compute/take/binview.rs deleted file mode 100644 index f3b4177ab86a..000000000000 --- a/src/common/arrow/src/arrow/compute/take/binview.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::BinaryViewArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::compute::take::primitive::take_values_and_validity_unchecked; -use crate::arrow::types::Index; - -/// # Safety -/// No bound checks -pub(super) unsafe fn take_binview_unchecked( - arr: &BinaryViewArray, - indices: &PrimitiveArray, -) -> BinaryViewArray { - let (views, validity) = - take_values_and_validity_unchecked(arr.views(), arr.validity(), indices); - - BinaryViewArray::new_unchecked_unknown_md( - arr.data_type().clone(), - views.into(), - arr.data_buffers().clone(), - validity, - Some(arr.total_buffer_len()), - ) - .maybe_gc() -} diff --git a/src/common/arrow/src/arrow/compute/take/boolean.rs b/src/common/arrow/src/arrow/compute/take/boolean.rs deleted file mode 100644 index 8e088918f7d5..000000000000 --- a/src/common/arrow/src/arrow/compute/take/boolean.rs +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Index; -use crate::arrow::array::Array; -use crate::arrow::array::BooleanArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; - -// take implementation when neither values nor indices contain nulls -fn take_no_validity(values: &Bitmap, indices: &[I]) -> (Bitmap, Option) { - let values = indices.iter().map(|index| values.get_bit(index.to_usize())); - let buffer = Bitmap::from_trusted_len_iter(values); - - (buffer, None) -} - -// take implementation when only values contain nulls -fn take_values_validity( - values: &BooleanArray, - indices: &[I], -) -> (Bitmap, Option) { - let validity_values = values.validity().unwrap(); - let validity = indices - .iter() - .map(|index| validity_values.get_bit(index.to_usize())); - let validity = Bitmap::from_trusted_len_iter(validity); - - let values_values = values.values(); - let values = indices - .iter() - .map(|index| values_values.get_bit(index.to_usize())); - let buffer = Bitmap::from_trusted_len_iter(values); - - (buffer, validity.into()) -} - -// take implementation when only indices contain nulls -fn take_indices_validity( - values: &Bitmap, - indices: &PrimitiveArray, -) -> (Bitmap, Option) { - let validity = indices.validity().unwrap(); - - let values = indices.values().iter().enumerate().map(|(i, index)| { - let index = index.to_usize(); - match values.get(index) { - Some(value) => value, - None => { - if !validity.get_bit(i) { - false - } else { - panic!("Out-of-bounds index {index}") - } - } - } - }); - - let buffer = Bitmap::from_trusted_len_iter(values); - - (buffer, indices.validity().cloned()) -} - -// take implementation when both values and indices contain nulls -fn take_values_indices_validity( - values: &BooleanArray, - indices: &PrimitiveArray, -) -> (Bitmap, Option) { - let mut validity = MutableBitmap::with_capacity(indices.len()); - - let values_validity = values.validity().unwrap(); - - let values_values = values.values(); - let values = indices.iter().map(|index| match index { - Some(index) => { - let index = index.to_usize(); - validity.push(values_validity.get_bit(index)); - values_values.get_bit(index) - } - None => { - validity.push(false); - false - } - }); - let values = Bitmap::from_trusted_len_iter(values); - (values, validity.into()) -} - -/// `take` implementation for boolean arrays -pub fn take(values: &BooleanArray, indices: &PrimitiveArray) -> BooleanArray { - let data_type = values.data_type().clone(); - let indices_has_validity = indices.null_count() > 0; - let values_has_validity = values.null_count() > 0; - - let (values, validity) = match (values_has_validity, indices_has_validity) { - (false, false) => take_no_validity(values.values(), indices.values()), - (true, false) => take_values_validity(values, indices.values()), - (false, true) => take_indices_validity(values.values(), indices), - (true, true) => take_values_indices_validity(values, indices), - }; - - BooleanArray::new(data_type, values, validity) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrow::array::Int32Array; - - fn _all_cases() -> Vec<(Int32Array, BooleanArray, BooleanArray)> { - vec![ - ( - Int32Array::from(&[Some(1), Some(0)]), - BooleanArray::from(vec![Some(true), Some(false)]), - BooleanArray::from(vec![Some(false), Some(true)]), - ), - ( - Int32Array::from(&[Some(1), None]), - BooleanArray::from(vec![Some(true), Some(false)]), - BooleanArray::from(vec![Some(false), None]), - ), - ( - Int32Array::from(&[Some(1), Some(0)]), - BooleanArray::from(vec![None, Some(false)]), - BooleanArray::from(vec![Some(false), None]), - ), - ( - Int32Array::from(&[Some(1), None, Some(0)]), - BooleanArray::from(vec![None, Some(false)]), - BooleanArray::from(vec![Some(false), None, None]), - ), - ] - } - - #[test] - fn all_cases() { - let cases = _all_cases(); - for (indices, input, expected) in cases { - let output = take(&input, &indices); - assert_eq!(expected, output); - } - } -} diff --git a/src/common/arrow/src/arrow/compute/take/dict.rs b/src/common/arrow/src/arrow/compute/take/dict.rs deleted file mode 100644 index e09262591f62..000000000000 --- a/src/common/arrow/src/arrow/compute/take/dict.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::primitive::take as take_primitive; -use super::Index; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::PrimitiveArray; - -/// `take` implementation for dictionary arrays -/// -/// applies `take` to the keys of the dictionary array and returns a new dictionary array -/// with the same dictionary values and reordered keys -pub fn take(values: &DictionaryArray, indices: &PrimitiveArray) -> DictionaryArray -where - K: DictionaryKey, - I: Index, -{ - let keys = take_primitive::(values.keys(), indices); - // safety - this operation takes a subset of keys and thus preserves the dictionary's invariant - unsafe { - DictionaryArray::::try_new_unchecked( - values.data_type().clone(), - keys, - values.values().clone(), - ) - .unwrap() - } -} diff --git a/src/common/arrow/src/arrow/compute/take/fixed_size_list.rs b/src/common/arrow/src/arrow/compute/take/fixed_size_list.rs deleted file mode 100644 index 39b640eda367..000000000000 --- a/src/common/arrow/src/arrow/compute/take/fixed_size_list.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Index; -use crate::arrow::array::growable::Growable; -use crate::arrow::array::growable::GrowableFixedSizeList; -use crate::arrow::array::FixedSizeListArray; -use crate::arrow::array::PrimitiveArray; - -/// `take` implementation for FixedSizeListArrays -pub fn take( - values: &FixedSizeListArray, - indices: &PrimitiveArray, -) -> FixedSizeListArray { - let mut capacity = 0; - let arrays = indices - .values() - .iter() - .map(|index| { - let index = index.to_usize(); - let slice = values.clone().sliced(index, 1); - capacity += slice.len(); - slice - }) - .collect::>(); - - let arrays = arrays.iter().collect(); - - if let Some(validity) = indices.validity() { - let mut growable: GrowableFixedSizeList = - GrowableFixedSizeList::new(arrays, true, capacity); - - for index in 0..indices.len() { - if validity.get_bit(index) { - growable.extend(index, 0, 1); - } else { - growable.extend_validity(1) - } - } - - growable.into() - } else { - let mut growable: GrowableFixedSizeList = - GrowableFixedSizeList::new(arrays, false, capacity); - for index in 0..indices.len() { - growable.extend(index, 0, 1); - } - - growable.into() - } -} diff --git a/src/common/arrow/src/arrow/compute/take/generic_binary.rs b/src/common/arrow/src/arrow/compute/take/generic_binary.rs deleted file mode 100644 index 3453f31ac8df..000000000000 --- a/src/common/arrow/src/arrow/compute/take/generic_binary.rs +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Index; -use crate::arrow::array::GenericBinaryArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::offset::OffsetsBuffer; - -pub fn take_values( - length: O, - starts: &[O], - offsets: &OffsetsBuffer, - values: &[u8], -) -> Buffer { - let new_len = length.to_usize(); - let mut buffer = Vec::with_capacity(new_len); - starts - .iter() - .map(|start| start.to_usize()) - .zip(offsets.lengths()) - .for_each(|(start, length)| { - let end = start + length; - buffer.extend_from_slice(&values[start..end]); - }); - buffer.into() -} - -// take implementation when neither values nor indices contain nulls -pub fn take_no_validity( - offsets: &OffsetsBuffer, - values: &[u8], - indices: &[I], -) -> (OffsetsBuffer, Buffer, Option) { - let mut buffer = Vec::::new(); - let lengths = indices.iter().map(|index| index.to_usize()).map(|index| { - let (start, end) = offsets.start_end(index); - // todo: remove this bound check - buffer.extend_from_slice(&values[start..end]); - end - start - }); - let offsets = Offsets::try_from_lengths(lengths).expect(""); - - (offsets.into(), buffer.into(), None) -} - -// take implementation when only values contain nulls -pub fn take_values_validity>( - values: &A, - indices: &[I], -) -> (OffsetsBuffer, Buffer, Option) { - let validity_values = values.validity().unwrap(); - let validity = indices - .iter() - .map(|index| validity_values.get_bit(index.to_usize())); - let validity = Bitmap::from_trusted_len_iter(validity); - - let mut length = O::default(); - - let offsets = values.offsets(); - let values_values = values.values(); - - let mut starts = Vec::::with_capacity(indices.len()); - let offsets = indices.iter().map(|index| { - let index = index.to_usize(); - let start = offsets[index]; - length += offsets[index + 1] - start; - starts.push(start); - length - }); - let offsets = std::iter::once(O::default()) - .chain(offsets) - .collect::>(); - // Safety: by construction offsets are monotonically increasing - let offsets = unsafe { Offsets::new_unchecked(offsets) }.into(); - - let buffer = take_values(length, starts.as_slice(), &offsets, values_values); - - (offsets, buffer, validity.into()) -} - -// take implementation when only indices contain nulls -pub fn take_indices_validity( - offsets: &OffsetsBuffer, - values: &[u8], - indices: &PrimitiveArray, -) -> (OffsetsBuffer, Buffer, Option) { - let mut length = O::default(); - - let offsets = offsets.buffer(); - - let mut starts = Vec::::with_capacity(indices.len()); - let offsets = indices.values().iter().map(|index| { - let index = index.to_usize(); - match offsets.get(index + 1) { - Some(&next) => { - let start = offsets[index]; - length += next - start; - starts.push(start); - } - None => starts.push(O::default()), - }; - length - }); - let offsets = std::iter::once(O::default()) - .chain(offsets) - .collect::>(); - // Safety: by construction offsets are monotonically increasing - let offsets = unsafe { Offsets::new_unchecked(offsets) }.into(); - - let buffer = take_values(length, &starts, &offsets, values); - - (offsets, buffer, indices.validity().cloned()) -} - -// take implementation when both indices and values contain nulls -pub fn take_values_indices_validity>( - values: &A, - indices: &PrimitiveArray, -) -> (OffsetsBuffer, Buffer, Option) { - let mut length = O::default(); - let mut validity = MutableBitmap::with_capacity(indices.len()); - - let values_validity = values.validity().unwrap(); - let offsets = values.offsets(); - let values_values = values.values(); - - let mut starts = Vec::::with_capacity(indices.len()); - let offsets = indices.iter().map(|index| { - match index { - Some(index) => { - let index = index.to_usize(); - if values_validity.get_bit(index) { - validity.push(true); - length += offsets[index + 1] - offsets[index]; - starts.push(offsets[index]); - } else { - validity.push(false); - starts.push(O::default()); - } - } - None => { - validity.push(false); - starts.push(O::default()); - } - }; - length - }); - let offsets = std::iter::once(O::default()) - .chain(offsets) - .collect::>(); - // Safety: by construction offsets are monotonically increasing - let offsets = unsafe { Offsets::new_unchecked(offsets) }.into(); - - let buffer = take_values(length, &starts, &offsets, values_values); - - (offsets, buffer, validity.into()) -} diff --git a/src/common/arrow/src/arrow/compute/take/list.rs b/src/common/arrow/src/arrow/compute/take/list.rs deleted file mode 100644 index c3a3a1130c33..000000000000 --- a/src/common/arrow/src/arrow/compute/take/list.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Index; -use crate::arrow::array::growable::Growable; -use crate::arrow::array::growable::GrowableList; -use crate::arrow::array::ListArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::offset::Offset; - -/// `take` implementation for ListArrays -pub fn take( - values: &ListArray, - indices: &PrimitiveArray, -) -> ListArray { - let mut capacity = 0; - let arrays = indices - .values() - .iter() - .map(|index| { - let index = index.to_usize(); - let slice = values.clone().sliced(index, 1); - capacity += slice.len(); - slice - }) - .collect::>>(); - - let arrays = arrays.iter().collect(); - - if let Some(validity) = indices.validity() { - let mut growable: GrowableList = GrowableList::new(arrays, true, capacity); - - for index in 0..indices.len() { - if validity.get_bit(index) { - growable.extend(index, 0, 1); - } else { - growable.extend_validity(1) - } - } - - growable.into() - } else { - let mut growable: GrowableList = GrowableList::new(arrays, false, capacity); - for index in 0..indices.len() { - growable.extend(index, 0, 1); - } - - growable.into() - } -} diff --git a/src/common/arrow/src/arrow/compute/take/mod.rs b/src/common/arrow/src/arrow/compute/take/mod.rs deleted file mode 100644 index 7c6df0c8c553..000000000000 --- a/src/common/arrow/src/arrow/compute/take/mod.rs +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Defines take kernel for [`Array`] - -use crate::arrow::array::new_empty_array; -use crate::arrow::array::Array; -use crate::arrow::array::NullArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::Utf8ViewArray; -use crate::arrow::compute::take::binview::take_binview_unchecked; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::types::Index; - -mod binary; -mod binview; -mod boolean; -mod dict; -mod fixed_size_list; -mod generic_binary; -mod list; -mod primitive; -mod structure; -mod utf8; - -/// Returns a new [`Array`] with only indices at `indices`. Null indices are taken as nulls. -/// The returned array has a length equal to `indices.len()`. -pub fn take(values: &dyn Array, indices: &PrimitiveArray) -> Result> { - if indices.is_empty() { - return Ok(new_empty_array(values.data_type().clone())); - } - - use crate::arrow::datatypes::PhysicalType::*; - match values.data_type().to_physical_type() { - Null => Ok(Box::new(NullArray::new( - values.data_type().clone(), - indices.len(), - ))), - Boolean => { - let values = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(boolean::take::(values, indices))) - } - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - let values = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(primitive::take::<$T, _>(&values, indices))) - }), - Utf8 => { - let values = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(utf8::take::(values, indices))) - } - LargeUtf8 => { - let values = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(utf8::take::(values, indices))) - } - Binary => { - let values = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(binary::take::(values, indices))) - } - LargeBinary => { - let values = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(binary::take::(values, indices))) - } - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - let values = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(dict::take::<$T, _>(&values, indices))) - }) - } - Struct => { - let array = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(structure::take::<_>(array, indices)?)) - } - List => { - let array = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(list::take::(array, indices))) - } - LargeList => { - let array = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(list::take::(array, indices))) - } - FixedSizeList => { - let array = values.as_any().downcast_ref().unwrap(); - Ok(Box::new(fixed_size_list::take::(array, indices))) - } - BinaryView => unsafe { - Ok(take_binview_unchecked(values.as_any().downcast_ref().unwrap(), indices).boxed()) - }, - Utf8View => unsafe { - let arr: &Utf8ViewArray = values.as_any().downcast_ref().unwrap(); - Ok(take_binview_unchecked(&arr.to_binview(), indices) - .to_utf8view_unchecked() - .boxed()) - }, - t => unimplemented!("Take not supported for data type {:?}", t), - } -} - -/// Checks if an array of type `datatype` can perform take operation -/// -/// # Examples -/// ``` -/// use arrow2::compute::take::can_take; -/// use arrow2::datatypes::DataType; -/// -/// let data_type = DataType::Int8; -/// assert_eq!(can_take(&data_type), true); -/// ``` -pub fn can_take(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Null - | DataType::Boolean - | DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::Date32 - | DataType::Time32(_) - | DataType::Interval(_) - | DataType::Int64 - | DataType::Date64 - | DataType::Time64(_) - | DataType::Duration(_) - | DataType::Timestamp(_, _) - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::Float16 - | DataType::Float32 - | DataType::Float64 - | DataType::Decimal(_, _) - | DataType::Utf8 - | DataType::LargeUtf8 - | DataType::Binary - | DataType::LargeBinary - | DataType::Struct(_) - | DataType::List(_) - | DataType::LargeList(_) - | DataType::FixedSizeList(_, _) - | DataType::Dictionary(..) - ) -} diff --git a/src/common/arrow/src/arrow/compute/take/primitive.rs b/src/common/arrow/src/arrow/compute/take/primitive.rs deleted file mode 100644 index 2b37ca8a4194..000000000000 --- a/src/common/arrow/src/arrow/compute/take/primitive.rs +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_buffer::bit_util::unset_bit_raw; - -use super::Index; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::types::NativeType; - -// take implementation when neither values nor indices contain nulls -fn take_no_validity( - values: &[T], - indices: &[I], -) -> (Buffer, Option) { - let values = indices - .iter() - .map(|index| values[index.to_usize()]) - .collect::>(); - - (values.into(), None) -} - -// take implementation when only values contain nulls -fn take_values_validity( - values: &PrimitiveArray, - indices: &[I], -) -> (Buffer, Option) { - let values_validity = values.validity().unwrap(); - - let validity = indices - .iter() - .map(|index| values_validity.get_bit(index.to_usize())); - let validity = MutableBitmap::from_trusted_len_iter(validity); - - let values_values = values.values(); - - let values = indices - .iter() - .map(|index| values_values[index.to_usize()]) - .collect::>(); - - (values.into(), validity.into()) -} - -// take implementation when only indices contain nulls -fn take_indices_validity( - values: &[T], - indices: &PrimitiveArray, -) -> (Buffer, Option) { - let validity = indices.validity().unwrap(); - let values = indices - .values() - .iter() - .enumerate() - .map(|(i, index)| { - let index = index.to_usize(); - match values.get(index) { - Some(value) => *value, - None => { - if !validity.get_bit(i) { - T::default() - } else { - panic!("Out-of-bounds index {index}") - } - } - } - }) - .collect::>(); - - (values.into(), indices.validity().cloned()) -} - -// take implementation when both values and indices contain nulls -fn take_values_indices_validity( - values: &PrimitiveArray, - indices: &PrimitiveArray, -) -> (Buffer, Option) { - let mut bitmap = MutableBitmap::with_capacity(indices.len()); - - let values_validity = values.validity().unwrap(); - - let values_values = values.values(); - let values = indices - .iter() - .map(|index| match index { - Some(index) => { - let index = index.to_usize(); - bitmap.push(values_validity.get_bit(index)); - values_values[index] - } - None => { - bitmap.push(false); - T::default() - } - }) - .collect::>(); - (values.into(), bitmap.into()) -} - -pub(super) unsafe fn take_values_and_validity_unchecked( - values: &[T], - validity_values: Option<&Bitmap>, - indices: &PrimitiveArray, -) -> (Vec, Option) { - let index_values = indices.values().as_slice(); - - let null_count = validity_values.map(|b| b.unset_bits()).unwrap_or(0); - - // first take the values, these are always needed - let values: Vec = if indices.null_count() == 0 { - index_values - .iter() - .map(|idx| *values.get_unchecked(idx.to_usize())) - .collect() - } else { - indices - .iter() - .map(|idx| match idx { - Some(idx) => *values.get_unchecked(idx.to_usize()), - None => T::default(), - }) - .collect() - }; - - if null_count > 0 { - let validity_values = validity_values.unwrap(); - // the validity buffer we will fill with all valid. And we unset the ones that are null - // in later checks - // this is in the assumption that most values will be valid. - // Maybe we could add another branch based on the null count - let mut validity = MutableBitmap::with_capacity(indices.len()); - validity.extend_constant(indices.len(), true); - let validity_ptr = validity.as_slice().as_ptr() as *mut u8; - - if let Some(validity_indices) = indices.validity().as_ref() { - index_values.iter().enumerate().for_each(|(i, idx)| { - // i is iteration count - // idx is the index that we take from the values array. - let idx = idx.to_usize(); - if !validity_indices.get_bit_unchecked(i) || !validity_values.get_bit_unchecked(idx) - { - unset_bit_raw(validity_ptr, i); - } - }); - } else { - index_values.iter().enumerate().for_each(|(i, idx)| { - let idx = idx.to_usize(); - if !validity_values.get_bit_unchecked(idx) { - unset_bit_raw(validity_ptr, i); - } - }); - }; - (values, Some(validity.freeze())) - } else { - (values, indices.validity().cloned()) - } -} - -/// `take` implementation for primitive arrays -pub fn take( - values: &PrimitiveArray, - indices: &PrimitiveArray, -) -> PrimitiveArray { - let indices_has_validity = indices.null_count() > 0; - let values_has_validity = values.null_count() > 0; - let (buffer, validity) = match (values_has_validity, indices_has_validity) { - (false, false) => take_no_validity::(values.values(), indices.values()), - (true, false) => take_values_validity::(values, indices.values()), - (false, true) => take_indices_validity::(values.values(), indices), - (true, true) => take_values_indices_validity::(values, indices), - }; - - PrimitiveArray::::new(values.data_type().clone(), buffer, validity) -} diff --git a/src/common/arrow/src/arrow/compute/take/structure.rs b/src/common/arrow/src/arrow/compute/take/structure.rs deleted file mode 100644 index 8f496607be90..000000000000 --- a/src/common/arrow/src/arrow/compute/take/structure.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Index; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::StructArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::error::Result; - -#[inline] -fn take_validity( - validity: Option<&Bitmap>, - indices: &PrimitiveArray, -) -> Result> { - let indices_validity = indices.validity(); - match (validity, indices_validity) { - (None, _) => Ok(indices_validity.cloned()), - (Some(validity), None) => { - let iter = indices.values().iter().map(|index| { - let index = index.to_usize(); - validity.get_bit(index) - }); - Ok(MutableBitmap::from_trusted_len_iter(iter).into()) - } - (Some(validity), _) => { - let iter = indices.iter().map(|x| match x { - Some(index) => { - let index = index.to_usize(); - validity.get_bit(index) - } - None => false, - }); - Ok(MutableBitmap::from_trusted_len_iter(iter).into()) - } - } -} - -pub fn take(array: &StructArray, indices: &PrimitiveArray) -> Result { - let values: Vec> = array - .values() - .iter() - .map(|a| super::take(a.as_ref(), indices)) - .collect::>()?; - let validity = take_validity(array.validity(), indices)?; - Ok(StructArray::new( - array.data_type().clone(), - values, - validity, - )) -} diff --git a/src/common/arrow/src/arrow/compute/take/utf8.rs b/src/common/arrow/src/arrow/compute/take/utf8.rs deleted file mode 100644 index 103993da73af..000000000000 --- a/src/common/arrow/src/arrow/compute/take/utf8.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::generic_binary::*; -use super::Index; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::offset::Offset; - -/// `take` implementation for utf8 arrays -pub fn take( - values: &Utf8Array, - indices: &PrimitiveArray, -) -> Utf8Array { - let data_type = values.data_type().clone(); - let indices_has_validity = indices.null_count() > 0; - let values_has_validity = values.null_count() > 0; - - let (offsets, values, validity) = match (values_has_validity, indices_has_validity) { - (false, false) => { - take_no_validity::(values.offsets(), values.values(), indices.values()) - } - (true, false) => take_values_validity(values, indices.values()), - (false, true) => take_indices_validity(values.offsets(), values.values(), indices), - (true, true) => take_values_indices_validity(values, indices), - }; - unsafe { Utf8Array::::new_unchecked(data_type, offsets, values, validity) } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrow::array::Int32Array; - - fn _all_cases() -> Vec<(Int32Array, Utf8Array, Utf8Array)> { - vec![ - ( - Int32Array::from(&[Some(1), Some(0)]), - Utf8Array::::from(vec![Some("one"), Some("two")]), - Utf8Array::::from(vec![Some("two"), Some("one")]), - ), - ( - Int32Array::from(&[Some(1), None]), - Utf8Array::::from(vec![Some("one"), Some("two")]), - Utf8Array::::from(vec![Some("two"), None]), - ), - ( - Int32Array::from(&[Some(1), Some(0)]), - Utf8Array::::from(vec![None, Some("two")]), - Utf8Array::::from(vec![Some("two"), None]), - ), - ( - Int32Array::from(&[Some(1), None, Some(0)]), - Utf8Array::::from(vec![None, Some("two")]), - Utf8Array::::from(vec![Some("two"), None, None]), - ), - ] - } - - #[test] - fn all_cases() { - let cases = _all_cases::(); - for (indices, input, expected) in cases { - let output = take(&input, &indices); - assert_eq!(expected, output); - } - let cases = _all_cases::(); - for (indices, input, expected) in cases { - let output = take(&input, &indices); - assert_eq!(expected, output); - } - } -} diff --git a/src/common/arrow/src/arrow/error.rs b/src/common/arrow/src/arrow/error.rs index 7c8e9d4e9e83..8841b8ce40df 100644 --- a/src/common/arrow/src/arrow/error.rs +++ b/src/common/arrow/src/arrow/error.rs @@ -81,6 +81,12 @@ impl From for Error { } } +impl From for Error { + fn from(error: serde_json::Error) -> Self { + Error::External("".to_string(), Box::new(error)) + } +} + impl From for Error { fn from(_: std::collections::TryReserveError) -> Error { Error::Overflow diff --git a/src/common/arrow/src/arrow/ffi/array.rs b/src/common/arrow/src/arrow/ffi/array.rs deleted file mode 100644 index 15e3a8fbef22..000000000000 --- a/src/common/arrow/src/arrow/ffi/array.rs +++ /dev/null @@ -1,618 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains functionality to load an ArrayData from the C Data Interface -use std::sync::Arc; - -use super::ArrowArray; -use crate::arrow::array::*; -use crate::arrow::bitmap::utils::bytes_for; -use crate::arrow::bitmap::utils::count_zeros; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::buffer::Bytes; -use crate::arrow::buffer::BytesAllocator; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::ffi::schema::get_child; -use crate::arrow::types::NativeType; -use crate::match_integer_type; -use crate::with_match_primitive_type; - -/// Reads a valid `ffi` interface into a `Box` -/// # Errors -/// If and only if: -/// * the interface is not valid (e.g. a null pointer) -pub unsafe fn try_from(array: A) -> Result> { - use PhysicalType::*; - Ok(match array.data_type().to_physical_type() { - Null => Box::new(NullArray::try_from_ffi(array)?), - Boolean => Box::new(BooleanArray::try_from_ffi(array)?), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - Box::new(PrimitiveArray::<$T>::try_from_ffi(array)?) - }), - Utf8 => Box::new(Utf8Array::::try_from_ffi(array)?), - LargeUtf8 => Box::new(Utf8Array::::try_from_ffi(array)?), - Binary => Box::new(BinaryArray::::try_from_ffi(array)?), - LargeBinary => Box::new(BinaryArray::::try_from_ffi(array)?), - FixedSizeBinary => Box::new(FixedSizeBinaryArray::try_from_ffi(array)?), - List => Box::new(ListArray::::try_from_ffi(array)?), - LargeList => Box::new(ListArray::::try_from_ffi(array)?), - FixedSizeList => Box::new(FixedSizeListArray::try_from_ffi(array)?), - Struct => Box::new(StructArray::try_from_ffi(array)?), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - Box::new(DictionaryArray::<$T>::try_from_ffi(array)?) - }) - } - Union => Box::new(UnionArray::try_from_ffi(array)?), - Map => Box::new(MapArray::try_from_ffi(array)?), - BinaryView => Box::new(BinaryViewArray::try_from_ffi(array)?), - Utf8View => Box::new(Utf8ViewArray::try_from_ffi(array)?), - }) -} - -// Sound because the arrow specification does not allow multiple implementations -// to change this struct -// This is intrinsically impossible to prove because the implementations agree -// on this as part of the Arrow specification -unsafe impl Send for ArrowArray {} -unsafe impl Sync for ArrowArray {} - -impl Drop for ArrowArray { - fn drop(&mut self) { - match self.release { - None => (), - Some(release) => unsafe { release(self) }, - }; - } -} - -// callback used to drop [ArrowArray] when it is exported -unsafe extern "C" fn c_release_array(array: *mut ArrowArray) { - if array.is_null() { - return; - } - let array = &mut *array; - - // take ownership of `private_data`, therefore dropping it - let private = Box::from_raw(array.private_data as *mut PrivateData); - for child in private.children_ptr.iter() { - let _ = Box::from_raw(*child); - } - - if let Some(ptr) = private.dictionary_ptr { - let _ = Box::from_raw(ptr); - } - - array.release = None; -} - -#[allow(dead_code)] -struct PrivateData { - array: Box, - buffers_ptr: Box<[*const std::os::raw::c_void]>, - children_ptr: Box<[*mut ArrowArray]>, - dictionary_ptr: Option<*mut ArrowArray>, -} - -impl ArrowArray { - /// creates a new `ArrowArray` from existing data. - /// # Safety - /// This method releases `buffers`. Consumers of this struct *must* call `release` before - /// releasing this struct, or contents in `buffers` leak. - pub(crate) fn new(array: Box) -> Self { - let (offset, buffers, children, dictionary) = - offset_buffers_children_dictionary(array.as_ref()); - - let buffers_ptr = buffers - .iter() - .map(|maybe_buffer| match maybe_buffer { - Some(b) => *b as *const std::os::raw::c_void, - None => std::ptr::null(), - }) - .collect::>(); - let n_buffers = buffers.len() as i64; - - let children_ptr = children - .into_iter() - .map(|child| Box::into_raw(Box::new(ArrowArray::new(child)))) - .collect::>(); - let n_children = children_ptr.len() as i64; - - let dictionary_ptr = - dictionary.map(|array| Box::into_raw(Box::new(ArrowArray::new(array)))); - - let length = array.len() as i64; - let null_count = array.null_count() as i64; - - let mut private_data = Box::new(PrivateData { - array, - buffers_ptr, - children_ptr, - dictionary_ptr, - }); - - Self { - length, - null_count, - offset: offset as i64, - n_buffers, - n_children, - buffers: private_data.buffers_ptr.as_mut_ptr(), - children: private_data.children_ptr.as_mut_ptr(), - dictionary: private_data.dictionary_ptr.unwrap_or(std::ptr::null_mut()), - release: Some(c_release_array), - private_data: Box::into_raw(private_data) as *mut ::std::os::raw::c_void, - } - } - - /// creates an empty [`ArrowArray`], which can be used to import data into - pub fn empty() -> Self { - Self { - length: 0, - null_count: 0, - offset: 0, - n_buffers: 0, - n_children: 0, - buffers: std::ptr::null_mut(), - children: std::ptr::null_mut(), - dictionary: std::ptr::null_mut(), - release: None, - private_data: std::ptr::null_mut(), - } - } - - /// the length of the array - pub(crate) fn len(&self) -> usize { - self.length as usize - } - - /// the offset of the array - pub(crate) fn offset(&self) -> usize { - self.offset as usize - } - - /// the null count of the array - pub(crate) fn null_count(&self) -> usize { - self.null_count as usize - } -} - -/// # Safety -/// The caller must ensure that the buffer at index `i` is not mutably shared. -unsafe fn get_buffer_ptr( - array: &ArrowArray, - data_type: &DataType, - index: usize, -) -> Result<*mut T> { - if array.buffers.is_null() { - return Err(Error::oos(format!( - "An ArrowArray of type {data_type:?} must have non-null buffers" - ))); - } - - if array - .buffers - .align_offset(std::mem::align_of::<*mut *const u8>()) - != 0 - { - return Err(Error::oos(format!( - "An ArrowArray of type {data_type:?} - must have buffer {index} aligned to type {}", - std::any::type_name::<*mut *const u8>() - ))); - } - let buffers = array.buffers as *mut *const u8; - - if index >= array.n_buffers as usize { - return Err(Error::oos(format!( - "An ArrowArray of type {data_type:?} - must have buffer {index}." - ))); - } - - let ptr = *buffers.add(index); - if ptr.is_null() { - return Err(Error::oos(format!( - "An array of type {data_type:?} - must have a non-null buffer {index}" - ))); - } - - // note: we can't prove that this pointer is not mutably shared - part of the safety invariant - Ok(ptr as *mut T) -} - -unsafe fn create_buffer_known_len( - array: &ArrowArray, - data_type: &DataType, - owner: InternalArrowArray, - len: usize, - index: usize, -) -> Result> { - if len == 0 { - return Ok(Buffer::new()); - } - let ptr: *mut T = get_buffer_ptr(array, data_type, index)?; - let bytes = Bytes::from_foreign(ptr, len, BytesAllocator::InternalArrowArray(owner)); - Ok(Buffer::from_bytes(bytes)) -} - -/// returns the buffer `i` of `array` interpreted as a [`Buffer`]. -/// # Safety -/// This function is safe iff: -/// * the buffers up to position `index` are valid for the declared length -/// * the buffers' pointers are not mutably shared for the lifetime of `owner` -unsafe fn create_buffer( - array: &ArrowArray, - data_type: &DataType, - owner: InternalArrowArray, - index: usize, -) -> Result> { - let len = buffer_len(array, data_type, index)?; - - if len == 0 { - return Ok(Buffer::new()); - } - - let offset = buffer_offset(array, data_type, index); - let ptr: *mut T = get_buffer_ptr(array, data_type, index)?; - - // We have to check alignment. - // This is the zero-copy path. - if ptr.align_offset(std::mem::align_of::()) == 0 { - let bytes = Bytes::from_foreign(ptr, len, BytesAllocator::InternalArrowArray(owner)); - Ok(Buffer::from_bytes(bytes).sliced(offset, len - offset)) - } - // This is the path where alignment isn't correct. - // We copy the data to a new vec - else { - let buf = std::slice::from_raw_parts(ptr, len - offset).to_vec(); - Ok(Buffer::from(buf)) - } -} - -/// returns the buffer `i` of `array` interpreted as a [`Bitmap`]. -/// # Safety -/// This function is safe iff: -/// * the buffer at position `index` is valid for the declared length -/// * the buffers' pointer is not mutable for the lifetime of `owner` -unsafe fn create_bitmap( - array: &ArrowArray, - data_type: &DataType, - owner: InternalArrowArray, - index: usize, - // if this is the validity bitmap - // we can use the null count directly - is_validity: bool, -) -> Result { - let len: usize = array.length.try_into().expect("length to fit in `usize`"); - if len == 0 { - return Ok(Bitmap::new()); - } - let ptr = get_buffer_ptr(array, data_type, index)?; - - // Pointer of u8 has alignment 1, so we don't have to check alignment. - - let offset: usize = array.offset.try_into().expect("offset to fit in `usize`"); - let bytes_len = bytes_for(offset + len); - let bytes = Bytes::from_foreign(ptr, bytes_len, BytesAllocator::InternalArrowArray(owner)); - - let null_count: usize = if is_validity { - array.null_count() - } else { - count_zeros(bytes.as_ref(), offset, len) - }; - Bitmap::from_inner(Arc::new(bytes), offset, len, null_count) -} - -fn buffer_offset(array: &ArrowArray, data_type: &DataType, i: usize) -> usize { - use PhysicalType::*; - match (data_type.to_physical_type(), i) { - (LargeUtf8, 2) | (LargeBinary, 2) | (Utf8, 2) | (Binary, 2) => 0, - (FixedSizeBinary, 1) => { - if let DataType::FixedSizeBinary(size) = data_type.to_logical_type() { - let offset: usize = array.offset.try_into().expect("Offset to fit in `usize`"); - offset * *size - } else { - unreachable!() - } - } - _ => array.offset.try_into().expect("Offset to fit in `usize`"), - } -} - -/// Returns the length, in slots, of the buffer `i` (indexed according to the C data interface) -unsafe fn buffer_len(array: &ArrowArray, data_type: &DataType, i: usize) -> Result { - Ok(match (data_type.to_physical_type(), i) { - (PhysicalType::FixedSizeBinary, 1) => { - if let DataType::FixedSizeBinary(size) = data_type.to_logical_type() { - *size * (array.offset as usize + array.length as usize) - } else { - unreachable!() - } - } - (PhysicalType::FixedSizeList, 1) => { - if let DataType::FixedSizeList(_, size) = data_type.to_logical_type() { - *size * (array.offset as usize + array.length as usize) - } else { - unreachable!() - } - } - (PhysicalType::Utf8, 1) - | (PhysicalType::LargeUtf8, 1) - | (PhysicalType::Binary, 1) - | (PhysicalType::LargeBinary, 1) - | (PhysicalType::List, 1) - | (PhysicalType::LargeList, 1) - | (PhysicalType::Map, 1) => { - // the len of the offset buffer (buffer 1) equals length + 1 - array.offset as usize + array.length as usize + 1 - } - (PhysicalType::Utf8, 2) | (PhysicalType::Binary, 2) => { - // the len of the data buffer (buffer 2) equals the last value of the offset buffer (buffer 1) - let len = buffer_len(array, data_type, 1)?; - // first buffer is the null buffer => add(1) - let offset_buffer = unsafe { *(array.buffers as *mut *const u8).add(1) }; - // interpret as i32 - let offset_buffer = offset_buffer as *const i32; - // get last offset - - (unsafe { *offset_buffer.add(len - 1) }) as usize - } - (PhysicalType::LargeUtf8, 2) | (PhysicalType::LargeBinary, 2) => { - // the len of the data buffer (buffer 2) equals the last value of the offset buffer (buffer 1) - let len = buffer_len(array, data_type, 1)?; - // first buffer is the null buffer => add(1) - let offset_buffer = unsafe { *(array.buffers as *mut *const u8).add(1) }; - // interpret as i64 - let offset_buffer = offset_buffer as *const i64; - // get last offset - (unsafe { *offset_buffer.add(len - 1) }) as usize - } - // buffer len of primitive types - _ => array.offset as usize + array.length as usize, - }) -} - -/// Safety -/// This function is safe iff: -/// * `array.children` at `index` is valid -/// * `array.children` is not mutably shared for the lifetime of `parent` -/// * the pointer of `array.children` at `index` is valid -/// * the pointer of `array.children` at `index` is not mutably shared for the lifetime of `parent` -unsafe fn create_child( - array: &ArrowArray, - data_type: &DataType, - parent: InternalArrowArray, - index: usize, -) -> Result> { - let data_type = get_child(data_type, index)?; - - // catch what we can - if array.children.is_null() { - return Err(Error::oos(format!( - "An ArrowArray of type {data_type:?} must have non-null children" - ))); - } - - if index >= array.n_children as usize { - return Err(Error::oos(format!( - "An ArrowArray of type {data_type:?} - must have child {index}." - ))); - } - - // Safety - part of the invariant - let arr_ptr = unsafe { *array.children.add(index) }; - - // catch what we can - if arr_ptr.is_null() { - return Err(Error::oos(format!( - "An array of type {data_type:?} - must have a non-null child {index}" - ))); - } - - // Safety - invariant of this function - let arr_ptr = unsafe { &*arr_ptr }; - Ok(ArrowArrayChild::new(arr_ptr, data_type, parent)) -} - -/// Safety -/// This function is safe iff: -/// * `array.dictionary` is valid -/// * `array.dictionary` is not mutably shared for the lifetime of `parent` -unsafe fn create_dictionary( - array: &ArrowArray, - data_type: &DataType, - parent: InternalArrowArray, -) -> Result>> { - if let DataType::Dictionary(_, values, _) = data_type { - let data_type = values.as_ref().clone(); - // catch what we can - if array.dictionary.is_null() { - return Err(Error::oos(format!( - "An array of type {data_type:?} - must have a non-null dictionary" - ))); - } - - // safety: part of the invariant - let array = unsafe { &*array.dictionary }; - Ok(Some(ArrowArrayChild::new(array, data_type, parent))) - } else { - Ok(None) - } -} - -pub trait ArrowArrayRef: std::fmt::Debug { - fn owner(&self) -> InternalArrowArray { - (*self.parent()).clone() - } - - /// returns the null bit buffer. - /// Rust implementation uses a buffer that is not part of the array of buffers. - /// The C Data interface's null buffer is part of the array of buffers. - /// # Safety - /// The caller must guarantee that the buffer `index` corresponds to a bitmap. - /// This function assumes that the bitmap created from FFI is valid; this is impossible to prove. - unsafe fn validity(&self) -> Result> { - if self.array().null_count() == 0 { - Ok(None) - } else { - create_bitmap(self.array(), self.data_type(), self.owner(), 0, true).map(Some) - } - } - - /// # Safety - /// The caller must guarantee that the buffer `index` corresponds to a buffer. - /// This function assumes that the buffer created from FFI is valid; this is impossible to prove. - unsafe fn buffer(&self, index: usize) -> Result> { - create_buffer::(self.array(), self.data_type(), self.owner(), index) - } - - /// # Safety - /// The caller must guarantee that the buffer `index` corresponds to a buffer. - /// This function assumes that the buffer created from FFI is valid; this is impossible to prove. - unsafe fn buffer_known_len( - &self, - index: usize, - len: usize, - ) -> Result> { - create_buffer_known_len::(self.array(), self.data_type(), self.owner(), len, index) - } - - /// # Safety - /// This function is safe iff: - /// * the buffer at position `index` is valid for the declared length - /// * the buffers' pointer is not mutable for the lifetime of `owner` - unsafe fn bitmap(&self, index: usize) -> Result { - create_bitmap(self.array(), self.data_type(), self.owner(), index, false) - } - - /// # Safety - /// * `array.children` at `index` is valid - /// * `array.children` is not mutably shared for the lifetime of `parent` - /// * the pointer of `array.children` at `index` is valid - /// * the pointer of `array.children` at `index` is not mutably shared for the lifetime of `parent` - unsafe fn child(&self, index: usize) -> Result { - create_child(self.array(), self.data_type(), self.parent().clone(), index) - } - - unsafe fn dictionary(&self) -> Result> { - create_dictionary(self.array(), self.data_type(), self.parent().clone()) - } - - fn n_buffers(&self) -> usize; - - fn parent(&self) -> &InternalArrowArray; - fn array(&self) -> &ArrowArray; - fn data_type(&self) -> &DataType; -} - -/// Struct used to move an Array from and to the C Data Interface. -/// Its main responsibility is to expose functionality that requires -/// both [ArrowArray] and [ArrowSchema]. -/// -/// This struct has two main paths: -/// -/// ## Import from the C Data Interface -/// * [InternalArrowArray::empty] to allocate memory to be filled by an external call -/// * [InternalArrowArray::try_from_raw] to consume two non-null allocated pointers -/// ## Export to the C Data Interface -/// * [InternalArrowArray::try_new] to create a new [InternalArrowArray] from Rust-specific information -/// * [InternalArrowArray::into_raw] to expose two pointers for [ArrowArray] and [ArrowSchema]. -/// -/// # Safety -/// Whoever creates this struct is responsible for releasing their resources. Specifically, -/// consumers *must* call [InternalArrowArray::into_raw] and take ownership of the individual pointers, -/// calling [ArrowArray::release] and [ArrowSchema::release] accordingly. -/// -/// Furthermore, this struct assumes that the incoming data agrees with the C data interface. -#[derive(Debug, Clone)] -pub struct InternalArrowArray { - // Arc is used for sharability since this is immutable - array: Arc, - // Arced to reduce cost of cloning - data_type: Arc, -} - -impl InternalArrowArray { - pub fn new(array: ArrowArray, data_type: DataType) -> Self { - Self { - array: Arc::new(array), - data_type: Arc::new(data_type), - } - } -} - -impl ArrowArrayRef for InternalArrowArray { - /// the data_type as declared in the schema - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn parent(&self) -> &InternalArrowArray { - self - } - - fn array(&self) -> &ArrowArray { - self.array.as_ref() - } - - fn n_buffers(&self) -> usize { - self.array.n_buffers as usize - } -} - -#[derive(Debug)] -pub struct ArrowArrayChild<'a> { - array: &'a ArrowArray, - data_type: DataType, - parent: InternalArrowArray, -} - -impl<'a> ArrowArrayRef for ArrowArrayChild<'a> { - /// the data_type as declared in the schema - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn parent(&self) -> &InternalArrowArray { - &self.parent - } - - fn array(&self) -> &ArrowArray { - self.array - } - - fn n_buffers(&self) -> usize { - self.array.n_buffers as usize - } -} - -impl<'a> ArrowArrayChild<'a> { - fn new(array: &'a ArrowArray, data_type: DataType, parent: InternalArrowArray) -> Self { - Self { - array, - data_type, - parent, - } - } -} diff --git a/src/common/arrow/src/arrow/ffi/bridge.rs b/src/common/arrow/src/arrow/ffi/bridge.rs deleted file mode 100644 index 85c9aa2e1cf5..000000000000 --- a/src/common/arrow/src/arrow/ffi/bridge.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::*; -use crate::match_integer_type; -use crate::with_match_primitive_type; - -macro_rules! ffi_dyn { - ($array:expr, $ty:ty) => {{ - let a = $array.as_any().downcast_ref::<$ty>().unwrap(); - if a.offset().is_some() { - $array - } else { - Box::new(a.to_ffi_aligned()) - } - }}; -} - -pub fn align_to_c_data_interface(array: Box) -> Box { - use crate::arrow::datatypes::PhysicalType::*; - match array.data_type().to_physical_type() { - Null => ffi_dyn!(array, NullArray), - Boolean => ffi_dyn!(array, BooleanArray), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - ffi_dyn!(array, PrimitiveArray<$T>) - }), - Binary => ffi_dyn!(array, BinaryArray), - LargeBinary => ffi_dyn!(array, BinaryArray), - FixedSizeBinary => ffi_dyn!(array, FixedSizeBinaryArray), - Utf8 => ffi_dyn!(array, Utf8Array::), - LargeUtf8 => ffi_dyn!(array, Utf8Array::), - List => ffi_dyn!(array, ListArray::), - LargeList => ffi_dyn!(array, ListArray::), - FixedSizeList => ffi_dyn!(array, FixedSizeListArray), - Struct => ffi_dyn!(array, StructArray), - Union => ffi_dyn!(array, UnionArray), - Map => ffi_dyn!(array, MapArray), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - ffi_dyn!(array, DictionaryArray<$T>) - }) - } - BinaryView => ffi_dyn!(array, BinaryViewArray), - Utf8View => ffi_dyn!(array, Utf8ViewArray), - } -} diff --git a/src/common/arrow/src/arrow/ffi/generated.rs b/src/common/arrow/src/arrow/ffi/generated.rs deleted file mode 100644 index 939a7110b912..000000000000 --- a/src/common/arrow/src/arrow/ffi/generated.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// automatically generated by rust-bindgen 0.59.2 - -/// ABI-compatible struct for [`ArrowSchema`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) -#[repr(C)] -#[derive(Debug)] -pub struct ArrowSchema { - pub(super) format: *const ::std::os::raw::c_char, - pub(super) name: *const ::std::os::raw::c_char, - pub(super) metadata: *const ::std::os::raw::c_char, - pub(super) flags: i64, - pub(super) n_children: i64, - pub(super) children: *mut *mut ArrowSchema, - pub(super) dictionary: *mut ArrowSchema, - pub(super) release: ::std::option::Option, - pub(super) private_data: *mut ::std::os::raw::c_void, -} - -/// ABI-compatible struct for [`ArrowArray`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) -#[repr(C)] -#[derive(Debug)] -pub struct ArrowArray { - pub(super) length: i64, - pub(super) null_count: i64, - pub(super) offset: i64, - pub(super) n_buffers: i64, - pub(super) n_children: i64, - pub(super) buffers: *mut *const ::std::os::raw::c_void, - pub(super) children: *mut *mut ArrowArray, - pub(super) dictionary: *mut ArrowArray, - pub(super) release: ::std::option::Option, - pub(super) private_data: *mut ::std::os::raw::c_void, -} - -/// ABI-compatible struct for [`ArrowArrayStream`](https://arrow.apache.org/docs/format/CStreamInterface.html). -#[repr(C)] -#[derive(Debug)] -pub struct ArrowArrayStream { - pub(super) get_schema: ::std::option::Option< - unsafe extern "C" fn( - arg1: *mut ArrowArrayStream, - out: *mut ArrowSchema, - ) -> ::std::os::raw::c_int, - >, - pub(super) get_next: ::std::option::Option< - unsafe extern "C" fn( - arg1: *mut ArrowArrayStream, - out: *mut ArrowArray, - ) -> ::std::os::raw::c_int, - >, - pub(super) get_last_error: ::std::option::Option< - unsafe extern "C" fn(arg1: *mut ArrowArrayStream) -> *const ::std::os::raw::c_char, - >, - pub(super) release: ::std::option::Option, - pub(super) private_data: *mut ::std::os::raw::c_void, -} diff --git a/src/common/arrow/src/arrow/ffi/mmap.rs b/src/common/arrow/src/arrow/ffi/mmap.rs deleted file mode 100644 index 931f35e16063..000000000000 --- a/src/common/arrow/src/arrow/ffi/mmap.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Functionality to mmap in-memory data regions. -use std::sync::Arc; - -use super::ArrowArray; -use super::InternalArrowArray; -use crate::arrow::array::BooleanArray; -use crate::arrow::array::FromFfi; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::types::NativeType; - -#[allow(dead_code)] -struct PrivateData { - // the owner of the pointers' regions - data: T, - buffers_ptr: Box<[*const std::os::raw::c_void]>, - children_ptr: Box<[*mut ArrowArray]>, - dictionary_ptr: Option<*mut ArrowArray>, -} - -pub(crate) unsafe fn create_array< - T: AsRef<[u8]>, - I: Iterator>, - II: Iterator, ->( - data: Arc, - num_rows: usize, - null_count: usize, - buffers: I, - children: II, - dictionary: Option, - offset: Option, -) -> ArrowArray { - let buffers_ptr = buffers - .map(|maybe_buffer| match maybe_buffer { - Some(b) => b as *const std::os::raw::c_void, - None => std::ptr::null(), - }) - .collect::>(); - let n_buffers = buffers_ptr.len() as i64; - - let children_ptr = children - .map(|child| Box::into_raw(Box::new(child))) - .collect::>(); - let n_children = children_ptr.len() as i64; - - let dictionary_ptr = dictionary.map(|array| Box::into_raw(Box::new(array))); - - let mut private_data = Box::new(PrivateData::> { - data, - buffers_ptr, - children_ptr, - dictionary_ptr, - }); - - ArrowArray { - length: num_rows as i64, - null_count: null_count as i64, - offset: offset.unwrap_or(0) as i64, // Unwrap: IPC files are by definition not offset - n_buffers, - n_children, - buffers: private_data.buffers_ptr.as_mut_ptr(), - children: private_data.children_ptr.as_mut_ptr(), - dictionary: private_data.dictionary_ptr.unwrap_or(std::ptr::null_mut()), - release: Some(release::>), - private_data: Box::into_raw(private_data) as *mut ::std::os::raw::c_void, - } -} - -/// callback used to drop [`ArrowArray`] when it is exported specified for [`PrivateData`]. -unsafe extern "C" fn release(array: *mut ArrowArray) { - if array.is_null() { - return; - } - let array = &mut *array; - - // take ownership of `private_data`, therefore dropping it - let private = Box::from_raw(array.private_data as *mut PrivateData); - for child in private.children_ptr.iter() { - let _ = Box::from_raw(*child); - } - - if let Some(ptr) = private.dictionary_ptr { - let _ = Box::from_raw(ptr); - } - - array.release = None; -} - -/// Creates a (non-null) [`PrimitiveArray`] from a slice of values. -/// This does not have memcopy and is the fastest way to create a [`PrimitiveArray`]. -/// -/// This can be useful if you want to apply arrow kernels on slices without incurring -/// a memcopy cost. -/// -/// # Safety -/// -/// Using this function is not unsafe, but the returned PrimitiveArray's lifetime is bound to the lifetime -/// of the slice. The returned [`PrimitiveArray`] _must not_ outlive the passed slice. -pub unsafe fn slice(slice: &[T]) -> PrimitiveArray { - let num_rows = slice.len(); - let null_count = 0; - let validity = None; - - let data: &[u8] = bytemuck::cast_slice(slice); - let ptr = data.as_ptr(); - let data = Arc::new(data); - - // safety: the underlying assumption of this function: the array will not be used - // beyond the - let array = create_array( - data, - num_rows, - null_count, - [validity, Some(ptr)].into_iter(), - [].into_iter(), - None, - None, - ); - let array = InternalArrowArray::new(array, T::PRIMITIVE.into()); - - // safety: we just created a valid array - unsafe { PrimitiveArray::::try_from_ffi(array) }.unwrap() -} - -/// Creates a (non-null) [`BooleanArray`] from a slice of bits. -/// This does not have memcopy and is the fastest way to create a [`BooleanArray`]. -/// -/// This can be useful if you want to apply arrow kernels on slices without incurring -/// a memcopy cost. -/// -/// The `offset` indicates where the first bit starts in the first byte. -/// -/// # Safety -/// -/// Using this function is not unsafe, but the returned BooleanArrays's lifetime is bound to the lifetime -/// of the slice. The returned [`BooleanArray`] _must not_ outlive the passed slice. -pub unsafe fn bitmap(data: &[u8], offset: usize, length: usize) -> Result { - if offset >= 8 { - return Err(Error::InvalidArgumentError("offset should be < 8".into())); - }; - if length > data.len() * 8 - offset { - return Err(Error::InvalidArgumentError("given length is oob".into())); - } - let null_count = 0; - let validity = None; - - let ptr = data.as_ptr(); - let data = Arc::new(data); - - // safety: the underlying assumption of this function: the array will not be used - // beyond the - let array = create_array( - data, - length, - null_count, - [validity, Some(ptr)].into_iter(), - [].into_iter(), - None, - Some(offset), - ); - let array = InternalArrowArray::new(array, DataType::Boolean); - - // safety: we just created a valid array - Ok(unsafe { BooleanArray::try_from_ffi(array) }.unwrap()) -} diff --git a/src/common/arrow/src/arrow/ffi/mod.rs b/src/common/arrow/src/arrow/ffi/mod.rs deleted file mode 100644 index 3c2fa2d56051..000000000000 --- a/src/common/arrow/src/arrow/ffi/mod.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! contains FFI bindings to import and export [`Array`](crate::arrow::array::Array) via -//! Arrow's [C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html) -mod array; -mod bridge; -mod generated; -pub mod mmap; -mod schema; -mod stream; - -pub(crate) use array::try_from; -pub(crate) use array::ArrowArrayRef; -pub(crate) use array::InternalArrowArray; -pub use generated::ArrowArray; -pub use generated::ArrowArrayStream; -pub use generated::ArrowSchema; -pub use stream::export_iterator; -pub use stream::ArrowArrayStreamReader; - -use self::schema::to_field; -use crate::arrow::array::Array; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Result; - -/// Exports an [`Box`] to the C data interface. -pub fn export_array_to_c(array: Box) -> ArrowArray { - ArrowArray::new(bridge::align_to_c_data_interface(array)) -} - -/// Exports a [`Field`] to the C data interface. -pub fn export_field_to_c(field: &Field) -> ArrowSchema { - ArrowSchema::new(field) -} - -/// Imports a [`Field`] from the C data interface. -/// # Safety -/// This function is intrinsically `unsafe` and relies on a [`ArrowSchema`] -/// being valid according to the [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) (FFI). -pub unsafe fn import_field_from_c(field: &ArrowSchema) -> Result { - to_field(field) -} - -/// Imports an [`Array`] from the C data interface. -/// # Safety -/// This function is intrinsically `unsafe` and relies on a [`ArrowArray`] -/// being valid according to the [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) (FFI). -pub unsafe fn import_array_from_c( - array: ArrowArray, - data_type: DataType, -) -> Result> { - try_from(InternalArrowArray::new(array, data_type)) -} diff --git a/src/common/arrow/src/arrow/ffi/schema.rs b/src/common/arrow/src/arrow/ffi/schema.rs deleted file mode 100644 index 7c47a49f205c..000000000000 --- a/src/common/arrow/src/arrow/ffi/schema.rs +++ /dev/null @@ -1,660 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::BTreeMap; -use std::convert::TryInto; -use std::ffi::CStr; -use std::ffi::CString; -use std::ptr; - -use super::ArrowSchema; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Extension; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::IntegerType; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::datatypes::Metadata; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::datatypes::UnionMode; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -#[allow(dead_code)] -struct SchemaPrivateData { - name: CString, - format: CString, - metadata: Option>, - children_ptr: Box<[*mut ArrowSchema]>, - dictionary: Option<*mut ArrowSchema>, -} - -// callback used to drop [ArrowSchema] when it is exported. -unsafe extern "C" fn c_release_schema(schema: *mut ArrowSchema) { - if schema.is_null() { - return; - } - let schema = &mut *schema; - - let private = Box::from_raw(schema.private_data as *mut SchemaPrivateData); - for child in private.children_ptr.iter() { - let _ = Box::from_raw(*child); - } - - if let Some(ptr) = private.dictionary { - let _ = Box::from_raw(ptr); - } - - schema.release = None; -} - -/// allocate (and hold) the children -fn schema_children(data_type: &DataType, flags: &mut i64) -> Box<[*mut ArrowSchema]> { - match data_type { - DataType::List(field) | DataType::FixedSizeList(field, _) | DataType::LargeList(field) => { - Box::new([Box::into_raw(Box::new(ArrowSchema::new(field.as_ref())))]) - } - DataType::Map(field, is_sorted) => { - *flags += (*is_sorted as i64) * 4; - Box::new([Box::into_raw(Box::new(ArrowSchema::new(field.as_ref())))]) - } - DataType::Struct(fields) | DataType::Union(fields, _, _) => fields - .iter() - .map(|field| Box::into_raw(Box::new(ArrowSchema::new(field)))) - .collect::>(), - DataType::Extension(_, inner, _) => schema_children(inner, flags), - _ => Box::new([]), - } -} - -impl ArrowSchema { - /// creates a new [ArrowSchema] - pub(crate) fn new(field: &Field) -> Self { - let format = to_format(field.data_type()); - let name = field.name.clone(); - - let mut flags = field.is_nullable as i64 * 2; - - // note: this cannot be done along with the above because the above is fallible and this op leaks. - let children_ptr = schema_children(field.data_type(), &mut flags); - let n_children = children_ptr.len() as i64; - - let dictionary = if let DataType::Dictionary(_, values, is_ordered) = field.data_type() { - flags += *is_ordered as i64; - // we do not store field info in the dict values, so can't recover it all :( - let field = Field::new("", values.as_ref().clone(), true); - Some(Box::new(ArrowSchema::new(&field))) - } else { - None - }; - - let metadata = &field.metadata; - - let metadata = if let DataType::Extension(name, _, extension_metadata) = field.data_type() { - // append extension information. - let mut metadata = metadata.clone(); - - // metadata - if let Some(extension_metadata) = extension_metadata { - metadata.insert( - "ARROW:extension:metadata".to_string(), - extension_metadata.clone(), - ); - } - - metadata.insert("ARROW:extension:name".to_string(), name.clone()); - - Some(metadata_to_bytes(&metadata)) - } else if !metadata.is_empty() { - Some(metadata_to_bytes(metadata)) - } else { - None - }; - - let name = CString::new(name).unwrap(); - let format = CString::new(format).unwrap(); - - let mut private = Box::new(SchemaPrivateData { - name, - format, - metadata, - children_ptr, - dictionary: dictionary.map(Box::into_raw), - }); - - // - Self { - format: private.format.as_ptr(), - name: private.name.as_ptr(), - metadata: private - .metadata - .as_ref() - .map(|x| x.as_ptr()) - .unwrap_or(std::ptr::null()) as *const ::std::os::raw::c_char, - flags, - n_children, - children: private.children_ptr.as_mut_ptr(), - dictionary: private.dictionary.unwrap_or(std::ptr::null_mut()), - release: Some(c_release_schema), - private_data: Box::into_raw(private) as *mut ::std::os::raw::c_void, - } - } - - /// create an empty [ArrowSchema] - pub fn empty() -> Self { - Self { - format: std::ptr::null_mut(), - name: std::ptr::null_mut(), - metadata: std::ptr::null_mut(), - flags: 0, - n_children: 0, - children: ptr::null_mut(), - dictionary: std::ptr::null_mut(), - release: None, - private_data: std::ptr::null_mut(), - } - } - - /// returns the format of this schema. - pub(crate) fn format(&self) -> &str { - assert!(!self.format.is_null()); - // safe because the lifetime of `self.format` equals `self` - unsafe { CStr::from_ptr(self.format) } - .to_str() - .expect("The external API has a non-utf8 as format") - } - - /// returns the name of this schema. - /// - /// Since this field is optional, `""` is returned if it is not set (as per the spec). - pub(crate) fn name(&self) -> &str { - if self.name.is_null() { - return ""; - } - // safe because the lifetime of `self.name` equals `self` - unsafe { CStr::from_ptr(self.name) }.to_str().unwrap() - } - - pub(crate) fn child(&self, index: usize) -> &'static Self { - assert!(index < self.n_children as usize); - unsafe { self.children.add(index).as_ref().unwrap().as_ref().unwrap() } - } - - pub(crate) fn dictionary(&self) -> Option<&'static Self> { - if self.dictionary.is_null() { - return None; - }; - Some(unsafe { self.dictionary.as_ref().unwrap() }) - } - - pub(crate) fn nullable(&self) -> bool { - (self.flags / 2) & 1 == 1 - } -} - -impl Drop for ArrowSchema { - fn drop(&mut self) { - match self.release { - None => (), - Some(release) => unsafe { release(self) }, - }; - } -} - -pub(crate) unsafe fn to_field(schema: &ArrowSchema) -> Result { - let dictionary = schema.dictionary(); - let data_type = if let Some(dictionary) = dictionary { - let indices = to_integer_type(schema.format())?; - let values = to_field(dictionary)?; - let is_ordered = schema.flags & 1 == 1; - DataType::Dictionary(indices, Box::new(values.data_type().clone()), is_ordered) - } else { - to_data_type(schema)? - }; - let (metadata, extension) = unsafe { metadata_from_bytes(schema.metadata) }; - - let data_type = if let Some((name, extension_metadata)) = extension { - DataType::Extension(name, Box::new(data_type), extension_metadata) - } else { - data_type - }; - - Ok(Field::new(schema.name(), data_type, schema.nullable()).with_metadata(metadata)) -} - -fn to_integer_type(format: &str) -> Result { - use IntegerType::*; - Ok(match format { - "c" => Int8, - "C" => UInt8, - "s" => Int16, - "S" => UInt16, - "i" => Int32, - "I" => UInt32, - "l" => Int64, - "L" => UInt64, - _ => { - return Err(Error::OutOfSpec( - "Dictionary indices can only be integers".to_string(), - )); - } - }) -} - -unsafe fn to_data_type(schema: &ArrowSchema) -> Result { - Ok(match schema.format() { - "n" => DataType::Null, - "b" => DataType::Boolean, - "c" => DataType::Int8, - "C" => DataType::UInt8, - "s" => DataType::Int16, - "S" => DataType::UInt16, - "i" => DataType::Int32, - "I" => DataType::UInt32, - "l" => DataType::Int64, - "L" => DataType::UInt64, - "e" => DataType::Float16, - "f" => DataType::Float32, - "g" => DataType::Float64, - "z" => DataType::Binary, - "Z" => DataType::LargeBinary, - "u" => DataType::Utf8, - "U" => DataType::LargeUtf8, - "tdD" => DataType::Date32, - "tdm" => DataType::Date64, - "tts" => DataType::Time32(TimeUnit::Second), - "ttm" => DataType::Time32(TimeUnit::Millisecond), - "ttu" => DataType::Time64(TimeUnit::Microsecond), - "ttn" => DataType::Time64(TimeUnit::Nanosecond), - "tDs" => DataType::Duration(TimeUnit::Second), - "tDm" => DataType::Duration(TimeUnit::Millisecond), - "tDu" => DataType::Duration(TimeUnit::Microsecond), - "tDn" => DataType::Duration(TimeUnit::Nanosecond), - "tiM" => DataType::Interval(IntervalUnit::YearMonth), - "tiD" => DataType::Interval(IntervalUnit::DayTime), - "vu" => DataType::Utf8View, - "vz" => DataType::BinaryView, - "+l" => { - let child = schema.child(0); - DataType::List(Box::new(to_field(child)?)) - } - "+L" => { - let child = schema.child(0); - DataType::LargeList(Box::new(to_field(child)?)) - } - "+m" => { - let child = schema.child(0); - - let is_sorted = (schema.flags & 4) != 0; - DataType::Map(Box::new(to_field(child)?), is_sorted) - } - "+s" => { - let children = (0..schema.n_children as usize) - .map(|x| to_field(schema.child(x))) - .collect::>>()?; - DataType::Struct(children) - } - other => { - match other.splitn(2, ':').collect::>()[..] { - // Timestamps with no timezone - ["tss", ""] => DataType::Timestamp(TimeUnit::Second, None), - ["tsm", ""] => DataType::Timestamp(TimeUnit::Millisecond, None), - ["tsu", ""] => DataType::Timestamp(TimeUnit::Microsecond, None), - ["tsn", ""] => DataType::Timestamp(TimeUnit::Nanosecond, None), - - // Timestamps with timezone - ["tss", tz] => DataType::Timestamp(TimeUnit::Second, Some(tz.to_string())), - ["tsm", tz] => DataType::Timestamp(TimeUnit::Millisecond, Some(tz.to_string())), - ["tsu", tz] => DataType::Timestamp(TimeUnit::Microsecond, Some(tz.to_string())), - ["tsn", tz] => DataType::Timestamp(TimeUnit::Nanosecond, Some(tz.to_string())), - - ["w", size_raw] => { - // Example: "w:42" fixed-width binary [42 bytes] - let size = size_raw - .parse::() - .map_err(|_| Error::OutOfSpec("size is not a valid integer".to_string()))?; - DataType::FixedSizeBinary(size) - } - ["+w", size_raw] => { - // Example: "+w:123" fixed-sized list [123 items] - let size = size_raw - .parse::() - .map_err(|_| Error::OutOfSpec("size is not a valid integer".to_string()))?; - let child = to_field(schema.child(0))?; - DataType::FixedSizeList(Box::new(child), size) - } - ["d", raw] => { - // Decimal - let (precision, scale) = match raw.split(',').collect::>()[..] { - [precision_raw, scale_raw] => { - // Example: "d:19,10" decimal128 [precision 19, scale 10] - (precision_raw, scale_raw) - } - [precision_raw, scale_raw, width_raw] => { - // Example: "d:19,10,NNN" decimal bitwidth = NNN [precision 19, scale 10] - // Only bitwdth of 128 currently supported - let bit_width = width_raw.parse::().map_err(|_| { - Error::OutOfSpec( - "Decimal bit width is not a valid integer".to_string(), - ) - })?; - if bit_width == 256 { - return Ok(DataType::Decimal256( - precision_raw.parse::().map_err(|_| { - Error::OutOfSpec( - "Decimal precision is not a valid integer".to_string(), - ) - })?, - scale_raw.parse::().map_err(|_| { - Error::OutOfSpec( - "Decimal scale is not a valid integer".to_string(), - ) - })?, - )); - } - (precision_raw, scale_raw) - } - _ => { - return Err(Error::OutOfSpec( - "Decimal must contain 2 or 3 comma-separated values".to_string(), - )); - } - }; - - DataType::Decimal( - precision.parse::().map_err(|_| { - Error::OutOfSpec("Decimal precision is not a valid integer".to_string()) - })?, - scale.parse::().map_err(|_| { - Error::OutOfSpec("Decimal scale is not a valid integer".to_string()) - })?, - ) - } - [union_type @ "+us", union_parts] | [union_type @ "+ud", union_parts] => { - // union, sparse - // Example "+us:I,J,..." sparse union with type ids I,J... - // Example: "+ud:I,J,..." dense union with type ids I,J... - let mode = UnionMode::sparse(union_type == "+us"); - let type_ids = union_parts - .split(',') - .map(|x| { - x.parse::().map_err(|_| { - Error::OutOfSpec("Union type id is not a valid integer".to_string()) - }) - }) - .collect::>>()?; - let fields = (0..schema.n_children as usize) - .map(|x| to_field(schema.child(x))) - .collect::>>()?; - DataType::Union(fields, Some(type_ids), mode) - } - _ => { - return Err(Error::OutOfSpec(format!( - "The datatype \"{other}\" is still not supported in Rust implementation", - ))); - } - } - } - }) -} - -/// the inverse of [to_field] -fn to_format(data_type: &DataType) -> String { - match data_type { - DataType::Null => "n".to_string(), - DataType::Boolean => "b".to_string(), - DataType::Int8 => "c".to_string(), - DataType::UInt8 => "C".to_string(), - DataType::Int16 => "s".to_string(), - DataType::UInt16 => "S".to_string(), - DataType::Int32 => "i".to_string(), - DataType::UInt32 => "I".to_string(), - DataType::Int64 => "l".to_string(), - DataType::UInt64 => "L".to_string(), - DataType::Float16 => "e".to_string(), - DataType::Float32 => "f".to_string(), - DataType::Float64 => "g".to_string(), - DataType::Binary => "z".to_string(), - DataType::LargeBinary => "Z".to_string(), - DataType::Utf8 => "u".to_string(), - DataType::LargeUtf8 => "U".to_string(), - DataType::Date32 => "tdD".to_string(), - DataType::Date64 => "tdm".to_string(), - DataType::Time32(TimeUnit::Second) => "tts".to_string(), - DataType::Time32(TimeUnit::Millisecond) => "ttm".to_string(), - DataType::Time32(_) => { - unreachable!("Time32 is only supported for seconds and milliseconds") - } - DataType::Time64(TimeUnit::Microsecond) => "ttu".to_string(), - DataType::Time64(TimeUnit::Nanosecond) => "ttn".to_string(), - DataType::Time64(_) => { - unreachable!("Time64 is only supported for micro and nanoseconds") - } - DataType::Duration(TimeUnit::Second) => "tDs".to_string(), - DataType::Duration(TimeUnit::Millisecond) => "tDm".to_string(), - DataType::Duration(TimeUnit::Microsecond) => "tDu".to_string(), - DataType::Duration(TimeUnit::Nanosecond) => "tDn".to_string(), - DataType::Interval(IntervalUnit::YearMonth) => "tiM".to_string(), - DataType::Interval(IntervalUnit::DayTime) => "tiD".to_string(), - DataType::Interval(IntervalUnit::MonthDayNano) => { - todo!("Spec for FFI for MonthDayNano still not defined.") - } - DataType::Timestamp(unit, tz) => { - let unit = match unit { - TimeUnit::Second => "s", - TimeUnit::Millisecond => "m", - TimeUnit::Microsecond => "u", - TimeUnit::Nanosecond => "n", - }; - format!( - "ts{}:{}", - unit, - tz.as_ref().map(|x| x.as_ref()).unwrap_or("") - ) - } - DataType::Utf8View => "vu".to_string(), - DataType::BinaryView => "vz".to_string(), - DataType::Decimal(precision, scale) => format!("d:{precision},{scale}"), - DataType::Decimal256(precision, scale) => format!("d:{precision},{scale},256"), - DataType::List(_) => "+l".to_string(), - DataType::LargeList(_) => "+L".to_string(), - DataType::Struct(_) => "+s".to_string(), - DataType::FixedSizeBinary(size) => format!("w:{size}"), - DataType::FixedSizeList(_, size) => format!("+w:{size}"), - DataType::Union(f, ids, mode) => { - let sparsness = if mode.is_sparse() { 's' } else { 'd' }; - let mut r = format!("+u{sparsness}:"); - let ids = if let Some(ids) = ids { - ids.iter() - .fold(String::new(), |a, b| a + &b.to_string() + ",") - } else { - (0..f.len()).fold(String::new(), |a, b| a + &b.to_string() + ",") - }; - let ids = &ids[..ids.len() - 1]; // take away last "," - r.push_str(ids); - r - } - DataType::Map(_, _) => "+m".to_string(), - DataType::Dictionary(index, _, _) => to_format(&(*index).into()), - DataType::Extension(_, inner, _) => to_format(inner.as_ref()), - } -} - -pub(super) fn get_child(data_type: &DataType, index: usize) -> Result { - match (index, data_type) { - (0, DataType::List(field)) => Ok(field.data_type().clone()), - (0, DataType::FixedSizeList(field, _)) => Ok(field.data_type().clone()), - (0, DataType::LargeList(field)) => Ok(field.data_type().clone()), - (0, DataType::Map(field, _)) => Ok(field.data_type().clone()), - (index, DataType::Struct(fields)) => Ok(fields[index].data_type().clone()), - (index, DataType::Union(fields, _, _)) => Ok(fields[index].data_type().clone()), - (index, DataType::Extension(_, subtype, _)) => get_child(subtype, index), - (child, data_type) => Err(Error::OutOfSpec(format!( - "Requested child {child} to type {data_type:?} that has no such child", - ))), - } -} - -fn metadata_to_bytes(metadata: &BTreeMap) -> Vec { - let a = (metadata.len() as i32).to_ne_bytes().to_vec(); - metadata.iter().fold(a, |mut acc, (key, value)| { - acc.extend((key.len() as i32).to_ne_bytes()); - acc.extend(key.as_bytes()); - acc.extend((value.len() as i32).to_ne_bytes()); - acc.extend(value.as_bytes()); - acc - }) -} - -unsafe fn read_ne_i32(ptr: *const u8) -> i32 { - let slice = std::slice::from_raw_parts(ptr, 4); - i32::from_ne_bytes(slice.try_into().unwrap()) -} - -unsafe fn read_bytes(ptr: *const u8, len: usize) -> &'static str { - let slice = std::slice::from_raw_parts(ptr, len); - simdutf8::basic::from_utf8(slice).unwrap() -} - -unsafe fn metadata_from_bytes(data: *const ::std::os::raw::c_char) -> (Metadata, Extension) { - #[allow(clippy::unnecessary_cast)] - let mut data = data as *const u8; // u8 = i8 - if data.is_null() { - return (Metadata::default(), None); - }; - let len = read_ne_i32(data); - data = data.add(4); - - let mut result = BTreeMap::new(); - let mut extension_name = None; - let mut extension_metadata = None; - for _ in 0..len { - let key_len = read_ne_i32(data) as usize; - data = data.add(4); - let key = read_bytes(data, key_len); - data = data.add(key_len); - let value_len = read_ne_i32(data) as usize; - data = data.add(4); - let value = read_bytes(data, value_len); - data = data.add(value_len); - match key { - "ARROW:extension:name" => { - extension_name = Some(value.to_string()); - } - "ARROW:extension:metadata" => { - extension_metadata = Some(value.to_string()); - } - _ => { - result.insert(key.to_string(), value.to_string()); - } - }; - } - let extension = extension_name.map(|name| (name, extension_metadata)); - (result, extension) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_all() { - let mut dts = vec![ - DataType::Null, - DataType::Boolean, - DataType::UInt8, - DataType::UInt16, - DataType::UInt32, - DataType::UInt64, - DataType::Int8, - DataType::Int16, - DataType::Int32, - DataType::Int64, - DataType::Float32, - DataType::Float64, - DataType::Date32, - DataType::Date64, - DataType::Time32(TimeUnit::Second), - DataType::Time32(TimeUnit::Millisecond), - DataType::Time64(TimeUnit::Microsecond), - DataType::Time64(TimeUnit::Nanosecond), - DataType::Decimal(5, 5), - DataType::Utf8, - DataType::LargeUtf8, - DataType::Binary, - DataType::LargeBinary, - DataType::FixedSizeBinary(2), - DataType::List(Box::new(Field::new("example", DataType::Boolean, false))), - DataType::FixedSizeList(Box::new(Field::new("example", DataType::Boolean, false)), 2), - DataType::LargeList(Box::new(Field::new("example", DataType::Boolean, false))), - DataType::Struct(vec![ - Field::new("a", DataType::Int64, true), - Field::new( - "b", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), - true, - ), - ]), - DataType::Map(Box::new(Field::new("a", DataType::Int64, true)), true), - DataType::Union( - vec![ - Field::new("a", DataType::Int64, true), - Field::new( - "b", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), - true, - ), - ], - Some(vec![1, 2]), - UnionMode::Dense, - ), - DataType::Union( - vec![ - Field::new("a", DataType::Int64, true), - Field::new( - "b", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), - true, - ), - ], - Some(vec![0, 1]), - UnionMode::Sparse, - ), - ]; - for time_unit in [ - TimeUnit::Second, - TimeUnit::Millisecond, - TimeUnit::Microsecond, - TimeUnit::Nanosecond, - ] { - dts.push(DataType::Timestamp(time_unit, None)); - dts.push(DataType::Timestamp(time_unit, Some("00:00".to_string()))); - dts.push(DataType::Duration(time_unit)); - } - for interval_type in [ - IntervalUnit::DayTime, - IntervalUnit::YearMonth, - // IntervalUnit::MonthDayNano, // not yet defined on the C data interface - ] { - dts.push(DataType::Interval(interval_type)); - } - - for expected in dts { - let field = Field::new("a", expected.clone(), true); - let schema = ArrowSchema::new(&field); - let result = unsafe { super::to_data_type(&schema).unwrap() }; - assert_eq!(result, expected); - } - } -} diff --git a/src/common/arrow/src/arrow/ffi/stream.rs b/src/common/arrow/src/arrow/ffi/stream.rs deleted file mode 100644 index 3b1820686db3..000000000000 --- a/src/common/arrow/src/arrow/ffi/stream.rs +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::ffi::CStr; -use std::ffi::CString; -use std::ops::DerefMut; - -use super::export_array_to_c; -use super::export_field_to_c; -use super::import_array_from_c; -use super::import_field_from_c; -use super::ArrowArray; -use super::ArrowArrayStream; -use super::ArrowSchema; -use crate::arrow::array::Array; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; - -impl Drop for ArrowArrayStream { - fn drop(&mut self) { - match self.release { - None => (), - Some(release) => unsafe { release(self) }, - }; - } -} - -impl ArrowArrayStream { - /// Creates an empty [`ArrowArrayStream`] used to import from a producer. - pub fn empty() -> Self { - Self { - get_schema: None, - get_next: None, - get_last_error: None, - release: None, - private_data: std::ptr::null_mut(), - } - } -} - -unsafe fn handle_error(iter: &mut ArrowArrayStream) -> Error { - let error = unsafe { (iter.get_last_error.unwrap())(&mut *iter) }; - - if error.is_null() { - return Error::External( - "C stream".to_string(), - Box::new(Error::ExternalFormat("an unspecified error".to_string())), - ); - } - - let error = unsafe { CStr::from_ptr(error) }; - Error::External( - "C stream".to_string(), - Box::new(Error::ExternalFormat(error.to_str().unwrap().to_string())), - ) -} - -/// Implements an iterator of [`Array`] consumed from the [C stream interface](https://arrow.apache.org/docs/format/CStreamInterface.html). -pub struct ArrowArrayStreamReader> { - iter: Iter, - field: Field, -} - -impl> ArrowArrayStreamReader { - /// Returns a new [`ArrowArrayStreamReader`] - /// # Error - /// Errors iff the [`ArrowArrayStream`] is out of specification, - /// or was already released prior to calling this function. - /// # Safety - /// This method is intrinsically `unsafe` since it assumes that the `ArrowArrayStream` - /// contains a valid Arrow C stream interface. - /// In particular: - /// * The `ArrowArrayStream` fulfills the invariants of the C stream interface - /// * The schema `get_schema` produces fulfills the C data interface - pub unsafe fn try_new(mut iter: Iter) -> Result { - if iter.release.is_none() { - return Err(Error::InvalidArgumentError( - "The C stream was already released".to_string(), - )); - }; - - if iter.get_next.is_none() { - return Err(Error::OutOfSpec( - "The C stream MUST contain a non-null get_next".to_string(), - )); - }; - - if iter.get_last_error.is_none() { - return Err(Error::OutOfSpec( - "The C stream MUST contain a non-null get_last_error".to_string(), - )); - }; - - let mut field = ArrowSchema::empty(); - let status = if let Some(f) = iter.get_schema { - unsafe { (f)(&mut *iter, &mut field) } - } else { - return Err(Error::OutOfSpec( - "The C stream MUST contain a non-null get_schema".to_string(), - )); - }; - - if status != 0 { - return Err(unsafe { handle_error(&mut iter) }); - } - - let field = unsafe { import_field_from_c(&field)? }; - - Ok(Self { iter, field }) - } - - /// Returns the field provided by the stream - pub fn field(&self) -> &Field { - &self.field - } - - /// Advances this iterator by one array - /// # Error - /// Errors iff: - /// * The C stream interface returns an error - /// * The C stream interface returns an invalid array (that we can identify, see Safety below) - /// # Safety - /// Calling this iterator's `next` assumes that the [`ArrowArrayStream`] produces arrow arrays - /// that fulfill the C data interface - pub unsafe fn next(&mut self) -> Option, Error>> { - let mut array = ArrowArray::empty(); - let status = unsafe { (self.iter.get_next.unwrap())(&mut *self.iter, &mut array) }; - - if status != 0 { - return Some(Err(unsafe { handle_error(&mut self.iter) })); - } - - // last paragraph of https://arrow.apache.org/docs/format/CStreamInterface.html#c.ArrowArrayStream.get_next - array.release?; - - // Safety: assumed from the C stream interface - unsafe { import_array_from_c(array, self.field.data_type.clone()) } - .map(Some) - .transpose() - } -} - -struct PrivateData { - iter: Box, Error>>>, - field: Field, - error: Option, -} - -unsafe extern "C" fn get_next(iter: *mut ArrowArrayStream, array: *mut ArrowArray) -> i32 { - if iter.is_null() { - return 2001; - } - let private = &mut *((*iter).private_data as *mut PrivateData); - - match private.iter.next() { - Some(Ok(item)) => { - // check that the array has the same data_type as field - let item_dt = item.data_type(); - let expected_dt = private.field.data_type(); - if item_dt != expected_dt { - private.error = Some(CString::new(format!("The iterator produced an item of data type {item_dt:?} but the producer expects data type {expected_dt:?}").as_bytes().to_vec()).unwrap()); - return 2001; // custom application specific error (since this is never a result of this interface) - } - - std::ptr::write(array, export_array_to_c(item)); - - private.error = None; - 0 - } - Some(Err(err)) => { - private.error = Some(CString::new(err.to_string().as_bytes().to_vec()).unwrap()); - 2001 // custom application specific error (since this is never a result of this interface) - } - None => { - let a = ArrowArray::empty(); - std::ptr::write_unaligned(array, a); - private.error = None; - 0 - } - } -} - -unsafe extern "C" fn get_schema(iter: *mut ArrowArrayStream, schema: *mut ArrowSchema) -> i32 { - if iter.is_null() { - return 2001; - } - let private = &mut *((*iter).private_data as *mut PrivateData); - - std::ptr::write(schema, export_field_to_c(&private.field)); - 0 -} - -unsafe extern "C" fn get_last_error(iter: *mut ArrowArrayStream) -> *const ::std::os::raw::c_char { - if iter.is_null() { - return std::ptr::null(); - } - let private = &mut *((*iter).private_data as *mut PrivateData); - - private - .error - .as_ref() - .map(|x| x.as_ptr()) - .unwrap_or(std::ptr::null()) -} - -unsafe extern "C" fn release(iter: *mut ArrowArrayStream) { - if iter.is_null() { - return; - } - let _ = Box::from_raw((*iter).private_data as *mut PrivateData); - (*iter).release = None; - // private drops automatically -} - -/// Exports an iterator to the [C stream interface](https://arrow.apache.org/docs/format/CStreamInterface.html) -pub fn export_iterator( - iter: Box, Error>>>, - field: Field, -) -> ArrowArrayStream { - let private_data = Box::new(PrivateData { - iter, - field, - error: None, - }); - - ArrowArrayStream { - get_schema: Some(get_schema), - get_next: Some(get_next), - get_last_error: Some(get_last_error), - release: Some(release), - private_data: Box::into_raw(private_data) as *mut ::std::os::raw::c_void, - } -} diff --git a/src/common/arrow/src/arrow/io/README.md b/src/common/arrow/src/arrow/io/README.md deleted file mode 100644 index 26be0b337a6e..000000000000 --- a/src/common/arrow/src/arrow/io/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# IO module - -This document describes the overall design of this module. - -## Rules: - -* Each directory in this module corresponds to a specific format such as `csv` and `json`. -* directories that depend on external dependencies MUST be feature gated, with a feature named with a prefix `io_`. -* modules MUST re-export any API of external dependencies they require as part of their public API. - E.g. - * if a module as an API `write(writer: &mut csv:Writer, ...)`, it MUST contain `pub use csv::Writer;`. - - The rational is that adding this crate to `cargo.toml` must be sufficient to use it. -* Each directory SHOULD contain two directories, `read` and `write`, corresponding - to functionality about reading from the format and writing to the format respectively. -* The base module SHOULD contain `use pub read;` and `use pub write;`. -* Implementations SHOULD separate reading of "data" from reading of "metadata". Examples: - * schema read or inference SHOULD be a separate function - * functions that read "data" SHOULD consume a schema typically pre-read. -* Implementations SHOULD separate IO-bounded operations from CPU-bounded operations. - I.e. implementations SHOULD: - * contain functions that consume a `Read` implementor and output a "raw" struct, i.e. a struct that is e.g. compressed and serialized - * contain functions that consume a "raw" struct and convert it into Arrow. - * offer each of these functions as independent public APIs, so that consumers can decide how to balance CPU-bounds and IO-bounds. diff --git a/src/common/arrow/src/arrow/io/flight/mod.rs b/src/common/arrow/src/arrow/io/flight/mod.rs deleted file mode 100644 index 65c570e621f7..000000000000 --- a/src/common/arrow/src/arrow/io/flight/mod.rs +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Serialization and deserialization to Arrow's flight protocol - -use arrow_format::flight::data::FlightData; -use arrow_format::flight::data::SchemaResult; -use arrow_format::ipc; -use arrow_format::ipc::planus::ReadAsRoot; - -use super::ipc::read::Dictionaries; -pub use super::ipc::write::default_ipc_fields; -use super::ipc::IpcField; -use super::ipc::IpcSchema; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::read; -use crate::arrow::io::ipc::write; -use crate::arrow::io::ipc::write::common::encode_chunk; -use crate::arrow::io::ipc::write::common::DictionaryTracker; -use crate::arrow::io::ipc::write::common::EncodedData; -pub use crate::arrow::io::ipc::write::common::WriteOptions; - -/// Serializes [`Chunk`] to a vector of [`FlightData`] representing the serialized dictionaries -/// and a [`FlightData`] representing the batch. -/// # Errors -/// This function errors iff `fields` is not consistent with `columns` -pub fn serialize_batch( - chunk: &Chunk>, - fields: &[IpcField], - options: &WriteOptions, -) -> Result<(Vec, FlightData)> { - if fields.len() != chunk.arrays().len() { - return Err(Error::InvalidArgumentError("The argument `fields` must be consistent with the columns' schema. Use e.g. &arrow2::io::flight::default_ipc_fields(&schema.fields)".to_string())); - } - - let mut dictionary_tracker = DictionaryTracker { - dictionaries: Default::default(), - cannot_replace: false, - }; - - let (encoded_dictionaries, encoded_batch) = - encode_chunk(chunk, fields, &mut dictionary_tracker, options) - .expect("DictionaryTracker configured above to not error on replacement"); - - let flight_dictionaries = encoded_dictionaries.into_iter().map(Into::into).collect(); - let flight_batch = encoded_batch.into(); - - Ok((flight_dictionaries, flight_batch)) -} - -impl From for FlightData { - fn from(data: EncodedData) -> Self { - FlightData { - data_header: data.ipc_message, - data_body: data.arrow_data, - ..Default::default() - } - } -} - -/// Serializes a [`Schema`] to [`SchemaResult`]. -pub fn serialize_schema_to_result( - schema: &Schema, - ipc_fields: Option<&[IpcField]>, -) -> SchemaResult { - SchemaResult { - schema: _serialize_schema(schema, ipc_fields), - } -} - -/// Serializes a [`Schema`] to [`FlightData`]. -pub fn serialize_schema(schema: &Schema, ipc_fields: Option<&[IpcField]>) -> FlightData { - FlightData { - data_header: _serialize_schema(schema, ipc_fields), - ..Default::default() - } -} - -/// Convert a [`Schema`] to bytes in the format expected in [`arrow_format::flight::data::FlightInfo`]. -pub fn serialize_schema_to_info( - schema: &Schema, - ipc_fields: Option<&[IpcField]>, -) -> Result> { - let encoded_data = if let Some(ipc_fields) = ipc_fields { - schema_as_encoded_data(schema, ipc_fields) - } else { - let ipc_fields = default_ipc_fields(&schema.fields); - schema_as_encoded_data(schema, &ipc_fields) - }; - - let mut schema = vec![]; - write::common_sync::write_message(&mut schema, &encoded_data)?; - Ok(schema) -} - -fn _serialize_schema(schema: &Schema, ipc_fields: Option<&[IpcField]>) -> Vec { - if let Some(ipc_fields) = ipc_fields { - write::schema_to_bytes(schema, ipc_fields) - } else { - let ipc_fields = default_ipc_fields(&schema.fields); - write::schema_to_bytes(schema, &ipc_fields) - } -} - -fn schema_as_encoded_data(schema: &Schema, ipc_fields: &[IpcField]) -> EncodedData { - EncodedData { - ipc_message: write::schema_to_bytes(schema, ipc_fields), - arrow_data: vec![], - } -} - -/// Deserialize an IPC message into [`Schema`], [`IpcSchema`]. -/// Use to deserialize [`FlightData::data_header`] and [`SchemaResult::schema`]. -pub fn deserialize_schemas(bytes: &[u8]) -> Result<(Schema, IpcSchema)> { - read::deserialize_schema(bytes) -} - -/// Deserializes [`FlightData`] representing a record batch message to [`Chunk`]. -pub fn deserialize_batch( - data: &FlightData, - fields: &[Field], - ipc_schema: &IpcSchema, - dictionaries: &read::Dictionaries, -) -> Result>> { - // check that the data_header is a record batch message - let message = arrow_format::ipc::MessageRef::read_as_root(&data.data_header) - .map_err(|err| Error::OutOfSpec(format!("Unable to get root as message: {err:?}")))?; - - let length = data.data_body.len(); - let mut reader = std::io::Cursor::new(&data.data_body); - - match message.header()?.ok_or_else(|| { - Error::oos("Unable to convert flight data header to a record batch".to_string()) - })? { - ipc::MessageHeaderRef::RecordBatch(batch) => read::read_record_batch( - batch, - fields, - ipc_schema, - None, - None, - dictionaries, - message.version()?, - &mut reader, - 0, - length as u64, - &mut Default::default(), - ), - _ => Err(Error::nyi( - "flight currently only supports reading RecordBatch messages", - )), - } -} - -/// Deserializes [`FlightData`], assuming it to be a dictionary message, into `dictionaries`. -pub fn deserialize_dictionary( - data: &FlightData, - fields: &[Field], - ipc_schema: &IpcSchema, - dictionaries: &mut read::Dictionaries, -) -> Result<()> { - let message = ipc::MessageRef::read_as_root(&data.data_header)?; - - let chunk = if let ipc::MessageHeaderRef::DictionaryBatch(chunk) = message - .header()? - .ok_or_else(|| Error::oos("Header is required"))? - { - chunk - } else { - return Ok(()); - }; - - let length = data.data_body.len(); - let mut reader = std::io::Cursor::new(&data.data_body); - read::read_dictionary( - chunk, - fields, - ipc_schema, - dictionaries, - &mut reader, - 0, - length as u64, - &mut Default::default(), - )?; - - Ok(()) -} - -/// Deserializes [`FlightData`] into either a [`Chunk`] (when the message is a record batch) -/// or by upserting into `dictionaries` (when the message is a dictionary) -pub fn deserialize_message( - data: &FlightData, - fields: &[Field], - ipc_schema: &IpcSchema, - dictionaries: &mut Dictionaries, -) -> Result>>> { - let FlightData { - data_header, - data_body, - .. - } = data; - - let message = arrow_format::ipc::MessageRef::read_as_root(data_header)?; - let header = message - .header()? - .ok_or_else(|| Error::oos("IPC Message must contain a header"))?; - - match header { - ipc::MessageHeaderRef::RecordBatch(batch) => { - let length = data_body.len(); - let mut reader = std::io::Cursor::new(data_body); - - let chunk = read::read_record_batch( - batch, - fields, - ipc_schema, - None, - None, - dictionaries, - arrow_format::ipc::MetadataVersion::V5, - &mut reader, - 0, - length as u64, - &mut Default::default(), - )?; - - Ok(chunk.into()) - } - ipc::MessageHeaderRef::DictionaryBatch(dict_batch) => { - let length = data_body.len(); - let mut reader = std::io::Cursor::new(data_body); - - read::read_dictionary( - dict_batch, - fields, - ipc_schema, - dictionaries, - &mut reader, - 0, - length as u64, - &mut Default::default(), - )?; - Ok(None) - } - t => Err(Error::nyi(format!( - "Reading types other than record batches not yet supported, unable to read {t:?}" - ))), - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/append/mod.rs b/src/common/arrow/src/arrow/io/ipc/append/mod.rs deleted file mode 100644 index 0883ad7e37ac..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/append/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! A struct adapter of Read+Seek+Write to append to IPC files -// read header and convert to writer information -// seek to first byte of header - 1 -// write new batch -// write new footer -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; -use std::io::Write; - -use super::endianness::is_native_little_endian; -use super::read::FileMetadata; -use super::read::{self}; -use super::write::common::DictionaryTracker; -use super::write::writer::*; -use super::write::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -impl FileWriter { - /// Creates a new [`FileWriter`] from an existing file, seeking to the last message - /// and appending new messages afterwards. Users call `finish` to write the footer (with both) - /// the existing and appended messages on it. - /// # Error - /// This function errors iff: - /// * the file's endianness is not the native endianness (not yet supported) - /// * the file is not a valid Arrow IPC file - pub fn try_from_file( - mut writer: R, - metadata: FileMetadata, - options: WriteOptions, - ) -> Result> { - if metadata.ipc_schema.is_little_endian != is_native_little_endian() { - return Err(Error::nyi( - "Appending to a file of a non-native endianness is still not supported", - )); - } - - let dictionaries = - read::read_file_dictionaries(&mut writer, &metadata, &mut Default::default())?; - - let last_block = metadata.blocks.last().ok_or_else(|| { - Error::oos("An Arrow IPC file must have at least 1 message (the schema message)") - })?; - let offset: u64 = last_block - .offset - .try_into() - .map_err(|_| Error::oos("The block's offset must be a positive number"))?; - let meta_data_length: u64 = last_block - .meta_data_length - .try_into() - .map_err(|_| Error::oos("The block's meta length must be a positive number"))?; - let body_length: u64 = last_block - .body_length - .try_into() - .map_err(|_| Error::oos("The block's body length must be a positive number"))?; - let offset: u64 = offset + meta_data_length + body_length; - - writer.seek(SeekFrom::Start(offset))?; - - Ok(FileWriter { - writer, - options, - schema: metadata.schema, - ipc_fields: metadata.ipc_schema.fields, - block_offsets: offset as usize, - dictionary_blocks: metadata.dictionaries.unwrap_or_default(), - record_blocks: metadata.blocks, - state: State::Started, // file already exists, so we are ready - dictionary_tracker: DictionaryTracker { - dictionaries, - cannot_replace: true, - }, - encoded_message: Default::default(), - }) - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/compression.rs b/src/common/arrow/src/arrow/io/ipc/compression.rs deleted file mode 100644 index df3fb30c65e5..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/compression.rs +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::error::Result; - -#[cfg(feature = "io_ipc_compression")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_ipc_compression")))] -pub fn decompress_lz4(input_buf: &[u8], output_buf: &mut [u8]) -> Result<()> { - use std::io::Read; - let mut decoder = lz4::Decoder::new(input_buf)?; - decoder.read_exact(output_buf).map_err(|e| e.into()) -} - -#[cfg(feature = "io_ipc_compression")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_ipc_compression")))] -pub fn decompress_zstd(input_buf: &[u8], output_buf: &mut [u8]) -> Result<()> { - use std::io::Read; - let mut decoder = zstd::Decoder::new(input_buf)?; - decoder.read_exact(output_buf).map_err(|e| e.into()) -} - -#[cfg(not(feature = "io_ipc_compression"))] -pub fn decompress_lz4(_input_buf: &[u8], _output_buf: &mut [u8]) -> Result<()> { - use crate::arrow::error::Error; - Err(Error::OutOfSpec("The crate was compiled without IPC compression. Use `io_ipc_compression` to read compressed IPC.".to_string())) -} - -#[cfg(not(feature = "io_ipc_compression"))] -pub fn decompress_zstd(_input_buf: &[u8], _output_buf: &mut [u8]) -> Result<()> { - use crate::arrow::error::Error; - Err(Error::OutOfSpec("The crate was compiled without IPC compression. Use `io_ipc_compression` to read compressed IPC.".to_string())) -} - -#[cfg(feature = "io_ipc_compression")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_ipc_compression")))] -pub fn compress_lz4(input_buf: &[u8], output_buf: &mut Vec) -> Result<()> { - use std::io::Write; - - use crate::arrow::error::Error; - let mut encoder = lz4::EncoderBuilder::new() - .build(output_buf) - .map_err(Error::from)?; - encoder.write_all(input_buf)?; - encoder.finish().1.map_err(|e| e.into()) -} - -#[cfg(feature = "io_ipc_compression")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_ipc_compression")))] -pub fn compress_zstd(input_buf: &[u8], output_buf: &mut Vec) -> Result<()> { - zstd::stream::copy_encode(input_buf, output_buf, 0).map_err(|e| e.into()) -} - -#[cfg(not(feature = "io_ipc_compression"))] -pub fn compress_lz4(_input_buf: &[u8], _output_buf: &[u8]) -> Result<()> { - use crate::arrow::error::Error; - Err(Error::OutOfSpec("The crate was compiled without IPC compression. Use `io_ipc_compression` to write compressed IPC.".to_string())) -} - -#[cfg(not(feature = "io_ipc_compression"))] -pub fn compress_zstd(_input_buf: &[u8], _output_buf: &[u8]) -> Result<()> { - use crate::arrow::error::Error; - Err(Error::OutOfSpec("The crate was compiled without IPC compression. Use `io_ipc_compression` to write compressed IPC.".to_string())) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[cfg(feature = "io_ipc_compression")] - #[test] - #[cfg_attr(miri, ignore)] // ZSTD uses foreign calls that miri does not support - fn round_trip_zstd() { - let data: Vec = (0..200u8).map(|x| x % 10).collect(); - let mut buffer = vec![]; - compress_zstd(&data, &mut buffer).unwrap(); - - let mut result = vec![0; 200]; - decompress_zstd(&buffer, &mut result).unwrap(); - assert_eq!(data, result); - } - - #[cfg(feature = "io_ipc_compression")] - #[test] - #[cfg_attr(miri, ignore)] // LZ4 uses foreign calls that miri does not support - fn round_trip_lz4() { - let data: Vec = (0..200u8).map(|x| x % 10).collect(); - let mut buffer = vec![]; - compress_lz4(&data, &mut buffer).unwrap(); - - let mut result = vec![0; 200]; - decompress_lz4(&buffer, &mut result).unwrap(); - assert_eq!(data, result); - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/endianness.rs b/src/common/arrow/src/arrow/io/ipc/endianness.rs deleted file mode 100644 index 75714492ca39..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/endianness.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(target_endian = "little")] -#[inline] -pub fn is_native_little_endian() -> bool { - true -} - -#[cfg(target_endian = "big")] -#[inline] -pub fn is_native_little_endian() -> bool { - false -} diff --git a/src/common/arrow/src/arrow/io/ipc/mod.rs b/src/common/arrow/src/arrow/io/ipc/mod.rs deleted file mode 100644 index ac296720c69c..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/mod.rs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to read from and write to Arrow's IPC format. -//! -//! Inter-process communication is a method through which different processes -//! share and pass data between them. Its use-cases include parallel -//! processing of chunks of data across different CPU cores, transferring -//! data between different Apache Arrow implementations in other languages and -//! more. Under the hood Apache Arrow uses [FlatBuffers](https://google.github.io/flatbuffers/) -//! as its binary protocol, so every Arrow-centered streaming or serialiation -//! problem that could be solved using FlatBuffers could probably be solved -//! using the more integrated approach that is exposed in this module. -//! -//! [Arrow's IPC protocol](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc) -//! allows only batch or dictionary columns to be passed -//! around due to its reliance on a pre-defined data scheme. This constraint -//! provides a large performance gain because serialized data will always have a -//! known structutre, i.e. the same fields and datatypes, with the only variance -//! being the number of rows and the actual data inside the Batch. This dramatically -//! increases the deserialization rate, as the bytes in the file or stream are already -//! structured "correctly". -//! -//! Reading and writing IPC messages is done using one of two variants - either -//! [`FileReader`](read::FileReader) <-> [`FileWriter`](struct@write::FileWriter) or -//! [`StreamReader`](read::StreamReader) <-> [`StreamWriter`](struct@write::StreamWriter). -//! These two variants wrap a type `T` that implements [`Read`](std::io::Read), and in -//! the case of the `File` variant it also implements [`Seek`](std::io::Seek). In -//! practice it means that `File`s can be arbitrarily accessed while `Stream`s are only -//! read in certain order - the one they were written in (first in, first out). -//! -//! # Examples -//! Read and write to a file: -//! ``` -//! use arrow2::io::ipc::read::read_file_metadata; -//! use arrow2::io::ipc::read::FileReader; -//! use arrow2::io::ipc::write::FileWriter; -//! use arrow2::io::ipc::write::WriteOptions; -//! # use std::fs::File; -//! # use arrow2::datatypes::{Field, Schema, DataType}; -//! # use arrow2::array::{Int32Array, Array}; -//! # use arrow2::chunk::Chunk; -//! # use arrow2::error::Error; -//! // Setup the writer -//! let path = "example.arrow".to_string(); -//! let mut file = File::create(&path)?; -//! let x_coord = Field::new("x", DataType::Int32, false); -//! let y_coord = Field::new("y", DataType::Int32, false); -//! let schema = Schema::from(vec![x_coord, y_coord]); -//! let options = WriteOptions { compression: None }; -//! let mut writer = FileWriter::try_new(file, schema, None, options)?; -//! -//! // Setup the data -//! let x_data = Int32Array::from_slice([-1i32, 1]); -//! let y_data = Int32Array::from_slice([1i32, -1]); -//! let chunk = Chunk::try_new(vec![x_data.boxed(), y_data.boxed()])?; -//! -//! // Write the messages and finalize the stream -//! for _ in 0..5 { -//! writer.write(&chunk, None); -//! } -//! writer.finish(); -//! -//! // Fetch some of the data and get the reader back -//! let mut reader = File::open(&path)?; -//! let metadata = read_file_metadata(&mut reader)?; -//! let mut reader = FileReader::new(reader, metadata, None, None); -//! let row1 = reader.next().unwrap(); // [[-1, 1], [1, -1]] -//! let row2 = reader.next().unwrap(); // [[-1, 1], [1, -1]] -//! let mut reader = reader.into_inner(); -//! // Do more stuff with the reader, like seeking ahead. -//! # Ok::<(), Error>(()) -//! ``` -//! -//! For further information and examples please consult the -//! [user guide](https://jorgecarleitao.github.io/arrow2/io/index.html). -//! For even more examples check the `examples` folder in the main repository -//! ([1](https://github.com/jorgecarleitao/arrow2/blob/main/examples/ipc_file_read.rs), -//! [2](https://github.com/jorgecarleitao/arrow2/blob/main/examples/ipc_file_write.rs), -//! [3](https://github.com/jorgecarleitao/arrow2/tree/main/examples/ipc_pyarrow)). - -mod compression; -mod endianness; - -pub mod append; -pub mod read; -pub mod write; - -const ARROW_MAGIC_V1: [u8; 4] = [b'F', b'E', b'A', b'1']; -const ARROW_MAGIC_V2: [u8; 6] = [b'A', b'R', b'R', b'O', b'W', b'1']; -pub(crate) const CONTINUATION_MARKER: [u8; 4] = [0xff; 4]; - -/// Struct containing `dictionary_id` and nested `IpcField`, allowing users -/// to specify the dictionary ids of the IPC fields when writing to IPC. -#[derive(Debug, Clone, PartialEq, Default)] -pub struct IpcField { - /// optional children - pub fields: Vec, - /// dictionary id - pub dictionary_id: Option, -} - -/// Struct containing fields and whether the file is written in little or big endian. -#[derive(Debug, Clone, PartialEq)] -pub struct IpcSchema { - /// The fields in the schema - pub fields: Vec, - /// Endianness of the file - pub is_little_endian: bool, -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/binary.rs b/src/common/arrow/src/arrow/io/ipc/read/array/binary.rs deleted file mode 100644 index 39b50eafe517..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/binary.rs +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use super::super::read_basic::*; -use super::super::Compression; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::OutOfSpecKind; -use crate::arrow::array::BinaryArray; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; - -#[allow(clippy::too_many_arguments)] -pub fn read_binary( - field_nodes: &mut VecDeque, - data_type: DataType, - buffers: &mut VecDeque, - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - scratch: &mut Vec, -) -> Result> { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let offsets: Buffer = read_buffer( - buffers, - 1 + length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - ) - // Older versions of the IPC format sometimes do not report an offset - .or_else(|_| Result::Ok(Buffer::::from(vec![O::default()])))?; - - let last_offset = offsets.last().unwrap().to_usize(); - let values = read_buffer( - buffers, - last_offset, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?; - - BinaryArray::::try_new(data_type, offsets.try_into()?, values, validity) -} - -pub fn skip_binary( - field_nodes: &mut VecDeque, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for binary. The file or stream is corrupted.") - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing offsets buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing values buffer."))?; - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/binview.rs b/src/common/arrow/src/arrow/io/ipc/read/array/binview.rs deleted file mode 100644 index c642dc88ec99..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/binview.rs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; -use std::sync::Arc; - -use crate::arrow::array::BinaryViewArrayGeneric; -use crate::arrow::array::View; -use crate::arrow::array::ViewType; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::read::read_basic::read_buffer; -use crate::arrow::io::ipc::read::read_basic::read_bytes; -use crate::arrow::io::ipc::read::read_basic::read_validity; -use crate::arrow::io::ipc::read::Compression; -use crate::arrow::io::ipc::read::IpcBuffer; -use crate::arrow::io::ipc::read::Node; -use crate::arrow::io::ipc::read::OutOfSpecKind; -use crate::ArrayRef; - -#[allow(clippy::too_many_arguments)] -pub fn read_binview( - field_nodes: &mut VecDeque, - variadic_buffer_counts: &mut VecDeque, - data_type: DataType, - buffers: &mut VecDeque, - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - scratch: &mut Vec, -) -> Result { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let views: Buffer = read_buffer( - buffers, - length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?; - - let n_variadic = variadic_buffer_counts.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the variadic buffers\n\nThe file or stream is corrupted.") - })?; - - let variadic_buffers = (0..n_variadic) - .map(|_| { - read_bytes( - buffers, - reader, - block_offset, - is_little_endian, - compression, - scratch, - ) - }) - .collect::>>>()?; - - BinaryViewArrayGeneric::::try_new(data_type, views, Arc::from(variadic_buffers), validity) - .map(|arr| arr.boxed()) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/boolean.rs b/src/common/arrow/src/arrow/io/ipc/read/array/boolean.rs deleted file mode 100644 index 8e42dee5480d..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/boolean.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use super::super::read_basic::*; -use super::super::Compression; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::OutOfSpecKind; -use crate::arrow::array::BooleanArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -#[allow(clippy::too_many_arguments)] -pub fn read_boolean( - field_nodes: &mut VecDeque, - data_type: DataType, - buffers: &mut VecDeque, - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - scratch: &mut Vec, -) -> Result { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let values = read_bitmap( - buffers, - length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?; - BooleanArray::try_new(data_type, values, validity) -} - -pub fn skip_boolean( - field_nodes: &mut VecDeque, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for boolean. The file or stream is corrupted.") - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing values buffer."))?; - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/dictionary.rs b/src/common/arrow/src/arrow/io/ipc/read/array/dictionary.rs deleted file mode 100644 index 45ebd160462b..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/dictionary.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashSet; -use std::collections::VecDeque; -use std::convert::TryInto; -use std::io::Read; -use std::io::Seek; - -use super::super::Compression; -use super::super::Dictionaries; -use super::super::IpcBuffer; -use super::super::Node; -use super::read_primitive; -use super::skip_primitive; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -#[allow(clippy::too_many_arguments)] -pub fn read_dictionary( - field_nodes: &mut VecDeque, - data_type: DataType, - id: Option, - buffers: &mut VecDeque, - reader: &mut R, - dictionaries: &Dictionaries, - block_offset: u64, - compression: Option, - limit: Option, - is_little_endian: bool, - scratch: &mut Vec, -) -> Result> -where - Vec: TryInto, -{ - let id = if let Some(id) = id { - id - } else { - return Err(Error::OutOfSpec("Dictionary has no id.".to_string())); - }; - let values = dictionaries - .get(&id) - .ok_or_else(|| { - let valid_ids = dictionaries.keys().collect::>(); - Error::OutOfSpec(format!( - "Dictionary id {id} not found. Valid ids: {valid_ids:?}" - )) - })? - .clone(); - - let keys = read_primitive( - field_nodes, - T::PRIMITIVE.into(), - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - DictionaryArray::::try_new(data_type, keys, values) -} - -pub fn skip_dictionary( - field_nodes: &mut VecDeque, - buffers: &mut VecDeque, -) -> Result<()> { - skip_primitive(field_nodes, buffers) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/fixed_size_binary.rs b/src/common/arrow/src/arrow/io/ipc/read/array/fixed_size_binary.rs deleted file mode 100644 index 0e205a8bc30d..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/fixed_size_binary.rs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use super::super::read_basic::*; -use super::super::Compression; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::OutOfSpecKind; -use crate::arrow::array::FixedSizeBinaryArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -#[allow(clippy::too_many_arguments)] -pub fn read_fixed_size_binary( - field_nodes: &mut VecDeque, - data_type: DataType, - buffers: &mut VecDeque, - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - scratch: &mut Vec, -) -> Result { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let length = length.saturating_mul(FixedSizeBinaryArray::maybe_get_size(&data_type)?); - let values = read_buffer( - buffers, - length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?; - - FixedSizeBinaryArray::try_new(data_type, values, validity) -} - -pub fn skip_fixed_size_binary( - field_nodes: &mut VecDeque, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos( - "IPC: unable to fetch the field for fixed-size binary. The file or stream is corrupted.", - ) - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing values buffer."))?; - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/fixed_size_list.rs b/src/common/arrow/src/arrow/io/ipc/read/array/fixed_size_list.rs deleted file mode 100644 index 3650f6a811c3..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/fixed_size_list.rs +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use super::super::super::IpcField; -use super::super::deserialize::read; -use super::super::deserialize::skip; -use super::super::read_basic::*; -use super::super::Compression; -use super::super::Dictionaries; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::Version; -use crate::arrow::array::FixedSizeListArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -#[allow(clippy::too_many_arguments)] -pub fn read_fixed_size_list( - field_nodes: &mut VecDeque, - variadic_buffer_counts: &mut VecDeque, - data_type: DataType, - ipc_field: &IpcField, - buffers: &mut VecDeque, - reader: &mut R, - dictionaries: &Dictionaries, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - version: Version, - scratch: &mut Vec, -) -> Result { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let (field, size) = FixedSizeListArray::get_child_and_size(&data_type); - - let limit = limit.map(|x| x.saturating_mul(size)); - - let values = read( - field_nodes, - variadic_buffer_counts, - field, - &ipc_field.fields[0], - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - limit, - version, - scratch, - )?; - FixedSizeListArray::try_new(data_type, values, validity) -} - -pub fn skip_fixed_size_list( - field_nodes: &mut VecDeque, - data_type: &DataType, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos( - "IPC: unable to fetch the field for fixed-size list. The file or stream is corrupted.", - ) - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - - let (field, _) = FixedSizeListArray::get_child_and_size(data_type); - - skip(field_nodes, field.data_type(), buffers) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/list.rs b/src/common/arrow/src/arrow/io/ipc/read/array/list.rs deleted file mode 100644 index d3a931cf7df2..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/list.rs +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::convert::TryInto; -use std::io::Read; -use std::io::Seek; - -use super::super::super::IpcField; -use super::super::deserialize::read; -use super::super::deserialize::skip; -use super::super::read_basic::*; -use super::super::Compression; -use super::super::Dictionaries; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::OutOfSpecKind; -use super::super::Version; -use crate::arrow::array::ListArray; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; - -#[allow(clippy::too_many_arguments)] -pub fn read_list( - field_nodes: &mut VecDeque, - variadic_buffer_counts: &mut VecDeque, - data_type: DataType, - ipc_field: &IpcField, - buffers: &mut VecDeque, - reader: &mut R, - dictionaries: &Dictionaries, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - version: Version, - scratch: &mut Vec, -) -> Result> -where - Vec: TryInto, -{ - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let offsets = read_buffer::( - buffers, - 1 + length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - ) - // Older versions of the IPC format sometimes do not report an offset - .or_else(|_| Result::Ok(Buffer::::from(vec![O::default()])))?; - - let last_offset = offsets.last().unwrap().to_usize(); - - let field = ListArray::::get_child_field(&data_type); - - let values = read( - field_nodes, - variadic_buffer_counts, - field, - &ipc_field.fields[0], - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - Some(last_offset), - version, - scratch, - )?; - ListArray::try_new(data_type, offsets.try_into()?, values, validity) -} - -pub fn skip_list( - field_nodes: &mut VecDeque, - data_type: &DataType, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for list. The file or stream is corrupted.") - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing offsets buffer."))?; - - let data_type = ListArray::::get_child_type(data_type); - - skip(field_nodes, data_type, buffers) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/map.rs b/src/common/arrow/src/arrow/io/ipc/read/array/map.rs deleted file mode 100644 index 9c9576455a87..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/map.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use super::super::super::IpcField; -use super::super::deserialize::read; -use super::super::deserialize::skip; -use super::super::read_basic::*; -use super::super::Compression; -use super::super::Dictionaries; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::OutOfSpecKind; -use super::super::Version; -use crate::arrow::array::MapArray; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -#[allow(clippy::too_many_arguments)] -pub fn read_map( - field_nodes: &mut VecDeque, - variadic_buffer_counts: &mut VecDeque, - data_type: DataType, - ipc_field: &IpcField, - buffers: &mut VecDeque, - reader: &mut R, - dictionaries: &Dictionaries, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - version: Version, - scratch: &mut Vec, -) -> Result { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let offsets = read_buffer::( - buffers, - 1 + length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - ) - // Older versions of the IPC format sometimes do not report an offset - .or_else(|_| Result::Ok(Buffer::::from(vec![0i32])))?; - - let field = MapArray::get_field(&data_type); - - let last_offset: usize = offsets.last().copied().unwrap() as usize; - - let field = read( - field_nodes, - variadic_buffer_counts, - field, - &ipc_field.fields[0], - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - Some(last_offset), - version, - scratch, - )?; - MapArray::try_new(data_type, offsets.try_into()?, field, validity) -} - -pub fn skip_map( - field_nodes: &mut VecDeque, - data_type: &DataType, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for map. The file or stream is corrupted.") - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing offsets buffer."))?; - - let data_type = MapArray::get_field(data_type).data_type(); - - skip(field_nodes, data_type, buffers) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/mod.rs b/src/common/arrow/src/arrow/io/ipc/read/array/mod.rs deleted file mode 100644 index fbef1718964b..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/mod.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod primitive; -pub use primitive::*; -mod boolean; -pub use boolean::*; -mod utf8; -pub use utf8::*; -mod binary; -pub use binary::*; -mod fixed_size_binary; -pub use fixed_size_binary::*; -mod list; -pub use list::*; -mod fixed_size_list; -pub use fixed_size_list::*; -mod struct_; -pub use struct_::*; -mod null; -pub use null::*; -mod dictionary; -pub use dictionary::*; -mod union; -pub use union::*; -mod binview; -pub use binview::*; -mod map; - -pub use map::*; diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/null.rs b/src/common/arrow/src/arrow/io/ipc/read/array/null.rs deleted file mode 100644 index bea4d76e1341..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/null.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use super::super::Node; -use super::super::OutOfSpecKind; -use crate::arrow::array::NullArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -pub fn read_null(field_nodes: &mut VecDeque, data_type: DataType) -> Result { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - NullArray::try_new(data_type, length) -} - -pub fn skip_null(field_nodes: &mut VecDeque) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for null. The file or stream is corrupted.") - })?; - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/primitive.rs b/src/common/arrow/src/arrow/io/ipc/read/array/primitive.rs deleted file mode 100644 index 88dce1f6ab93..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/primitive.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::convert::TryInto; -use std::io::Read; -use std::io::Seek; - -use super::super::read_basic::*; -use super::super::Compression; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::OutOfSpecKind; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; - -#[allow(clippy::too_many_arguments)] -pub fn read_primitive( - field_nodes: &mut VecDeque, - data_type: DataType, - buffers: &mut VecDeque, - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - scratch: &mut Vec, -) -> Result> -where - Vec: TryInto, -{ - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let values = read_buffer( - buffers, - length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?; - PrimitiveArray::::try_new(data_type, values, validity) -} - -pub fn skip_primitive( - field_nodes: &mut VecDeque, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for primitive. The file or stream is corrupted.") - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing values buffer."))?; - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/struct_.rs b/src/common/arrow/src/arrow/io/ipc/read/array/struct_.rs deleted file mode 100644 index d4e7fcf702a2..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/struct_.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use super::super::super::IpcField; -use super::super::deserialize::read; -use super::super::deserialize::skip; -use super::super::read_basic::*; -use super::super::Compression; -use super::super::Dictionaries; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::Version; -use crate::arrow::array::StructArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -#[allow(clippy::too_many_arguments)] -pub fn read_struct( - field_nodes: &mut VecDeque, - variadic_buffer_counts: &mut VecDeque, - data_type: DataType, - ipc_field: &IpcField, - buffers: &mut VecDeque, - reader: &mut R, - dictionaries: &Dictionaries, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - version: Version, - scratch: &mut Vec, -) -> Result { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let fields = StructArray::get_fields(&data_type); - - let values = fields - .iter() - .zip(ipc_field.fields.iter()) - .map(|(field, ipc_field)| { - read( - field_nodes, - variadic_buffer_counts, - field, - ipc_field, - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - limit, - version, - scratch, - ) - }) - .collect::>>()?; - - StructArray::try_new(data_type, values, validity) -} - -pub fn skip_struct( - field_nodes: &mut VecDeque, - data_type: &DataType, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for struct. The file or stream is corrupted.") - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - - let fields = StructArray::get_fields(data_type); - - fields - .iter() - .try_for_each(|field| skip(field_nodes, field.data_type(), buffers)) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/union.rs b/src/common/arrow/src/arrow/io/ipc/read/array/union.rs deleted file mode 100644 index f4e0ea8ea173..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/union.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use super::super::super::IpcField; -use super::super::deserialize::read; -use super::super::deserialize::skip; -use super::super::read_basic::*; -use super::super::Compression; -use super::super::Dictionaries; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::OutOfSpecKind; -use super::super::Version; -use crate::arrow::array::UnionArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::UnionMode::Dense; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -#[allow(clippy::too_many_arguments)] -pub fn read_union( - field_nodes: &mut VecDeque, - variadic_buffer_counts: &mut VecDeque, - data_type: DataType, - ipc_field: &IpcField, - buffers: &mut VecDeque, - reader: &mut R, - dictionaries: &Dictionaries, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - version: Version, - scratch: &mut Vec, -) -> Result { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - if version != Version::V5 { - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - }; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let types = read_buffer( - buffers, - length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?; - - let offsets = if let DataType::Union(_, _, mode) = data_type { - if !mode.is_sparse() { - Some(read_buffer( - buffers, - length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?) - } else { - None - } - } else { - unreachable!() - }; - - let fields = UnionArray::get_fields(&data_type); - - let fields = fields - .iter() - .zip(ipc_field.fields.iter()) - .map(|(field, ipc_field)| { - read( - field_nodes, - variadic_buffer_counts, - field, - ipc_field, - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - None, - version, - scratch, - ) - }) - .collect::>>()?; - - UnionArray::try_new(data_type, types, fields, offsets) -} - -pub fn skip_union( - field_nodes: &mut VecDeque, - data_type: &DataType, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for struct. The file or stream is corrupted.") - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - if let DataType::Union(_, _, Dense) = data_type { - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing offsets buffer."))?; - } else { - unreachable!() - }; - - let fields = UnionArray::get_fields(data_type); - - fields - .iter() - .try_for_each(|field| skip(field_nodes, field.data_type(), buffers)) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/array/utf8.rs b/src/common/arrow/src/arrow/io/ipc/read/array/utf8.rs deleted file mode 100644 index 580af874700b..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/array/utf8.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use super::super::read_basic::*; -use super::super::Compression; -use super::super::IpcBuffer; -use super::super::Node; -use super::super::OutOfSpecKind; -use crate::arrow::array::Utf8Array; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; - -#[allow(clippy::too_many_arguments)] -pub fn read_utf8( - field_nodes: &mut VecDeque, - data_type: DataType, - buffers: &mut VecDeque, - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - scratch: &mut Vec, -) -> Result> { - let field_node = field_nodes.pop_front().ok_or_else(|| { - Error::oos(format!( - "IPC: unable to fetch the field for {data_type:?}. The file or stream is corrupted." - )) - })?; - - let validity = read_validity( - buffers, - field_node, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - )?; - - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - let offsets: Buffer = read_buffer( - buffers, - 1 + length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - ) - // Older versions of the IPC format sometimes do not report an offset - .or_else(|_| Result::Ok(Buffer::::from(vec![O::default()])))?; - - let last_offset = offsets.last().unwrap().to_usize(); - let values = read_buffer( - buffers, - last_offset, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?; - - Utf8Array::::try_new(data_type, offsets.try_into()?, values, validity) -} - -pub fn skip_utf8( - field_nodes: &mut VecDeque, - buffers: &mut VecDeque, -) -> Result<()> { - let _ = field_nodes.pop_front().ok_or_else(|| { - Error::oos("IPC: unable to fetch the field for utf8. The file or stream is corrupted.") - })?; - - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing validity buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing offsets buffer."))?; - let _ = buffers - .pop_front() - .ok_or_else(|| Error::oos("IPC: missing values buffer."))?; - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/common.rs b/src/common/arrow/src/arrow/io/ipc/read/common.rs deleted file mode 100644 index 54fa59ca96c5..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/common.rs +++ /dev/null @@ -1,388 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use ahash::AHashMap; -use arrow_format; - -use super::deserialize::read; -use super::deserialize::skip; -use super::Dictionaries; -use crate::arrow::array::*; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::read::OutOfSpecKind; -use crate::arrow::io::ipc::IpcField; -use crate::arrow::io::ipc::IpcSchema; - -#[derive(Debug, Eq, PartialEq, Hash)] -enum ProjectionResult { - Selected(A), - NotSelected(A), -} - -/// An iterator adapter that will return `Some(x)` or `None` -/// # Panics -/// The iterator panics iff the `projection` is not strictly increasing. -struct ProjectionIter<'a, A, I: Iterator> { - projection: &'a [usize], - iter: I, - current_count: usize, - current_projection: usize, -} - -impl<'a, A, I: Iterator> ProjectionIter<'a, A, I> { - /// # Panics - /// iff `projection` is empty - pub fn new(projection: &'a [usize], iter: I) -> Self { - assert!(!projection.is_empty(), "projection cannot be empty"); - Self { - projection: &projection[1..], - iter, - current_count: 0, - current_projection: projection[0], - } - } -} - -impl<'a, A, I: Iterator> Iterator for ProjectionIter<'a, A, I> { - type Item = ProjectionResult; - - fn next(&mut self) -> Option { - if let Some(item) = self.iter.next() { - let result = if self.current_count == self.current_projection { - if !self.projection.is_empty() { - assert!(self.projection[0] > self.current_projection); - self.current_projection = self.projection[0]; - self.projection = &self.projection[1..]; - } else { - self.current_projection = 0 // a value that most likely already passed - }; - Some(ProjectionResult::Selected(item)) - } else { - Some(ProjectionResult::NotSelected(item)) - }; - self.current_count += 1; - result - } else { - None - } - } - - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -/// Returns a [`Chunk`] from a reader. -/// # Panic -/// Panics iff the projection is not in increasing order (e.g. `[1, 0]` nor `[0, 1, 1]` are valid) -#[allow(clippy::too_many_arguments)] -pub fn read_record_batch( - batch: arrow_format::ipc::RecordBatchRef, - fields: &[Field], - ipc_schema: &IpcSchema, - projection: Option<&[usize]>, - limit: Option, - dictionaries: &Dictionaries, - version: arrow_format::ipc::MetadataVersion, - reader: &mut R, - block_offset: u64, - file_size: u64, - scratch: &mut Vec, -) -> Result>> { - assert_eq!(fields.len(), ipc_schema.fields.len()); - let buffers = batch - .buffers() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferBuffers(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingMessageBuffers))?; - let mut variadic_buffer_counts = batch - .variadic_buffer_counts() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferRecordBatches(err)))? - .map(|v| v.iter().map(|v| v as usize).collect::>()) - .unwrap_or_else(VecDeque::new); - let mut buffers: VecDeque = buffers.iter().collect(); - - // check that the sum of the sizes of all buffers is <= than the size of the file - let buffers_size = buffers - .iter() - .map(|buffer| { - let buffer_size: u64 = buffer - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - Ok(buffer_size) - }) - .sum::>()?; - if buffers_size > file_size { - return Err(Error::from(OutOfSpecKind::InvalidBuffersLength { - buffers_size, - file_size, - })); - } - - let field_nodes = batch - .nodes() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferNodes(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingMessageNodes))?; - let mut field_nodes = field_nodes.iter().collect::>(); - - let columns = if let Some(projection) = projection { - let projection = - ProjectionIter::new(projection, fields.iter().zip(ipc_schema.fields.iter())); - - projection - .map(|maybe_field| match maybe_field { - ProjectionResult::Selected((field, ipc_field)) => Ok(Some(read( - &mut field_nodes, - &mut variadic_buffer_counts, - field, - ipc_field, - &mut buffers, - reader, - dictionaries, - block_offset, - ipc_schema.is_little_endian, - batch.compression().map_err(|err| { - Error::from(OutOfSpecKind::InvalidFlatbufferCompression(err)) - })?, - limit, - version, - scratch, - )?)), - ProjectionResult::NotSelected((field, _)) => { - skip(&mut field_nodes, &field.data_type, &mut buffers)?; - Ok(None) - } - }) - .filter_map(|x| x.transpose()) - .collect::>>()? - } else { - fields - .iter() - .zip(ipc_schema.fields.iter()) - .map(|(field, ipc_field)| { - read( - &mut field_nodes, - &mut variadic_buffer_counts, - field, - ipc_field, - &mut buffers, - reader, - dictionaries, - block_offset, - ipc_schema.is_little_endian, - batch.compression().map_err(|err| { - Error::from(OutOfSpecKind::InvalidFlatbufferCompression(err)) - })?, - limit, - version, - scratch, - ) - }) - .collect::>>()? - }; - Chunk::try_new(columns) -} - -fn find_first_dict_field_d<'a>( - id: i64, - data_type: &'a DataType, - ipc_field: &'a IpcField, -) -> Option<(&'a Field, &'a IpcField)> { - use DataType::*; - match data_type { - Dictionary(_, inner, _) => find_first_dict_field_d(id, inner.as_ref(), ipc_field), - List(field) | LargeList(field) | FixedSizeList(field, ..) | Map(field, ..) => { - find_first_dict_field(id, field.as_ref(), &ipc_field.fields[0]) - } - Union(fields, ..) | Struct(fields) => { - for (field, ipc_field) in fields.iter().zip(ipc_field.fields.iter()) { - if let Some(f) = find_first_dict_field(id, field, ipc_field) { - return Some(f); - } - } - None - } - _ => None, - } -} - -fn find_first_dict_field<'a>( - id: i64, - field: &'a Field, - ipc_field: &'a IpcField, -) -> Option<(&'a Field, &'a IpcField)> { - if let Some(field_id) = ipc_field.dictionary_id { - if id == field_id { - return Some((field, ipc_field)); - } - } - find_first_dict_field_d(id, &field.data_type, ipc_field) -} - -pub(crate) fn first_dict_field<'a>( - id: i64, - fields: &'a [Field], - ipc_fields: &'a [IpcField], -) -> Result<(&'a Field, &'a IpcField)> { - assert_eq!(fields.len(), ipc_fields.len()); - for (field, ipc_field) in fields.iter().zip(ipc_fields.iter()) { - if let Some(field) = find_first_dict_field(id, field, ipc_field) { - return Ok(field); - } - } - Err(Error::from(OutOfSpecKind::InvalidId { requested_id: id })) -} - -/// Reads a dictionary from the reader, -/// updating `dictionaries` with the resulting dictionary -#[allow(clippy::too_many_arguments)] -pub fn read_dictionary( - batch: arrow_format::ipc::DictionaryBatchRef, - fields: &[Field], - ipc_schema: &IpcSchema, - dictionaries: &mut Dictionaries, - reader: &mut R, - block_offset: u64, - file_size: u64, - scratch: &mut Vec, -) -> Result<()> { - if batch - .is_delta() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferIsDelta(err)))? - { - return Err(Error::NotYetImplemented( - "delta dictionary batches not supported".to_string(), - )); - } - - let id = batch - .id() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferId(err)))?; - let (first_field, first_ipc_field) = first_dict_field(id, fields, &ipc_schema.fields)?; - - let batch = batch - .data() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferData(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingData))?; - - let value_type = - if let DataType::Dictionary(_, value_type, _) = first_field.data_type.to_logical_type() { - value_type.as_ref() - } else { - return Err(Error::from(OutOfSpecKind::InvalidIdDataType { - requested_id: id, - })); - }; - - // Make a fake schema for the dictionary batch. - let fields = vec![Field::new("", value_type.clone(), false)]; - let ipc_schema = IpcSchema { - fields: vec![first_ipc_field.clone()], - is_little_endian: ipc_schema.is_little_endian, - }; - let chunk = read_record_batch( - batch, - &fields, - &ipc_schema, - None, - None, // we must read the whole dictionary - dictionaries, - arrow_format::ipc::MetadataVersion::V5, - reader, - block_offset, - file_size, - scratch, - )?; - - dictionaries.insert(id, chunk.into_arrays().pop().unwrap()); - - Ok(()) -} - -pub fn prepare_projection( - fields: &[Field], - mut projection: Vec, -) -> (Vec, AHashMap, Vec) { - let fields = projection.iter().map(|x| fields[*x].clone()).collect(); - - // todo: find way to do this more efficiently - let mut indices = (0..projection.len()).collect::>(); - indices.sort_unstable_by_key(|&i| &projection[i]); - let map = indices.iter().copied().enumerate().fold( - AHashMap::default(), - |mut acc, (index, new_index)| { - acc.insert(index, new_index); - acc - }, - ); - projection.sort_unstable(); - - // check unique - if !projection.is_empty() { - let mut previous = projection[0]; - - for &i in &projection[1..] { - assert!( - previous < i, - "The projection on IPC must not contain duplicates" - ); - previous = i; - } - } - - (projection, map, fields) -} - -pub fn apply_projection( - chunk: Chunk>, - map: &AHashMap, -) -> Chunk> { - // re-order according to projection - let arrays = chunk.into_arrays(); - let mut new_arrays = arrays.clone(); - - map.iter() - .for_each(|(old, new)| new_arrays[*new] = arrays[*old].clone()); - - Chunk::new(new_arrays) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn project_iter() { - let iter = 1..6; - let iter = ProjectionIter::new(&[0, 2, 4], iter); - let result: Vec<_> = iter.collect(); - use ProjectionResult::*; - assert_eq!(result, vec![ - Selected(1), - NotSelected(2), - Selected(3), - NotSelected(4), - Selected(5) - ]) - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/deserialize.rs b/src/common/arrow/src/arrow/io/ipc/read/deserialize.rs deleted file mode 100644 index 18038c0b9d75..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/deserialize.rs +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -use arrow_format::ipc::BodyCompressionRef; -use arrow_format::ipc::MetadataVersion; - -use super::array::*; -use super::Dictionaries; -use super::IpcBuffer; -use super::Node; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::IpcField; - -#[allow(clippy::too_many_arguments)] -pub fn read( - field_nodes: &mut VecDeque, - variadic_buffer_counts: &mut VecDeque, - field: &Field, - ipc_field: &IpcField, - buffers: &mut VecDeque, - reader: &mut R, - dictionaries: &Dictionaries, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - version: MetadataVersion, - scratch: &mut Vec, -) -> Result> { - use PhysicalType::*; - let data_type = field.data_type.clone(); - - match data_type.to_physical_type() { - Null => read_null(field_nodes, data_type).map(|x| x.boxed()), - Boolean => read_boolean( - field_nodes, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ) - .map(|x| x.boxed()), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - read_primitive::<$T, _>( - field_nodes, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ) - .map(|x| x.boxed()) - }), - Binary => read_binary::( - field_nodes, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ) - .map(|x| x.boxed()), - LargeBinary => read_binary::( - field_nodes, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ) - .map(|x| x.boxed()), - FixedSizeBinary => read_fixed_size_binary( - field_nodes, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ) - .map(|x| x.boxed()), - Utf8 => read_utf8::( - field_nodes, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ) - .map(|x| x.boxed()), - LargeUtf8 => read_utf8::( - field_nodes, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ) - .map(|x| x.boxed()), - List => read_list::( - field_nodes, - variadic_buffer_counts, - data_type, - ipc_field, - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - limit, - version, - scratch, - ) - .map(|x| x.boxed()), - LargeList => read_list::( - field_nodes, - variadic_buffer_counts, - data_type, - ipc_field, - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - limit, - version, - scratch, - ) - .map(|x| x.boxed()), - FixedSizeList => read_fixed_size_list( - field_nodes, - variadic_buffer_counts, - data_type, - ipc_field, - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - limit, - version, - scratch, - ) - .map(|x| x.boxed()), - Struct => read_struct( - field_nodes, - variadic_buffer_counts, - data_type, - ipc_field, - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - limit, - version, - scratch, - ) - .map(|x| x.boxed()), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - read_dictionary::<$T, _>( - field_nodes, - data_type, - ipc_field.dictionary_id, - buffers, - reader, - dictionaries, - block_offset, - compression, - limit, - is_little_endian, - scratch, - ) - .map(|x| x.boxed()) - }) - } - Union => read_union( - field_nodes, - variadic_buffer_counts, - data_type, - ipc_field, - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - limit, - version, - scratch, - ) - .map(|x| x.boxed()), - Map => read_map( - field_nodes, - variadic_buffer_counts, - data_type, - ipc_field, - buffers, - reader, - dictionaries, - block_offset, - is_little_endian, - compression, - limit, - version, - scratch, - ) - .map(|x| x.boxed()), - Utf8View => read_binview::( - field_nodes, - variadic_buffer_counts, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ), - BinaryView => read_binview::<[u8], _>( - field_nodes, - variadic_buffer_counts, - data_type, - buffers, - reader, - block_offset, - is_little_endian, - compression, - limit, - scratch, - ), - } -} - -pub fn skip( - field_nodes: &mut VecDeque, - data_type: &DataType, - buffers: &mut VecDeque, -) -> Result<()> { - use PhysicalType::*; - match data_type.to_physical_type() { - Null => skip_null(field_nodes), - Boolean => skip_boolean(field_nodes, buffers), - Primitive(_) => skip_primitive(field_nodes, buffers), - LargeBinary | Binary => skip_binary(field_nodes, buffers), - LargeUtf8 | Utf8 => skip_utf8(field_nodes, buffers), - FixedSizeBinary => skip_fixed_size_binary(field_nodes, buffers), - List => skip_list::(field_nodes, data_type, buffers), - LargeList => skip_list::(field_nodes, data_type, buffers), - FixedSizeList => skip_fixed_size_list(field_nodes, data_type, buffers), - Struct => skip_struct(field_nodes, data_type, buffers), - Dictionary(_) => skip_dictionary(field_nodes, buffers), - Union => skip_union(field_nodes, data_type, buffers), - Map => skip_map(field_nodes, data_type, buffers), - BinaryView | Utf8View => todo!(), - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/error.rs b/src/common/arrow/src/arrow/io/ipc/read/error.rs deleted file mode 100644 index e910de1021d2..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/error.rs +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::error::Error; - -/// The different types of errors that reading from IPC can cause -#[derive(Debug)] -#[non_exhaustive] -pub enum OutOfSpecKind { - /// The IPC file does not start with [b'A', b'R', b'R', b'O', b'W', b'1'] - InvalidHeader, - /// The IPC file does not end with [b'A', b'R', b'R', b'O', b'W', b'1'] - InvalidFooter, - /// The first 4 bytes of the last 10 bytes is < 0 - NegativeFooterLength, - /// The footer is an invalid flatbuffer - InvalidFlatbufferFooter(arrow_format::ipc::planus::Error), - /// The file's footer does not contain record batches - MissingRecordBatches, - /// The footer's record batches is an invalid flatbuffer - InvalidFlatbufferRecordBatches(arrow_format::ipc::planus::Error), - /// The file's footer does not contain a schema - MissingSchema, - /// The footer's schema is an invalid flatbuffer - InvalidFlatbufferSchema(arrow_format::ipc::planus::Error), - /// The file's schema does not contain fields - MissingFields, - /// The footer's dictionaries is an invalid flatbuffer - InvalidFlatbufferDictionaries(arrow_format::ipc::planus::Error), - /// The block is an invalid flatbuffer - InvalidFlatbufferBlock(arrow_format::ipc::planus::Error), - /// The dictionary message is an invalid flatbuffer - InvalidFlatbufferMessage(arrow_format::ipc::planus::Error), - /// The message does not contain a header - MissingMessageHeader, - /// The message's header is an invalid flatbuffer - InvalidFlatbufferHeader(arrow_format::ipc::planus::Error), - /// Relative positions in the file is < 0 - UnexpectedNegativeInteger, - /// dictionaries can only contain dictionary messages; record batches can only contain records - UnexpectedMessageType, - /// RecordBatch messages do not contain buffers - MissingMessageBuffers, - /// The message's buffers is an invalid flatbuffer - InvalidFlatbufferBuffers(arrow_format::ipc::planus::Error), - /// RecordBatch messages does not contain nodes - MissingMessageNodes, - /// The message's nodes is an invalid flatbuffer - InvalidFlatbufferNodes(arrow_format::ipc::planus::Error), - /// The message's body length is an invalid flatbuffer - InvalidFlatbufferBodyLength(arrow_format::ipc::planus::Error), - /// The message does not contain data - MissingData, - /// The message's data is an invalid flatbuffer - InvalidFlatbufferData(arrow_format::ipc::planus::Error), - /// The version is an invalid flatbuffer - InvalidFlatbufferVersion(arrow_format::ipc::planus::Error), - /// The compression is an invalid flatbuffer - InvalidFlatbufferCompression(arrow_format::ipc::planus::Error), - /// The record contains a number of buffers that does not match the required number by the data type - ExpectedBuffer, - /// A buffer's size is smaller than the required for the number of elements - InvalidBuffer { - /// Declared number of elements in the buffer - length: usize, - /// The name of the `NativeType` - type_name: &'static str, - /// Bytes required for the `length` and `type` - required_number_of_bytes: usize, - /// The size of the IPC buffer - buffer_length: usize, - }, - /// A buffer's size is larger than the file size - InvalidBuffersLength { - /// number of bytes of all buffers in the record - buffers_size: u64, - /// the size of the file - file_size: u64, - }, - /// A bitmap's size is smaller than the required for the number of elements - InvalidBitmap { - /// Declared length of the bitmap - length: usize, - /// Number of bits on the IPC buffer - number_of_bits: usize, - }, - /// The dictionary is_delta is an invalid flatbuffer - InvalidFlatbufferIsDelta(arrow_format::ipc::planus::Error), - /// The dictionary id is an invalid flatbuffer - InvalidFlatbufferId(arrow_format::ipc::planus::Error), - /// Invalid dictionary id - InvalidId { - /// The requested dictionary id - requested_id: i64, - }, - /// Field id is not a dictionary - InvalidIdDataType { - /// The requested dictionary id - requested_id: i64, - }, - /// FixedSizeBinaryArray has invalid datatype. - InvalidDataType, -} - -impl From for Error { - fn from(kind: OutOfSpecKind) -> Self { - Error::OutOfSpec(format!("{kind:?}")) - } -} - -impl From for Error { - fn from(error: arrow_format::ipc::planus::Error) -> Self { - Error::OutOfSpec(error.to_string()) - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/file.rs b/src/common/arrow/src/arrow/io/ipc/read/file.rs deleted file mode 100644 index c0b470091743..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/file.rs +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::convert::TryInto; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; - -use ahash::AHashMap; -use arrow_format::ipc::planus::ReadAsRoot; - -use super::super::ARROW_MAGIC_V1; -use super::super::ARROW_MAGIC_V2; -use super::super::CONTINUATION_MARKER; -use super::common::*; -use super::schema::fb_to_schema; -use super::Dictionaries; -use super::OutOfSpecKind; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::IpcSchema; - -/// Metadata of an Arrow IPC file, written in the footer of the file. -#[derive(Debug, Clone)] -pub struct FileMetadata { - /// The schema that is read from the file footer - pub schema: Schema, - - /// The files' [`IpcSchema`] - pub ipc_schema: IpcSchema, - - /// The blocks in the file - /// - /// A block indicates the regions in the file to read to get data - pub blocks: Vec, - - /// Dictionaries associated to each dict_id - pub(crate) dictionaries: Option>, - - /// The total size of the file in bytes - pub size: u64, -} - -fn read_dictionary_message( - reader: &mut R, - offset: u64, - data: &mut Vec, -) -> Result<()> { - let mut message_size: [u8; 4] = [0; 4]; - reader.seek(SeekFrom::Start(offset))?; - reader.read_exact(&mut message_size)?; - if message_size == CONTINUATION_MARKER { - reader.read_exact(&mut message_size)?; - }; - let message_length = i32::from_le_bytes(message_size); - - let message_length: usize = message_length - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - data.clear(); - data.try_reserve(message_length)?; - reader - .by_ref() - .take(message_length as u64) - .read_to_end(data)?; - - Ok(()) -} - -pub(crate) fn get_dictionary_batch<'a>( - message: &'a arrow_format::ipc::MessageRef, -) -> Result> { - let header = message - .header() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferHeader(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingMessageHeader))?; - match header { - arrow_format::ipc::MessageHeaderRef::DictionaryBatch(batch) => Ok(batch), - _ => Err(Error::from(OutOfSpecKind::UnexpectedMessageType)), - } -} - -fn read_dictionary_block( - reader: &mut R, - metadata: &FileMetadata, - block: &arrow_format::ipc::Block, - dictionaries: &mut Dictionaries, - message_scratch: &mut Vec, - dictionary_scratch: &mut Vec, -) -> Result<()> { - let offset: u64 = block - .offset - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::UnexpectedNegativeInteger))?; - let length: u64 = block - .meta_data_length - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::UnexpectedNegativeInteger))?; - read_dictionary_message(reader, offset, message_scratch)?; - - let message = arrow_format::ipc::MessageRef::read_as_root(message_scratch.as_ref()) - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferMessage(err)))?; - - let batch = get_dictionary_batch(&message)?; - - read_dictionary( - batch, - &metadata.schema.fields, - &metadata.ipc_schema, - dictionaries, - reader, - offset + length, - metadata.size, - dictionary_scratch, - ) -} - -/// Reads all file's dictionaries, if any -/// This function is IO-bounded -pub fn read_file_dictionaries( - reader: &mut R, - metadata: &FileMetadata, - scratch: &mut Vec, -) -> Result { - let mut dictionaries = Default::default(); - - let blocks = if let Some(blocks) = &metadata.dictionaries { - blocks - } else { - return Ok(AHashMap::new()); - }; - // use a temporary smaller scratch for the messages - let mut message_scratch = Default::default(); - - for block in blocks { - read_dictionary_block( - reader, - metadata, - block, - &mut dictionaries, - &mut message_scratch, - scratch, - )?; - } - Ok(dictionaries) -} - -/// Reads the footer's length and magic number in footer -fn read_footer_len(reader: &mut R) -> Result<(u64, usize)> { - // read footer length and magic number in footer - let end = reader.seek(SeekFrom::End(-10))? + 10; - - let mut footer: [u8; 10] = [0; 10]; - - reader.read_exact(&mut footer)?; - let footer_len = i32::from_le_bytes(footer[..4].try_into().unwrap()); - - if footer[4..] != ARROW_MAGIC_V2 { - return Err(Error::from(OutOfSpecKind::InvalidFooter)); - } - let footer_len = footer_len - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - Ok((end, footer_len)) -} - -pub(super) fn deserialize_footer(footer_data: &[u8], size: u64) -> Result { - let footer = arrow_format::ipc::FooterRef::read_as_root(footer_data) - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferFooter(err)))?; - - let blocks = footer - .record_batches() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferRecordBatches(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingRecordBatches))?; - - let blocks = blocks.iter().map(|block| block.into()).collect::>(); - - let ipc_schema = footer - .schema() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferSchema(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingSchema))?; - let (schema, ipc_schema) = fb_to_schema(ipc_schema)?; - - let dictionaries = footer - .dictionaries() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferDictionaries(err)))? - .map(|dictionaries| { - dictionaries - .into_iter() - .map(|block| block.into()) - .collect::>() - }); - - Ok(FileMetadata { - schema, - ipc_schema, - blocks, - dictionaries, - size, - }) -} - -/// Read the Arrow IPC file's metadata -pub fn read_file_metadata(reader: &mut R) -> Result { - // check if header contain the correct magic bytes - let mut magic_buffer: [u8; 6] = [0; 6]; - let start = reader.stream_position()?; - reader.read_exact(&mut magic_buffer)?; - if magic_buffer != ARROW_MAGIC_V2 { - if magic_buffer[..4] == ARROW_MAGIC_V1 { - return Err(Error::NotYetImplemented("feather v1 not supported".into())); - } - return Err(Error::from(OutOfSpecKind::InvalidHeader)); - } - - let (end, footer_len) = read_footer_len(reader)?; - - // read footer - reader.seek(SeekFrom::End(-10 - footer_len as i64))?; - - let mut serialized_footer = vec![]; - serialized_footer.try_reserve(footer_len)?; - reader - .by_ref() - .take(footer_len as u64) - .read_to_end(&mut serialized_footer)?; - - deserialize_footer(&serialized_footer, end - start) -} - -pub(crate) fn get_record_batch( - message: arrow_format::ipc::MessageRef, -) -> Result { - let header = message - .header() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferHeader(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingMessageHeader))?; - match header { - arrow_format::ipc::MessageHeaderRef::RecordBatch(batch) => Ok(batch), - _ => Err(Error::from(OutOfSpecKind::UnexpectedMessageType)), - } -} - -/// Reads the record batch at position `index` from the reader. -/// -/// This function is useful for random access to the file. For example, if -/// you have indexed the file somewhere else, this allows pruning -/// certain parts of the file. -/// # Panics -/// This function panics iff `index >= metadata.blocks.len()` -#[allow(clippy::too_many_arguments)] -pub fn read_batch( - reader: &mut R, - dictionaries: &Dictionaries, - metadata: &FileMetadata, - projection: Option<&[usize]>, - limit: Option, - index: usize, - message_scratch: &mut Vec, - data_scratch: &mut Vec, -) -> Result>> { - let block = metadata.blocks[index]; - - let offset: u64 = block - .offset - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - let length: u64 = block - .meta_data_length - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - // read length - reader.seek(SeekFrom::Start(offset))?; - let mut meta_buf = [0; 4]; - reader.read_exact(&mut meta_buf)?; - if meta_buf == CONTINUATION_MARKER { - // continuation marker encountered, read message next - reader.read_exact(&mut meta_buf)?; - } - let meta_len = i32::from_le_bytes(meta_buf) - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::UnexpectedNegativeInteger))?; - - message_scratch.clear(); - message_scratch.try_reserve(meta_len)?; - reader - .by_ref() - .take(meta_len as u64) - .read_to_end(message_scratch)?; - - let message = arrow_format::ipc::MessageRef::read_as_root(message_scratch.as_ref()) - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferMessage(err)))?; - - let batch = get_record_batch(message)?; - - read_record_batch( - batch, - &metadata.schema.fields, - &metadata.ipc_schema, - projection, - limit, - dictionaries, - message - .version() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferVersion(err)))?, - reader, - offset + length, - metadata.size, - data_scratch, - ) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/file_async.rs b/src/common/arrow/src/arrow/io/ipc/read/file_async.rs deleted file mode 100644 index dbbad6015197..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/file_async.rs +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Async reader for Arrow IPC files -use std::io::SeekFrom; - -use ahash::AHashMap; -use arrow_format::ipc::planus::ReadAsRoot; -use arrow_format::ipc::Block; -use arrow_format::ipc::MessageHeaderRef; -use futures::stream::BoxStream; -use futures::AsyncRead; -use futures::AsyncReadExt; -use futures::AsyncSeek; -use futures::AsyncSeekExt; -use futures::Stream; -use futures::StreamExt; - -use super::common::apply_projection; -use super::common::prepare_projection; -use super::common::read_dictionary; -use super::common::read_record_batch; -use super::file::deserialize_footer; -use super::file::get_record_batch; -use super::Dictionaries; -use super::FileMetadata; -use super::OutOfSpecKind; -use crate::arrow::array::*; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::IpcSchema; -use crate::arrow::io::ipc::ARROW_MAGIC_V2; -use crate::arrow::io::ipc::CONTINUATION_MARKER; - -/// Async reader for Arrow IPC files -pub struct FileStream<'a> { - stream: BoxStream<'a, Result>>>, - schema: Option, - metadata: FileMetadata, -} - -impl<'a> FileStream<'a> { - /// Create a new IPC file reader. - /// - /// # Examples - /// See [`FileSink`](crate::io::ipc::write::file_async::FileSink). - pub fn new( - reader: R, - metadata: FileMetadata, - projection: Option>, - limit: Option, - ) -> Self - where - R: AsyncRead + AsyncSeek + Unpin + Send + 'a, - { - let (projection, schema) = if let Some(projection) = projection { - let (p, h, fields) = prepare_projection(&metadata.schema.fields, projection); - let schema = Schema { - fields, - metadata: metadata.schema.metadata.clone(), - }; - (Some((p, h)), Some(schema)) - } else { - (None, None) - }; - - let stream = Self::stream(reader, None, metadata.clone(), projection, limit); - Self { - stream, - metadata, - schema, - } - } - - /// Get the metadata from the IPC file. - pub fn metadata(&self) -> &FileMetadata { - &self.metadata - } - - /// Get the projected schema from the IPC file. - pub fn schema(&self) -> &Schema { - self.schema.as_ref().unwrap_or(&self.metadata.schema) - } - - fn stream( - mut reader: R, - mut dictionaries: Option, - metadata: FileMetadata, - projection: Option<(Vec, AHashMap)>, - limit: Option, - ) -> BoxStream<'a, Result>>> - where - R: AsyncRead + AsyncSeek + Unpin + Send + 'a, - { - async_stream::try_stream! { - // read dictionaries - cached_read_dictionaries(&mut reader, &metadata, &mut dictionaries).await?; - - let mut meta_buffer = Default::default(); - let mut block_buffer = Default::default(); - let mut scratch = Default::default(); - let mut remaining = limit.unwrap_or(usize::MAX); - for block in 0..metadata.blocks.len() { - let chunk = read_batch( - &mut reader, - dictionaries.as_mut().unwrap(), - &metadata, - projection.as_ref().map(|x| x.0.as_ref()), - Some(remaining), - block, - &mut meta_buffer, - &mut block_buffer, - &mut scratch - ).await?; - remaining -= chunk.len(); - - let chunk = if let Some((_, map)) = &projection { - // re-order according to projection - apply_projection(chunk, map) - } else { - chunk - }; - - yield chunk; - } - } - .boxed() - } -} - -impl<'a> Stream for FileStream<'a> { - type Item = Result>>; - - fn poll_next( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.get_mut().stream.poll_next_unpin(cx) - } -} - -/// Reads the footer's length and magic number in footer -async fn read_footer_len(reader: &mut R) -> Result { - // read footer length and magic number in footer - reader.seek(SeekFrom::End(-10)).await?; - let mut footer: [u8; 10] = [0; 10]; - - reader.read_exact(&mut footer).await?; - let footer_len = i32::from_le_bytes(footer[..4].try_into().unwrap()); - - if footer[4..] != ARROW_MAGIC_V2 { - return Err(Error::from(OutOfSpecKind::InvalidFooter)); - } - footer_len - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength)) -} - -/// Read the metadata from an IPC file. -pub async fn read_file_metadata_async(reader: &mut R) -> Result -where R: AsyncRead + AsyncSeek + Unpin { - let footer_size = read_footer_len(reader).await?; - // Read footer - reader.seek(SeekFrom::End(-10 - footer_size as i64)).await?; - - let mut footer = vec![]; - footer.try_reserve(footer_size)?; - reader - .take(footer_size as u64) - .read_to_end(&mut footer) - .await?; - - deserialize_footer(&footer, u64::MAX) -} - -#[allow(clippy::too_many_arguments)] -async fn read_batch( - mut reader: R, - dictionaries: &mut Dictionaries, - metadata: &FileMetadata, - projection: Option<&[usize]>, - limit: Option, - block: usize, - meta_buffer: &mut Vec, - block_buffer: &mut Vec, - scratch: &mut Vec, -) -> Result>> -where - R: AsyncRead + AsyncSeek + Unpin, -{ - let block = metadata.blocks[block]; - - let offset: u64 = block - .offset - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - reader.seek(SeekFrom::Start(offset)).await?; - let mut meta_buf = [0; 4]; - reader.read_exact(&mut meta_buf).await?; - if meta_buf == CONTINUATION_MARKER { - reader.read_exact(&mut meta_buf).await?; - } - - let meta_len = i32::from_le_bytes(meta_buf) - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::UnexpectedNegativeInteger))?; - - meta_buffer.clear(); - meta_buffer.try_reserve(meta_len)?; - (&mut reader) - .take(meta_len as u64) - .read_to_end(meta_buffer) - .await?; - - let message = arrow_format::ipc::MessageRef::read_as_root(meta_buffer) - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferMessage(err)))?; - - let batch = get_record_batch(message)?; - - let block_length: usize = message - .body_length() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferBodyLength(err)))? - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::UnexpectedNegativeInteger))?; - - block_buffer.clear(); - block_buffer.try_reserve(block_length)?; - reader - .take(block_length as u64) - .read_to_end(block_buffer) - .await?; - - let mut cursor = std::io::Cursor::new(&block_buffer); - - read_record_batch( - batch, - &metadata.schema.fields, - &metadata.ipc_schema, - projection, - limit, - dictionaries, - message - .version() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferVersion(err)))?, - &mut cursor, - 0, - metadata.size, - scratch, - ) -} - -async fn read_dictionaries( - mut reader: R, - fields: &[Field], - ipc_schema: &IpcSchema, - blocks: &[Block], - scratch: &mut Vec, -) -> Result -where - R: AsyncRead + AsyncSeek + Unpin, -{ - let mut dictionaries = Default::default(); - let mut data: Vec = vec![]; - let mut buffer: Vec = vec![]; - - for block in blocks { - let offset: u64 = block - .offset - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - let length: usize = block - .body_length - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - read_dictionary_message(&mut reader, offset, &mut data).await?; - - let message = arrow_format::ipc::MessageRef::read_as_root(data.as_ref()) - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferMessage(err)))?; - - let header = message - .header() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferHeader(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingMessageHeader))?; - - match header { - MessageHeaderRef::DictionaryBatch(batch) => { - buffer.clear(); - buffer.try_reserve(length)?; - (&mut reader) - .take(length as u64) - .read_to_end(&mut buffer) - .await?; - let mut cursor = std::io::Cursor::new(&buffer); - read_dictionary( - batch, - fields, - ipc_schema, - &mut dictionaries, - &mut cursor, - 0, - u64::MAX, - scratch, - )?; - } - _ => return Err(Error::from(OutOfSpecKind::UnexpectedMessageType)), - } - } - Ok(dictionaries) -} - -async fn read_dictionary_message(mut reader: R, offset: u64, data: &mut Vec) -> Result<()> -where R: AsyncRead + AsyncSeek + Unpin { - let mut message_size = [0; 4]; - reader.seek(SeekFrom::Start(offset)).await?; - reader.read_exact(&mut message_size).await?; - if message_size == CONTINUATION_MARKER { - reader.read_exact(&mut message_size).await?; - } - let footer_size = i32::from_le_bytes(message_size); - - let footer_size: usize = footer_size - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - data.clear(); - data.try_reserve(footer_size)?; - (&mut reader) - .take(footer_size as u64) - .read_to_end(data) - .await?; - - Ok(()) -} - -async fn cached_read_dictionaries( - reader: &mut R, - metadata: &FileMetadata, - dictionaries: &mut Option, -) -> Result<()> { - match (&dictionaries, metadata.dictionaries.as_deref()) { - (None, Some(blocks)) => { - let new_dictionaries = read_dictionaries( - reader, - &metadata.schema.fields, - &metadata.ipc_schema, - blocks, - &mut Default::default(), - ) - .await?; - *dictionaries = Some(new_dictionaries); - } - (None, None) => { - *dictionaries = Some(Default::default()); - } - _ => {} - }; - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/mod.rs b/src/common/arrow/src/arrow/io/ipc/read/mod.rs deleted file mode 100644 index b0261925e0fc..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/mod.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to read Arrow's IPC format. -//! -//! The two important structs here are the [`FileReader`](reader::FileReader), -//! which provides arbitrary access to any of its messages, and the -//! [`StreamReader`](stream::StreamReader), which only supports reading -//! data in the order it was written in. -use ahash::AHashMap; - -use crate::arrow::array::Array; - -mod array; -mod common; -mod deserialize; -mod error; -pub(crate) mod file; -mod read_basic; -mod reader; -mod schema; -mod stream; - -#[cfg(feature = "io_flight")] -pub(crate) use common::read_dictionary; -#[cfg(feature = "io_flight")] -pub(crate) use common::read_record_batch; -pub use error::OutOfSpecKind; -pub use file::read_batch; -pub use file::read_file_dictionaries; -pub use file::read_file_metadata; -pub use file::FileMetadata; -pub use reader::FileReader; -pub use schema::deserialize_schema; -pub use stream::read_stream_metadata; -pub use stream::StreamMetadata; -pub use stream::StreamReader; -pub use stream::StreamState; - -/// how dictionaries are tracked in this crate -pub type Dictionaries = AHashMap>; - -pub(crate) type Node<'a> = arrow_format::ipc::FieldNodeRef<'a>; -pub(crate) type IpcBuffer<'a> = arrow_format::ipc::BufferRef<'a>; -pub(crate) type Compression<'a> = arrow_format::ipc::BodyCompressionRef<'a>; -pub(crate) type Version = arrow_format::ipc::MetadataVersion; diff --git a/src/common/arrow/src/arrow/io/ipc/read/read_basic.rs b/src/common/arrow/src/arrow/io/ipc/read/read_basic.rs deleted file mode 100644 index 1cf059968747..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/read_basic.rs +++ /dev/null @@ -1,386 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::convert::TryInto; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; - -use super::super::compression; -use super::super::endianness::is_native_little_endian; -use super::Compression; -use super::IpcBuffer; -use super::Node; -use super::OutOfSpecKind; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; - -fn read_swapped( - reader: &mut R, - length: usize, - buffer: &mut Vec, - is_little_endian: bool, -) -> Result<()> { - // slow case where we must reverse bits - let mut slice = vec![0u8; length * std::mem::size_of::()]; - reader.read_exact(&mut slice)?; - - let chunks = slice.chunks_exact(std::mem::size_of::()); - if !is_little_endian { - // machine is little endian, file is big endian - buffer - .as_mut_slice() - .iter_mut() - .zip(chunks) - .try_for_each(|(slot, chunk)| { - let a: T::Bytes = match chunk.try_into() { - Ok(a) => a, - Err(_) => unreachable!(), - }; - *slot = T::from_be_bytes(a); - Result::Ok(()) - })?; - } else { - // machine is big endian, file is little endian - return Err(Error::NotYetImplemented( - "Reading little endian files from big endian machines".to_string(), - )); - } - Ok(()) -} - -fn read_uncompressed_bytes( - reader: &mut R, - buffer_length: usize, - is_little_endian: bool, -) -> Result> { - if is_native_little_endian() == is_little_endian { - let mut buffer = Vec::with_capacity(buffer_length); - let _ = reader - .take(buffer_length as u64) - .read_to_end(&mut buffer) - .unwrap(); - Ok(buffer) - } else { - unreachable!() - } -} - -fn read_uncompressed_buffer( - reader: &mut R, - buffer_length: usize, - length: usize, - is_little_endian: bool, -) -> Result> { - let required_number_of_bytes = length.saturating_mul(std::mem::size_of::()); - if required_number_of_bytes > buffer_length { - return Err(Error::from(OutOfSpecKind::InvalidBuffer { - length, - type_name: std::any::type_name::(), - required_number_of_bytes, - buffer_length, - })); - // todo: move this to the error's Display - // return Err(Error::OutOfSpec( - // format!("The slots of the array times the physical size must \ - // be smaller or equal to the length of the IPC buffer. \ - // However, this array reports {} slots, which, for physical type \"{}\", corresponds to {} bytes, \ - // which is larger than the buffer length {}", - // length, - // std::any::type_name::(), - // bytes, - // buffer_length, - // ), - // )); - } - - // it is undefined behavior to call read_exact on un-initialized, https://doc.rust-lang.org/std/io/trait.Read.html#tymethod.read - // see also https://github.com/MaikKlein/ash/issues/354#issue-781730580 - let mut buffer = vec![T::default(); length]; - - if is_native_little_endian() == is_little_endian { - // fast case where we can just copy the contents - let slice = bytemuck::cast_slice_mut(&mut buffer); - reader.read_exact(slice)?; - } else { - read_swapped(reader, length, &mut buffer, is_little_endian)?; - } - Ok(buffer) -} - -fn read_compressed_buffer( - reader: &mut R, - buffer_length: usize, - length: usize, - is_little_endian: bool, - compression: Compression, - scratch: &mut Vec, -) -> Result> { - if is_little_endian != is_native_little_endian() { - return Err(Error::NotYetImplemented( - "Reading compressed and big endian IPC".to_string(), - )); - } - - // it is undefined behavior to call read_exact on un-initialized, https://doc.rust-lang.org/std/io/trait.Read.html#tymethod.read - // see also https://github.com/MaikKlein/ash/issues/354#issue-781730580 - let mut buffer = vec![T::default(); length]; - - // decompress first - scratch.clear(); - scratch.try_reserve(buffer_length)?; - reader - .by_ref() - .take(buffer_length as u64) - .read_to_end(scratch)?; - - let out_slice = bytemuck::cast_slice_mut(&mut buffer); - - let compression = compression - .codec() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferCompression(err)))?; - - match compression { - arrow_format::ipc::CompressionType::Lz4Frame => { - compression::decompress_lz4(&scratch[8..], out_slice)?; - } - arrow_format::ipc::CompressionType::Zstd => { - compression::decompress_zstd(&scratch[8..], out_slice)?; - } - } - Ok(buffer) -} - -fn read_compressed_bytes( - reader: &mut R, - buffer_length: usize, - is_little_endian: bool, - compression: Compression, - scratch: &mut Vec, -) -> Result> { - read_compressed_buffer::( - reader, - buffer_length, - buffer_length, - is_little_endian, - compression, - scratch, - ) -} - -pub fn read_bytes( - buf: &mut VecDeque, - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - scratch: &mut Vec, -) -> Result> { - let buf = buf - .pop_front() - .ok_or_else(|| Error::oos(format!("out-of-spec: {:?}", OutOfSpecKind::ExpectedBuffer)))?; - - let offset: u64 = buf.offset().try_into().map_err(|_| { - Error::oos(format!( - "out-of-spec: {:?}", - OutOfSpecKind::NegativeFooterLength - )) - })?; - - let buffer_length: usize = buf.length().try_into().map_err(|_| { - Error::oos(format!( - "out-of-spec: {:?}", - OutOfSpecKind::NegativeFooterLength - )) - })?; - - reader.seek(SeekFrom::Start(block_offset + offset))?; - - if let Some(compression) = compression { - Ok(read_compressed_bytes( - reader, - buffer_length, - is_little_endian, - compression, - scratch, - )? - .into()) - } else { - Ok(read_uncompressed_bytes(reader, buffer_length, is_little_endian)?.into()) - } -} - -pub fn read_buffer( - buf: &mut VecDeque, - length: usize, // in slots - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - scratch: &mut Vec, -) -> Result> { - let buf = buf - .pop_front() - .ok_or_else(|| Error::from(OutOfSpecKind::ExpectedBuffer))?; - - let offset: u64 = buf - .offset() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - let buffer_length: usize = buf - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - reader.seek(SeekFrom::Start(block_offset + offset))?; - - if let Some(compression) = compression { - Ok(read_compressed_buffer( - reader, - buffer_length, - length, - is_little_endian, - compression, - scratch, - )? - .into()) - } else { - Ok(read_uncompressed_buffer(reader, buffer_length, length, is_little_endian)?.into()) - } -} - -fn read_uncompressed_bitmap( - length: usize, - bytes: usize, - reader: &mut R, -) -> Result> { - if length > bytes * 8 { - return Err(Error::from(OutOfSpecKind::InvalidBitmap { - length, - number_of_bits: bytes * 8, - })); - } - - let mut buffer = vec![]; - buffer.try_reserve(bytes)?; - reader - .by_ref() - .take(bytes as u64) - .read_to_end(&mut buffer)?; - - Ok(buffer) -} - -fn read_compressed_bitmap( - length: usize, - bytes: usize, - compression: Compression, - reader: &mut R, - scratch: &mut Vec, -) -> Result> { - let mut buffer = vec![0; (length + 7) / 8]; - - scratch.clear(); - scratch.try_reserve(bytes)?; - reader.by_ref().take(bytes as u64).read_to_end(scratch)?; - - let compression = compression - .codec() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferCompression(err)))?; - - match compression { - arrow_format::ipc::CompressionType::Lz4Frame => { - compression::decompress_lz4(&scratch[8..], &mut buffer)?; - } - arrow_format::ipc::CompressionType::Zstd => { - compression::decompress_zstd(&scratch[8..], &mut buffer)?; - } - } - Ok(buffer) -} - -pub fn read_bitmap( - buf: &mut VecDeque, - length: usize, - reader: &mut R, - block_offset: u64, - _: bool, - compression: Option, - scratch: &mut Vec, -) -> Result { - let buf = buf - .pop_front() - .ok_or_else(|| Error::from(OutOfSpecKind::ExpectedBuffer))?; - - let offset: u64 = buf - .offset() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - let bytes: usize = buf - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - reader.seek(SeekFrom::Start(block_offset + offset))?; - - let buffer = if let Some(compression) = compression { - read_compressed_bitmap(length, bytes, compression, reader, scratch) - } else { - read_uncompressed_bitmap(length, bytes, reader) - }?; - - Bitmap::try_new(buffer, length) -} - -#[allow(clippy::too_many_arguments)] -pub fn read_validity( - buffers: &mut VecDeque, - field_node: Node, - reader: &mut R, - block_offset: u64, - is_little_endian: bool, - compression: Option, - limit: Option, - scratch: &mut Vec, -) -> Result> { - let length: usize = field_node - .length() - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let length = limit.map(|limit| limit.min(length)).unwrap_or(length); - - Ok(if field_node.null_count() > 0 { - Some(read_bitmap( - buffers, - length, - reader, - block_offset, - is_little_endian, - compression, - scratch, - )?) - } else { - let _ = buffers - .pop_front() - .ok_or_else(|| Error::from(OutOfSpecKind::ExpectedBuffer))?; - None - }) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/reader.rs b/src/common/arrow/src/arrow/io/ipc/read/reader.rs deleted file mode 100644 index 7a21b7007a92..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/reader.rs +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Read; -use std::io::Seek; - -use ahash::AHashMap; - -use super::common::*; -use super::read_batch; -use super::read_file_dictionaries; -use super::Dictionaries; -use super::FileMetadata; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Result; - -/// An iterator of [`Chunk`]s from an Arrow IPC file. -pub struct FileReader { - reader: R, - metadata: FileMetadata, - // the dictionaries are going to be read - dictionaries: Option, - current_block: usize, - projection: Option<(Vec, AHashMap, Schema)>, - remaining: usize, - data_scratch: Vec, - message_scratch: Vec, -} - -impl FileReader { - /// Creates a new [`FileReader`]. Use `projection` to only take certain columns. - /// # Panic - /// Panics iff the projection is not in increasing order (e.g. `[1, 0]` nor `[0, 1, 1]` are valid) - pub fn new( - reader: R, - metadata: FileMetadata, - projection: Option>, - limit: Option, - ) -> Self { - let projection = projection.map(|projection| { - let (p, h, fields) = prepare_projection(&metadata.schema.fields, projection); - let schema = Schema { - fields, - metadata: metadata.schema.metadata.clone(), - }; - (p, h, schema) - }); - Self { - reader, - metadata, - dictionaries: Default::default(), - projection, - remaining: limit.unwrap_or(usize::MAX), - current_block: 0, - data_scratch: Default::default(), - message_scratch: Default::default(), - } - } - - /// Return the schema of the file - pub fn schema(&self) -> &Schema { - self.projection - .as_ref() - .map(|x| &x.2) - .unwrap_or(&self.metadata.schema) - } - - /// Returns the [`FileMetadata`] - pub fn metadata(&self) -> &FileMetadata { - &self.metadata - } - - /// Consumes this FileReader, returning the underlying reader - pub fn into_inner(self) -> R { - self.reader - } - - /// Get the inner memory scratches so they can be reused in a new writer. - /// This can be utilized to save memory allocations for performance reasons. - pub fn get_scratches(&mut self) -> (Vec, Vec) { - ( - std::mem::take(&mut self.data_scratch), - std::mem::take(&mut self.message_scratch), - ) - } - - /// Set the inner memory scratches so they can be reused in a new writer. - /// This can be utilized to save memory allocations for performance reasons. - pub fn set_scratches(&mut self, scratches: (Vec, Vec)) { - (self.data_scratch, self.message_scratch) = scratches; - } - - fn read_dictionaries(&mut self) -> Result<()> { - if self.dictionaries.is_none() { - self.dictionaries = Some(read_file_dictionaries( - &mut self.reader, - &self.metadata, - &mut self.data_scratch, - )?); - }; - Ok(()) - } -} - -impl Iterator for FileReader { - type Item = Result>>; - - fn next(&mut self) -> Option { - // get current block - if self.current_block == self.metadata.blocks.len() { - return None; - } - - match self.read_dictionaries() { - Ok(_) => {} - Err(e) => return Some(Err(e)), - }; - - let block = self.current_block; - self.current_block += 1; - - let chunk = read_batch( - &mut self.reader, - self.dictionaries.as_ref().unwrap(), - &self.metadata, - self.projection.as_ref().map(|x| x.0.as_ref()), - Some(self.remaining), - block, - &mut self.message_scratch, - &mut self.data_scratch, - ); - self.remaining -= chunk.as_ref().map(|x| x.len()).unwrap_or_default(); - - let chunk = if let Some((_, map, _)) = &self.projection { - // re-order according to projection - chunk.map(|chunk| apply_projection(chunk, map)) - } else { - chunk - }; - Some(chunk) - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/schema.rs b/src/common/arrow/src/arrow/io/ipc/read/schema.rs deleted file mode 100644 index 690cca728ba4..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/schema.rs +++ /dev/null @@ -1,446 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_format::ipc::planus::ReadAsRoot; -use arrow_format::ipc::FieldRef; -use arrow_format::ipc::FixedSizeListRef; -use arrow_format::ipc::MapRef; -use arrow_format::ipc::TimeRef; -use arrow_format::ipc::TimestampRef; -use arrow_format::ipc::UnionRef; - -use super::super::IpcField; -use super::super::IpcSchema; -use super::OutOfSpecKind; -use super::StreamMetadata; -use crate::arrow::datatypes::get_extension; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Extension; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::IntegerType; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::datatypes::Metadata; -use crate::arrow::datatypes::Schema; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::datatypes::UnionMode; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -fn try_unzip_vec>>(iter: I) -> Result<(Vec, Vec)> { - let mut a = vec![]; - let mut b = vec![]; - for maybe_item in iter { - let (a_i, b_i) = maybe_item?; - a.push(a_i); - b.push(b_i); - } - - Ok((a, b)) -} - -fn deserialize_field(ipc_field: arrow_format::ipc::FieldRef) -> Result<(Field, IpcField)> { - let metadata = read_metadata(&ipc_field)?; - - let extension = get_extension(&metadata); - - let (data_type, ipc_field_) = get_data_type(ipc_field, extension, true)?; - - let field = Field { - name: ipc_field - .name()? - .ok_or_else(|| Error::oos("Every field in IPC must have a name"))? - .to_string(), - data_type, - is_nullable: ipc_field.nullable()?, - metadata, - }; - - Ok((field, ipc_field_)) -} - -fn read_metadata(field: &arrow_format::ipc::FieldRef) -> Result { - Ok(if let Some(list) = field.custom_metadata()? { - let mut metadata_map = Metadata::new(); - for kv in list { - let kv = kv?; - if let (Some(k), Some(v)) = (kv.key()?, kv.value()?) { - metadata_map.insert(k.to_string(), v.to_string()); - } - } - metadata_map - } else { - Metadata::default() - }) -} - -fn deserialize_integer(int: arrow_format::ipc::IntRef) -> Result { - Ok(match (int.bit_width()?, int.is_signed()?) { - (8, true) => IntegerType::Int8, - (8, false) => IntegerType::UInt8, - (16, true) => IntegerType::Int16, - (16, false) => IntegerType::UInt16, - (32, true) => IntegerType::Int32, - (32, false) => IntegerType::UInt32, - (64, true) => IntegerType::Int64, - (64, false) => IntegerType::UInt64, - _ => return Err(Error::oos("IPC: indexType can only be 8, 16, 32 or 64.")), - }) -} - -fn deserialize_timeunit(time_unit: arrow_format::ipc::TimeUnit) -> Result { - use arrow_format::ipc::TimeUnit::*; - Ok(match time_unit { - Second => TimeUnit::Second, - Millisecond => TimeUnit::Millisecond, - Microsecond => TimeUnit::Microsecond, - Nanosecond => TimeUnit::Nanosecond, - }) -} - -fn deserialize_time(time: TimeRef) -> Result<(DataType, IpcField)> { - let unit = deserialize_timeunit(time.unit()?)?; - - let data_type = match (time.bit_width()?, unit) { - (32, TimeUnit::Second) => DataType::Time32(TimeUnit::Second), - (32, TimeUnit::Millisecond) => DataType::Time32(TimeUnit::Millisecond), - (64, TimeUnit::Microsecond) => DataType::Time64(TimeUnit::Microsecond), - (64, TimeUnit::Nanosecond) => DataType::Time64(TimeUnit::Nanosecond), - (bits, precision) => { - return Err(Error::nyi(format!( - "Time type with bit width of {bits} and unit of {precision:?}" - ))); - } - }; - Ok((data_type, IpcField::default())) -} - -fn deserialize_timestamp(timestamp: TimestampRef) -> Result<(DataType, IpcField)> { - let timezone = timestamp.timezone()?.map(|tz| tz.to_string()); - let time_unit = deserialize_timeunit(timestamp.unit()?)?; - Ok(( - DataType::Timestamp(time_unit, timezone), - IpcField::default(), - )) -} - -fn deserialize_union(union_: UnionRef, field: FieldRef) -> Result<(DataType, IpcField)> { - let mode = UnionMode::sparse(union_.mode()? == arrow_format::ipc::UnionMode::Sparse); - let ids = union_.type_ids()?.map(|x| x.iter().collect()); - - let fields = field - .children()? - .ok_or_else(|| Error::oos("IPC: Union must contain children"))?; - if fields.is_empty() { - return Err(Error::oos("IPC: Union must contain at least one child")); - } - - let (fields, ipc_fields) = try_unzip_vec(fields.iter().map(|field| { - let (field, fields) = deserialize_field(field?)?; - Ok((field, fields)) - }))?; - let ipc_field = IpcField { - fields: ipc_fields, - dictionary_id: None, - }; - Ok((DataType::Union(fields, ids, mode), ipc_field)) -} - -fn deserialize_map(map: MapRef, field: FieldRef) -> Result<(DataType, IpcField)> { - let is_sorted = map.keys_sorted()?; - - let children = field - .children()? - .ok_or_else(|| Error::oos("IPC: Map must contain children"))?; - let inner = children - .get(0) - .ok_or_else(|| Error::oos("IPC: Map must contain one child"))??; - let (field, ipc_field) = deserialize_field(inner)?; - - let data_type = DataType::Map(Box::new(field), is_sorted); - Ok((data_type, IpcField { - fields: vec![ipc_field], - dictionary_id: None, - })) -} - -fn deserialize_struct(field: FieldRef) -> Result<(DataType, IpcField)> { - let fields = field - .children()? - .ok_or_else(|| Error::oos("IPC: Struct must contain children"))?; - if fields.is_empty() { - return Err(Error::oos("IPC: Struct must contain at least one child")); - } - let (fields, ipc_fields) = try_unzip_vec(fields.iter().map(|field| { - let (field, fields) = deserialize_field(field?)?; - Ok((field, fields)) - }))?; - let ipc_field = IpcField { - fields: ipc_fields, - dictionary_id: None, - }; - Ok((DataType::Struct(fields), ipc_field)) -} - -fn deserialize_list(field: FieldRef) -> Result<(DataType, IpcField)> { - let children = field - .children()? - .ok_or_else(|| Error::oos("IPC: List must contain children"))?; - let inner = children - .get(0) - .ok_or_else(|| Error::oos("IPC: List must contain one child"))??; - let (field, ipc_field) = deserialize_field(inner)?; - - Ok((DataType::List(Box::new(field)), IpcField { - fields: vec![ipc_field], - dictionary_id: None, - })) -} - -fn deserialize_large_list(field: FieldRef) -> Result<(DataType, IpcField)> { - let children = field - .children()? - .ok_or_else(|| Error::oos("IPC: List must contain children"))?; - let inner = children - .get(0) - .ok_or_else(|| Error::oos("IPC: List must contain one child"))??; - let (field, ipc_field) = deserialize_field(inner)?; - - Ok((DataType::LargeList(Box::new(field)), IpcField { - fields: vec![ipc_field], - dictionary_id: None, - })) -} - -fn deserialize_fixed_size_list( - list: FixedSizeListRef, - field: FieldRef, -) -> Result<(DataType, IpcField)> { - let children = field - .children()? - .ok_or_else(|| Error::oos("IPC: FixedSizeList must contain children"))?; - let inner = children - .get(0) - .ok_or_else(|| Error::oos("IPC: FixedSizeList must contain one child"))??; - let (field, ipc_field) = deserialize_field(inner)?; - - let size = list - .list_size()? - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - Ok((DataType::FixedSizeList(Box::new(field), size), IpcField { - fields: vec![ipc_field], - dictionary_id: None, - })) -} - -/// Get the Arrow data type from the flatbuffer Field table -fn get_data_type( - field: arrow_format::ipc::FieldRef, - extension: Extension, - may_be_dictionary: bool, -) -> Result<(DataType, IpcField)> { - if let Some(dictionary) = field.dictionary()? { - if may_be_dictionary { - let int = dictionary - .index_type()? - .ok_or_else(|| Error::oos("indexType is mandatory in Dictionary."))?; - let index_type = deserialize_integer(int)?; - let (inner, mut ipc_field) = get_data_type(field, extension, false)?; - ipc_field.dictionary_id = Some(dictionary.id()?); - return Ok(( - DataType::Dictionary(index_type, Box::new(inner), dictionary.is_ordered()?), - ipc_field, - )); - } - } - - if let Some(extension) = extension { - let (name, metadata) = extension; - let (data_type, fields) = get_data_type(field, None, false)?; - return Ok(( - DataType::Extension(name, Box::new(data_type), metadata), - fields, - )); - } - - let type_ = field - .type_()? - .ok_or_else(|| Error::oos("IPC: field type is mandatory"))?; - - use arrow_format::ipc::TypeRef::*; - Ok(match type_ { - Null(_) => (DataType::Null, IpcField::default()), - Bool(_) => (DataType::Boolean, IpcField::default()), - Int(int) => { - let data_type = deserialize_integer(int)?.into(); - (data_type, IpcField::default()) - } - Binary(_) => (DataType::Binary, IpcField::default()), - LargeBinary(_) => (DataType::LargeBinary, IpcField::default()), - Utf8(_) => (DataType::Utf8, IpcField::default()), - LargeUtf8(_) => (DataType::LargeUtf8, IpcField::default()), - BinaryView(_) => (DataType::BinaryView, IpcField::default()), - Utf8View(_) => (DataType::Utf8View, IpcField::default()), - FixedSizeBinary(fixed) => ( - DataType::FixedSizeBinary( - fixed - .byte_width()? - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?, - ), - IpcField::default(), - ), - FloatingPoint(float) => { - let data_type = match float.precision()? { - arrow_format::ipc::Precision::Half => DataType::Float16, - arrow_format::ipc::Precision::Single => DataType::Float32, - arrow_format::ipc::Precision::Double => DataType::Float64, - }; - (data_type, IpcField::default()) - } - Date(date) => { - let data_type = match date.unit()? { - arrow_format::ipc::DateUnit::Day => DataType::Date32, - arrow_format::ipc::DateUnit::Millisecond => DataType::Date64, - }; - (data_type, IpcField::default()) - } - Time(time) => deserialize_time(time)?, - Timestamp(timestamp) => deserialize_timestamp(timestamp)?, - Interval(interval) => { - let data_type = match interval.unit()? { - arrow_format::ipc::IntervalUnit::YearMonth => { - DataType::Interval(IntervalUnit::YearMonth) - } - arrow_format::ipc::IntervalUnit::DayTime => { - DataType::Interval(IntervalUnit::DayTime) - } - arrow_format::ipc::IntervalUnit::MonthDayNano => { - DataType::Interval(IntervalUnit::MonthDayNano) - } - }; - (data_type, IpcField::default()) - } - Duration(duration) => { - let time_unit = deserialize_timeunit(duration.unit()?)?; - (DataType::Duration(time_unit), IpcField::default()) - } - Decimal(decimal) => { - let bit_width: usize = decimal - .bit_width()? - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let precision: usize = decimal - .precision()? - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - let scale: usize = decimal - .scale()? - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - let data_type = match bit_width { - 128 => DataType::Decimal(precision, scale), - 256 => DataType::Decimal256(precision, scale), - _ => return Err(Error::from(OutOfSpecKind::NegativeFooterLength)), - }; - - (data_type, IpcField::default()) - } - List(_) => deserialize_list(field)?, - LargeList(_) => deserialize_large_list(field)?, - FixedSizeList(list) => deserialize_fixed_size_list(list, field)?, - Struct(_) => deserialize_struct(field)?, - Union(union_) => deserialize_union(union_, field)?, - Map(map) => deserialize_map(map, field)?, - RunEndEncoded(_) | LargeListView(_) | ListView(_) => unimplemented!(), - }) -} - -/// Deserialize an flatbuffers-encoded Schema message into [`Schema`] and [`IpcSchema`]. -pub fn deserialize_schema(message: &[u8]) -> Result<(Schema, IpcSchema)> { - let message = arrow_format::ipc::MessageRef::read_as_root(message) - .map_err(|err| Error::oos(format!("Unable deserialize message: {err:?}")))?; - - let schema = match message - .header()? - .ok_or_else(|| Error::oos("Unable to convert header to a schema".to_string()))? - { - arrow_format::ipc::MessageHeaderRef::Schema(schema) => Ok(schema), - _ => Err(Error::nyi("The message is expected to be a Schema message")), - }?; - - fb_to_schema(schema) -} - -/// Deserialize the raw Schema table from IPC format to Schema data type -pub(super) fn fb_to_schema(schema: arrow_format::ipc::SchemaRef) -> Result<(Schema, IpcSchema)> { - let fields = schema - .fields()? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingFields))?; - let (fields, ipc_fields) = try_unzip_vec(fields.iter().map(|field| { - let (field, fields) = deserialize_field(field?)?; - Ok((field, fields)) - }))?; - - let is_little_endian = match schema.endianness()? { - arrow_format::ipc::Endianness::Little => true, - arrow_format::ipc::Endianness::Big => false, - }; - - let mut metadata = Metadata::default(); - if let Some(md_fields) = schema.custom_metadata()? { - for kv in md_fields { - let kv = kv?; - let k_str = kv.key()?; - let v_str = kv.value()?; - if let Some(k) = k_str { - if let Some(v) = v_str { - metadata.insert(k.to_string(), v.to_string()); - } - } - } - } - - Ok((Schema { fields, metadata }, IpcSchema { - fields: ipc_fields, - is_little_endian, - })) -} - -pub(super) fn deserialize_stream_metadata(meta: &[u8]) -> Result { - let message = arrow_format::ipc::MessageRef::read_as_root(meta) - .map_err(|err| Error::OutOfSpec(format!("Unable to get root as message: {err:?}")))?; - let version = message.version()?; - // message header is a Schema, so read it - let header = message - .header()? - .ok_or_else(|| Error::oos("Unable to read the first IPC message"))?; - let schema = if let arrow_format::ipc::MessageHeaderRef::Schema(schema) = header { - schema - } else { - return Err(Error::oos( - "The first IPC message of the stream must be a schema", - )); - }; - let (schema, ipc_schema) = fb_to_schema(schema)?; - - Ok(StreamMetadata { - schema, - version, - ipc_schema, - }) -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/stream.rs b/src/common/arrow/src/arrow/io/ipc/read/stream.rs deleted file mode 100644 index e3c9c083fefc..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/stream.rs +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Read; - -use ahash::AHashMap; -use arrow_format; -use arrow_format::ipc::planus::ReadAsRoot; - -use super::super::CONTINUATION_MARKER; -use super::common::*; -use super::schema::deserialize_stream_metadata; -use super::Dictionaries; -use super::OutOfSpecKind; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::IpcSchema; - -/// Metadata of an Arrow IPC stream, written at the start of the stream -#[derive(Debug, Clone)] -pub struct StreamMetadata { - /// The schema that is read from the stream's first message - pub schema: Schema, - - /// The IPC version of the stream - pub version: arrow_format::ipc::MetadataVersion, - - /// The IPC fields tracking dictionaries - pub ipc_schema: IpcSchema, -} - -/// Reads the metadata of the stream -pub fn read_stream_metadata(reader: &mut R) -> Result { - // determine metadata length - let mut meta_size: [u8; 4] = [0; 4]; - reader.read_exact(&mut meta_size)?; - let meta_length = { - // If a continuation marker is encountered, skip over it and read - // the size from the next four bytes. - if meta_size == CONTINUATION_MARKER { - reader.read_exact(&mut meta_size)?; - } - i32::from_le_bytes(meta_size) - }; - - let length: usize = meta_length - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - let mut buffer = vec![]; - buffer.try_reserve(length)?; - reader - .by_ref() - .take(length as u64) - .read_to_end(&mut buffer)?; - - deserialize_stream_metadata(&buffer) -} - -/// Encodes the stream's status after each read. -/// -/// A stream is an iterator, and an iterator returns `Option`. The `Item` -/// type in the [`StreamReader`] case is `StreamState`, which means that an Arrow -/// stream may yield one of three values: (1) `None`, which signals that the stream -/// is done; (2) [`StreamState::Some`], which signals that there was -/// data waiting in the stream and we read it; and finally (3) -/// [`Some(StreamState::Waiting)`], which means that the stream is still "live", it -/// just doesn't hold any data right now. -pub enum StreamState { - /// A live stream without data - Waiting, - /// Next item in the stream - Some(Chunk>), -} - -impl StreamState { - /// Return the data inside this wrapper. - /// - /// # Panics - /// - /// If the `StreamState` was `Waiting`. - pub fn unwrap(self) -> Chunk> { - if let StreamState::Some(batch) = self { - batch - } else { - panic!("The batch is not available") - } - } -} - -/// Reads the next item, yielding `None` if the stream is done, -/// and a [`StreamState`] otherwise. -fn read_next( - reader: &mut R, - metadata: &StreamMetadata, - dictionaries: &mut Dictionaries, - message_buffer: &mut Vec, - data_buffer: &mut Vec, - projection: &Option<(Vec, AHashMap, Schema)>, - scratch: &mut Vec, -) -> Result> { - // determine metadata length - let mut meta_length: [u8; 4] = [0; 4]; - - match reader.read_exact(&mut meta_length) { - Ok(()) => (), - Err(e) => { - return if e.kind() == std::io::ErrorKind::UnexpectedEof { - // Handle EOF without the "0xFFFFFFFF 0x00000000" - // valid according to: - // https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format - Ok(Some(StreamState::Waiting)) - } else { - Err(Error::from(e)) - }; - } - } - - let meta_length = { - // If a continuation marker is encountered, skip over it and read - // the size from the next four bytes. - if meta_length == CONTINUATION_MARKER { - reader.read_exact(&mut meta_length)?; - } - i32::from_le_bytes(meta_length) - }; - - let meta_length: usize = meta_length - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - if meta_length == 0 { - // the stream has ended, mark the reader as finished - return Ok(None); - } - - message_buffer.clear(); - message_buffer.try_reserve(meta_length)?; - reader - .by_ref() - .take(meta_length as u64) - .read_to_end(message_buffer)?; - - let message = arrow_format::ipc::MessageRef::read_as_root(message_buffer.as_ref()) - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferMessage(err)))?; - - let header = message - .header() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferHeader(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingMessageHeader))?; - - let block_length: usize = message - .body_length() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferBodyLength(err)))? - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::UnexpectedNegativeInteger))?; - - match header { - arrow_format::ipc::MessageHeaderRef::RecordBatch(batch) => { - data_buffer.clear(); - data_buffer.try_reserve(block_length)?; - reader - .by_ref() - .take(block_length as u64) - .read_to_end(data_buffer)?; - - let file_size = data_buffer.len() as u64; - - let mut reader = std::io::Cursor::new(data_buffer); - - let chunk = read_record_batch( - batch, - &metadata.schema.fields, - &metadata.ipc_schema, - projection.as_ref().map(|x| x.0.as_ref()), - None, - dictionaries, - metadata.version, - &mut reader, - 0, - file_size, - scratch, - ); - - if let Some((_, map, _)) = projection { - // re-order according to projection - chunk - .map(|chunk| apply_projection(chunk, map)) - .map(|x| Some(StreamState::Some(x))) - } else { - chunk.map(|x| Some(StreamState::Some(x))) - } - } - arrow_format::ipc::MessageHeaderRef::DictionaryBatch(batch) => { - data_buffer.clear(); - data_buffer.try_reserve(block_length)?; - reader - .by_ref() - .take(block_length as u64) - .read_to_end(data_buffer)?; - - let file_size = data_buffer.len() as u64; - let mut dict_reader = std::io::Cursor::new(&data_buffer); - - read_dictionary( - batch, - &metadata.schema.fields, - &metadata.ipc_schema, - dictionaries, - &mut dict_reader, - 0, - file_size, - scratch, - )?; - - // read the next message until we encounter a RecordBatch message - read_next( - reader, - metadata, - dictionaries, - message_buffer, - data_buffer, - projection, - scratch, - ) - } - _ => Err(Error::from(OutOfSpecKind::UnexpectedMessageType)), - } -} - -/// Arrow Stream reader. -/// -/// An [`Iterator`] over an Arrow stream that yields a result of [`StreamState`]s. -/// This is the recommended way to read an arrow stream (by iterating over its data). -/// -/// For a more thorough walkthrough consult [this example](https://github.com/jorgecarleitao/arrow2/tree/main/examples/ipc_pyarrow). -pub struct StreamReader { - reader: R, - metadata: StreamMetadata, - dictionaries: Dictionaries, - finished: bool, - data_buffer: Vec, - message_buffer: Vec, - projection: Option<(Vec, AHashMap, Schema)>, - scratch: Vec, -} - -impl StreamReader { - /// Try to create a new stream reader - /// - /// The first message in the stream is the schema, the reader will fail if it does not - /// encounter a schema. - /// To check if the reader is done, use `is_finished(self)` - pub fn new(reader: R, metadata: StreamMetadata, projection: Option>) -> Self { - let projection = projection.map(|projection| { - let (p, h, fields) = prepare_projection(&metadata.schema.fields, projection); - let schema = Schema { - fields, - metadata: metadata.schema.metadata.clone(), - }; - (p, h, schema) - }); - - Self { - reader, - metadata, - dictionaries: Default::default(), - finished: false, - data_buffer: Default::default(), - message_buffer: Default::default(), - projection, - scratch: Default::default(), - } - } - - /// Return the schema of the stream - pub fn metadata(&self) -> &StreamMetadata { - &self.metadata - } - - /// Return the schema of the file - pub fn schema(&self) -> &Schema { - self.projection - .as_ref() - .map(|x| &x.2) - .unwrap_or(&self.metadata.schema) - } - - /// Check if the stream is finished - pub fn is_finished(&self) -> bool { - self.finished - } - - fn maybe_next(&mut self) -> Result> { - if self.finished { - return Ok(None); - } - let batch = read_next( - &mut self.reader, - &self.metadata, - &mut self.dictionaries, - &mut self.message_buffer, - &mut self.data_buffer, - &self.projection, - &mut self.scratch, - )?; - if batch.is_none() { - self.finished = true; - } - Ok(batch) - } -} - -impl Iterator for StreamReader { - type Item = Result; - - fn next(&mut self) -> Option { - self.maybe_next().transpose() - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/read/stream_async.rs b/src/common/arrow/src/arrow/io/ipc/read/stream_async.rs deleted file mode 100644 index 30d9b9e737b1..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/read/stream_async.rs +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to read Arrow streams asynchronously - -use arrow_format::ipc::planus::ReadAsRoot; -use futures::future::BoxFuture; -use futures::AsyncRead; -use futures::AsyncReadExt; -use futures::FutureExt; -use futures::Stream; - -use super::super::CONTINUATION_MARKER; -use super::common::read_dictionary; -use super::common::read_record_batch; -use super::schema::deserialize_stream_metadata; -use super::Dictionaries; -use super::OutOfSpecKind; -use super::StreamMetadata; -use crate::arrow::array::*; -use crate::arrow::chunk::Chunk; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// A (private) state of stream messages -struct ReadState { - pub reader: R, - pub metadata: StreamMetadata, - pub dictionaries: Dictionaries, - /// The internal buffer to read data inside the messages (records and dictionaries) to - pub data_buffer: Vec, - /// The internal buffer to read messages to - pub message_buffer: Vec, -} - -/// The state of an Arrow stream -enum StreamState { - /// The stream does not contain new chunks (and it has not been closed) - Waiting(ReadState), - /// The stream contain a new chunk - Some((ReadState, Chunk>)), -} - -/// Reads the [`StreamMetadata`] of the Arrow stream asynchronously -pub async fn read_stream_metadata_async( - reader: &mut R, -) -> Result { - // determine metadata length - let mut meta_size: [u8; 4] = [0; 4]; - reader.read_exact(&mut meta_size).await?; - let meta_len = { - // If a continuation marker is encountered, skip over it and read - // the size from the next four bytes. - if meta_size == CONTINUATION_MARKER { - reader.read_exact(&mut meta_size).await?; - } - i32::from_le_bytes(meta_size) - }; - - let meta_len: usize = meta_len - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - let mut meta_buffer = vec![]; - meta_buffer.try_reserve(meta_len)?; - reader - .take(meta_len as u64) - .read_to_end(&mut meta_buffer) - .await?; - - deserialize_stream_metadata(&meta_buffer) -} - -/// Reads the next item, yielding `None` if the stream has been closed, -/// or a [`StreamState`] otherwise. -async fn maybe_next( - mut state: ReadState, -) -> Result>> { - let mut scratch = Default::default(); - // determine metadata length - let mut meta_length: [u8; 4] = [0; 4]; - - match state.reader.read_exact(&mut meta_length).await { - Ok(()) => (), - Err(e) => { - return if e.kind() == std::io::ErrorKind::UnexpectedEof { - // Handle EOF without the "0xFFFFFFFF 0x00000000" - // valid according to: - // https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format - Ok(Some(StreamState::Waiting(state))) - } else { - Err(Error::from(e)) - }; - } - } - - let meta_length = { - // If a continuation marker is encountered, skip over it and read - // the size from the next four bytes. - if meta_length == CONTINUATION_MARKER { - state.reader.read_exact(&mut meta_length).await?; - } - i32::from_le_bytes(meta_length) - }; - - let meta_length: usize = meta_length - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; - - if meta_length == 0 { - // the stream has ended, mark the reader as finished - return Ok(None); - } - - state.message_buffer.clear(); - state.message_buffer.try_reserve(meta_length)?; - (&mut state.reader) - .take(meta_length as u64) - .read_to_end(&mut state.message_buffer) - .await?; - - let message = arrow_format::ipc::MessageRef::read_as_root(state.message_buffer.as_ref()) - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferMessage(err)))?; - - let header = message - .header() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferHeader(err)))? - .ok_or_else(|| Error::from(OutOfSpecKind::MissingMessageHeader))?; - - let block_length: usize = message - .body_length() - .map_err(|err| Error::from(OutOfSpecKind::InvalidFlatbufferBodyLength(err)))? - .try_into() - .map_err(|_| Error::from(OutOfSpecKind::UnexpectedNegativeInteger))?; - - match header { - arrow_format::ipc::MessageHeaderRef::RecordBatch(batch) => { - state.data_buffer.clear(); - state.data_buffer.try_reserve(block_length)?; - (&mut state.reader) - .take(block_length as u64) - .read_to_end(&mut state.data_buffer) - .await?; - - read_record_batch( - batch, - &state.metadata.schema.fields, - &state.metadata.ipc_schema, - None, - None, - &state.dictionaries, - state.metadata.version, - &mut std::io::Cursor::new(&state.data_buffer), - 0, - state.data_buffer.len() as u64, - &mut scratch, - ) - .map(|chunk| Some(StreamState::Some((state, chunk)))) - } - arrow_format::ipc::MessageHeaderRef::DictionaryBatch(batch) => { - state.data_buffer.clear(); - state.data_buffer.try_reserve(block_length)?; - (&mut state.reader) - .take(block_length as u64) - .read_to_end(&mut state.data_buffer) - .await?; - - let file_size = state.data_buffer.len() as u64; - - let mut dict_reader = std::io::Cursor::new(&state.data_buffer); - - read_dictionary( - batch, - &state.metadata.schema.fields, - &state.metadata.ipc_schema, - &mut state.dictionaries, - &mut dict_reader, - 0, - file_size, - &mut scratch, - )?; - - // read the next message until we encounter a Chunk> message - Ok(Some(StreamState::Waiting(state))) - } - _ => Err(Error::from(OutOfSpecKind::UnexpectedMessageType)), - } -} - -/// A [`Stream`] over an Arrow IPC stream that asynchronously yields [`Chunk`]s. -pub struct AsyncStreamReader<'a, R: AsyncRead + Unpin + Send + 'a> { - metadata: StreamMetadata, - future: Option>>>>, -} - -impl<'a, R: AsyncRead + Unpin + Send + 'a> AsyncStreamReader<'a, R> { - /// Creates a new [`AsyncStreamReader`] - pub fn new(reader: R, metadata: StreamMetadata) -> Self { - let state = ReadState { - reader, - metadata: metadata.clone(), - dictionaries: Default::default(), - data_buffer: Default::default(), - message_buffer: Default::default(), - }; - let future = Some(maybe_next(state).boxed()); - Self { metadata, future } - } - - /// Return the schema of the stream - pub fn metadata(&self) -> &StreamMetadata { - &self.metadata - } -} - -impl<'a, R: AsyncRead + Unpin + Send> Stream for AsyncStreamReader<'a, R> { - type Item = Result>>; - - fn poll_next( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - use std::pin::Pin; - use std::task::Poll; - let me = Pin::into_inner(self); - - match &mut me.future { - Some(fut) => match fut.as_mut().poll(cx) { - Poll::Ready(Ok(None)) => { - me.future = None; - Poll::Ready(None) - } - Poll::Ready(Ok(Some(StreamState::Some((state, batch))))) => { - me.future = Some(Box::pin(maybe_next(state))); - Poll::Ready(Some(Ok(batch))) - } - Poll::Ready(Ok(Some(StreamState::Waiting(_)))) => Poll::Pending, - Poll::Ready(Err(err)) => { - me.future = None; - Poll::Ready(Some(Err(err))) - } - Poll::Pending => Poll::Pending, - }, - None => Poll::Ready(None), - } - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/common.rs b/src/common/arrow/src/arrow/io/ipc/write/common.rs deleted file mode 100644 index fe55287c6c30..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/common.rs +++ /dev/null @@ -1,518 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::borrow::Borrow; -use std::borrow::Cow; - -use arrow_format::ipc::planus::Builder; - -use super::super::IpcField; -use super::write; -use super::write_dictionary; -use crate::arrow::array::*; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::endianness::is_native_little_endian; -use crate::arrow::io::ipc::read::Dictionaries; - -/// Compression codec -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Compression { - /// LZ4 (framed) - LZ4, - /// ZSTD - ZSTD, -} - -/// Options declaring the behaviour of writing to IPC -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub struct WriteOptions { - /// Whether the buffers should be compressed and which codec to use. - /// Note: to use compression the crate must be compiled with feature `io_ipc_compression`. - pub compression: Option, -} - -fn encode_dictionary( - field: &IpcField, - array: &dyn Array, - options: &WriteOptions, - dictionary_tracker: &mut DictionaryTracker, - encoded_dictionaries: &mut Vec, -) -> Result<()> { - use PhysicalType::*; - match array.data_type().to_physical_type() { - Utf8 | LargeUtf8 | Binary | LargeBinary | Primitive(_) | Boolean | Null - | FixedSizeBinary | BinaryView | Utf8View => Ok(()), - Dictionary(key_type) => match_integer_type!(key_type, |$T| { - let dict_id = field.dictionary_id - .ok_or_else(|| Error::InvalidArgumentError("Dictionaries must have an associated id".to_string()))?; - - let emit = dictionary_tracker.insert(dict_id, array)?; - - let array = array.as_any().downcast_ref::>().unwrap(); - let values = array.values(); - encode_dictionary(field, - values.as_ref(), - options, - dictionary_tracker, - encoded_dictionaries - )?; - - if emit { - encoded_dictionaries.push(dictionary_batch_to_bytes::<$T>( - dict_id, - array, - options, - is_native_little_endian(), - )); - }; - Ok(()) - }), - Struct => { - let array = array.as_any().downcast_ref::().unwrap(); - let fields = field.fields.as_slice(); - if array.fields().len() != fields.len() { - return Err(Error::InvalidArgumentError( - "The number of fields in a struct must equal the number of children in IpcField".to_string(), - )); - } - fields - .iter() - .zip(array.values().iter()) - .try_for_each(|(field, values)| { - encode_dictionary( - field, - values.as_ref(), - options, - dictionary_tracker, - encoded_dictionaries, - ) - }) - } - List => { - let values = array - .as_any() - .downcast_ref::>() - .unwrap() - .values(); - let field = &field.fields[0]; // todo: error instead - encode_dictionary( - field, - values.as_ref(), - options, - dictionary_tracker, - encoded_dictionaries, - ) - } - LargeList => { - let values = array - .as_any() - .downcast_ref::>() - .unwrap() - .values(); - let field = &field.fields[0]; // todo: error instead - encode_dictionary( - field, - values.as_ref(), - options, - dictionary_tracker, - encoded_dictionaries, - ) - } - FixedSizeList => { - let values = array - .as_any() - .downcast_ref::() - .unwrap() - .values(); - let field = &field.fields[0]; // todo: error instead - encode_dictionary( - field, - values.as_ref(), - options, - dictionary_tracker, - encoded_dictionaries, - ) - } - Union => { - let values = array - .as_any() - .downcast_ref::() - .unwrap() - .fields(); - let fields = &field.fields[..]; // todo: error instead - if values.len() != fields.len() { - return Err(Error::InvalidArgumentError( - "The number of fields in a union must equal the number of children in IpcField" - .to_string(), - )); - } - fields - .iter() - .zip(values.iter()) - .try_for_each(|(field, values)| { - encode_dictionary( - field, - values.as_ref(), - options, - dictionary_tracker, - encoded_dictionaries, - ) - }) - } - Map => { - let values = array.as_any().downcast_ref::().unwrap().field(); - let field = &field.fields[0]; // todo: error instead - encode_dictionary( - field, - values.as_ref(), - options, - dictionary_tracker, - encoded_dictionaries, - ) - } - } -} - -pub fn encode_chunk( - chunk: &Chunk>, - fields: &[IpcField], - dictionary_tracker: &mut DictionaryTracker, - options: &WriteOptions, -) -> Result<(Vec, EncodedData)> { - let mut encoded_message = EncodedData::default(); - let encoded_dictionaries = encode_chunk_amortized( - chunk, - fields, - dictionary_tracker, - options, - &mut encoded_message, - )?; - Ok((encoded_dictionaries, encoded_message)) -} - -// Amortizes `EncodedData` allocation. -pub fn encode_chunk_amortized( - chunk: &Chunk>, - fields: &[IpcField], - dictionary_tracker: &mut DictionaryTracker, - options: &WriteOptions, - encoded_message: &mut EncodedData, -) -> Result> { - let mut encoded_dictionaries = vec![]; - - for (field, array) in fields.iter().zip(chunk.as_ref()) { - encode_dictionary( - field, - array.as_ref(), - options, - dictionary_tracker, - &mut encoded_dictionaries, - )?; - } - - chunk_to_bytes_amortized(chunk, options, encoded_message); - - Ok(encoded_dictionaries) -} - -fn serialize_compression( - compression: Option, -) -> Option> { - if let Some(compression) = compression { - let codec = match compression { - Compression::LZ4 => arrow_format::ipc::CompressionType::Lz4Frame, - Compression::ZSTD => arrow_format::ipc::CompressionType::Zstd, - }; - Some(Box::new(arrow_format::ipc::BodyCompression { - codec, - method: arrow_format::ipc::BodyCompressionMethod::Buffer, - })) - } else { - None - } -} - -fn set_variadic_buffer_counts(counts: &mut Vec, array: &dyn Array) { - match array.data_type() { - DataType::Utf8View => { - let array = array.as_any().downcast_ref::().unwrap(); - counts.push(array.data_buffers().len() as i64); - } - DataType::BinaryView => { - let array = array.as_any().downcast_ref::().unwrap(); - counts.push(array.data_buffers().len() as i64); - } - DataType::Struct(_) => { - let array = array.as_any().downcast_ref::().unwrap(); - for array in array.values() { - set_variadic_buffer_counts(counts, array.as_ref()) - } - } - DataType::LargeList(_) => { - let array = array.as_any().downcast_ref::>().unwrap(); - set_variadic_buffer_counts(counts, array.values().as_ref()) - } - DataType::FixedSizeList(_, _) => { - let array = array.as_any().downcast_ref::().unwrap(); - set_variadic_buffer_counts(counts, array.values().as_ref()) - } - DataType::Dictionary(_, _, _) => { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - set_variadic_buffer_counts(counts, array.values().as_ref()) - } - _ => (), - } -} - -/// Write [`Chunk`] into two sets of bytes, one for the header (ipc::Schema::Message) and the -/// other for the batch's data -fn chunk_to_bytes_amortized( - chunk: &Chunk>, - options: &WriteOptions, - encoded_message: &mut EncodedData, -) { - let mut nodes: Vec = vec![]; - let mut buffers: Vec = vec![]; - let mut arrow_data = std::mem::take(&mut encoded_message.arrow_data); - arrow_data.clear(); - - let mut offset = 0; - let mut variadic_buffer_counts = vec![]; - for array in chunk.arrays() { - set_variadic_buffer_counts(&mut variadic_buffer_counts, array.as_ref()); - write( - array.as_ref(), - &mut buffers, - &mut arrow_data, - &mut nodes, - &mut offset, - is_native_little_endian(), - options.compression, - ) - } - - let variadic_buffer_counts = if variadic_buffer_counts.is_empty() { - None - } else { - Some(variadic_buffer_counts) - }; - - let compression = serialize_compression(options.compression); - - let message = arrow_format::ipc::Message { - version: arrow_format::ipc::MetadataVersion::V5, - header: Some(arrow_format::ipc::MessageHeader::RecordBatch(Box::new( - arrow_format::ipc::RecordBatch { - length: chunk.len() as i64, - nodes: Some(nodes), - buffers: Some(buffers), - compression, - variadic_buffer_counts, - }, - ))), - body_length: arrow_data.len() as i64, - custom_metadata: None, - }; - - let mut builder = Builder::new(); - let ipc_message = builder.finish(&message, None); - encoded_message.ipc_message = ipc_message.to_vec(); - encoded_message.arrow_data = arrow_data -} - -/// Write dictionary values into two sets of bytes, one for the header (ipc::Schema::Message) and the -/// other for the data -fn dictionary_batch_to_bytes( - dict_id: i64, - array: &DictionaryArray, - options: &WriteOptions, - is_little_endian: bool, -) -> EncodedData { - let mut nodes: Vec = vec![]; - let mut buffers: Vec = vec![]; - let mut arrow_data: Vec = vec![]; - - let mut variadic_buffer_counts = vec![]; - set_variadic_buffer_counts(&mut variadic_buffer_counts, array.values().as_ref()); - - let variadic_buffer_counts = if variadic_buffer_counts.is_empty() { - None - } else { - Some(variadic_buffer_counts) - }; - - let length = write_dictionary( - array, - &mut buffers, - &mut arrow_data, - &mut nodes, - &mut 0, - is_little_endian, - options.compression, - false, - ); - - let compression = serialize_compression(options.compression); - - let message = arrow_format::ipc::Message { - version: arrow_format::ipc::MetadataVersion::V5, - header: Some(arrow_format::ipc::MessageHeader::DictionaryBatch(Box::new( - arrow_format::ipc::DictionaryBatch { - id: dict_id, - data: Some(Box::new(arrow_format::ipc::RecordBatch { - length: length as i64, - nodes: Some(nodes), - buffers: Some(buffers), - compression, - variadic_buffer_counts, - })), - is_delta: false, - }, - ))), - body_length: arrow_data.len() as i64, - custom_metadata: None, - }; - - let mut builder = Builder::new(); - let ipc_message = builder.finish(&message, None); - - EncodedData { - ipc_message: ipc_message.to_vec(), - arrow_data, - } -} - -/// Keeps track of dictionaries that have been written, to avoid emitting the same dictionary -/// multiple times. Can optionally error if an update to an existing dictionary is attempted, which -/// isn't allowed in the `FileWriter`. -pub struct DictionaryTracker { - pub dictionaries: Dictionaries, - pub cannot_replace: bool, -} - -impl DictionaryTracker { - /// Keep track of the dictionary with the given ID and values. Behavior: - /// - /// * If this ID has been written already and has the same data, return `Ok(false)` to indicate - /// that the dictionary was not actually inserted (because it's already been seen). - /// * If this ID has been written already but with different data, and this tracker is - /// configured to return an error, return an error. - /// * If the tracker has not been configured to error on replacement or this dictionary - /// has never been seen before, return `Ok(true)` to indicate that the dictionary was just - /// inserted. - pub fn insert(&mut self, dict_id: i64, array: &dyn Array) -> Result { - let values = match array.data_type() { - DataType::Dictionary(key_type, _, _) => { - match_integer_type!(key_type, |$T| { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - array.values() - }) - } - _ => unreachable!(), - }; - - // If a dictionary with this id was already emitted, check if it was the same. - if let Some(last) = self.dictionaries.get(&dict_id) { - if last.as_ref() == values.as_ref() { - // Same dictionary values => no need to emit it again - return Ok(false); - } else if self.cannot_replace { - return Err(Error::InvalidArgumentError( - "Dictionary replacement detected when writing IPC file format. \ - Arrow IPC files only support a single dictionary for a given field \ - across all batches." - .to_string(), - )); - } - }; - - self.dictionaries.insert(dict_id, values.clone()); - Ok(true) - } -} - -/// Stores the encoded data, which is an ipc::Schema::Message, and optional Arrow data -#[derive(Debug, Default)] -pub struct EncodedData { - /// An encoded ipc::Schema::Message - pub ipc_message: Vec, - /// Arrow buffers to be written, should be an empty vec for schema messages - pub arrow_data: Vec, -} - -/// Calculate an 8-byte boundary and return the number of bytes needed to pad to 8 bytes -#[inline] -pub(crate) fn pad_to_64(len: usize) -> usize { - ((len + 63) & !63) - len -} - -/// An array [`Chunk`] with optional accompanying IPC fields. -#[derive(Debug, Clone, PartialEq)] -pub struct Record<'a> { - columns: Cow<'a, Chunk>>, - fields: Option>, -} - -impl<'a> Record<'a> { - /// Get the IPC fields for this record. - pub fn fields(&self) -> Option<&[IpcField]> { - self.fields.as_deref() - } - - /// Get the Arrow columns in this record. - pub fn columns(&self) -> &Chunk> { - self.columns.borrow() - } -} - -impl From>> for Record<'static> { - fn from(columns: Chunk>) -> Self { - Self { - columns: Cow::Owned(columns), - fields: None, - } - } -} - -impl<'a, F> From<(Chunk>, Option)> for Record<'a> -where F: Into> -{ - fn from((columns, fields): (Chunk>, Option)) -> Self { - Self { - columns: Cow::Owned(columns), - fields: fields.map(|f| f.into()), - } - } -} - -impl<'a, F> From<(&'a Chunk>, Option)> for Record<'a> -where F: Into> -{ - fn from((columns, fields): (&'a Chunk>, Option)) -> Self { - Self { - columns: Cow::Borrowed(columns), - fields: fields.map(|f| f.into()), - } - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/common_async.rs b/src/common/arrow/src/arrow/io/ipc/write/common_async.rs deleted file mode 100644 index c8067ebdb05c..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/common_async.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use futures::AsyncWrite; -use futures::AsyncWriteExt; - -use super::super::CONTINUATION_MARKER; -use super::common::pad_to_64; -use super::common::EncodedData; -use crate::arrow::error::Result; - -/// Write a message's IPC data and buffers, returning metadata and buffer data lengths written -pub async fn write_message( - mut writer: W, - encoded: EncodedData, -) -> Result<(usize, usize)> { - let arrow_data_len = encoded.arrow_data.len(); - - let a = 64 - 1; - let buffer = encoded.ipc_message; - let flatbuf_size = buffer.len(); - let prefix_size = 8; // the message length - let aligned_size = (flatbuf_size + prefix_size + a) & !a; - let padding_bytes = aligned_size - flatbuf_size - prefix_size; - - write_continuation(&mut writer, (aligned_size - prefix_size) as i32).await?; - - // write the flatbuf - if flatbuf_size > 0 { - writer.write_all(&buffer).await?; - } - // write padding - writer.write_all(&vec![0; padding_bytes]).await?; - - // write arrow data - let body_len = if arrow_data_len > 0 { - write_body_buffers(writer, &encoded.arrow_data).await? - } else { - 0 - }; - - Ok((aligned_size, body_len)) -} - -/// Write a record batch to the writer, writing the message size before the message -/// if the record batch is being written to a stream -pub async fn write_continuation( - mut writer: W, - total_len: i32, -) -> Result { - writer.write_all(&CONTINUATION_MARKER).await?; - writer.write_all(&total_len.to_le_bytes()[..]).await?; - Ok(8) -} - -async fn write_body_buffers( - mut writer: W, - data: &[u8], -) -> Result { - let len = data.len(); - let pad_len = pad_to_64(data.len()); - let total_len = len + pad_len; - - // write body buffer - writer.write_all(data).await?; - if pad_len > 0 { - writer.write_all(&vec![0u8; pad_len][..]).await?; - } - - Ok(total_len) -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/common_sync.rs b/src/common/arrow/src/arrow/io/ipc/write/common_sync.rs deleted file mode 100644 index ccf286048893..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/common_sync.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -use super::super::CONTINUATION_MARKER; -use super::common::pad_to_64; -use super::common::EncodedData; -use crate::arrow::error::Result; - -/// Write a message's IPC data and buffers, returning metadata and buffer data lengths written -pub fn write_message(writer: &mut W, encoded: &EncodedData) -> Result<(usize, usize)> { - let arrow_data_len = encoded.arrow_data.len(); - - let a = 8 - 1; - let buffer = &encoded.ipc_message; - let flatbuf_size = buffer.len(); - let prefix_size = 8; - let aligned_size = (flatbuf_size + prefix_size + a) & !a; - let padding_bytes = aligned_size - flatbuf_size - prefix_size; - - write_continuation(writer, (aligned_size - prefix_size) as i32)?; - - // write the flatbuf - if flatbuf_size > 0 { - writer.write_all(buffer)?; - } - // write padding - // aligned to a 8 byte boundary, so maximum is [u8;8] - const PADDING_MAX: [u8; 8] = [0u8; 8]; - writer.write_all(&PADDING_MAX[..padding_bytes])?; - - // write arrow data - let body_len = if arrow_data_len > 0 { - write_body_buffers(writer, &encoded.arrow_data)? - } else { - 0 - }; - - Ok((aligned_size, body_len)) -} - -fn write_body_buffers(mut writer: W, data: &[u8]) -> Result { - let len = data.len(); - let pad_len = pad_to_64(data.len()); - let total_len = len + pad_len; - - // write body buffer - writer.write_all(data)?; - if pad_len > 0 { - writer.write_all(&vec![0u8; pad_len][..])?; - } - - Ok(total_len) -} - -/// Write a record batch to the writer, writing the message size before the message -/// if the record batch is being written to a stream -pub fn write_continuation(writer: &mut W, total_len: i32) -> Result { - writer.write_all(&CONTINUATION_MARKER)?; - writer.write_all(&total_len.to_le_bytes()[..])?; - Ok(8) -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/file_async.rs b/src/common/arrow/src/arrow/io/ipc/write/file_async.rs deleted file mode 100644 index db8d2f7925cd..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/file_async.rs +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Async writer for IPC files. - -use std::task::Poll; - -use arrow_format::ipc::planus::Builder; -use arrow_format::ipc::Block; -use arrow_format::ipc::Footer; -use arrow_format::ipc::MetadataVersion; -use futures::future::BoxFuture; -use futures::AsyncWrite; -use futures::AsyncWriteExt; -use futures::FutureExt; -use futures::Sink; - -use super::common::encode_chunk; -use super::common::DictionaryTracker; -use super::common::EncodedData; -use super::common::WriteOptions; -use super::common_async::write_continuation; -use super::common_async::write_message; -use super::default_ipc_fields; -use super::schema::serialize_schema; -use super::schema_to_bytes; -use super::Record; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::IpcField; -use crate::arrow::io::ipc::ARROW_MAGIC_V2; - -type WriteOutput = (usize, Option, Vec, Option); - -/// Sink that writes array [`chunks`](crate::chunk::Chunk) as an IPC file. -/// -/// The file header is automatically written before writing the first chunk, and the file footer is -/// automatically written when the sink is closed. -/// -/// # Examples -/// -/// ``` -/// use arrow2::array::Array; -/// use arrow2::array::Int32Array; -/// use arrow2::chunk::Chunk; -/// use arrow2::datatypes::DataType; -/// use arrow2::datatypes::Field; -/// use arrow2::datatypes::Schema; -/// use arrow2::io::ipc::read::file_async::read_file_metadata_async; -/// use arrow2::io::ipc::read::file_async::FileStream; -/// use arrow2::io::ipc::write::file_async::FileSink; -/// use futures::io::Cursor; -/// use futures::SinkExt; -/// use futures::TryStreamExt; -/// # futures::executor::block_on(async move { -/// let schema = Schema::from(vec![Field::new("values", DataType::Int32, true)]); -/// -/// let mut buffer = Cursor::new(vec![]); -/// let mut sink = FileSink::new(&mut buffer, schema, None, Default::default()); -/// -/// // Write chunks to file -/// for i in 0..3 { -/// let values = Int32Array::from(&[Some(i), None]); -/// let chunk = Chunk::new(vec![values.boxed()]); -/// sink.feed(chunk.into()).await?; -/// } -/// sink.close().await?; -/// drop(sink); -/// -/// // Read chunks from file -/// buffer.set_position(0); -/// let metadata = read_file_metadata_async(&mut buffer).await?; -/// let mut stream = FileStream::new(buffer, metadata, None, None); -/// let chunks = stream.try_collect::>().await?; -/// # arrow2::error::Result::Ok(()) -/// # }).unwrap(); -/// ``` -pub struct FileSink<'a, W: AsyncWrite + Unpin + Send + 'a> { - writer: Option, - task: Option>>>, - options: WriteOptions, - dictionary_tracker: DictionaryTracker, - offset: usize, - fields: Vec, - record_blocks: Vec, - dictionary_blocks: Vec, - schema: Schema, -} - -impl<'a, W> FileSink<'a, W> -where W: AsyncWrite + Unpin + Send + 'a -{ - /// Create a new file writer. - pub fn new( - writer: W, - schema: Schema, - ipc_fields: Option>, - options: WriteOptions, - ) -> Self { - let fields = ipc_fields.unwrap_or_else(|| default_ipc_fields(&schema.fields)); - let encoded = EncodedData { - ipc_message: schema_to_bytes(&schema, &fields), - arrow_data: vec![], - }; - let task = Some(Self::start(writer, encoded).boxed()); - Self { - writer: None, - task, - options, - fields, - offset: 0, - schema, - dictionary_tracker: DictionaryTracker { - dictionaries: Default::default(), - cannot_replace: true, - }, - record_blocks: vec![], - dictionary_blocks: vec![], - } - } - - async fn start(mut writer: W, encoded: EncodedData) -> Result> { - writer.write_all(&ARROW_MAGIC_V2[..]).await?; - writer.write_all(&[0, 0]).await?; - let (meta, data) = write_message(&mut writer, encoded).await?; - - Ok((meta + data + 8, None, vec![], Some(writer))) - } - - async fn write( - mut writer: W, - mut offset: usize, - record: EncodedData, - dictionaries: Vec, - ) -> Result> { - let mut dict_blocks = vec![]; - for dict in dictionaries { - let (meta, data) = write_message(&mut writer, dict).await?; - let block = Block { - offset: offset as i64, - meta_data_length: meta as i32, - body_length: data as i64, - }; - dict_blocks.push(block); - offset += meta + data; - } - let (meta, data) = write_message(&mut writer, record).await?; - let block = Block { - offset: offset as i64, - meta_data_length: meta as i32, - body_length: data as i64, - }; - offset += meta + data; - Ok((offset, Some(block), dict_blocks, Some(writer))) - } - - async fn finish(mut writer: W, footer: Footer) -> Result> { - write_continuation(&mut writer, 0).await?; - let footer = { - let mut builder = Builder::new(); - builder.finish(&footer, None).to_owned() - }; - writer.write_all(&footer[..]).await?; - writer - .write_all(&(footer.len() as i32).to_le_bytes()) - .await?; - writer.write_all(&ARROW_MAGIC_V2).await?; - writer.close().await?; - - Ok((0, None, vec![], None)) - } - - fn poll_write(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - if let Some(task) = &mut self.task { - match futures::ready!(task.poll_unpin(cx)) { - Ok((offset, record, mut dictionaries, writer)) => { - self.task = None; - self.writer = writer; - self.offset = offset; - if let Some(block) = record { - self.record_blocks.push(block); - } - self.dictionary_blocks.append(&mut dictionaries); - Poll::Ready(Ok(())) - } - Err(error) => { - self.task = None; - Poll::Ready(Err(error)) - } - } - } else { - Poll::Ready(Ok(())) - } - } -} - -impl<'a, W> Sink> for FileSink<'a, W> -where W: AsyncWrite + Unpin + Send + 'a -{ - type Error = Error; - - fn poll_ready( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.get_mut().poll_write(cx) - } - - fn start_send(self: std::pin::Pin<&mut Self>, item: Record<'_>) -> Result<()> { - let this = self.get_mut(); - - if let Some(writer) = this.writer.take() { - let fields = item.fields().unwrap_or_else(|| &this.fields[..]); - - let (dictionaries, record) = encode_chunk( - item.columns(), - fields, - &mut this.dictionary_tracker, - &this.options, - )?; - - this.task = Some(Self::write(writer, this.offset, record, dictionaries).boxed()); - Ok(()) - } else { - Err(Error::Io(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "writer is closed", - ))) - } - } - - fn poll_flush( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.get_mut().poll_write(cx) - } - - fn poll_close( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - let this = self.get_mut(); - match futures::ready!(this.poll_write(cx)) { - Ok(()) => { - if let Some(writer) = this.writer.take() { - let schema = serialize_schema(&this.schema, &this.fields); - let footer = Footer { - version: MetadataVersion::V5, - schema: Some(Box::new(schema)), - dictionaries: Some(std::mem::take(&mut this.dictionary_blocks)), - record_batches: Some(std::mem::take(&mut this.record_blocks)), - custom_metadata: None, - }; - this.task = Some(Self::finish(writer, footer).boxed()); - this.poll_write(cx) - } else { - Poll::Ready(Ok(())) - } - } - Err(error) => Poll::Ready(Err(error)), - } - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/mod.rs b/src/common/arrow/src/arrow/io/ipc/write/mod.rs deleted file mode 100644 index e6d2dfb03588..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/mod.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to write to Arrow's IPC format. -pub(crate) mod common; -mod schema; -mod serialize; -mod stream; -pub(crate) mod writer; - -pub use common::Compression; -pub use common::Record; -pub use common::WriteOptions; -pub use schema::schema_to_bytes; -pub use serialize::write; -use serialize::write_dictionary; -pub use stream::StreamWriter; -pub use writer::FileWriter; - -pub(crate) mod common_sync; - -use super::IpcField; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; - -fn default_ipc_field(data_type: &DataType, current_id: &mut i64) -> IpcField { - use crate::arrow::datatypes::DataType::*; - match data_type.to_logical_type() { - // single child => recurse - Map(inner, ..) | FixedSizeList(inner, _) | LargeList(inner) | List(inner) => IpcField { - fields: vec![default_ipc_field(inner.data_type(), current_id)], - dictionary_id: None, - }, - // multiple children => recurse - Union(fields, ..) | Struct(fields) => IpcField { - fields: fields - .iter() - .map(|f| default_ipc_field(f.data_type(), current_id)) - .collect(), - dictionary_id: None, - }, - // dictionary => current_id - Dictionary(_, data_type, _) => { - let dictionary_id = Some(*current_id); - *current_id += 1; - IpcField { - fields: vec![default_ipc_field(data_type, current_id)], - dictionary_id, - } - } - // no children => do nothing - _ => IpcField { - fields: vec![], - dictionary_id: None, - }, - } -} - -/// Assigns every dictionary field a unique ID -pub fn default_ipc_fields(fields: &[Field]) -> Vec { - let mut dictionary_id = 0i64; - fields - .iter() - .map(|field| default_ipc_field(field.data_type().to_logical_type(), &mut dictionary_id)) - .collect() -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/schema.rs b/src/common/arrow/src/arrow/io/ipc/write/schema.rs deleted file mode 100644 index 4f2740843f6c..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/schema.rs +++ /dev/null @@ -1,357 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_format::ipc::planus::Builder; - -use super::super::IpcField; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::IntegerType; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::datatypes::Metadata; -use crate::arrow::datatypes::Schema; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::datatypes::UnionMode; -use crate::arrow::io::ipc::endianness::is_native_little_endian; - -/// Converts a [Schema] and [IpcField]s to a flatbuffers-encoded [arrow_format::ipc::Message]. -pub fn schema_to_bytes(schema: &Schema, ipc_fields: &[IpcField]) -> Vec { - let schema = serialize_schema(schema, ipc_fields); - - let message = arrow_format::ipc::Message { - version: arrow_format::ipc::MetadataVersion::V5, - header: Some(arrow_format::ipc::MessageHeader::Schema(Box::new(schema))), - body_length: 0, - custom_metadata: None, // todo: allow writing custom metadata - }; - let mut builder = Builder::new(); - let footer_data = builder.finish(&message, None); - footer_data.to_vec() -} - -pub fn serialize_schema(schema: &Schema, ipc_fields: &[IpcField]) -> arrow_format::ipc::Schema { - let endianness = if is_native_little_endian() { - arrow_format::ipc::Endianness::Little - } else { - arrow_format::ipc::Endianness::Big - }; - - let fields = schema - .fields - .iter() - .zip(ipc_fields.iter()) - .map(|(field, ipc_field)| serialize_field(field, ipc_field)) - .collect::>(); - - let mut custom_metadata = vec![]; - for (key, value) in &schema.metadata { - custom_metadata.push(arrow_format::ipc::KeyValue { - key: Some(key.clone()), - value: Some(value.clone()), - }); - } - let custom_metadata = if custom_metadata.is_empty() { - None - } else { - Some(custom_metadata) - }; - - arrow_format::ipc::Schema { - endianness, - fields: Some(fields), - custom_metadata, - features: None, // todo add this one - } -} - -fn write_metadata(metadata: &Metadata, kv_vec: &mut Vec) { - for (k, v) in metadata { - if k != "ARROW:extension:name" && k != "ARROW:extension:metadata" { - let entry = arrow_format::ipc::KeyValue { - key: Some(k.clone()), - value: Some(v.clone()), - }; - kv_vec.push(entry); - } - } -} - -fn write_extension( - name: &str, - metadata: &Option, - kv_vec: &mut Vec, -) { - // metadata - if let Some(metadata) = metadata { - let entry = arrow_format::ipc::KeyValue { - key: Some("ARROW:extension:metadata".to_string()), - value: Some(metadata.clone()), - }; - kv_vec.push(entry); - } - - // name - let entry = arrow_format::ipc::KeyValue { - key: Some("ARROW:extension:name".to_string()), - value: Some(name.to_string()), - }; - kv_vec.push(entry); -} - -/// Create an IPC Field from an Arrow Field -pub(crate) fn serialize_field(field: &Field, ipc_field: &IpcField) -> arrow_format::ipc::Field { - // custom metadata. - let mut kv_vec = vec![]; - if let DataType::Extension(name, _, metadata) = field.data_type() { - write_extension(name, metadata, &mut kv_vec); - } - - let type_ = serialize_type(field.data_type()); - let children = serialize_children(field.data_type(), ipc_field); - - let dictionary = if let DataType::Dictionary(index_type, inner, is_ordered) = field.data_type() - { - if let DataType::Extension(name, _, metadata) = inner.as_ref() { - write_extension(name, metadata, &mut kv_vec); - } - Some(serialize_dictionary( - index_type, - ipc_field - .dictionary_id - .expect("All Dictionary types have `dict_id`"), - *is_ordered, - )) - } else { - None - }; - - write_metadata(&field.metadata, &mut kv_vec); - - let custom_metadata = if !kv_vec.is_empty() { - Some(kv_vec) - } else { - None - }; - - arrow_format::ipc::Field { - name: Some(field.name.clone()), - nullable: field.is_nullable, - type_: Some(type_), - dictionary: dictionary.map(Box::new), - children: Some(children), - custom_metadata, - } -} - -fn serialize_time_unit(unit: &TimeUnit) -> arrow_format::ipc::TimeUnit { - match unit { - TimeUnit::Second => arrow_format::ipc::TimeUnit::Second, - TimeUnit::Millisecond => arrow_format::ipc::TimeUnit::Millisecond, - TimeUnit::Microsecond => arrow_format::ipc::TimeUnit::Microsecond, - TimeUnit::Nanosecond => arrow_format::ipc::TimeUnit::Nanosecond, - } -} - -fn serialize_type(data_type: &DataType) -> arrow_format::ipc::Type { - use arrow_format::ipc; - use DataType::*; - match data_type { - Null => ipc::Type::Null(Box::new(ipc::Null {})), - Boolean => ipc::Type::Bool(Box::new(ipc::Bool {})), - UInt8 => ipc::Type::Int(Box::new(ipc::Int { - bit_width: 8, - is_signed: false, - })), - UInt16 => ipc::Type::Int(Box::new(ipc::Int { - bit_width: 16, - is_signed: false, - })), - UInt32 => ipc::Type::Int(Box::new(ipc::Int { - bit_width: 32, - is_signed: false, - })), - UInt64 => ipc::Type::Int(Box::new(ipc::Int { - bit_width: 64, - is_signed: false, - })), - Int8 => ipc::Type::Int(Box::new(ipc::Int { - bit_width: 8, - is_signed: true, - })), - Int16 => ipc::Type::Int(Box::new(ipc::Int { - bit_width: 16, - is_signed: true, - })), - Int32 => ipc::Type::Int(Box::new(ipc::Int { - bit_width: 32, - is_signed: true, - })), - Int64 => ipc::Type::Int(Box::new(ipc::Int { - bit_width: 64, - is_signed: true, - })), - Float16 => ipc::Type::FloatingPoint(Box::new(ipc::FloatingPoint { - precision: ipc::Precision::Half, - })), - Float32 => ipc::Type::FloatingPoint(Box::new(ipc::FloatingPoint { - precision: ipc::Precision::Single, - })), - Float64 => ipc::Type::FloatingPoint(Box::new(ipc::FloatingPoint { - precision: ipc::Precision::Double, - })), - Decimal(precision, scale) => ipc::Type::Decimal(Box::new(ipc::Decimal { - precision: *precision as i32, - scale: *scale as i32, - bit_width: 128, - })), - Decimal256(precision, scale) => ipc::Type::Decimal(Box::new(ipc::Decimal { - precision: *precision as i32, - scale: *scale as i32, - bit_width: 256, - })), - Binary => ipc::Type::Binary(Box::new(ipc::Binary {})), - LargeBinary => ipc::Type::LargeBinary(Box::new(ipc::LargeBinary {})), - Utf8 => ipc::Type::Utf8(Box::new(ipc::Utf8 {})), - LargeUtf8 => ipc::Type::LargeUtf8(Box::new(ipc::LargeUtf8 {})), - FixedSizeBinary(size) => ipc::Type::FixedSizeBinary(Box::new(ipc::FixedSizeBinary { - byte_width: *size as i32, - })), - Date32 => ipc::Type::Date(Box::new(ipc::Date { - unit: ipc::DateUnit::Day, - })), - Date64 => ipc::Type::Date(Box::new(ipc::Date { - unit: ipc::DateUnit::Millisecond, - })), - Duration(unit) => ipc::Type::Duration(Box::new(ipc::Duration { - unit: serialize_time_unit(unit), - })), - Time32(unit) => ipc::Type::Time(Box::new(ipc::Time { - unit: serialize_time_unit(unit), - bit_width: 32, - })), - Time64(unit) => ipc::Type::Time(Box::new(ipc::Time { - unit: serialize_time_unit(unit), - bit_width: 64, - })), - Timestamp(unit, tz) => ipc::Type::Timestamp(Box::new(ipc::Timestamp { - unit: serialize_time_unit(unit), - timezone: tz.as_ref().cloned(), - })), - Interval(unit) => ipc::Type::Interval(Box::new(ipc::Interval { - unit: match unit { - IntervalUnit::YearMonth => ipc::IntervalUnit::YearMonth, - IntervalUnit::DayTime => ipc::IntervalUnit::DayTime, - IntervalUnit::MonthDayNano => ipc::IntervalUnit::MonthDayNano, - }, - })), - List(_) => ipc::Type::List(Box::new(ipc::List {})), - LargeList(_) => ipc::Type::LargeList(Box::new(ipc::LargeList {})), - FixedSizeList(_, size) => ipc::Type::FixedSizeList(Box::new(ipc::FixedSizeList { - list_size: *size as i32, - })), - Union(_, type_ids, mode) => ipc::Type::Union(Box::new(ipc::Union { - mode: match mode { - UnionMode::Dense => ipc::UnionMode::Dense, - UnionMode::Sparse => ipc::UnionMode::Sparse, - }, - type_ids: type_ids.clone(), - })), - Map(_, keys_sorted) => ipc::Type::Map(Box::new(ipc::Map { - keys_sorted: *keys_sorted, - })), - Struct(_) => ipc::Type::Struct(Box::new(ipc::Struct {})), - Dictionary(_, v, _) => serialize_type(v), - Extension(_, v, _) => serialize_type(v), - Utf8View => ipc::Type::Utf8View(Box::new(ipc::Utf8View {})), - BinaryView => ipc::Type::BinaryView(Box::new(ipc::BinaryView {})), - } -} - -fn serialize_children(data_type: &DataType, ipc_field: &IpcField) -> Vec { - use DataType::*; - match data_type { - Null - | Boolean - | Int8 - | Int16 - | Int32 - | Int64 - | UInt8 - | UInt16 - | UInt32 - | UInt64 - | Float16 - | Float32 - | Float64 - | Timestamp(_, _) - | Date32 - | Date64 - | Time32(_) - | Time64(_) - | Duration(_) - | Interval(_) - | Binary - | FixedSizeBinary(_) - | LargeBinary - | Utf8 - | LargeUtf8 - | Utf8View - | BinaryView - | Decimal(_, _) - | Decimal256(_, _) => vec![], - FixedSizeList(inner, _) | LargeList(inner) | List(inner) | Map(inner, _) => { - vec![serialize_field(inner, &ipc_field.fields[0])] - } - Union(fields, _, _) | Struct(fields) => fields - .iter() - .zip(ipc_field.fields.iter()) - .map(|(field, ipc)| serialize_field(field, ipc)) - .collect(), - Dictionary(_, inner, _) => serialize_children(inner, ipc_field), - Extension(_, inner, _) => serialize_children(inner, ipc_field), - } -} - -/// Create an IPC dictionary encoding -pub(crate) fn serialize_dictionary( - index_type: &IntegerType, - dict_id: i64, - dict_is_ordered: bool, -) -> arrow_format::ipc::DictionaryEncoding { - use IntegerType::*; - let is_signed = match index_type { - Int8 | Int16 | Int32 | Int64 => true, - UInt8 | UInt16 | UInt32 | UInt64 => false, - }; - - let bit_width = match index_type { - Int8 | UInt8 => 8, - Int16 | UInt16 => 16, - Int32 | UInt32 => 32, - Int64 | UInt64 => 64, - }; - - let index_type = arrow_format::ipc::Int { - bit_width, - is_signed, - }; - - arrow_format::ipc::DictionaryEncoding { - id: dict_id, - index_type: Some(Box::new(index_type)), - is_ordered: dict_is_ordered, - dictionary_kind: arrow_format::ipc::DictionaryKind::DenseArray, - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/serialize.rs b/src/common/arrow/src/arrow/io/ipc/write/serialize.rs deleted file mode 100644 index 21879ff2c8cd..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/serialize.rs +++ /dev/null @@ -1,833 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![allow(clippy::ptr_arg)] // false positive in clippy, see https://github.com/rust-lang/rust-clippy/issues/8463 -use arrow_format::ipc; - -use super::super::compression; -use super::super::endianness::is_native_little_endian; -use super::common::pad_to_64; -use super::common::Compression; -use crate::arrow::array::*; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::offset::Offset; -use crate::arrow::offset::OffsetsBuffer; -use crate::arrow::trusted_len::TrustedLen; -use crate::arrow::types::NativeType; - -fn write_primitive( - array: &PrimitiveArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - write_bitmap( - array.validity(), - array.len(), - buffers, - arrow_data, - offset, - compression, - ); - - write_buffer( - array.values(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ) -} - -fn write_boolean( - array: &BooleanArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - _: bool, - compression: Option, -) { - write_bitmap( - array.validity(), - array.len(), - buffers, - arrow_data, - offset, - compression, - ); - write_bitmap( - Some(&array.values().clone()), - array.len(), - buffers, - arrow_data, - offset, - compression, - ); -} - -#[allow(clippy::too_many_arguments)] -fn write_generic_binary( - validity: Option<&Bitmap>, - offsets: &OffsetsBuffer, - values: &[u8], - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - let offsets = offsets.buffer(); - write_bitmap( - validity, - offsets.len() - 1, - buffers, - arrow_data, - offset, - compression, - ); - - let first = *offsets.first().unwrap(); - let last = *offsets.last().unwrap(); - if first == O::default() { - write_buffer( - offsets, - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - } else { - write_buffer_from_iter( - offsets.iter().map(|x| *x - first), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - } - - write_bytes( - &values[first.to_usize()..last.to_usize()], - buffers, - arrow_data, - offset, - compression, - ); -} - -fn write_binary( - array: &BinaryArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - write_generic_binary( - array.validity(), - array.offsets(), - array.values(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); -} - -fn write_utf8( - array: &Utf8Array, - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - write_generic_binary( - array.validity(), - array.offsets(), - array.values(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); -} - -fn write_fixed_size_binary( - array: &FixedSizeBinaryArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - _is_little_endian: bool, - compression: Option, -) { - write_bitmap( - array.validity(), - array.len(), - buffers, - arrow_data, - offset, - compression, - ); - write_bytes(array.values(), buffers, arrow_data, offset, compression); -} - -fn write_list( - array: &ListArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - nodes: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - let offsets = array.offsets().buffer(); - let validity = array.validity(); - - write_bitmap( - validity, - offsets.len() - 1, - buffers, - arrow_data, - offset, - compression, - ); - - let first = *offsets.first().unwrap(); - let last = *offsets.last().unwrap(); - if first == O::zero() { - write_buffer( - offsets, - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - } else { - write_buffer_from_iter( - offsets.iter().map(|x| *x - first), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - } - - write( - array - .values() - .sliced(first.to_usize(), last.to_usize() - first.to_usize()) - .as_ref(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ); -} - -pub fn write_struct( - array: &StructArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - nodes: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - write_bitmap( - array.validity(), - array.len(), - buffers, - arrow_data, - offset, - compression, - ); - array.values().iter().for_each(|array| { - write( - array.as_ref(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ); - }); -} - -pub fn write_union( - array: &UnionArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - nodes: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - write_buffer( - array.types(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - - if let Some(offsets) = array.offsets() { - write_buffer( - offsets, - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - } - array.fields().iter().for_each(|array| { - write( - array.as_ref(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ) - }); -} - -fn write_map( - array: &MapArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - nodes: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - let offsets = array.offsets().buffer(); - let validity = array.validity(); - - write_bitmap( - validity, - offsets.len() - 1, - buffers, - arrow_data, - offset, - compression, - ); - - let first = *offsets.first().unwrap(); - let last = *offsets.last().unwrap(); - if first == 0 { - write_buffer( - offsets, - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - } else { - write_buffer_from_iter( - offsets.iter().map(|x| *x - first), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - } - - write( - array - .field() - .sliced(first as usize, last as usize - first as usize) - .as_ref(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ); -} - -fn write_fixed_size_list( - array: &FixedSizeListArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - nodes: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - write_bitmap( - array.validity(), - array.len(), - buffers, - arrow_data, - offset, - compression, - ); - write( - array.values().as_ref(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ); -} - -// use `write_keys` to either write keys or values -#[allow(clippy::too_many_arguments)] -pub(super) fn write_dictionary( - array: &DictionaryArray, - buffers: &mut Vec, - arrow_data: &mut Vec, - nodes: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, - write_keys: bool, -) -> usize { - if write_keys { - write_primitive( - array.keys(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - array.keys().len() - } else { - write( - array.values().as_ref(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ); - array.values().len() - } -} - -#[allow(clippy::too_many_arguments)] -pub(super) fn write_binview( - array: &BinaryViewArrayGeneric, - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - let array = if array.is_sliced() { - array.clone().maybe_gc() - } else { - array.clone() - }; - write_bitmap( - array.validity(), - Array::len(&array), - buffers, - arrow_data, - offset, - compression, - ); - - write_buffer( - array.views(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ); - - for data in array.data_buffers().as_ref() { - write_bytes(data, buffers, arrow_data, offset, compression); - } -} - -/// Writes an [`Array`] to `arrow_data` -pub fn write( - array: &dyn Array, - buffers: &mut Vec, - arrow_data: &mut Vec, - nodes: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - nodes.push(ipc::FieldNode { - length: array.len() as i64, - null_count: array.null_count() as i64, - }); - use PhysicalType::*; - match array.data_type().to_physical_type() { - Null => (), - Boolean => write_boolean( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - let array = array.as_any().downcast_ref().unwrap(); - write_primitive::<$T>(array, buffers, arrow_data, offset, is_little_endian, compression) - }), - Binary => write_binary::( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ), - LargeBinary => write_binary::( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ), - FixedSizeBinary => write_fixed_size_binary( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ), - Utf8 => write_utf8::( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ), - LargeUtf8 => write_utf8::( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ), - List => write_list::( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ), - LargeList => write_list::( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ), - FixedSizeList => write_fixed_size_list( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ), - Struct => write_struct( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ), - Dictionary(key_type) => match_integer_type!(key_type, |$T| { - write_dictionary::<$T>( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - true, - ); - }), - Union => { - write_union( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ); - } - Map => { - write_map( - array.as_any().downcast_ref().unwrap(), - buffers, - arrow_data, - nodes, - offset, - is_little_endian, - compression, - ); - } - Utf8View => write_binview( - array.as_any().downcast_ref::().unwrap(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ), - BinaryView => write_binview( - array.as_any().downcast_ref::().unwrap(), - buffers, - arrow_data, - offset, - is_little_endian, - compression, - ), - } -} - -#[inline] -fn pad_buffer_to_64(buffer: &mut Vec, length: usize) { - let pad_len = pad_to_64(length); - buffer.extend_from_slice(&vec![0u8; pad_len]); -} - -/// writes `bytes` to `arrow_data` updating `buffers` and `offset` and guaranteeing a 8 byte boundary. -fn write_bytes( - bytes: &[u8], - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - compression: Option, -) { - let start = arrow_data.len(); - if let Some(compression) = compression { - arrow_data.extend_from_slice(&(bytes.len() as i64).to_le_bytes()); - match compression { - Compression::LZ4 => { - compression::compress_lz4(bytes, arrow_data).unwrap(); - } - Compression::ZSTD => { - compression::compress_zstd(bytes, arrow_data).unwrap(); - } - } - } else { - arrow_data.extend_from_slice(bytes); - }; - - buffers.push(finish_buffer(arrow_data, start, offset)); -} - -fn write_bitmap( - bitmap: Option<&Bitmap>, - length: usize, - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - compression: Option, -) { - match bitmap { - Some(bitmap) => { - assert_eq!(bitmap.len(), length); - let (slice, slice_offset, _) = bitmap.as_slice(); - if slice_offset != 0 { - // case where we can't slice the bitmap as the offsets are not multiple of 8 - let bytes = Bitmap::from_trusted_len_iter(bitmap.iter()); - let (slice, _, _) = bytes.as_slice(); - write_bytes(slice, buffers, arrow_data, offset, compression) - } else { - write_bytes(slice, buffers, arrow_data, offset, compression) - } - } - None => { - buffers.push(ipc::Buffer { - offset: *offset, - length: 0, - }); - } - } -} - -/// writes `bytes` to `arrow_data` updating `buffers` and `offset` and guaranteeing a 8 byte boundary. -fn write_buffer( - buffer: &[T], - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - let start = arrow_data.len(); - if let Some(compression) = compression { - _write_compressed_buffer(buffer, arrow_data, is_little_endian, compression); - } else { - _write_buffer(buffer, arrow_data, is_little_endian); - }; - - buffers.push(finish_buffer(arrow_data, start, offset)); -} - -#[inline] -fn _write_buffer_from_iter>( - buffer: I, - arrow_data: &mut Vec, - is_little_endian: bool, -) { - let len = buffer.size_hint().0; - arrow_data.reserve(len * std::mem::size_of::()); - if is_little_endian { - buffer - .map(|x| T::to_le_bytes(&x)) - .for_each(|x| arrow_data.extend_from_slice(x.as_ref())) - } else { - buffer - .map(|x| T::to_be_bytes(&x)) - .for_each(|x| arrow_data.extend_from_slice(x.as_ref())) - } -} - -#[inline] -fn _write_compressed_buffer_from_iter>( - buffer: I, - arrow_data: &mut Vec, - is_little_endian: bool, - compression: Compression, -) { - let len = buffer.size_hint().0; - let mut swapped = Vec::with_capacity(len * std::mem::size_of::()); - if is_little_endian { - buffer - .map(|x| T::to_le_bytes(&x)) - .for_each(|x| swapped.extend_from_slice(x.as_ref())); - } else { - buffer - .map(|x| T::to_be_bytes(&x)) - .for_each(|x| swapped.extend_from_slice(x.as_ref())) - }; - arrow_data.extend_from_slice(&(swapped.len() as i64).to_le_bytes()); - match compression { - Compression::LZ4 => { - compression::compress_lz4(&swapped, arrow_data).unwrap(); - } - Compression::ZSTD => { - compression::compress_zstd(&swapped, arrow_data).unwrap(); - } - } -} - -fn _write_buffer(buffer: &[T], arrow_data: &mut Vec, is_little_endian: bool) { - if is_little_endian == is_native_little_endian() { - // in native endianness we can use the bytes directly. - let buffer = bytemuck::cast_slice(buffer); - arrow_data.extend_from_slice(buffer); - } else { - _write_buffer_from_iter(buffer.iter().copied(), arrow_data, is_little_endian) - } -} - -fn _write_compressed_buffer( - buffer: &[T], - arrow_data: &mut Vec, - is_little_endian: bool, - compression: Compression, -) { - if is_little_endian == is_native_little_endian() { - let bytes = bytemuck::cast_slice(buffer); - arrow_data.extend_from_slice(&(bytes.len() as i64).to_le_bytes()); - match compression { - Compression::LZ4 => { - compression::compress_lz4(bytes, arrow_data).unwrap(); - } - Compression::ZSTD => { - compression::compress_zstd(bytes, arrow_data).unwrap(); - } - } - } else { - todo!() - } -} - -/// writes `bytes` to `arrow_data` updating `buffers` and `offset` and guaranteeing a 8 byte boundary. -#[inline] -fn write_buffer_from_iter>( - buffer: I, - buffers: &mut Vec, - arrow_data: &mut Vec, - offset: &mut i64, - is_little_endian: bool, - compression: Option, -) { - let start = arrow_data.len(); - - if let Some(compression) = compression { - _write_compressed_buffer_from_iter(buffer, arrow_data, is_little_endian, compression); - } else { - _write_buffer_from_iter(buffer, arrow_data, is_little_endian); - } - - buffers.push(finish_buffer(arrow_data, start, offset)); -} - -fn finish_buffer(arrow_data: &mut Vec, start: usize, offset: &mut i64) -> ipc::Buffer { - let buffer_len = (arrow_data.len() - start) as i64; - - pad_buffer_to_64(arrow_data, arrow_data.len() - start); - let total_len = (arrow_data.len() - start) as i64; - - let buffer = ipc::Buffer { - offset: *offset, - length: buffer_len, - }; - *offset += total_len; - buffer -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/stream.rs b/src/common/arrow/src/arrow/io/ipc/write/stream.rs deleted file mode 100644 index 6683c3de53de..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/stream.rs +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Arrow IPC File and Stream Writers -//! -//! The `FileWriter` and `StreamWriter` have similar interfaces, -//! however the `FileWriter` expects a reader that supports `Seek`ing - -use std::io::Write; - -use super::super::IpcField; -use super::common::encode_chunk; -use super::common::DictionaryTracker; -use super::common::EncodedData; -use super::common::WriteOptions; -use super::common_sync::write_continuation; -use super::common_sync::write_message; -use super::default_ipc_fields; -use super::schema_to_bytes; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// Arrow stream writer -/// -/// The data written by this writer must be read in order. To signal that no more -/// data is arriving through the stream call [`self.finish()`](StreamWriter::finish); -/// -/// For a usage walkthrough consult [this example](https://github.com/jorgecarleitao/arrow2/tree/main/examples/ipc_pyarrow). -pub struct StreamWriter { - /// The object to write to - writer: W, - /// IPC write options - write_options: WriteOptions, - /// Whether the stream has been finished - finished: bool, - /// Keeps track of dictionaries that have been written - dictionary_tracker: DictionaryTracker, - - ipc_fields: Option>, -} - -impl StreamWriter { - /// Creates a new [`StreamWriter`] - pub fn new(writer: W, write_options: WriteOptions) -> Self { - Self { - writer, - write_options, - finished: false, - dictionary_tracker: DictionaryTracker { - dictionaries: Default::default(), - cannot_replace: false, - }, - ipc_fields: None, - } - } - - /// Starts the stream by writing a Schema message to it. - /// Use `ipc_fields` to declare dictionary ids in the schema, for dictionary-reuse - pub fn start(&mut self, schema: &Schema, ipc_fields: Option>) -> Result<()> { - self.ipc_fields = Some(if let Some(ipc_fields) = ipc_fields { - ipc_fields - } else { - default_ipc_fields(&schema.fields) - }); - - let encoded_message = EncodedData { - ipc_message: schema_to_bytes(schema, self.ipc_fields.as_ref().unwrap()), - arrow_data: vec![], - }; - write_message(&mut self.writer, &encoded_message)?; - Ok(()) - } - - /// Writes [`Chunk`] to the stream - pub fn write( - &mut self, - columns: &Chunk>, - ipc_fields: Option<&[IpcField]>, - ) -> Result<()> { - if self.finished { - return Err(Error::Io(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "Cannot write to a finished stream".to_string(), - ))); - } - - // we can't make it a closure because it borrows (and it can't borrow mut and non-mut below) - #[allow(clippy::or_fun_call)] - let fields = ipc_fields.unwrap_or(self.ipc_fields.as_ref().unwrap()); - - let (encoded_dictionaries, encoded_message) = encode_chunk( - columns, - fields, - &mut self.dictionary_tracker, - &self.write_options, - )?; - - for encoded_dictionary in encoded_dictionaries { - write_message(&mut self.writer, &encoded_dictionary)?; - } - - write_message(&mut self.writer, &encoded_message)?; - Ok(()) - } - - /// Write continuation bytes, and mark the stream as done - pub fn finish(&mut self) -> Result<()> { - write_continuation(&mut self.writer, 0)?; - - self.finished = true; - - Ok(()) - } - - /// Consumes itself, returning the inner writer. - pub fn into_inner(self) -> W { - self.writer - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/stream_async.rs b/src/common/arrow/src/arrow/io/ipc/write/stream_async.rs deleted file mode 100644 index 94ee8971d5b9..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/stream_async.rs +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! `async` writing of arrow streams - -use std::pin::Pin; -use std::task::Poll; - -use futures::future::BoxFuture; -use futures::AsyncWrite; -use futures::AsyncWriteExt; -use futures::FutureExt; -use futures::Sink; - -use super::super::IpcField; -use super::common::encode_chunk; -use super::common::DictionaryTracker; -use super::common::EncodedData; -pub use super::common::WriteOptions; -use super::common_async::write_continuation; -use super::common_async::write_message; -use super::default_ipc_fields; -use super::schema_to_bytes; -use super::Record; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// A sink that writes array [`chunks`](crate::chunk::Chunk) as an IPC stream. -/// -/// The stream header is automatically written before writing the first chunk. -/// -/// # Examples -/// -/// ``` -/// use arrow2::array::Array; -/// use arrow2::array::Int32Array; -/// use arrow2::chunk::Chunk; -/// use arrow2::datatypes::DataType; -/// use arrow2::datatypes::Field; -/// use arrow2::datatypes::Schema; -/// use futures::SinkExt; -/// # use arrow2::io::ipc::write::stream_async::StreamSink; -/// # futures::executor::block_on(async move { -/// let schema = Schema::from(vec![Field::new("values", DataType::Int32, true)]); -/// -/// let mut buffer = vec![]; -/// let mut sink = StreamSink::new(&mut buffer, &schema, None, Default::default()); -/// -/// for i in 0..3 { -/// let values = Int32Array::from(&[Some(i), None]); -/// let chunk = Chunk::new(vec![values.boxed()]); -/// sink.feed(chunk.into()).await?; -/// } -/// sink.close().await?; -/// # arrow2::error::Result::Ok(()) -/// # }).unwrap(); -/// ``` -pub struct StreamSink<'a, W: AsyncWrite + Unpin + Send + 'a> { - writer: Option, - task: Option>>>, - options: WriteOptions, - dictionary_tracker: DictionaryTracker, - fields: Vec, -} - -impl<'a, W> StreamSink<'a, W> -where W: AsyncWrite + Unpin + Send + 'a -{ - /// Create a new [`StreamSink`]. - pub fn new( - writer: W, - schema: &Schema, - ipc_fields: Option>, - write_options: WriteOptions, - ) -> Self { - let fields = ipc_fields.unwrap_or_else(|| default_ipc_fields(&schema.fields)); - let task = Some(Self::start(writer, schema, &fields[..])); - Self { - writer: None, - task, - fields, - dictionary_tracker: DictionaryTracker { - dictionaries: Default::default(), - cannot_replace: false, - }, - options: write_options, - } - } - - fn start( - mut writer: W, - schema: &Schema, - ipc_fields: &[IpcField], - ) -> BoxFuture<'a, Result>> { - let message = EncodedData { - ipc_message: schema_to_bytes(schema, ipc_fields), - arrow_data: vec![], - }; - async move { - write_message(&mut writer, message).await?; - Ok(Some(writer)) - } - .boxed() - } - - fn write(&mut self, record: Record<'_>) -> Result<()> { - let fields = record.fields().unwrap_or(&self.fields[..]); - let (dictionaries, message) = encode_chunk( - record.columns(), - fields, - &mut self.dictionary_tracker, - &self.options, - )?; - - if let Some(mut writer) = self.writer.take() { - self.task = Some( - async move { - for d in dictionaries { - write_message(&mut writer, d).await?; - } - write_message(&mut writer, message).await?; - Ok(Some(writer)) - } - .boxed(), - ); - Ok(()) - } else { - Err(Error::Io(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "writer closed".to_string(), - ))) - } - } - - fn poll_complete(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - if let Some(task) = &mut self.task { - match futures::ready!(task.poll_unpin(cx)) { - Ok(writer) => { - self.writer = writer; - self.task = None; - Poll::Ready(Ok(())) - } - Err(error) => { - self.task = None; - Poll::Ready(Err(error)) - } - } - } else { - Poll::Ready(Ok(())) - } - } -} - -impl<'a, W> Sink> for StreamSink<'a, W> -where W: AsyncWrite + Unpin + Send -{ - type Error = Error; - - fn poll_ready(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { - self.get_mut().poll_complete(cx) - } - - fn start_send(self: Pin<&mut Self>, item: Record<'_>) -> Result<()> { - self.get_mut().write(item) - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { - self.get_mut().poll_complete(cx) - } - - fn poll_close(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { - let this = self.get_mut(); - match this.poll_complete(cx) { - Poll::Ready(Ok(())) => { - if let Some(mut writer) = this.writer.take() { - this.task = Some( - async move { - write_continuation(&mut writer, 0).await?; - writer.flush().await?; - writer.close().await?; - Ok(None) - } - .boxed(), - ); - this.poll_complete(cx) - } else { - Poll::Ready(Ok(())) - } - } - res => res, - } - } -} diff --git a/src/common/arrow/src/arrow/io/ipc/write/writer.rs b/src/common/arrow/src/arrow/io/ipc/write/writer.rs deleted file mode 100644 index cc85482816ad..000000000000 --- a/src/common/arrow/src/arrow/io/ipc/write/writer.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -use arrow_format::ipc::planus::Builder; - -use super::super::IpcField; -use super::super::ARROW_MAGIC_V2; -use super::common::DictionaryTracker; -use super::common::EncodedData; -use super::common::WriteOptions; -use super::common_sync::write_continuation; -use super::common_sync::write_message; -use super::default_ipc_fields; -use super::schema; -use super::schema_to_bytes; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::write::common::encode_chunk_amortized; - -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum State { - None, - Started, - Finished, -} - -/// Arrow file writer -pub struct FileWriter { - /// The object to write to - pub(crate) writer: W, - /// IPC write options - pub(crate) options: WriteOptions, - /// A reference to the schema, used in validating record batches - pub(crate) schema: Schema, - pub(crate) ipc_fields: Vec, - /// The number of bytes between each block of bytes, as an offset for random access - pub(crate) block_offsets: usize, - /// Dictionary blocks that will be written as part of the IPC footer - pub(crate) dictionary_blocks: Vec, - /// Record blocks that will be written as part of the IPC footer - pub(crate) record_blocks: Vec, - /// Whether the writer footer has been written, and the writer is finished - pub(crate) state: State, - /// Keeps track of dictionaries that have been written - pub(crate) dictionary_tracker: DictionaryTracker, - /// Buffer/scratch that is reused between writes - pub(crate) encoded_message: EncodedData, -} - -impl FileWriter { - /// Creates a new [`FileWriter`] and writes the header to `writer` - pub fn try_new( - writer: W, - schema: Schema, - ipc_fields: Option>, - options: WriteOptions, - ) -> Result { - let mut slf = Self::new(writer, schema, ipc_fields, options); - slf.start()?; - - Ok(slf) - } - - /// Creates a new [`FileWriter`]. - pub fn new( - writer: W, - schema: Schema, - ipc_fields: Option>, - options: WriteOptions, - ) -> Self { - let ipc_fields = if let Some(ipc_fields) = ipc_fields { - ipc_fields - } else { - default_ipc_fields(&schema.fields) - }; - - Self { - writer, - options, - schema, - ipc_fields, - block_offsets: 0, - dictionary_blocks: vec![], - record_blocks: vec![], - state: State::None, - dictionary_tracker: DictionaryTracker { - dictionaries: Default::default(), - cannot_replace: true, - }, - encoded_message: Default::default(), - } - } - - /// Consumes itself into the inner writer - pub fn into_inner(self) -> W { - self.writer - } - - /// Get the inner memory scratches so they can be reused in a new writer. - /// This can be utilized to save memory allocations for performance reasons. - pub fn get_scratches(&mut self) -> EncodedData { - std::mem::take(&mut self.encoded_message) - } - /// Set the inner memory scratches so they can be reused in a new writer. - /// This can be utilized to save memory allocations for performance reasons. - pub fn set_scratches(&mut self, scratches: EncodedData) { - self.encoded_message = scratches; - } - - /// Writes the header and first (schema) message to the file. - /// # Errors - /// Errors if the file has been started or has finished. - pub fn start(&mut self) -> Result<()> { - if self.state != State::None { - return Err(Error::oos("The IPC file can only be started once")); - } - // write magic to header - self.writer.write_all(&ARROW_MAGIC_V2[..])?; - // create an 8-byte boundary after the header - self.writer.write_all(&[0, 0])?; - // write the schema, set the written bytes to the schema - - let encoded_message = EncodedData { - ipc_message: schema_to_bytes(&self.schema, &self.ipc_fields), - arrow_data: vec![], - }; - - let (meta, data) = write_message(&mut self.writer, &encoded_message)?; - self.block_offsets += meta + data + 8; // 8 <=> arrow magic + 2 bytes for alignment - self.state = State::Started; - Ok(()) - } - - /// Writes [`Chunk`] to the file - pub fn write( - &mut self, - chunk: &Chunk>, - ipc_fields: Option<&[IpcField]>, - ) -> Result<()> { - if self.state != State::Started { - return Err(Error::oos( - "The IPC file must be started before it can be written to. Call `start` before `write`", - )); - } - - let ipc_fields = if let Some(ipc_fields) = ipc_fields { - ipc_fields - } else { - self.ipc_fields.as_ref() - }; - let encoded_dictionaries = encode_chunk_amortized( - chunk, - ipc_fields, - &mut self.dictionary_tracker, - &self.options, - &mut self.encoded_message, - )?; - - // add all dictionaries - for encoded_dictionary in encoded_dictionaries { - let (meta, data) = write_message(&mut self.writer, &encoded_dictionary)?; - - let block = arrow_format::ipc::Block { - offset: self.block_offsets as i64, - meta_data_length: meta as i32, - body_length: data as i64, - }; - self.dictionary_blocks.push(block); - self.block_offsets += meta + data; - } - - let (meta, data) = write_message(&mut self.writer, &self.encoded_message)?; - // add a record block for the footer - let block = arrow_format::ipc::Block { - offset: self.block_offsets as i64, - meta_data_length: meta as i32, // TODO: is this still applicable? - body_length: data as i64, - }; - self.record_blocks.push(block); - self.block_offsets += meta + data; - Ok(()) - } - - /// Write footer and closing tag, then mark the writer as done - pub fn finish(&mut self) -> Result<()> { - if self.state != State::Started { - return Err(Error::oos( - "The IPC file must be started before it can be finished. Call `start` before `finish`", - )); - } - - // write EOS - write_continuation(&mut self.writer, 0)?; - - let schema = schema::serialize_schema(&self.schema, &self.ipc_fields); - - let root = arrow_format::ipc::Footer { - version: arrow_format::ipc::MetadataVersion::V5, - schema: Some(Box::new(schema)), - dictionaries: Some(std::mem::take(&mut self.dictionary_blocks)), - record_batches: Some(std::mem::take(&mut self.record_blocks)), - custom_metadata: None, - }; - let mut builder = Builder::new(); - let footer_data = builder.finish(&root, None); - self.writer.write_all(footer_data)?; - self.writer - .write_all(&(footer_data.len() as i32).to_le_bytes())?; - self.writer.write_all(&ARROW_MAGIC_V2)?; - self.writer.flush()?; - self.state = State::Finished; - - Ok(()) - } -} diff --git a/src/common/arrow/src/arrow/io/iterator.rs b/src/common/arrow/src/arrow/io/iterator.rs deleted file mode 100644 index 772b8b1f9d00..000000000000 --- a/src/common/arrow/src/arrow/io/iterator.rs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub use streaming_iterator::StreamingIterator; - -/// A [`StreamingIterator`] with an internal buffer of [`Vec`] used to efficiently -/// present items of type `T` as `&[u8]`. -/// It is generic over the type `T` and the transformation `F: T -> &[u8]`. -pub struct BufStreamingIterator -where - I: Iterator, - F: FnMut(T, &mut Vec), -{ - iterator: I, - f: F, - buffer: Vec, - is_valid: bool, -} - -impl BufStreamingIterator -where - I: Iterator, - F: FnMut(T, &mut Vec), -{ - #[inline] - pub fn new(iterator: I, f: F, buffer: Vec) -> Self { - Self { - iterator, - f, - buffer, - is_valid: false, - } - } -} - -impl StreamingIterator for BufStreamingIterator -where - I: Iterator, - F: FnMut(T, &mut Vec), -{ - type Item = [u8]; - - #[inline] - fn advance(&mut self) { - let a = self.iterator.next(); - if let Some(a) = a { - self.is_valid = true; - self.buffer.clear(); - (self.f)(a, &mut self.buffer); - } else { - self.is_valid = false; - } - } - - #[inline] - fn get(&self) -> Option<&Self::Item> { - if self.is_valid { - Some(&self.buffer) - } else { - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iterator.size_hint() - } -} diff --git a/src/common/arrow/src/arrow/io/mod.rs b/src/common/arrow/src/arrow/io/mod.rs deleted file mode 100644 index 6807314c8347..000000000000 --- a/src/common/arrow/src/arrow/io/mod.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![forbid(unsafe_code)] -//! Contains modules to interface with other formats such as [`csv`], -//! [`parquet`], [`json`], [`ipc`], [`mod@print`] and [`avro`]. - -#[cfg(feature = "io_ipc")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_ipc")))] -pub mod ipc; - -#[cfg(feature = "io_flight")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_flight")))] -pub mod flight; - -#[cfg(feature = "io_parquet")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_parquet")))] -pub mod parquet; diff --git a/src/common/arrow/src/arrow/io/parquet/mod.rs b/src/common/arrow/src/arrow/io/parquet/mod.rs deleted file mode 100644 index da0118773942..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/mod.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to read from and write to Parquet format. -use crate::arrow::error::Error; - -pub mod read; -pub mod write; - -#[cfg(feature = "io_parquet_bloom_filter")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_parquet_bloom_filter")))] -pub use parquet2::bloom_filter; - -const ARROW_SCHEMA_META_KEY: &str = "ARROW:schema"; - -impl From for Error { - fn from(error: parquet2::error::Error) -> Self { - match error { - parquet2::error::Error::FeatureNotActive(_, _) => { - let message = "Failed to read a compressed parquet file. \ - Use the cargo feature \"io_parquet_compression\" to read compressed parquet files." - .to_string(); - Error::ExternalFormat(message) - } - _ => Error::ExternalFormat(error.to_string()), - } - } -} - -impl From for parquet2::error::Error { - fn from(error: Error) -> Self { - parquet2::error::Error::OutOfSpec(error.to_string()) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/README.md b/src/common/arrow/src/arrow/io/parquet/read/README.md deleted file mode 100644 index 37bf71d5fd61..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/README.md +++ /dev/null @@ -1,34 +0,0 @@ -## Observations - -### LSB equivalence between definition levels and bitmaps - -When the maximum repetition level is 0 and the maximum definition level is 1, -the RLE-encoded definition levels correspond exactly to Arrow's bitmap and can be -memcopied without further transformations. - -## Nested parquet groups are deserialized recursively - -Reading a parquet nested field is done by reading each primitive -column sequentially, and build the nested struct recursively. - -Rows of nested parquet groups are encoded in the repetition and definition levels. -In arrow, they correspond to: -* list's offsets and validity -* struct's validity - -The implementation in this module leverages this observation: - -Nested parquet fields are initially recursed over to gather -whether the type is a Struct or List, and whether it is required or optional, which we store -in `nested_info: Vec>`. `Nested` is a trait object that receives definition -and repetition levels depending on the type and nullability of the nested item. -We process the definition and repetition levels into `nested_info`. - -When we finish a field, we recursively pop from `nested_info` as we build -the `StructArray` or `ListArray`. - -With this approach, the only difference vs flat is: -1. we do not leverage the bitmap optimization, and instead need to deserialize the repetition - and definition levels to `i32`. -2. we deserialize definition levels twice, once to extend the values/nullability and - one to extend `nested_info`. diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/README.md b/src/common/arrow/src/arrow/io/parquet/read/deserialize/README.md deleted file mode 100644 index 347424fca2f8..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Design - -## Non-nested types - -Let's start with the design used for non-nested arrays. The (private) entry point of this -module for non-nested arrays is `simple::page_iter_to_arrays`. - -This function expects - -* a (fallible) streaming iterator of decompressed and encoded pages, `Pages` -* the source (parquet) column type, including its logical information -* the target (arrow) `DataType` -* the chunk size - -and returns an iterator of `Array`, `ArrayIter`. - -This design is shared among _all_ `(parquet, arrow)` implemented tuples. Their main -difference is how they are deserialized, which depends on the source and target types. - -When the array iterator is pulled the first time, the following happens: -* a page from `Pages` is pulled -* a `PageState<'a>` is built from the page -* the `PageState` is consumed into a mutable array: - * if `chunk_size` is larger than the number of rows in the page, the mutable array state is preserved and a new page is pulled and the process repeated until we fill a chunk. - * if `chunk_size` is smaller than the number of rows in the page, the mutable array state - is returned and the remaining of the page is consumed into multiple mutable arrays of length `chunk_size` into a FIFO queue. - -Subsequent pulls of arrays will first try to pull from the FIFO queue. Once the queue is empty, the -a new page is pulled. - -### `PageState` - -As mentioned above, the iterator leverages the idea that we attach a state to a page. Recall -that a page is essentially `[header][data]`. The `data` part contains encoded -`[rep levels][def levels][non-null values]`. Some pages have an associated dictionary page, -in which case the `non-null values` represent the indices. - -Irrespectively of the physical type, the main idea is to split the page in two iterators: - -* An iterator over `def levels` -* An iterator over `non-null values` - -and progress the iterators as needed. In particular, for non-nested types, `def levels` is -a bitmap with the same representation as Arrow, in which case the validity is extended directly. - -The `non-null values` are "expanded" by filling null values with the default value of each physical -type. - -## Nested types - -For nested type with N+1 levels (1 is the primitive), we need to build the nest information of each -N levels + the non-nested Arrow array. - -This is done by first transversing the parquet types and using it to initialize, per chunk, the N levels. - -The per-chunk execution is then similar but `chunk_size` only drives the number of retrieved -rows from the outermost parquet group (the field). Each of these pulls knows how many items need -to be pulled from the inner groups, all the way to the primitive type. This works because -in parquet a row cannot be split between two pages and thus each page is guaranteed -to contain a full row. - -The `PageState` of nested types is composed by 4 iterators: - -* A (zipped) iterator over `rep levels` and `def levels` -* An iterator over `def levels` -* An iterator over `non-null values` - -The idea is that an iterator of `rep, def` contain all the information to decode the -nesting structure of an arrow array. The other two iterators are equivalent to the non-nested -types with the exception that `def levels` are no equivalent to arrow bitmaps. diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/basic.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/basic.rs deleted file mode 100644 index 57d1511c3b9d..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/basic.rs +++ /dev/null @@ -1,549 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::default::Default; - -use parquet2::deserialize::SliceFilteredIter; -use parquet2::encoding::delta_length_byte_array; -use parquet2::encoding::hybrid_rle; -use parquet2::encoding::Encoding; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; - -use super::super::utils; -use super::super::utils::extend_from_decoder; -use super::super::utils::get_selected_rows; -use super::super::utils::next; -use super::super::utils::DecodedState; -use super::super::utils::FilteredOptionalPageValidity; -use super::super::utils::MaybeNext; -use super::super::utils::OptionalPageValidity; -use super::super::Pages; -use super::utils::*; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; - -#[derive(Debug)] -pub(crate) struct Required<'a> { - pub values: SizedBinaryIter<'a>, -} - -impl<'a> Required<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, _, values) = split_buffer(page)?; - let values = SizedBinaryIter::new(values, page.num_values()); - - Ok(Self { values }) - } - - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(crate) struct Delta<'a> { - pub lengths: std::vec::IntoIter, - pub values: &'a [u8], -} - -impl<'a> Delta<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, _, values) = split_buffer(page)?; - - let mut lengths_iter = delta_length_byte_array::Decoder::try_new(values)?; - - #[allow(clippy::needless_collect)] // we need to consume it to get the values - let lengths = lengths_iter - .by_ref() - .map(|x| x.map(|x| x as usize).map_err(Error::from)) - .collect::>>()?; - - let values = lengths_iter.into_values(); - Ok(Self { - lengths: lengths.into_iter(), - values, - }) - } - - pub fn len(&self) -> usize { - self.lengths.size_hint().0 - } -} - -impl<'a> Iterator for Delta<'a> { - type Item = &'a [u8]; - - #[inline] - fn next(&mut self) -> Option { - let length = self.lengths.next()?; - let (item, remaining) = self.values.split_at(length); - self.values = remaining; - Some(item) - } - - fn size_hint(&self) -> (usize, Option) { - self.lengths.size_hint() - } -} - -#[derive(Debug)] -pub(crate) struct FilteredRequired<'a> { - pub values: SliceFilteredIter>, -} - -impl<'a> FilteredRequired<'a> { - pub fn new(page: &'a DataPage) -> Self { - let values = SizedBinaryIter::new(page.buffer(), page.num_values()); - - let rows = get_selected_rows(page); - let values = SliceFilteredIter::new(values, rows); - - Self { values } - } - - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(crate) struct FilteredDelta<'a> { - pub values: SliceFilteredIter>, -} - -impl<'a> FilteredDelta<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let values = Delta::try_new(page)?; - - let rows = get_selected_rows(page); - let values = SliceFilteredIter::new(values, rows); - - Ok(Self { values }) - } - - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -pub(crate) type BinaryDict = Vec>; - -#[derive(Debug)] -pub(crate) struct RequiredDictionary<'a> { - pub values: hybrid_rle::HybridRleDecoder<'a>, - pub dict: &'a BinaryDict, -} - -impl<'a> RequiredDictionary<'a> { - pub fn try_new(page: &'a DataPage, dict: &'a BinaryDict) -> Result { - let values = utils::dict_indices_decoder(page)?; - - Ok(Self { dict, values }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(crate) struct FilteredRequiredDictionary<'a> { - pub values: SliceFilteredIter>, - pub dict: &'a BinaryDict, -} - -impl<'a> FilteredRequiredDictionary<'a> { - pub fn try_new(page: &'a DataPage, dict: &'a BinaryDict) -> Result { - let values = utils::dict_indices_decoder(page)?; - - let rows = get_selected_rows(page); - let values = SliceFilteredIter::new(values, rows); - - Ok(Self { values, dict }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(crate) struct ValuesDictionary<'a> { - pub values: hybrid_rle::HybridRleDecoder<'a>, - pub dict: &'a BinaryDict, -} - -impl<'a> ValuesDictionary<'a> { - pub fn try_new(page: &'a DataPage, dict: &'a BinaryDict) -> Result { - let values = utils::dict_indices_decoder(page)?; - - Ok(Self { dict, values }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(crate) enum BinaryState<'a> { - Optional(OptionalPageValidity<'a>, BinaryIter<'a>), - Required(Required<'a>), - RequiredDictionary(RequiredDictionary<'a>), - OptionalDictionary(OptionalPageValidity<'a>, ValuesDictionary<'a>), - Delta(Delta<'a>), - OptionalDelta(OptionalPageValidity<'a>, Delta<'a>), - FilteredRequired(FilteredRequired<'a>), - FilteredDelta(FilteredDelta<'a>), - FilteredOptionalDelta(FilteredOptionalPageValidity<'a>, Delta<'a>), - FilteredOptional(FilteredOptionalPageValidity<'a>, BinaryIter<'a>), - FilteredRequiredDictionary(FilteredRequiredDictionary<'a>), - FilteredOptionalDictionary(FilteredOptionalPageValidity<'a>, ValuesDictionary<'a>), -} - -impl<'a> utils::PageState<'a> for BinaryState<'a> { - fn len(&self) -> usize { - match self { - BinaryState::Optional(validity, _) => validity.len(), - BinaryState::Required(state) => state.len(), - BinaryState::Delta(state) => state.len(), - BinaryState::OptionalDelta(state, _) => state.len(), - BinaryState::RequiredDictionary(values) => values.len(), - BinaryState::OptionalDictionary(optional, _) => optional.len(), - BinaryState::FilteredRequired(state) => state.len(), - BinaryState::FilteredOptional(validity, _) => validity.len(), - BinaryState::FilteredDelta(state) => state.len(), - BinaryState::FilteredOptionalDelta(state, _) => state.len(), - BinaryState::FilteredRequiredDictionary(values) => values.len(), - BinaryState::FilteredOptionalDictionary(optional, _) => optional.len(), - } - } -} - -impl DecodedState for (Binary, MutableBitmap) { - fn len(&self) -> usize { - self.0.len() - } -} - -#[derive(Debug, Default)] -struct BinaryDecoder { - phantom_o: std::marker::PhantomData, -} - -impl<'a, O: Offset> utils::Decoder<'a> for BinaryDecoder { - type State = BinaryState<'a>; - type Dict = BinaryDict; - type DecodedState = (Binary, MutableBitmap); - - fn build_state(&self, page: &'a DataPage, dict: Option<&'a Self::Dict>) -> Result { - build_binary_state(page, dict) - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - Binary::::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn extend_from_state( - &self, - state: &mut Self::State, - decoded: &mut Self::DecodedState, - additional: usize, - ) { - let (values, validity) = decoded; - match state { - BinaryState::Optional(page_validity, page_values) => extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - page_values, - ), - BinaryState::Required(page) => { - for x in page.values.by_ref().take(additional) { - values.push(x) - } - } - BinaryState::Delta(page) => { - values.extend_lengths(page.lengths.by_ref().take(additional), &mut page.values); - } - BinaryState::OptionalDelta(page_validity, page_values) => { - let Binary { - offsets, - values: values_, - } = values; - - let last_offset = *offsets.last(); - extend_from_decoder( - validity, - page_validity, - Some(additional), - offsets, - page_values.lengths.by_ref(), - ); - - let length = *offsets.last() - last_offset; - - let (consumed, remaining) = page_values.values.split_at(length.to_usize()); - page_values.values = remaining; - values_.extend_from_slice(consumed); - } - BinaryState::FilteredRequired(page) => { - for x in page.values.by_ref().take(additional) { - values.push(x) - } - } - BinaryState::FilteredDelta(page) => { - for x in page.values.by_ref().take(additional) { - values.push(x) - } - } - BinaryState::OptionalDictionary(page_validity, page_values) => { - let page_dict = &page_values.dict; - utils::extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - &mut page_values - .values - .by_ref() - .map(|index| page_dict[index.unwrap() as usize].as_ref()), - ) - } - BinaryState::RequiredDictionary(page) => { - let page_dict = &page.dict; - - for x in page - .values - .by_ref() - .map(|index| page_dict[index.unwrap() as usize].as_ref()) - .take(additional) - { - values.push(x) - } - } - BinaryState::FilteredOptional(page_validity, page_values) => { - utils::extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - page_values.by_ref(), - ); - } - BinaryState::FilteredOptionalDelta(page_validity, page_values) => { - utils::extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - page_values.by_ref(), - ); - } - BinaryState::FilteredRequiredDictionary(page) => { - let page_dict = &page.dict; - for x in page - .values - .by_ref() - .map(|index| page_dict[index.unwrap() as usize].as_ref()) - .take(additional) - { - values.push(x) - } - } - BinaryState::FilteredOptionalDictionary(page_validity, page_values) => { - let page_dict = &page_values.dict; - utils::extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - &mut page_values - .values - .by_ref() - .map(|index| page_dict[index.unwrap() as usize].as_ref()), - ) - } - } - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dict { - deserialize_plain(&page.buffer, page.num_values) - } -} - -pub(super) fn finish( - data_type: &DataType, - mut values: Binary, - mut validity: MutableBitmap, -) -> Result> { - values.offsets.shrink_to_fit(); - values.values.shrink_to_fit(); - validity.shrink_to_fit(); - - match data_type.to_physical_type() { - PhysicalType::Binary | PhysicalType::LargeBinary => BinaryArray::::try_new( - data_type.clone(), - values.offsets.into(), - values.values.into(), - validity.into(), - ) - .map(|x| x.boxed()), - PhysicalType::Utf8 | PhysicalType::LargeUtf8 => Utf8Array::::try_new( - data_type.clone(), - values.offsets.into(), - values.values.into(), - validity.into(), - ) - .map(|x| x.boxed()), - _ => unreachable!(), - } -} - -pub struct Iter { - iter: I, - data_type: DataType, - items: VecDeque<(Binary, MutableBitmap)>, - dict: Option, - chunk_size: Option, - remaining: usize, -} - -impl Iter { - pub fn new(iter: I, data_type: DataType, chunk_size: Option, num_rows: usize) -> Self { - Self { - iter, - data_type, - items: VecDeque::new(), - dict: None, - chunk_size, - remaining: num_rows, - } - } -} - -impl Iterator for Iter { - type Item = Result>; - - fn next(&mut self) -> Option { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - self.chunk_size, - &BinaryDecoder::::default(), - ); - match maybe_state { - MaybeNext::Some(Ok((values, validity))) => { - Some(finish(&self.data_type, values, validity)) - } - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} - -pub(crate) fn deserialize_plain(values: &[u8], num_values: usize) -> BinaryDict { - SizedBinaryIter::new(values, num_values) - .map(|x| x.to_vec()) - .collect() -} - -pub(crate) fn build_binary_state<'a>( - page: &'a DataPage, - dict: Option<&'a BinaryDict>, -) -> Result> { - let is_optional = page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), dict, is_optional, is_filtered) { - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false, false) => Ok( - BinaryState::RequiredDictionary(RequiredDictionary::try_new(page, dict)?), - ), - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true, false) => { - Ok(BinaryState::OptionalDictionary( - OptionalPageValidity::try_new(page)?, - ValuesDictionary::try_new(page, dict)?, - )) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false, true) => { - FilteredRequiredDictionary::try_new(page, dict) - .map(BinaryState::FilteredRequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true, true) => { - Ok(BinaryState::FilteredOptionalDictionary( - FilteredOptionalPageValidity::try_new(page)?, - ValuesDictionary::try_new(page, dict)?, - )) - } - (Encoding::Plain, _, true, false) => { - let (_, _, values) = split_buffer(page)?; - - let values = BinaryIter::new(values); - - Ok(BinaryState::Optional( - OptionalPageValidity::try_new(page)?, - values, - )) - } - (Encoding::Plain, _, false, false) => Ok(BinaryState::Required(Required::try_new(page)?)), - (Encoding::Plain, _, false, true) => { - Ok(BinaryState::FilteredRequired(FilteredRequired::new(page))) - } - (Encoding::Plain, _, true, true) => { - let (_, _, values) = split_buffer(page)?; - - Ok(BinaryState::FilteredOptional( - FilteredOptionalPageValidity::try_new(page)?, - BinaryIter::new(values), - )) - } - (Encoding::DeltaLengthByteArray, _, false, false) => { - Delta::try_new(page).map(BinaryState::Delta) - } - (Encoding::DeltaLengthByteArray, _, true, false) => Ok(BinaryState::OptionalDelta( - OptionalPageValidity::try_new(page)?, - Delta::try_new(page)?, - )), - (Encoding::DeltaLengthByteArray, _, false, true) => { - FilteredDelta::try_new(page).map(BinaryState::FilteredDelta) - } - (Encoding::DeltaLengthByteArray, _, true, true) => Ok(BinaryState::FilteredOptionalDelta( - FilteredOptionalPageValidity::try_new(page)?, - Delta::try_new(page)?, - )), - _ => Err(utils::not_implemented(page)), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/dictionary.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/dictionary.rs deleted file mode 100644 index 45e998209afe..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/dictionary.rs +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::page::DictPage; - -use super::super::dictionary::*; -use super::super::utils::MaybeNext; -use super::super::Pages; -use super::utils::Binary; -use super::utils::SizedBinaryIter; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::Utf8Array; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::deserialize::nested_utils::InitNested; -use crate::arrow::io::parquet::read::deserialize::nested_utils::NestedState; -use crate::arrow::offset::Offset; - -/// An iterator adapter over [`Pages`] assumed to be encoded as parquet's dictionary-encoded binary representation -#[derive(Debug)] -pub struct DictIter -where - I: Pages, - O: Offset, - K: DictionaryKey, -{ - iter: I, - data_type: DataType, - values: Option>, - items: VecDeque<(Vec, MutableBitmap)>, - remaining: usize, - chunk_size: Option, - phantom: std::marker::PhantomData, -} - -impl DictIter -where - K: DictionaryKey, - O: Offset, - I: Pages, -{ - pub fn new(iter: I, data_type: DataType, num_rows: usize, chunk_size: Option) -> Self { - Self { - iter, - data_type, - values: None, - items: VecDeque::new(), - remaining: num_rows, - chunk_size, - phantom: std::marker::PhantomData, - } - } -} - -fn read_dict(data_type: DataType, dict: &DictPage) -> Box { - let data_type = match data_type { - DataType::Dictionary(_, values, _) => *values, - _ => data_type, - }; - - let values = SizedBinaryIter::new(&dict.buffer, dict.num_values); - - let mut data = Binary::::with_capacity(dict.num_values); - data.values = Vec::with_capacity(dict.buffer.len() - 4 * dict.num_values); - for item in values { - data.push(item) - } - - match data_type.to_physical_type() { - PhysicalType::Utf8 | PhysicalType::LargeUtf8 => { - Utf8Array::::new(data_type, data.offsets.into(), data.values.into(), None).boxed() - } - PhysicalType::Binary | PhysicalType::LargeBinary => { - BinaryArray::::new(data_type, data.offsets.into(), data.values.into(), None).boxed() - } - _ => unreachable!(), - } -} - -impl Iterator for DictIter -where - I: Pages, - O: Offset, - K: DictionaryKey, -{ - type Item = Result>; - - fn next(&mut self) -> Option { - let maybe_state = next_dict( - &mut self.iter, - &mut self.items, - &mut self.values, - self.data_type.clone(), - &mut self.remaining, - self.chunk_size, - |dict| read_dict::(self.data_type.clone(), dict), - ); - match maybe_state { - MaybeNext::Some(Ok(dict)) => Some(Ok(dict)), - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} - -/// An iterator adapter that converts [`DataPages`] into an [`Iterator`] of [`DictionaryArray`] -#[derive(Debug)] -pub struct NestedDictIter -where - I: Pages, - O: Offset, - K: DictionaryKey, -{ - iter: I, - init: Vec, - data_type: DataType, - values: Option>, - items: VecDeque<(NestedState, (Vec, MutableBitmap))>, - remaining: usize, - chunk_size: Option, - phantom: std::marker::PhantomData, -} - -impl NestedDictIter -where - I: Pages, - O: Offset, - K: DictionaryKey, -{ - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - ) -> Self { - Self { - iter, - init, - data_type, - values: None, - items: VecDeque::new(), - remaining: num_rows, - chunk_size, - phantom: Default::default(), - } - } -} - -impl Iterator for NestedDictIter -where - I: Pages, - O: Offset, - K: DictionaryKey, -{ - type Item = Result<(NestedState, DictionaryArray)>; - - fn next(&mut self) -> Option { - let maybe_state = nested_next_dict( - &mut self.iter, - &mut self.items, - &mut self.remaining, - &self.init, - &mut self.values, - self.data_type.clone(), - self.chunk_size, - |dict| read_dict::(self.data_type.clone(), dict), - ); - match maybe_state { - MaybeNext::Some(Ok(dict)) => Some(Ok(dict)), - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/mod.rs deleted file mode 100644 index 9b0e7ad8bbdf..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub(super) mod basic; -mod dictionary; -pub(super) mod nested; -mod utils; - -pub use basic::Iter; -pub use dictionary::DictIter; -pub use dictionary::NestedDictIter; -pub use nested::NestedIter; -pub use utils::SizedBinaryIter; diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/nested.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/nested.rs deleted file mode 100644 index dee1ea5b5afc..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/nested.rs +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::encoding::Encoding; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; - -use super::super::nested_utils::*; -use super::super::utils; -use super::super::utils::MaybeNext; -use super::basic::deserialize_plain; -use super::basic::finish; -use super::basic::BinaryDict; -use super::basic::ValuesDictionary; -use super::utils::*; -use crate::arrow::array::Array; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::Pages; -use crate::arrow::offset::Offset; - -#[derive(Debug)] -pub(crate) enum BinaryNestedState<'a> { - Optional(BinaryIter<'a>), - Required(BinaryIter<'a>), - RequiredDictionary(ValuesDictionary<'a>), - OptionalDictionary(ValuesDictionary<'a>), -} - -impl<'a> utils::PageState<'a> for BinaryNestedState<'a> { - fn len(&self) -> usize { - match self { - BinaryNestedState::Optional(validity) => validity.size_hint().0, - BinaryNestedState::Required(state) => state.size_hint().0, - BinaryNestedState::RequiredDictionary(required) => required.len(), - BinaryNestedState::OptionalDictionary(optional) => optional.len(), - } - } -} - -#[derive(Debug, Default)] -struct BinaryDecoder { - phantom_o: std::marker::PhantomData, -} - -impl<'a, O: Offset> NestedDecoder<'a> for BinaryDecoder { - type State = BinaryNestedState<'a>; - type Dictionary = BinaryDict; - type DecodedState = (Binary, MutableBitmap); - - fn build_state( - &self, - page: &'a DataPage, - dict: Option<&'a Self::Dictionary>, - ) -> Result { - build_nested_state(page, dict) - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - Binary::::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn push_valid(&self, state: &mut Self::State, decoded: &mut Self::DecodedState) -> Result<()> { - let (values, validity) = decoded; - match state { - BinaryNestedState::Optional(page) => { - let value = page.next().unwrap_or_default(); - values.push(value); - validity.push(true); - } - BinaryNestedState::Required(page) => { - let value = page.next().unwrap_or_default(); - values.push(value); - } - BinaryNestedState::RequiredDictionary(page) => { - let dict_values = &page.dict; - let item = page - .values - .next() - .map(|index| dict_values[index.unwrap() as usize].as_ref()) - .unwrap_or_default(); - values.push(item); - } - BinaryNestedState::OptionalDictionary(page) => { - let dict_values = &page.dict; - let item = page - .values - .next() - .map(|index| dict_values[index.unwrap() as usize].as_ref()) - .unwrap_or_default(); - values.push(item); - validity.push(true); - } - } - Ok(()) - } - - fn push_null(&self, decoded: &mut Self::DecodedState) { - let (values, validity) = decoded; - values.push(&[]); - validity.push(false); - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dictionary { - deserialize_plain(&page.buffer, page.num_values) - } -} - -pub(crate) fn build_nested_state<'a>( - page: &'a DataPage, - dict: Option<&'a BinaryDict>, -) -> Result> { - let is_optional = page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), dict, is_optional, is_filtered) { - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false, false) => { - ValuesDictionary::try_new(page, dict).map(BinaryNestedState::RequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true, false) => { - ValuesDictionary::try_new(page, dict).map(BinaryNestedState::OptionalDictionary) - } - (Encoding::Plain, _, true, false) => { - let (_, _, values) = split_buffer(page)?; - - let values = BinaryIter::new(values); - - Ok(BinaryNestedState::Optional(values)) - } - (Encoding::Plain, _, false, false) => { - let (_, _, values) = split_buffer(page)?; - - let values = BinaryIter::new(values); - - Ok(BinaryNestedState::Required(values)) - } - _ => Err(utils::not_implemented(page)), - } -} - -pub struct NestedIter { - iter: I, - data_type: DataType, - init: Vec, - items: VecDeque<(NestedState, (Binary, MutableBitmap))>, - dict: Option, - chunk_size: Option, - remaining: usize, -} - -impl NestedIter { - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - ) -> Self { - Self { - iter, - data_type, - init, - items: VecDeque::new(), - dict: None, - chunk_size, - remaining: num_rows, - } - } -} - -impl Iterator for NestedIter { - type Item = Result<(NestedState, Box)>; - - fn next(&mut self) -> Option { - loop { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - &self.init, - self.chunk_size, - &BinaryDecoder::::default(), - ); - match maybe_state { - MaybeNext::Some(Ok((nested, decoded))) => { - return Some( - finish(&self.data_type, decoded.0, decoded.1).map(|array| (nested, array)), - ); - } - MaybeNext::Some(Err(e)) => return Some(Err(e)), - MaybeNext::None => return None, - MaybeNext::More => continue, /* Using continue in a loop instead of calling next helps prevent stack overflow. */ - } - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/utils.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/utils.rs deleted file mode 100644 index ccdb13eb055d..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binary/utils.rs +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::utils::Pushable; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; - -/// [`Pushable`] for variable length binary data. -#[derive(Debug)] -pub struct Binary { - pub offsets: Offsets, - pub values: Vec, -} - -impl Pushable for Offsets { - fn reserve(&mut self, additional: usize) { - self.reserve(additional) - } - #[inline] - fn len(&self) -> usize { - self.len_proxy() - } - - #[inline] - fn push(&mut self, value: usize) { - self.try_push_usize(value).unwrap() - } - - #[inline] - fn push_null(&mut self) { - self.extend_constant(1); - } - - #[inline] - fn extend_constant(&mut self, additional: usize, _: usize) { - self.extend_constant(additional) - } -} - -impl Binary { - #[inline] - pub fn with_capacity(capacity: usize) -> Self { - Self { - offsets: Offsets::with_capacity(capacity), - values: Vec::with_capacity(capacity.min(100) * 24), - } - } - - #[inline] - pub fn push(&mut self, v: &[u8]) { - if self.offsets.len_proxy() == 100 && self.offsets.capacity() > 100 { - let bytes_per_row = self.values.len() / 100 + 1; - let bytes_estimate = bytes_per_row * self.offsets.capacity(); - - if bytes_estimate > self.values.capacity() && bytes_estimate < 10 * 1024 * 1024 { - self.values.reserve(bytes_estimate - self.values.capacity()); - } - } - - self.values.extend(v); - self.offsets.try_push_usize(v.len()).unwrap() - } - - #[inline] - pub fn extend_constant(&mut self, additional: usize) { - self.offsets.extend_constant(additional); - } - - #[inline] - pub fn len(&self) -> usize { - self.offsets.len_proxy() - } - - #[inline] - pub fn extend_lengths>(&mut self, lengths: I, values: &mut &[u8]) { - let current_offset = *self.offsets.last(); - self.offsets.try_extend_from_lengths(lengths).unwrap(); - let new_offset = *self.offsets.last(); - let length = new_offset.to_usize() - current_offset.to_usize(); - let (consumed, remaining) = values.split_at(length); - *values = remaining; - self.values.extend_from_slice(consumed); - } -} - -impl<'a, O: Offset> Pushable<&'a [u8]> for Binary { - #[inline] - fn reserve(&mut self, additional: usize) { - let avg_len = self.values.len() / std::cmp::max(self.offsets.last().to_usize(), 1); - self.values.reserve(additional * avg_len); - self.offsets.reserve(additional); - } - #[inline] - fn len(&self) -> usize { - self.len() - } - - #[inline] - fn push_null(&mut self) { - self.push(&[]) - } - - #[inline] - fn push(&mut self, value: &[u8]) { - self.push(value) - } - - #[inline] - fn extend_constant(&mut self, additional: usize, value: &[u8]) { - assert_eq!(value.len(), 0); - self.extend_constant(additional) - } -} - -#[derive(Debug)] -pub struct BinaryIter<'a> { - values: &'a [u8], -} - -impl<'a> BinaryIter<'a> { - pub fn new(values: &'a [u8]) -> Self { - Self { values } - } -} - -impl<'a> Iterator for BinaryIter<'a> { - type Item = &'a [u8]; - - #[inline] - fn next(&mut self) -> Option { - if self.values.is_empty() { - return None; - } - let (length, remaining) = self.values.split_at(4); - let length = u32::from_le_bytes(length.try_into().unwrap()) as usize; - let (result, remaining) = remaining.split_at(length); - self.values = remaining; - Some(result) - } -} - -#[derive(Debug)] -pub struct SizedBinaryIter<'a> { - iter: BinaryIter<'a>, - remaining: usize, -} - -impl<'a> SizedBinaryIter<'a> { - pub fn new(values: &'a [u8], size: usize) -> Self { - let iter = BinaryIter::new(values); - Self { - iter, - remaining: size, - } - } -} - -impl<'a> Iterator for SizedBinaryIter<'a> { - type Item = &'a [u8]; - - #[inline] - fn next(&mut self) -> Option { - if self.remaining == 0 { - return None; - } else { - self.remaining -= 1 - }; - self.iter.next() - } - - fn size_hint(&self) -> (usize, Option) { - (self.remaining, Some(self.remaining)) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/basic.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/basic.rs deleted file mode 100644 index fe3edd0fa90f..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/basic.rs +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::cell::Cell; -use std::collections::VecDeque; - -use parquet2::page::DataPage; -use parquet2::page::DictPage; - -use super::super::binary::basic::deserialize_plain; -use super::super::binary::basic::BinaryDict; -use super::super::binary::basic::BinaryState; -use super::super::utils; -use super::super::utils::extend_from_decoder; -use super::super::utils::next; -use super::super::utils::DecodedState; -use super::super::utils::MaybeNext; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryViewArray; -use crate::arrow::array::MutableBinaryViewArray; -use crate::arrow::array::Utf8ViewArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::deserialize::binary::basic::build_binary_state; -use crate::arrow::io::parquet::read::Pages; -use crate::ArrayRef; - -type DecodedStateTuple = (MutableBinaryViewArray<[u8]>, MutableBitmap); - -#[derive(Default)] -struct BinViewDecoder { - check_utf8: Cell, -} - -impl DecodedState for DecodedStateTuple { - fn len(&self) -> usize { - self.0.len() - } -} - -impl<'a> utils::Decoder<'a> for BinViewDecoder { - type State = BinaryState<'a>; - type Dict = BinaryDict; - type DecodedState = DecodedStateTuple; - - fn build_state(&self, page: &'a DataPage, dict: Option<&'a Self::Dict>) -> Result { - build_binary_state(page, dict) - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - MutableBinaryViewArray::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn extend_from_state( - &self, - state: &mut Self::State, - decoded: &mut Self::DecodedState, - additional: usize, - ) { - let (values, validity) = decoded; - let mut validate_utf8 = self.check_utf8.take(); - - match state { - BinaryState::Optional(page_validity, page_values) => extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - page_values, - ), - BinaryState::Required(page) => { - for x in page.values.by_ref().take(additional) { - values.push_value_ignore_validity(x) - } - } - BinaryState::Delta(page) => { - for value in page { - values.push_value_ignore_validity(value) - } - } - BinaryState::OptionalDelta(page_validity, page_values) => { - extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - page_values, - ); - } - BinaryState::FilteredRequired(page) => { - for x in page.values.by_ref().take(additional) { - values.push_value_ignore_validity(x) - } - } - BinaryState::FilteredDelta(page) => { - for x in page.values.by_ref().take(additional) { - values.push_value_ignore_validity(x) - } - } - BinaryState::OptionalDictionary(page_validity, page_values) => { - // Already done on the dict. - validate_utf8 = false; - let page_dict = &page_values.dict; - utils::extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - &mut page_values - .values - .by_ref() - .map(|index| page_dict[index.unwrap() as usize].as_ref()), - ) - } - BinaryState::RequiredDictionary(page) => { - // Already done on the dict. - validate_utf8 = false; - let page_dict = &page.dict; - - for x in page - .values - .by_ref() - .map(|index| page_dict[index.unwrap() as usize].as_ref()) - .take(additional) - { - values.push_value_ignore_validity::<&[u8]>(x) - } - } - BinaryState::FilteredOptional(page_validity, page_values) => { - extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - page_values.by_ref(), - ); - } - BinaryState::FilteredOptionalDelta(page_validity, page_values) => { - extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - page_values.by_ref(), - ); - } - BinaryState::FilteredRequiredDictionary(page) => { - // TODO! directly set the dict as buffers and only insert the proper views. - // This will save a lot of memory. - // Already done on the dict. - validate_utf8 = false; - let page_dict = &page.dict; - for x in page - .values - .by_ref() - .map(|index| page_dict[index.unwrap() as usize].as_ref()) - .take(additional) - { - values.push_value_ignore_validity::<&[u8]>(x) - } - } - BinaryState::FilteredOptionalDictionary(page_validity, page_values) => { - // Already done on the dict. - validate_utf8 = false; - // TODO! directly set the dict as buffers and only insert the proper views. - // This will save a lot of memory. - let page_dict = &page_values.dict; - extend_from_decoder( - validity, - page_validity, - Some(additional), - values, - &mut page_values - .values - .by_ref() - .map(|index| page_dict[index.unwrap() as usize].as_ref()), - ) - } - } - - if validate_utf8 { - values.validate_utf8().expect("Not an utf8 string buffer.") - } - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dict { - deserialize_plain(&page.buffer, page.num_values) - } -} - -pub struct BinaryViewArrayIter { - iter: I, - data_type: DataType, - items: VecDeque, - dict: Option, - chunk_size: Option, - remaining: usize, -} -impl BinaryViewArrayIter { - pub fn new(iter: I, data_type: DataType, chunk_size: Option, num_rows: usize) -> Self { - Self { - iter, - data_type, - items: VecDeque::new(), - dict: None, - chunk_size, - remaining: num_rows, - } - } -} - -impl Iterator for BinaryViewArrayIter { - type Item = Result; - - fn next(&mut self) -> Option { - let decoder = BinViewDecoder::default(); - loop { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - self.chunk_size, - &decoder, - ); - match maybe_state { - MaybeNext::Some(Ok((values, validity))) => { - return Some(finish(&self.data_type, values, validity)); - } - MaybeNext::Some(Err(e)) => return Some(Err(e)), - MaybeNext::None => return None, - MaybeNext::More => continue, - } - } - } -} - -pub(super) fn finish( - data_type: &DataType, - values: MutableBinaryViewArray<[u8]>, - validity: MutableBitmap, -) -> Result> { - let mut array: BinaryViewArray = values.into(); - let validity: Bitmap = validity.into(); - - if validity.unset_bits() != validity.len() { - array = array.with_validity(Some(validity)) - } - - match data_type.to_physical_type() { - PhysicalType::BinaryView => Ok(BinaryViewArray::new_unchecked( - data_type.clone(), - array.views().clone(), - array.data_buffers().clone(), - array.validity().cloned(), - array.total_bytes_len(), - array.total_buffer_len(), - ) - .boxed()), - PhysicalType::Utf8View => { - // Safety: we already checked utf8 - Ok(Utf8ViewArray::new_unchecked( - data_type.clone(), - array.views().clone(), - array.data_buffers().clone(), - array.validity().cloned(), - array.total_bytes_len(), - array.total_buffer_len(), - ) - .boxed()) - } - _ => unreachable!(), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/dictionary.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/dictionary.rs deleted file mode 100644 index 1c1106a981da..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/dictionary.rs +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::page::DictPage; - -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::MutableBinaryViewArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::deserialize::binary::SizedBinaryIter; -use crate::arrow::io::parquet::read::deserialize::dictionary::nested_next_dict; -use crate::arrow::io::parquet::read::deserialize::dictionary::next_dict; -use crate::arrow::io::parquet::read::deserialize::utils::MaybeNext; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; -use crate::arrow::io::parquet::read::Pages; - -// An iterator adapter over [`PagesIter`] assumed to be encoded as parquet's dictionary-encoded binary representation -#[derive(Debug)] -pub struct DictIter -where - I: Pages, - K: DictionaryKey, -{ - iter: I, - data_type: DataType, - values: Option>, - items: VecDeque<(Vec, MutableBitmap)>, - remaining: usize, - chunk_size: Option, -} - -impl DictIter -where - K: DictionaryKey, - I: Pages, -{ - pub fn new(iter: I, data_type: DataType, num_rows: usize, chunk_size: Option) -> Self { - Self { - iter, - data_type, - values: None, - items: VecDeque::new(), - remaining: num_rows, - chunk_size, - } - } -} - -fn read_dict(data_type: DataType, dict: &DictPage) -> Box { - let data_type = match data_type { - DataType::Dictionary(_, values, _) => *values, - _ => data_type, - }; - - let values = SizedBinaryIter::new(&dict.buffer, dict.num_values); - - let mut data = MutableBinaryViewArray::<[u8]>::with_capacity(dict.num_values); - for item in values { - data.push_value(item) - } - - match data_type.to_physical_type() { - PhysicalType::Utf8View => data.freeze().to_utf8view().unwrap().boxed(), - PhysicalType::BinaryView => data.freeze().boxed(), - _ => unreachable!(), - } -} - -impl Iterator for DictIter -where - I: Pages, - K: DictionaryKey, -{ - type Item = Result>; - - fn next(&mut self) -> Option { - let maybe_state = next_dict( - &mut self.iter, - &mut self.items, - &mut self.values, - self.data_type.clone(), - &mut self.remaining, - self.chunk_size, - |dict| read_dict(self.data_type.clone(), dict), - ); - match maybe_state { - MaybeNext::Some(Ok(dict)) => Some(Ok(dict)), - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} - -/// An iterator adapter that converts [`DataPages`] into an [`Iterator`] of [`DictionaryArray`] -#[derive(Debug)] -pub struct NestedDictIter -where - I: Pages, - K: DictionaryKey, -{ - iter: I, - init: Vec, - data_type: DataType, - values: Option>, - items: VecDeque<(NestedState, (Vec, MutableBitmap))>, - remaining: usize, - chunk_size: Option, -} - -impl NestedDictIter -where - I: Pages, - K: DictionaryKey, -{ - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - ) -> Self { - Self { - iter, - init, - data_type, - values: None, - items: VecDeque::new(), - remaining: num_rows, - chunk_size, - } - } -} - -impl Iterator for NestedDictIter -where - I: Pages, - K: DictionaryKey, -{ - type Item = Result<(NestedState, DictionaryArray)>; - - fn next(&mut self) -> Option { - loop { - let maybe_state = nested_next_dict( - &mut self.iter, - &mut self.items, - &mut self.remaining, - &self.init, - &mut self.values, - self.data_type.clone(), - self.chunk_size, - |dict| read_dict(self.data_type.clone(), dict), - ); - match maybe_state { - MaybeNext::Some(Ok(dict)) => return Some(Ok(dict)), - MaybeNext::Some(Err(e)) => return Some(Err(e)), - MaybeNext::None => return None, - MaybeNext::More => continue, - } - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/mod.rs deleted file mode 100644 index 0fec4ed0c30f..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod dictionary; -mod nested; - -pub use basic::BinaryViewArrayIter; -pub use dictionary::DictIter; -pub use dictionary::NestedDictIter; -pub use nested::NestedIter; diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/nested.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/nested.rs deleted file mode 100644 index 34649622364a..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/binview/nested.rs +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::page::DataPage; -use parquet2::page::DictPage; - -use crate::arrow::array::MutableBinaryViewArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::deserialize::binary::basic::deserialize_plain; -use crate::arrow::io::parquet::read::deserialize::binary::basic::BinaryDict; -use crate::arrow::io::parquet::read::deserialize::binary::nested::build_nested_state; -use crate::arrow::io::parquet::read::deserialize::binary::nested::BinaryNestedState; -use crate::arrow::io::parquet::read::deserialize::binview::basic::finish; -use crate::arrow::io::parquet::read::deserialize::nested_utils::next; -use crate::arrow::io::parquet::read::deserialize::nested_utils::NestedDecoder; -use crate::arrow::io::parquet::read::deserialize::utils::MaybeNext; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; -use crate::arrow::io::parquet::read::Pages; -use crate::ArrayRef; - -#[derive(Debug, Default)] -struct BinViewDecoder {} - -type DecodedStateTuple = (MutableBinaryViewArray<[u8]>, MutableBitmap); - -impl<'a> NestedDecoder<'a> for BinViewDecoder { - type State = BinaryNestedState<'a>; - type Dictionary = BinaryDict; - type DecodedState = DecodedStateTuple; - - fn build_state( - &self, - page: &'a DataPage, - dict: Option<&'a Self::Dictionary>, - ) -> Result { - build_nested_state(page, dict) - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - MutableBinaryViewArray::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn push_valid(&self, state: &mut Self::State, decoded: &mut Self::DecodedState) -> Result<()> { - let (values, validity) = decoded; - match state { - BinaryNestedState::Optional(page) => { - let value = page.next().unwrap_or_default(); - values.push_value_ignore_validity(value); - validity.push(true); - } - BinaryNestedState::Required(page) => { - let value = page.next().unwrap_or_default(); - values.push_value_ignore_validity(value); - } - BinaryNestedState::RequiredDictionary(page) => { - let dict_values = &page.dict; - let item = page - .values - .next() - .map(|index| dict_values[index.unwrap() as usize].as_ref()) - .unwrap_or_default(); - values.push_value_ignore_validity::<&[u8]>(item); - } - BinaryNestedState::OptionalDictionary(page) => { - let dict_values = &page.dict; - let item = page - .values - .next() - .map(|index| dict_values[index.unwrap() as usize].as_ref()) - .unwrap_or_default(); - values.push_value_ignore_validity::<&[u8]>(item); - validity.push(true); - } - } - Ok(()) - } - - fn push_null(&self, decoded: &mut Self::DecodedState) { - let (values, validity) = decoded; - values.push_null(); - validity.push(false); - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dictionary { - deserialize_plain(&page.buffer, page.num_values) - } -} - -pub struct NestedIter { - iter: I, - data_type: DataType, - init: Vec, - items: VecDeque<(NestedState, DecodedStateTuple)>, - dict: Option, - chunk_size: Option, - remaining: usize, -} - -impl NestedIter { - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - ) -> Self { - Self { - iter, - data_type, - init, - items: VecDeque::new(), - dict: None, - chunk_size, - remaining: num_rows, - } - } -} - -impl Iterator for NestedIter { - type Item = Result<(NestedState, ArrayRef)>; - - fn next(&mut self) -> Option { - loop { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - &self.init, - self.chunk_size, - &BinViewDecoder::default(), - ); - match maybe_state { - MaybeNext::Some(Ok((nested, decoded))) => { - return Some( - finish(&self.data_type, decoded.0, decoded.1).map(|array| (nested, array)), - ); - } - MaybeNext::Some(Err(e)) => return Some(Err(e)), - MaybeNext::None => return None, - MaybeNext::More => continue, /* Using continue in a loop instead of calling next helps prevent stack overflow. */ - } - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/basic.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/basic.rs deleted file mode 100644 index e7e2c77e335e..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/basic.rs +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::deserialize::SliceFilteredIter; -use parquet2::encoding::Encoding; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; - -use super::super::utils; -use super::super::utils::extend_from_decoder; -use super::super::utils::get_selected_rows; -use super::super::utils::next; -use super::super::utils::DecodedState; -use super::super::utils::Decoder; -use super::super::utils::FilteredOptionalPageValidity; -use super::super::utils::MaybeNext; -use super::super::utils::OptionalPageValidity; -use super::super::Pages; -use crate::arrow::array::BooleanArray; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; - -#[derive(Debug)] -struct Values<'a>(BitmapIter<'a>); - -impl<'a> Values<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, _, values) = split_buffer(page)?; - - Ok(Self(BitmapIter::new(values, 0, values.len() * 8))) - } -} - -// The state of a required DataPage with a boolean physical type -#[derive(Debug)] -struct Required<'a> { - values: &'a [u8], - // invariant: offset <= length; - offset: usize, - length: usize, -} - -impl<'a> Required<'a> { - pub fn new(page: &'a DataPage) -> Self { - Self { - values: page.buffer(), - offset: 0, - length: page.num_values(), - } - } -} - -#[derive(Debug)] -struct FilteredRequired<'a> { - values: SliceFilteredIter>, -} - -impl<'a> FilteredRequired<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, _, values) = split_buffer(page)?; - // todo: replace this by an iterator over slices, for faster deserialization - let values = BitmapIter::new(values, 0, page.num_values()); - - let rows = get_selected_rows(page); - let values = SliceFilteredIter::new(values, rows); - - Ok(Self { values }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -// The state of a `DataPage` of `Boolean` parquet boolean type -#[derive(Debug)] -enum State<'a> { - Optional(OptionalPageValidity<'a>, Values<'a>), - Required(Required<'a>), - FilteredRequired(FilteredRequired<'a>), - FilteredOptional(FilteredOptionalPageValidity<'a>, Values<'a>), -} - -impl<'a> State<'a> { - pub fn len(&self) -> usize { - match self { - State::Optional(validity, _) => validity.len(), - State::Required(page) => page.length - page.offset, - State::FilteredRequired(page) => page.len(), - State::FilteredOptional(optional, _) => optional.len(), - } - } -} - -impl<'a> utils::PageState<'a> for State<'a> { - fn len(&self) -> usize { - self.len() - } -} - -impl DecodedState for (MutableBitmap, MutableBitmap) { - fn len(&self) -> usize { - self.0.len() - } -} - -#[derive(Default)] -struct BooleanDecoder {} - -impl<'a> Decoder<'a> for BooleanDecoder { - type State = State<'a>; - type Dict = (); - type DecodedState = (MutableBitmap, MutableBitmap); - - fn build_state(&self, page: &'a DataPage, _: Option<&'a Self::Dict>) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), is_optional, is_filtered) { - (Encoding::Plain, true, false) => Ok(State::Optional( - OptionalPageValidity::try_new(page)?, - Values::try_new(page)?, - )), - (Encoding::Plain, false, false) => Ok(State::Required(Required::new(page))), - (Encoding::Plain, true, true) => Ok(State::FilteredOptional( - FilteredOptionalPageValidity::try_new(page)?, - Values::try_new(page)?, - )), - (Encoding::Plain, false, true) => { - Ok(State::FilteredRequired(FilteredRequired::try_new(page)?)) - } - _ => Err(utils::not_implemented(page)), - } - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - MutableBitmap::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn extend_from_state( - &self, - state: &mut Self::State, - decoded: &mut Self::DecodedState, - remaining: usize, - ) { - let (values, validity) = decoded; - match state { - State::Optional(page_validity, page_values) => extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - &mut page_values.0, - ), - State::Required(page) => { - let remaining = remaining.min(page.length - page.offset); - values.extend_from_slice(page.values, page.offset, remaining); - page.offset += remaining; - } - State::FilteredRequired(page) => { - values.reserve(remaining); - for item in page.values.by_ref().take(remaining) { - values.push(item) - } - } - State::FilteredOptional(page_validity, page_values) => { - utils::extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - page_values.0.by_ref(), - ); - } - } - } - - fn deserialize_dict(&self, _: &DictPage) -> Self::Dict {} -} - -fn finish(data_type: &DataType, values: MutableBitmap, validity: MutableBitmap) -> BooleanArray { - BooleanArray::new(data_type.clone(), values.into(), validity.into()) -} - -/// An iterator adapter over [`Pages`] assumed to be encoded as boolean arrays -#[derive(Debug)] -pub struct Iter { - iter: I, - data_type: DataType, - items: VecDeque<(MutableBitmap, MutableBitmap)>, - chunk_size: Option, - remaining: usize, -} - -impl Iter { - pub fn new(iter: I, data_type: DataType, chunk_size: Option, num_rows: usize) -> Self { - Self { - iter, - data_type, - items: VecDeque::new(), - chunk_size, - remaining: num_rows, - } - } -} - -impl Iterator for Iter { - type Item = Result; - - fn next(&mut self) -> Option { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut None, - &mut self.remaining, - self.chunk_size, - &BooleanDecoder::default(), - ); - match maybe_state { - MaybeNext::Some(Ok((values, validity))) => { - Some(Ok(finish(&self.data_type, values, validity))) - } - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/mod.rs deleted file mode 100644 index 496ccb87784a..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod nested; - -pub use nested::NestedIter; - -pub use self::basic::Iter; diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/nested.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/nested.rs deleted file mode 100644 index 6ff8a31906b6..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/boolean/nested.rs +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::encoding::Encoding; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; - -use super::super::nested_utils::*; -use super::super::utils; -use super::super::utils::MaybeNext; -use super::super::Pages; -use crate::arrow::array::BooleanArray; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; - -// The state of a `DataPage` of `Boolean` parquet boolean type -#[allow(clippy::large_enum_variant)] -#[derive(Debug)] -enum State<'a> { - Optional(BitmapIter<'a>), - Required(BitmapIter<'a>), -} - -impl<'a> State<'a> { - pub fn len(&self) -> usize { - match self { - State::Optional(iter) => iter.size_hint().0, - State::Required(iter) => iter.size_hint().0, - } - } -} - -impl<'a> utils::PageState<'a> for State<'a> { - fn len(&self) -> usize { - self.len() - } -} - -#[derive(Default)] -struct BooleanDecoder {} - -impl<'a> NestedDecoder<'a> for BooleanDecoder { - type State = State<'a>; - type Dictionary = (); - type DecodedState = (MutableBitmap, MutableBitmap); - - fn build_state( - &self, - page: &'a DataPage, - _: Option<&'a Self::Dictionary>, - ) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), is_optional, is_filtered) { - (Encoding::Plain, true, false) => { - let (_, _, values) = split_buffer(page)?; - let values = BitmapIter::new(values, 0, values.len() * 8); - - Ok(State::Optional(values)) - } - (Encoding::Plain, false, false) => { - let (_, _, values) = split_buffer(page)?; - let values = BitmapIter::new(values, 0, values.len() * 8); - - Ok(State::Required(values)) - } - _ => Err(utils::not_implemented(page)), - } - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - MutableBitmap::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn push_valid(&self, state: &mut State, decoded: &mut Self::DecodedState) -> Result<()> { - let (values, validity) = decoded; - match state { - State::Optional(page_values) => { - let value = page_values.next().unwrap_or_default(); - values.push(value); - validity.push(true); - } - State::Required(page_values) => { - let value = page_values.next().unwrap_or_default(); - values.push(value); - } - } - Ok(()) - } - - fn push_null(&self, decoded: &mut Self::DecodedState) { - let (values, validity) = decoded; - values.push(false); - validity.push(false); - } - - fn deserialize_dict(&self, _: &DictPage) -> Self::Dictionary {} -} - -/// An iterator adapter over [`Pages`] assumed to be encoded as boolean arrays -#[derive(Debug)] -pub struct NestedIter { - iter: I, - init: Vec, - items: VecDeque<(NestedState, (MutableBitmap, MutableBitmap))>, - remaining: usize, - chunk_size: Option, -} - -impl NestedIter { - pub fn new(iter: I, init: Vec, num_rows: usize, chunk_size: Option) -> Self { - Self { - iter, - init, - items: VecDeque::new(), - remaining: num_rows, - chunk_size, - } - } -} - -fn finish(data_type: &DataType, values: MutableBitmap, validity: MutableBitmap) -> BooleanArray { - BooleanArray::new(data_type.clone(), values.into(), validity.into()) -} - -impl Iterator for NestedIter { - type Item = Result<(NestedState, BooleanArray)>; - - fn next(&mut self) -> Option { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut None, - &mut self.remaining, - &self.init, - self.chunk_size, - &BooleanDecoder::default(), - ); - match maybe_state { - MaybeNext::Some(Ok((nested, (values, validity)))) => { - Some(Ok((nested, finish(&DataType::Boolean, values, validity)))) - } - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/dictionary/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/dictionary/mod.rs deleted file mode 100644 index a38ce30bb6d4..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/dictionary/mod.rs +++ /dev/null @@ -1,337 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod nested; - -use std::collections::VecDeque; - -use parquet2::deserialize::SliceFilteredIter; -use parquet2::encoding::hybrid_rle::HybridRleDecoder; -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::page::Page; -use parquet2::schema::Repetition; - -use super::utils::dict_indices_decoder; -use super::utils::extend_from_decoder; -use super::utils::get_selected_rows; -use super::utils::DecodedState; -use super::utils::Decoder; -use super::utils::FilteredOptionalPageValidity; -use super::utils::MaybeNext; -use super::utils::OptionalPageValidity; -use super::utils::{self}; -use super::Pages; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -// The state of a `DataPage` of `Primitive` parquet primitive type -#[derive(Debug)] -pub enum State<'a> { - Optional(Optional<'a>), - Required(Required<'a>), - FilteredRequired(FilteredRequired<'a>), - FilteredOptional(FilteredOptionalPageValidity<'a>, HybridRleDecoder<'a>), -} - -#[derive(Debug)] -pub struct Required<'a> { - values: HybridRleDecoder<'a>, -} - -impl<'a> Required<'a> { - fn try_new(page: &'a DataPage) -> Result { - let values = dict_indices_decoder(page)?; - Ok(Self { values }) - } -} - -#[derive(Debug)] -pub struct FilteredRequired<'a> { - values: SliceFilteredIter>, -} - -impl<'a> FilteredRequired<'a> { - fn try_new(page: &'a DataPage) -> Result { - let values = dict_indices_decoder(page)?; - - let rows = get_selected_rows(page); - let values = SliceFilteredIter::new(values, rows); - - Ok(Self { values }) - } -} - -#[derive(Debug)] -pub struct Optional<'a> { - values: HybridRleDecoder<'a>, - validity: OptionalPageValidity<'a>, -} - -impl<'a> Optional<'a> { - fn try_new(page: &'a DataPage) -> Result { - let values = dict_indices_decoder(page)?; - - Ok(Self { - values, - validity: OptionalPageValidity::try_new(page)?, - }) - } -} - -impl<'a> utils::PageState<'a> for State<'a> { - fn len(&self) -> usize { - match self { - State::Optional(optional) => optional.validity.len(), - State::Required(required) => required.values.size_hint().0, - State::FilteredRequired(required) => required.values.size_hint().0, - State::FilteredOptional(validity, _) => validity.len(), - } - } -} - -#[derive(Debug)] -pub struct PrimitiveDecoder -where K: DictionaryKey -{ - phantom_k: std::marker::PhantomData, -} - -impl Default for PrimitiveDecoder -where K: DictionaryKey -{ - #[inline] - fn default() -> Self { - Self { - phantom_k: std::marker::PhantomData, - } - } -} - -impl<'a, K> utils::Decoder<'a> for PrimitiveDecoder -where K: DictionaryKey -{ - type State = State<'a>; - type Dict = (); - type DecodedState = (Vec, MutableBitmap); - - fn build_state(&self, page: &'a DataPage, _: Option<&'a Self::Dict>) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), is_optional, is_filtered) { - (Encoding::PlainDictionary | Encoding::RleDictionary, false, false) => { - Required::try_new(page).map(State::Required) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, true, false) => { - Optional::try_new(page).map(State::Optional) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, false, true) => { - FilteredRequired::try_new(page).map(State::FilteredRequired) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, true, true) => { - Ok(State::FilteredOptional( - FilteredOptionalPageValidity::try_new(page)?, - dict_indices_decoder(page)?, - )) - } - _ => Err(utils::not_implemented(page)), - } - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - Vec::::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn extend_from_state( - &self, - state: &mut Self::State, - decoded: &mut Self::DecodedState, - remaining: usize, - ) { - let (values, validity) = decoded; - match state { - State::Optional(page) => extend_from_decoder( - validity, - &mut page.validity, - Some(remaining), - values, - &mut page.values.by_ref().map(|x| { - // todo: rm unwrap - let x: usize = x.unwrap().try_into().unwrap(); - match x.try_into() { - Ok(key) => key, - // todo: convert this to an error. - Err(_) => panic!("The maximum key is too small"), - } - }), - ), - State::Required(page) => { - values.extend( - page.values - .by_ref() - .map(|x| { - // todo: rm unwrap - let x: usize = x.unwrap().try_into().unwrap(); - let x: K = match x.try_into() { - Ok(key) => key, - // todo: convert this to an error. - Err(_) => { - panic!("The maximum key is too small") - } - }; - x - }) - .take(remaining), - ); - } - State::FilteredOptional(page_validity, page_values) => extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - &mut page_values.by_ref().map(|x| { - // todo: rm unwrap - let x: usize = x.unwrap().try_into().unwrap(); - let x: K = match x.try_into() { - Ok(key) => key, - // todo: convert this to an error. - Err(_) => { - panic!("The maximum key is too small") - } - }; - x - }), - ), - State::FilteredRequired(page) => { - values.extend( - page.values - .by_ref() - .map(|x| { - // todo: rm unwrap - let x: usize = x.unwrap().try_into().unwrap(); - let x: K = match x.try_into() { - Ok(key) => key, - // todo: convert this to an error. - Err(_) => { - panic!("The maximum key is too small") - } - }; - x - }) - .take(remaining), - ); - } - } - } - - fn deserialize_dict(&self, _: &DictPage) -> Self::Dict {} -} - -fn finish_key(values: Vec, validity: MutableBitmap) -> PrimitiveArray { - PrimitiveArray::new(K::PRIMITIVE.into(), values.into(), validity.into()) -} - -#[inline] -pub(super) fn next_dict Box>( - iter: &mut I, - items: &mut VecDeque<(Vec, MutableBitmap)>, - dict: &mut Option>, - data_type: DataType, - remaining: &mut usize, - chunk_size: Option, - read_dict: F, -) -> MaybeNext>> { - if items.len() > 1 { - let (values, validity) = items.pop_front().unwrap(); - let keys = finish_key(values, validity); - return MaybeNext::Some(DictionaryArray::try_new( - data_type, - keys, - dict.clone().unwrap(), - )); - } - match iter.next() { - Err(e) => MaybeNext::Some(Err(e.into())), - Ok(Some(page)) => { - let (page, dict) = match (&dict, page) { - (None, Page::Data(_)) => { - return MaybeNext::Some(Err(Error::nyi( - "dictionary arrays from non-dict-encoded pages", - ))); - } - (_, Page::Dict(dict_page)) => { - *dict = Some(read_dict(dict_page)); - return next_dict( - iter, items, dict, data_type, remaining, chunk_size, read_dict, - ); - } - (Some(dict), Page::Data(page)) => (page, dict), - }; - - // there is a new page => consume the page from the start - let maybe_page = PrimitiveDecoder::::default().build_state(page, None); - let page = match maybe_page { - Ok(page) => page, - Err(e) => return MaybeNext::Some(Err(e)), - }; - - utils::extend_from_new_page( - page, - chunk_size, - items, - remaining, - &PrimitiveDecoder::::default(), - ); - - if items.front().unwrap().len() < chunk_size.unwrap_or(usize::MAX) { - MaybeNext::More - } else { - let (values, validity) = items.pop_front().unwrap(); - let keys = finish_key(values, validity); - MaybeNext::Some(DictionaryArray::try_new(data_type, keys, dict.clone())) - } - } - Ok(None) => { - if let Some((values, validity)) = items.pop_front() { - // we have a populated item and no more pages - // the only case where an item's length may be smaller than chunk_size - debug_assert!(values.len() <= chunk_size.unwrap_or(usize::MAX)); - - let keys = finish_key(values, validity); - MaybeNext::Some(DictionaryArray::try_new( - data_type, - keys, - dict.clone().unwrap(), - )) - } else { - MaybeNext::None - } - } - } -} - -pub use nested::next_dict as nested_next_dict; diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/dictionary/nested.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/dictionary/nested.rs deleted file mode 100644 index 8b7a72c99314..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/dictionary/nested.rs +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::encoding::hybrid_rle::HybridRleDecoder; -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::page::Page; -use parquet2::schema::Repetition; - -use super::super::super::Pages; -use super::super::nested_utils::*; -use super::super::utils::dict_indices_decoder; -use super::super::utils::not_implemented; -use super::super::utils::MaybeNext; -use super::super::utils::PageState; -use super::finish_key; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -// The state of a required DataPage with a boolean physical type -#[derive(Debug)] -pub struct Required<'a> { - values: HybridRleDecoder<'a>, - length: usize, -} - -impl<'a> Required<'a> { - fn try_new(page: &'a DataPage) -> Result { - let values = dict_indices_decoder(page)?; - let length = page.num_values(); - Ok(Self { values, length }) - } -} - -// The state of a `DataPage` of a `Dictionary` type -#[allow(clippy::large_enum_variant)] -#[derive(Debug)] -pub enum State<'a> { - Optional(HybridRleDecoder<'a>), - Required(Required<'a>), -} - -impl<'a> State<'a> { - pub fn len(&self) -> usize { - match self { - State::Optional(page) => page.len(), - State::Required(page) => page.length, - } - } -} - -impl<'a> PageState<'a> for State<'a> { - fn len(&self) -> usize { - self.len() - } -} - -#[derive(Debug)] -pub struct DictionaryDecoder -where K: DictionaryKey -{ - phantom_k: std::marker::PhantomData, -} - -impl Default for DictionaryDecoder -where K: DictionaryKey -{ - #[inline] - fn default() -> Self { - Self { - phantom_k: std::marker::PhantomData, - } - } -} - -impl<'a, K: DictionaryKey> NestedDecoder<'a> for DictionaryDecoder { - type State = State<'a>; - type Dictionary = (); - type DecodedState = (Vec, MutableBitmap); - - fn build_state( - &self, - page: &'a DataPage, - _: Option<&'a Self::Dictionary>, - ) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), is_optional, is_filtered) { - (Encoding::RleDictionary | Encoding::PlainDictionary, true, false) => { - dict_indices_decoder(page).map(State::Optional) - } - (Encoding::RleDictionary | Encoding::PlainDictionary, false, false) => { - Required::try_new(page).map(State::Required) - } - _ => Err(not_implemented(page)), - } - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - Vec::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn push_valid(&self, state: &mut Self::State, decoded: &mut Self::DecodedState) -> Result<()> { - let (values, validity) = decoded; - match state { - State::Optional(page_values) => { - let key = page_values.next().transpose()?; - // todo: convert unwrap to error - let key = match K::try_from(key.unwrap_or_default() as usize) { - Ok(key) => key, - Err(_) => todo!(), - }; - values.push(key); - validity.push(true); - } - State::Required(page_values) => { - let key = page_values.values.next().transpose()?; - let key = match K::try_from(key.unwrap_or_default() as usize) { - Ok(key) => key, - Err(_) => todo!(), - }; - values.push(key); - } - } - Ok(()) - } - - fn push_null(&self, decoded: &mut Self::DecodedState) { - let (values, validity) = decoded; - values.push(K::default()); - validity.push(false) - } - - fn deserialize_dict(&self, _: &DictPage) -> Self::Dictionary {} -} - -#[allow(clippy::too_many_arguments)] -pub fn next_dict Box>( - iter: &mut I, - items: &mut VecDeque<(NestedState, (Vec, MutableBitmap))>, - remaining: &mut usize, - init: &[InitNested], - dict: &mut Option>, - data_type: DataType, - chunk_size: Option, - read_dict: F, -) -> MaybeNext)>> { - if items.len() > 1 { - let (nested, (values, validity)) = items.pop_front().unwrap(); - let keys = finish_key(values, validity); - let dict = DictionaryArray::try_new(data_type, keys, dict.clone().unwrap()); - return MaybeNext::Some(dict.map(|dict| (nested, dict))); - } - match iter.next() { - Err(e) => MaybeNext::Some(Err(e.into())), - Ok(Some(page)) => { - let (page, dict) = match (&dict, page) { - (None, Page::Data(_)) => { - return MaybeNext::Some(Err(Error::nyi( - "dictionary arrays from non-dict-encoded pages", - ))); - } - (_, Page::Dict(dict_page)) => { - *dict = Some(read_dict(dict_page)); - return next_dict( - iter, items, remaining, init, dict, data_type, chunk_size, read_dict, - ); - } - (Some(dict), Page::Data(page)) => (page, dict), - }; - - let error = extend( - page, - init, - items, - None, - remaining, - &DictionaryDecoder::::default(), - chunk_size, - ); - match error { - Ok(_) => {} - Err(e) => return MaybeNext::Some(Err(e)), - }; - - if items.front().unwrap().0.len() < chunk_size.unwrap_or(usize::MAX) { - MaybeNext::More - } else { - let (nested, (values, validity)) = items.pop_front().unwrap(); - let keys = finish_key(values, validity); - let dict = DictionaryArray::try_new(data_type, keys, dict.clone()); - MaybeNext::Some(dict.map(|dict| (nested, dict))) - } - } - Ok(None) => { - if let Some((nested, (values, validity))) = items.pop_front() { - // we have a populated item and no more pages - // the only case where an item's length may be smaller than chunk_size - debug_assert!(values.len() <= chunk_size.unwrap_or(usize::MAX)); - - let keys = finish_key(values, validity); - let dict = DictionaryArray::try_new(data_type, keys, dict.clone().unwrap()); - MaybeNext::Some(dict.map(|dict| (nested, dict))) - } else { - MaybeNext::None - } - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/basic.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/basic.rs deleted file mode 100644 index 297ac9ade67c..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/basic.rs +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::deserialize::SliceFilteredIter; -use parquet2::encoding::hybrid_rle; -use parquet2::encoding::Encoding; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; - -use super::super::utils::dict_indices_decoder; -use super::super::utils::extend_from_decoder; -use super::super::utils::get_selected_rows; -use super::super::utils::next; -use super::super::utils::not_implemented; -use super::super::utils::DecodedState; -use super::super::utils::Decoder; -use super::super::utils::FilteredOptionalPageValidity; -use super::super::utils::MaybeNext; -use super::super::utils::OptionalPageValidity; -use super::super::utils::PageState; -use super::super::utils::Pushable; -use super::super::Pages; -use super::utils::FixedSizeBinary; -use crate::arrow::array::FixedSizeBinaryArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; - -pub(super) type Dict = Vec; - -#[derive(Debug)] -pub(super) struct Optional<'a> { - pub(super) values: std::slice::ChunksExact<'a, u8>, - pub(super) validity: OptionalPageValidity<'a>, -} - -impl<'a> Optional<'a> { - pub(super) fn try_new(page: &'a DataPage, size: usize) -> Result { - let (_, _, values) = split_buffer(page)?; - - let values = values.chunks_exact(size); - - Ok(Self { - values, - validity: OptionalPageValidity::try_new(page)?, - }) - } -} - -#[derive(Debug)] -pub(super) struct Required<'a> { - pub values: std::slice::ChunksExact<'a, u8>, -} - -impl<'a> Required<'a> { - pub(super) fn try_new(page: &'a DataPage, size: usize) -> Result { - let (_, _, values) = split_buffer(page)?; - assert_eq!(values.len() % size, 0); - let values = values.chunks_exact(size); - Ok(Self { values }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(super) struct FilteredRequired<'a> { - pub values: SliceFilteredIter>, -} - -impl<'a> FilteredRequired<'a> { - fn new(page: &'a DataPage, size: usize) -> Self { - let values = page.buffer(); - assert_eq!(values.len() % size, 0); - let values = values.chunks_exact(size); - - let rows = get_selected_rows(page); - let values = SliceFilteredIter::new(values, rows); - - Self { values } - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(super) struct RequiredDictionary<'a> { - pub values: hybrid_rle::HybridRleDecoder<'a>, - pub dict: &'a Dict, -} - -impl<'a> RequiredDictionary<'a> { - pub(super) fn try_new(page: &'a DataPage, dict: &'a Dict) -> Result { - let values = dict_indices_decoder(page)?; - - Ok(Self { dict, values }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(super) struct OptionalDictionary<'a> { - pub(super) values: hybrid_rle::HybridRleDecoder<'a>, - pub(super) validity: OptionalPageValidity<'a>, - pub(super) dict: &'a Dict, -} - -impl<'a> OptionalDictionary<'a> { - pub(super) fn try_new(page: &'a DataPage, dict: &'a Dict) -> Result { - let values = dict_indices_decoder(page)?; - - Ok(Self { - values, - validity: OptionalPageValidity::try_new(page)?, - dict, - }) - } -} - -#[derive(Debug)] -enum State<'a> { - Optional(Optional<'a>), - Required(Required<'a>), - RequiredDictionary(RequiredDictionary<'a>), - OptionalDictionary(OptionalDictionary<'a>), - FilteredRequired(FilteredRequired<'a>), - FilteredOptional( - FilteredOptionalPageValidity<'a>, - std::slice::ChunksExact<'a, u8>, - ), -} - -impl<'a> PageState<'a> for State<'a> { - fn len(&self) -> usize { - match self { - State::Optional(state) => state.validity.len(), - State::Required(state) => state.len(), - State::RequiredDictionary(state) => state.len(), - State::OptionalDictionary(state) => state.validity.len(), - State::FilteredRequired(state) => state.len(), - State::FilteredOptional(state, _) => state.len(), - } - } -} - -struct BinaryDecoder { - size: usize, -} - -impl DecodedState for (FixedSizeBinary, MutableBitmap) { - fn len(&self) -> usize { - self.0.len() - } -} - -impl<'a> Decoder<'a> for BinaryDecoder { - type State = State<'a>; - type Dict = Dict; - type DecodedState = (FixedSizeBinary, MutableBitmap); - - fn build_state(&self, page: &'a DataPage, dict: Option<&'a Self::Dict>) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), dict, is_optional, is_filtered) { - (Encoding::Plain, _, true, false) => { - Ok(State::Optional(Optional::try_new(page, self.size)?)) - } - (Encoding::Plain, _, false, false) => { - Ok(State::Required(Required::try_new(page, self.size)?)) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false, false) => { - RequiredDictionary::try_new(page, dict).map(State::RequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true, false) => { - OptionalDictionary::try_new(page, dict).map(State::OptionalDictionary) - } - (Encoding::Plain, None, false, true) => Ok(State::FilteredRequired( - FilteredRequired::new(page, self.size), - )), - (Encoding::Plain, _, true, true) => { - let (_, _, values) = split_buffer(page)?; - - Ok(State::FilteredOptional( - FilteredOptionalPageValidity::try_new(page)?, - values.chunks_exact(self.size), - )) - } - _ => Err(not_implemented(page)), - } - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - FixedSizeBinary::with_capacity(capacity, self.size), - MutableBitmap::with_capacity(capacity), - ) - } - - fn extend_from_state( - &self, - state: &mut Self::State, - decoded: &mut Self::DecodedState, - - remaining: usize, - ) { - let (values, validity) = decoded; - match state { - State::Optional(page) => extend_from_decoder( - validity, - &mut page.validity, - Some(remaining), - values, - &mut page.values, - ), - State::Required(page) => { - for x in page.values.by_ref().take(remaining) { - values.push(x) - } - } - State::FilteredRequired(page) => { - for x in page.values.by_ref().take(remaining) { - values.push(x) - } - } - State::OptionalDictionary(page) => extend_from_decoder( - validity, - &mut page.validity, - Some(remaining), - values, - page.values.by_ref().map(|index| { - let index = index.unwrap() as usize; - &page.dict[index * self.size..(index + 1) * self.size] - }), - ), - State::RequiredDictionary(page) => { - for x in page - .values - .by_ref() - .map(|index| { - let index = index.unwrap() as usize; - &page.dict[index * self.size..(index + 1) * self.size] - }) - .take(remaining) - { - values.push(x) - } - } - State::FilteredOptional(page_validity, page_values) => { - extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - page_values.by_ref(), - ); - } - } - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dict { - page.buffer.clone() - } -} - -pub fn finish( - data_type: &DataType, - values: FixedSizeBinary, - validity: MutableBitmap, -) -> FixedSizeBinaryArray { - FixedSizeBinaryArray::new(data_type.clone(), values.values.into(), validity.into()) -} - -pub struct Iter { - iter: I, - data_type: DataType, - size: usize, - items: VecDeque<(FixedSizeBinary, MutableBitmap)>, - dict: Option, - chunk_size: Option, - remaining: usize, -} - -impl Iter { - pub fn new(iter: I, data_type: DataType, num_rows: usize, chunk_size: Option) -> Self { - let size = FixedSizeBinaryArray::get_size(&data_type); - Self { - iter, - data_type, - size, - items: VecDeque::new(), - dict: None, - chunk_size, - remaining: num_rows, - } - } -} - -impl Iterator for Iter { - type Item = Result; - - fn next(&mut self) -> Option { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - self.chunk_size, - &BinaryDecoder { size: self.size }, - ); - match maybe_state { - MaybeNext::Some(Ok((values, validity))) => { - Some(Ok(finish(&self.data_type, values, validity))) - } - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/dictionary.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/dictionary.rs deleted file mode 100644 index be3e909da634..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/dictionary.rs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::page::DictPage; - -use super::super::dictionary::*; -use super::super::utils::MaybeNext; -use super::super::Pages; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::FixedSizeBinaryArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::deserialize::nested_utils::InitNested; -use crate::arrow::io::parquet::read::deserialize::nested_utils::NestedState; - -/// An iterator adapter over [`Pages`] assumed to be encoded as parquet's dictionary-encoded binary representation -#[derive(Debug)] -pub struct DictIter -where - I: Pages, - K: DictionaryKey, -{ - iter: I, - data_type: DataType, - values: Option>, - items: VecDeque<(Vec, MutableBitmap)>, - remaining: usize, - chunk_size: Option, -} - -impl DictIter -where - K: DictionaryKey, - I: Pages, -{ - pub fn new(iter: I, data_type: DataType, num_rows: usize, chunk_size: Option) -> Self { - Self { - iter, - data_type, - values: None, - items: VecDeque::new(), - remaining: num_rows, - chunk_size, - } - } -} - -fn read_dict(data_type: DataType, dict: &DictPage) -> Box { - let data_type = match data_type { - DataType::Dictionary(_, values, _) => *values, - _ => data_type, - }; - - let values = dict.buffer.clone(); - - FixedSizeBinaryArray::try_new(data_type, values.into(), None) - .unwrap() - .boxed() -} - -impl Iterator for DictIter -where - I: Pages, - K: DictionaryKey, -{ - type Item = Result>; - - fn next(&mut self) -> Option { - let maybe_state = next_dict( - &mut self.iter, - &mut self.items, - &mut self.values, - self.data_type.clone(), - &mut self.remaining, - self.chunk_size, - |dict| read_dict(self.data_type.clone(), dict), - ); - match maybe_state { - MaybeNext::Some(Ok(dict)) => Some(Ok(dict)), - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} - -/// An iterator adapter that converts [`DataPages`] into an [`Iterator`] of [`DictionaryArray`]. -#[derive(Debug)] -pub struct NestedDictIter -where - I: Pages, - K: DictionaryKey, -{ - iter: I, - init: Vec, - data_type: DataType, - values: Option>, - items: VecDeque<(NestedState, (Vec, MutableBitmap))>, - remaining: usize, - chunk_size: Option, -} - -impl NestedDictIter -where - I: Pages, - K: DictionaryKey, -{ - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - ) -> Self { - Self { - iter, - init, - data_type, - values: None, - remaining: num_rows, - items: VecDeque::new(), - chunk_size, - } - } -} - -impl Iterator for NestedDictIter -where - I: Pages, - K: DictionaryKey, -{ - type Item = Result<(NestedState, DictionaryArray)>; - - fn next(&mut self) -> Option { - let maybe_state = nested_next_dict( - &mut self.iter, - &mut self.items, - &mut self.remaining, - &self.init, - &mut self.values, - self.data_type.clone(), - self.chunk_size, - |dict| read_dict(self.data_type.clone(), dict), - ); - match maybe_state { - MaybeNext::Some(Ok(dict)) => Some(Ok(dict)), - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/mod.rs deleted file mode 100644 index 527e390d1a5b..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod dictionary; -mod nested; -mod utils; - -pub use basic::Iter; -pub use dictionary::DictIter; -pub use dictionary::NestedDictIter; -pub use nested::NestedIter; diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/nested.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/nested.rs deleted file mode 100644 index 87c31f28e2de..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/nested.rs +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; - -use super::super::utils::not_implemented; -use super::super::utils::MaybeNext; -use super::super::utils::PageState; -use super::utils::FixedSizeBinary; -use crate::arrow::array::FixedSizeBinaryArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::deserialize::fixed_size_binary::basic::finish; -use crate::arrow::io::parquet::read::deserialize::fixed_size_binary::basic::Dict; -use crate::arrow::io::parquet::read::deserialize::fixed_size_binary::basic::Optional; -use crate::arrow::io::parquet::read::deserialize::fixed_size_binary::basic::OptionalDictionary; -use crate::arrow::io::parquet::read::deserialize::fixed_size_binary::basic::Required; -use crate::arrow::io::parquet::read::deserialize::fixed_size_binary::basic::RequiredDictionary; -use crate::arrow::io::parquet::read::deserialize::nested_utils::next; -use crate::arrow::io::parquet::read::deserialize::nested_utils::NestedDecoder; -use crate::arrow::io::parquet::read::deserialize::utils::Pushable; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; -use crate::arrow::io::parquet::read::Pages; - -#[derive(Debug)] -enum State<'a> { - Optional(Optional<'a>), - Required(Required<'a>), - RequiredDictionary(RequiredDictionary<'a>), - OptionalDictionary(OptionalDictionary<'a>), -} - -impl<'a> PageState<'a> for State<'a> { - fn len(&self) -> usize { - match self { - State::Optional(state) => state.validity.len(), - State::Required(state) => state.len(), - State::RequiredDictionary(state) => state.len(), - State::OptionalDictionary(state) => state.validity.len(), - } - } -} - -#[derive(Debug, Default)] -struct BinaryDecoder { - size: usize, -} - -impl<'a> NestedDecoder<'a> for BinaryDecoder { - type State = State<'a>; - type Dictionary = Dict; - type DecodedState = (FixedSizeBinary, MutableBitmap); - - fn build_state( - &self, - page: &'a DataPage, - dict: Option<&'a Self::Dictionary>, - ) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), dict, is_optional, is_filtered) { - (Encoding::Plain, _, true, false) => { - Ok(State::Optional(Optional::try_new(page, self.size)?)) - } - (Encoding::Plain, _, false, false) => { - Ok(State::Required(Required::try_new(page, self.size)?)) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false, false) => { - RequiredDictionary::try_new(page, dict).map(State::RequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true, false) => { - OptionalDictionary::try_new(page, dict).map(State::OptionalDictionary) - } - _ => Err(not_implemented(page)), - } - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - FixedSizeBinary::with_capacity(capacity, self.size), - MutableBitmap::with_capacity(capacity), - ) - } - - fn push_valid(&self, state: &mut Self::State, decoded: &mut Self::DecodedState) -> Result<()> { - let (values, validity) = decoded; - match state { - State::Optional(page) => { - let value = page.values.by_ref().next().unwrap_or_default(); - values.push(value); - validity.push(true); - } - State::Required(page) => { - let value = page.values.by_ref().next().unwrap_or_default(); - values.push(value); - } - State::RequiredDictionary(page) => { - let item = page - .values - .by_ref() - .next() - .map(|index| { - let index = index.unwrap() as usize; - &page.dict[index * self.size..(index + 1) * self.size] - }) - .unwrap_or_default(); - values.push(item); - } - State::OptionalDictionary(page) => { - let item = page - .values - .by_ref() - .next() - .map(|index| { - let index = index.unwrap() as usize; - &page.dict[index * self.size..(index + 1) * self.size] - }) - .unwrap_or_default(); - values.push(item); - validity.push(true); - } - } - Ok(()) - } - - fn push_null(&self, decoded: &mut Self::DecodedState) { - let (values, validity) = decoded; - values.push_null(); - validity.push(false); - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dictionary { - page.buffer.clone() - } -} - -pub struct NestedIter { - iter: I, - data_type: DataType, - size: usize, - init: Vec, - items: VecDeque<(NestedState, (FixedSizeBinary, MutableBitmap))>, - dict: Option, - chunk_size: Option, - remaining: usize, -} - -impl NestedIter { - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - ) -> Self { - let size = FixedSizeBinaryArray::get_size(&data_type); - Self { - iter, - data_type, - size, - init, - items: VecDeque::new(), - dict: None, - chunk_size, - remaining: num_rows, - } - } -} - -impl Iterator for NestedIter { - type Item = Result<(NestedState, FixedSizeBinaryArray)>; - - fn next(&mut self) -> Option { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - &self.init, - self.chunk_size, - &BinaryDecoder { size: self.size }, - ); - match maybe_state { - MaybeNext::Some(Ok((nested, decoded))) => { - Some(Ok((nested, finish(&self.data_type, decoded.0, decoded.1)))) - } - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/utils.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/utils.rs deleted file mode 100644 index 9f2ca66847e5..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/fixed_size_binary/utils.rs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::utils::Pushable; - -/// A [`Pushable`] for fixed sized binary data -#[derive(Debug)] -pub struct FixedSizeBinary { - pub values: Vec, - pub size: usize, -} - -impl FixedSizeBinary { - #[inline] - pub fn with_capacity(capacity: usize, size: usize) -> Self { - Self { - values: Vec::with_capacity(capacity * size), - size, - } - } - - #[inline] - pub fn push(&mut self, value: &[u8]) { - debug_assert_eq!(value.len(), self.size); - self.values.extend(value); - } - - #[inline] - pub fn extend_constant(&mut self, additional: usize) { - self.values - .resize(self.values.len() + additional * self.size, 0); - } -} - -impl<'a> Pushable<&'a [u8]> for FixedSizeBinary { - #[inline] - fn reserve(&mut self, additional: usize) { - self.values.reserve(additional * self.size); - } - #[inline] - fn push(&mut self, value: &[u8]) { - debug_assert_eq!(value.len(), self.size); - self.push(value); - } - - #[inline] - fn push_null(&mut self) { - self.values.extend(std::iter::repeat(0).take(self.size)) - } - - #[inline] - fn extend_constant(&mut self, additional: usize, value: &[u8]) { - assert_eq!(value.len(), 0); - self.extend_constant(additional) - } - - #[inline] - fn len(&self) -> usize { - self.values.len() / self.size - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/mod.rs deleted file mode 100644 index 3ba94725cb98..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/mod.rs +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to read from Parquet format. -mod binary; -mod binview; -mod boolean; -mod dictionary; -mod fixed_size_binary; -mod nested; -mod nested_utils; -mod null; -mod primitive; -mod simple; -mod struct_; -mod utils; - -use parquet2::read::get_page_iterator as _get_page_iterator; -use parquet2::schema::types::PrimitiveType; -use simple::page_iter_to_arrays; - -pub use self::nested_utils::init_nested; -pub use self::nested_utils::InitNested; -pub use self::nested_utils::NestedArrayIter; -pub use self::nested_utils::NestedState; -pub use self::struct_::StructIterator; -use super::*; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::FixedSizeListArray; -use crate::arrow::array::ListArray; -use crate::arrow::array::MapArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::error::Result; -use crate::arrow::offset::Offsets; - -/// Creates a new iterator of compressed pages. -pub fn get_page_iterator( - column_metadata: &ColumnChunkMetaData, - reader: R, - pages_filter: Option, - buffer: Vec, - max_header_size: usize, -) -> Result> { - Ok(_get_page_iterator( - column_metadata, - reader, - pages_filter, - buffer, - max_header_size, - )?) -} - -/// Creates a new [`ListArray`] or [`FixedSizeListArray`]. -pub fn create_list( - data_type: DataType, - nested: &mut NestedState, - values: Box, -) -> Box { - let (mut offsets, validity) = nested.nested.pop().unwrap().inner(); - match data_type.to_logical_type() { - DataType::List(_) => { - offsets.push(values.len() as i64); - - let offsets = offsets.iter().map(|x| *x as i32).collect::>(); - - let offsets: Offsets = offsets - .try_into() - .expect("i64 offsets do not fit in i32 offsets"); - - Box::new(ListArray::::new( - data_type, - offsets.into(), - values, - validity.and_then(|x| x.into()), - )) - } - DataType::LargeList(_) => { - offsets.push(values.len() as i64); - - Box::new(ListArray::::new( - data_type, - offsets.try_into().expect("List too large"), - values, - validity.and_then(|x| x.into()), - )) - } - DataType::FixedSizeList(_, _) => Box::new(FixedSizeListArray::new( - data_type, - values, - validity.and_then(|x| x.into()), - )), - _ => unreachable!(), - } -} - -/// Creates a new [`MapArray`]. -pub fn create_map( - data_type: DataType, - nested: &mut NestedState, - values: Box, -) -> Box { - let (mut offsets, validity) = nested.nested.pop().unwrap().inner(); - match data_type.to_logical_type() { - DataType::Map(_, _) => { - offsets.push(values.len() as i64); - let offsets = offsets.iter().map(|x| *x as i32).collect::>(); - - let offsets: Offsets = offsets - .try_into() - .expect("i64 offsets do not fit in i32 offsets"); - - Box::new(MapArray::new( - data_type, - offsets.into(), - values, - validity.and_then(|x| x.into()), - )) - } - _ => unreachable!(), - } -} - -fn is_primitive(data_type: &DataType) -> bool { - matches!( - data_type.to_physical_type(), - crate::arrow::datatypes::PhysicalType::Primitive(_) - | crate::arrow::datatypes::PhysicalType::Null - | crate::arrow::datatypes::PhysicalType::Boolean - | crate::arrow::datatypes::PhysicalType::Utf8 - | crate::arrow::datatypes::PhysicalType::LargeUtf8 - | crate::arrow::datatypes::PhysicalType::Binary - | crate::arrow::datatypes::PhysicalType::LargeBinary - | crate::arrow::datatypes::PhysicalType::BinaryView - | crate::arrow::datatypes::PhysicalType::Utf8View - | crate::arrow::datatypes::PhysicalType::FixedSizeBinary - | crate::arrow::datatypes::PhysicalType::Dictionary(_) - ) -} - -fn columns_to_iter_recursive<'a, I>( - mut columns: Vec, - mut types: Vec<&PrimitiveType>, - field: Field, - init: Vec, - num_rows: usize, - chunk_size: Option, -) -> Result> -where - I: Pages + 'a, -{ - if init.is_empty() && is_primitive(&field.data_type) { - return Ok(Box::new( - page_iter_to_arrays( - columns.pop().unwrap(), - types.pop().unwrap(), - field.data_type, - chunk_size, - num_rows, - )? - .map(|x| Ok((NestedState::new(vec![]), x?))), - )); - } - - nested::columns_to_iter_recursive(columns, types, field, init, num_rows, chunk_size) -} - -/// Returns the number of (parquet) columns that a [`DataType`] contains. -pub fn n_columns(data_type: &DataType) -> usize { - use crate::arrow::datatypes::PhysicalType::*; - match data_type.to_physical_type() { - Null | Boolean | Primitive(_) | Binary | FixedSizeBinary | LargeBinary | Utf8 - | Dictionary(_) | LargeUtf8 | BinaryView | Utf8View => 1, - List | FixedSizeList | LargeList => { - let a = data_type.to_logical_type(); - if let DataType::List(inner) = a { - n_columns(&inner.data_type) - } else if let DataType::LargeList(inner) = a { - n_columns(&inner.data_type) - } else if let DataType::FixedSizeList(inner, _) = a { - n_columns(&inner.data_type) - } else { - unreachable!() - } - } - Map => { - let a = data_type.to_logical_type(); - if let DataType::Map(inner, _) = a { - n_columns(&inner.data_type) - } else { - unreachable!() - } - } - Struct => { - if let DataType::Struct(fields) = data_type.to_logical_type() { - fields.iter().map(|inner| n_columns(&inner.data_type)).sum() - } else { - unreachable!() - } - } - _ => todo!(), - } -} - -/// An iterator adapter that maps multiple iterators of [`Pages`] into an iterator of [`Array`]s. -/// -/// For a non-nested datatypes such as [`DataType::Int32`], this function requires a single element in `columns` and `types`. -/// For nested types, `columns` must be composed by all parquet columns with associated types `types`. -/// -/// The arrays are guaranteed to be at most of size `chunk_size` and data type `field.data_type`. -pub fn column_iter_to_arrays<'a, I>( - columns: Vec, - types: Vec<&PrimitiveType>, - field: Field, - chunk_size: Option, - num_rows: usize, -) -> Result> -where - I: Pages + 'a, -{ - Ok(Box::new( - columns_to_iter_recursive(columns, types, field, vec![], num_rows, chunk_size)? - .map(|x| x.map(|x| x.1)), - )) -} - -/// Basically the same as `column_iter_to_arrays`, with the addition of the `init` parameter -/// to read the inner columns of the nested type directly, instead of reading the entire nested type. -pub fn nested_column_iter_to_arrays<'a, I>( - columns: Vec, - types: Vec<&PrimitiveType>, - field: Field, - init: Vec, - chunk_size: Option, - num_rows: usize, -) -> Result> -where - I: Pages + 'a, -{ - Ok(Box::new( - nested::columns_to_iter_recursive(columns, types, field, init, num_rows, chunk_size)? - .map(|x| x.map(|x| x.1)), - )) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/nested.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/nested.rs deleted file mode 100644 index fbac10b47cf7..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/nested.rs +++ /dev/null @@ -1,620 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use ethnum::I256; -use parquet2::schema::types::PrimitiveType; - -use super::nested_utils::InitNested; -use super::nested_utils::NestedArrayIter; -use super::*; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// Converts an iterator of arrays to a trait object returning trait objects -#[inline] -fn remove_nested<'a, I>(iter: I) -> NestedArrayIter<'a> -where I: Iterator)>> + Send + Sync + 'a { - Box::new(iter.map(|x| { - x.map(|(mut nested, array)| { - let _ = nested.nested.pop().unwrap(); // the primitive - (nested, array) - }) - })) -} - -/// Converts an iterator of arrays to a trait object returning trait objects -#[inline] -fn primitive<'a, A, I>(iter: I) -> NestedArrayIter<'a> -where - A: Array, - I: Iterator> + Send + Sync + 'a, -{ - Box::new(iter.map(|x| { - x.map(|(mut nested, array)| { - let _ = nested.nested.pop().unwrap(); // the primitive - (nested, Box::new(array) as _) - }) - })) -} - -pub fn columns_to_iter_recursive<'a, I>( - mut columns: Vec, - mut types: Vec<&PrimitiveType>, - field: Field, - mut init: Vec, - num_rows: usize, - chunk_size: Option, -) -> Result> -where - I: Pages + 'a, -{ - use crate::arrow::datatypes::PhysicalType::*; - use crate::arrow::datatypes::PrimitiveType::*; - - Ok(match field.data_type().to_physical_type() { - Null => { - // physical type is i32 - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(null::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - )) - } - Boolean => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(boolean::NestedIter::new( - columns.pop().unwrap(), - init, - num_rows, - chunk_size, - )) - } - Primitive(Int8) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i32| x as i8, - )) - } - Primitive(Int16) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i32| x as i16, - )) - } - Primitive(Int32) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i32| x, - )) - } - Primitive(Int64) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i64| x, - )) - } - Primitive(UInt8) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i32| x as u8, - )) - } - Primitive(UInt16) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i32| x as u16, - )) - } - Primitive(UInt32) => { - init.push(InitNested::Primitive(field.is_nullable)); - let type_ = types.pop().unwrap(); - match type_.physical_type { - PhysicalType::Int32 => primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i32| x as u32, - )), - // some implementations of parquet write arrow's u32 into i64. - PhysicalType::Int64 => primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i64| x as u32, - )), - other => { - return Err(Error::nyi(format!( - "Deserializing UInt32 from {other:?}'s parquet" - ))); - } - } - } - Primitive(UInt64) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: i64| x as u64, - )) - } - Primitive(Float32) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: f32| x, - )) - } - Primitive(Float64) => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - |x: f64| x, - )) - } - Binary | Utf8 => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - remove_nested(binary::NestedIter::::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - )) - } - LargeBinary | LargeUtf8 => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - remove_nested(binary::NestedIter::::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - )) - } - BinaryView | Utf8View => { - init.push(InitNested::Primitive(field.is_nullable)); - types.pop(); - remove_nested(binview::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type().clone(), - num_rows, - chunk_size, - )) - } - _ => match field.data_type().to_logical_type() { - DataType::Dictionary(key_type, _, _) => { - init.push(InitNested::Primitive(field.is_nullable)); - let type_ = types.pop().unwrap(); - let iter = columns.pop().unwrap(); - let data_type = field.data_type().clone(); - match_integer_type!(key_type, |$K| { - dict_read::<$K, _>(iter, init, type_, data_type, num_rows, chunk_size) - })? - } - DataType::List(inner) - | DataType::LargeList(inner) - | DataType::FixedSizeList(inner, _) => { - init.push(InitNested::List(field.is_nullable)); - let iter = columns_to_iter_recursive( - columns, - types, - inner.as_ref().clone(), - init, - num_rows, - chunk_size, - )?; - let iter = iter.map(move |x| { - let (mut nested, array) = x?; - let array = create_list(field.data_type().clone(), &mut nested, array); - Ok((nested, array)) - }); - Box::new(iter) as _ - } - DataType::Decimal(_, _) => { - init.push(InitNested::Primitive(field.is_nullable)); - let type_ = types.pop().unwrap(); - match type_.physical_type { - PhysicalType::Int32 => primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type.clone(), - num_rows, - chunk_size, - |x: i32| x as i128, - )), - PhysicalType::Int64 => primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type.clone(), - num_rows, - chunk_size, - |x: i64| x as i128, - )), - PhysicalType::FixedLenByteArray(n) if n > 16 => { - return Err(Error::InvalidArgumentError(format!( - "Can't decode Decimal128 type from `FixedLenByteArray` of len {n}" - ))); - } - PhysicalType::FixedLenByteArray(n) => { - let iter = fixed_size_binary::NestedIter::new( - columns.pop().unwrap(), - init, - DataType::FixedSizeBinary(n), - num_rows, - chunk_size, - ); - // Convert the fixed length byte array to Decimal. - let iter = iter.map(move |x| { - let (mut nested, array) = x?; - let values = array - .values() - .chunks_exact(n) - .map(|value: &[u8]| super::super::convert_i128(value, n)) - .collect::>(); - let validity = array.validity().cloned(); - - let array: Box = Box::new(PrimitiveArray::::try_new( - field.data_type.clone(), - values.into(), - validity, - )?); - - let _ = nested.nested.pop().unwrap(); // the primitive - - Ok((nested, array)) - }); - Box::new(iter) - } - _ => { - return Err(Error::nyi(format!( - "Deserializing type for Decimal {:?} from parquet", - type_.physical_type - ))); - } - } - } - DataType::Decimal256(_, _) => { - init.push(InitNested::Primitive(field.is_nullable)); - let type_ = types.pop().unwrap(); - match type_.physical_type { - PhysicalType::Int32 => primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type.clone(), - num_rows, - chunk_size, - |x: i32| i256(I256::new(x as i128)), - )), - PhysicalType::Int64 => primitive(primitive::NestedIter::new( - columns.pop().unwrap(), - init, - field.data_type.clone(), - num_rows, - chunk_size, - |x: i64| i256(I256::new(x as i128)), - )), - PhysicalType::FixedLenByteArray(n) if n <= 16 => { - let iter = fixed_size_binary::NestedIter::new( - columns.pop().unwrap(), - init, - DataType::FixedSizeBinary(n), - num_rows, - chunk_size, - ); - // Convert the fixed length byte array to Decimal. - let iter = iter.map(move |x| { - let (mut nested, array) = x?; - let values = array - .values() - .chunks_exact(n) - .map(|value| i256(I256::new(super::super::convert_i128(value, n)))) - .collect::>(); - let validity = array.validity().cloned(); - - let array: Box = Box::new(PrimitiveArray::::try_new( - field.data_type.clone(), - values.into(), - validity, - )?); - - let _ = nested.nested.pop().unwrap(); // the primitive - - Ok((nested, array)) - }); - Box::new(iter) as _ - } - - PhysicalType::FixedLenByteArray(n) if n <= 32 => { - let iter = fixed_size_binary::NestedIter::new( - columns.pop().unwrap(), - init, - DataType::FixedSizeBinary(n), - num_rows, - chunk_size, - ); - // Convert the fixed length byte array to Decimal. - let iter = iter.map(move |x| { - let (mut nested, array) = x?; - let values = array - .values() - .chunks_exact(n) - .map(super::super::convert_i256) - .collect::>(); - let validity = array.validity().cloned(); - - let array: Box = Box::new(PrimitiveArray::::try_new( - field.data_type.clone(), - values.into(), - validity, - )?); - - let _ = nested.nested.pop().unwrap(); // the primitive - - Ok((nested, array)) - }); - Box::new(iter) as _ - } - PhysicalType::FixedLenByteArray(n) => { - return Err(Error::InvalidArgumentError(format!( - "Can't decode Decimal256 type from from `FixedLenByteArray` of len {n}" - ))); - } - _ => { - return Err(Error::nyi(format!( - "Deserializing type for Decimal {:?} from parquet", - type_.physical_type - ))); - } - } - } - DataType::Struct(fields) => { - let columns = fields - .iter() - .rev() - .map(|f| { - let mut init = init.clone(); - init.push(InitNested::Struct(field.is_nullable)); - let n = n_columns(&f.data_type); - let columns = columns.drain(columns.len() - n..).collect(); - let types = types.drain(types.len() - n..).collect(); - columns_to_iter_recursive( - columns, - types, - f.clone(), - init, - num_rows, - chunk_size, - ) - }) - .collect::>>()?; - let columns = columns.into_iter().rev().collect(); - Box::new(struct_::StructIterator::new(columns, fields.clone())) - } - DataType::Map(inner, _) => { - init.push(InitNested::List(field.is_nullable)); - let iter = columns_to_iter_recursive( - columns, - types, - inner.as_ref().clone(), - init, - num_rows, - chunk_size, - )?; - let iter = iter.map(move |x| { - let (mut nested, array) = x?; - let array = create_map(field.data_type().clone(), &mut nested, array); - Ok((nested, array)) - }); - Box::new(iter) as _ - } - other => { - return Err(Error::nyi(format!( - "Deserializing type {other:?} from parquet" - ))); - } - }, - }) -} - -fn dict_read<'a, K: DictionaryKey, I: 'a + Pages>( - iter: I, - init: Vec, - _type_: &PrimitiveType, - data_type: DataType, - num_rows: usize, - chunk_size: Option, -) -> Result> { - use DataType::*; - let values_data_type = if let Dictionary(_, v, _) = &data_type { - v.as_ref() - } else { - panic!() - }; - - Ok(match values_data_type.to_logical_type() { - UInt8 => primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: i32| x as u8, - )), - UInt16 => primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: i32| x as u16, - )), - UInt32 => primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: i32| x as u32, - )), - Int8 => primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: i32| x as i8, - )), - Int16 => primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: i32| x as i16, - )), - Int32 | Date32 | Time32(_) | Interval(IntervalUnit::YearMonth) => { - primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: i32| x, - )) - } - Int64 | Date64 | Time64(_) | Duration(_) => { - primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: i64| x as i32, - )) - } - Float32 => primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: f32| x, - )), - Float64 => primitive(primitive::NestedDictIter::::new( - iter, - init, - data_type, - num_rows, - chunk_size, - |x: f64| x, - )), - Utf8 | Binary => primitive(binary::NestedDictIter::::new( - iter, init, data_type, num_rows, chunk_size, - )), - LargeUtf8 | LargeBinary => primitive(binary::NestedDictIter::::new( - iter, init, data_type, num_rows, chunk_size, - )), - Utf8View | BinaryView => primitive(binview::NestedDictIter::::new( - iter, init, data_type, num_rows, chunk_size, - )), - FixedSizeBinary(_) => primitive(fixed_size_binary::NestedDictIter::::new( - iter, init, data_type, num_rows, chunk_size, - )), - /* - - Timestamp(time_unit, _) => { - let time_unit = *time_unit; - return timestamp_dict::( - iter, - physical_type, - logical_type, - data_type, - chunk_size, - time_unit, - ); - } - */ - other => { - return Err(Error::nyi(format!( - "Reading nested dictionaries of type {other:?}" - ))) - } - }) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/nested_utils.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/nested_utils.rs deleted file mode 100644 index 1af156c52a56..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/nested_utils.rs +++ /dev/null @@ -1,586 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::encoding::hybrid_rle::HybridRleDecoder; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::page::Page; -use parquet2::read::levels::get_bit_width; - -use super::super::Pages; -use super::utils::DecodedState; -use super::utils::MaybeNext; -use super::utils::PageState; -use crate::arrow::array::Array; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::error::Result; - -/// trait describing deserialized repetition and definition levels -pub trait Nested: std::fmt::Debug + Send + Sync { - fn inner(&mut self) -> (Vec, Option); - - fn push(&mut self, length: i64, is_valid: bool); - - fn is_nullable(&self) -> bool; - - fn is_repeated(&self) -> bool { - false - } - - // Whether the Arrow container requires all items to be filled. - fn is_required(&self) -> bool; - - /// number of rows - fn len(&self) -> usize; - - /// Returns `true` if the number of rows is 0. - fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// number of values associated to the primitive type this nested tracks - fn num_values(&self) -> usize; -} - -#[derive(Debug, Default)] -pub struct NestedPrimitive { - is_nullable: bool, - length: usize, -} - -impl NestedPrimitive { - pub fn new(is_nullable: bool) -> Self { - Self { - is_nullable, - length: 0, - } - } -} - -impl Nested for NestedPrimitive { - fn inner(&mut self) -> (Vec, Option) { - (Default::default(), Default::default()) - } - - fn is_nullable(&self) -> bool { - self.is_nullable - } - - fn is_required(&self) -> bool { - false - } - - fn push(&mut self, _value: i64, _is_valid: bool) { - self.length += 1 - } - - fn len(&self) -> usize { - self.length - } - - fn num_values(&self) -> usize { - self.length - } -} - -#[derive(Debug, Default)] -pub struct NestedOptional { - pub validity: MutableBitmap, - pub offsets: Vec, -} - -impl Nested for NestedOptional { - fn inner(&mut self) -> (Vec, Option) { - let offsets = std::mem::take(&mut self.offsets); - let validity = std::mem::take(&mut self.validity); - (offsets, Some(validity)) - } - - fn is_nullable(&self) -> bool { - true - } - - fn is_repeated(&self) -> bool { - true - } - - fn is_required(&self) -> bool { - // it may be for FixedSizeList - false - } - - fn push(&mut self, value: i64, is_valid: bool) { - self.offsets.push(value); - self.validity.push(is_valid); - } - - fn len(&self) -> usize { - self.offsets.len() - } - - fn num_values(&self) -> usize { - self.offsets.last().copied().unwrap_or(0) as usize - } -} - -impl NestedOptional { - pub fn with_capacity(capacity: usize) -> Self { - let offsets = Vec::::with_capacity(capacity + 1); - let validity = MutableBitmap::with_capacity(capacity); - Self { validity, offsets } - } -} - -#[derive(Debug, Default)] -pub struct NestedValid { - pub offsets: Vec, -} - -impl Nested for NestedValid { - fn inner(&mut self) -> (Vec, Option) { - let offsets = std::mem::take(&mut self.offsets); - (offsets, None) - } - - fn is_nullable(&self) -> bool { - false - } - - fn is_repeated(&self) -> bool { - true - } - - fn is_required(&self) -> bool { - // it may be for FixedSizeList - false - } - - fn push(&mut self, value: i64, _is_valid: bool) { - self.offsets.push(value); - } - - fn len(&self) -> usize { - self.offsets.len() - } - - fn num_values(&self) -> usize { - self.offsets.last().copied().unwrap_or(0) as usize - } -} - -impl NestedValid { - pub fn with_capacity(capacity: usize) -> Self { - let offsets = Vec::::with_capacity(capacity + 1); - Self { offsets } - } -} - -#[derive(Debug, Default)] -pub struct NestedStructValid { - length: usize, -} - -impl NestedStructValid { - pub fn new() -> Self { - Self { length: 0 } - } -} - -impl Nested for NestedStructValid { - fn inner(&mut self) -> (Vec, Option) { - (Default::default(), None) - } - - fn is_nullable(&self) -> bool { - false - } - - fn is_required(&self) -> bool { - true - } - - fn push(&mut self, _value: i64, _is_valid: bool) { - self.length += 1; - } - - fn len(&self) -> usize { - self.length - } - - fn num_values(&self) -> usize { - self.length - } -} - -#[derive(Debug, Default)] -pub struct NestedStruct { - validity: MutableBitmap, -} - -impl NestedStruct { - pub fn with_capacity(capacity: usize) -> Self { - Self { - validity: MutableBitmap::with_capacity(capacity), - } - } -} - -impl Nested for NestedStruct { - fn inner(&mut self) -> (Vec, Option) { - (Default::default(), Some(std::mem::take(&mut self.validity))) - } - - fn is_nullable(&self) -> bool { - true - } - - fn is_required(&self) -> bool { - true - } - - fn push(&mut self, _value: i64, is_valid: bool) { - self.validity.push(is_valid) - } - - fn len(&self) -> usize { - self.validity.len() - } - - fn num_values(&self) -> usize { - self.validity.len() - } -} - -/// A decoder that knows how to map `State` -> Array -pub(super) trait NestedDecoder<'a> { - type State: PageState<'a>; - type Dictionary; - type DecodedState: DecodedState; - - fn build_state( - &self, - page: &'a DataPage, - dict: Option<&'a Self::Dictionary>, - ) -> Result; - - /// Initializes a new state - fn with_capacity(&self, capacity: usize) -> Self::DecodedState; - - fn push_valid(&self, state: &mut Self::State, decoded: &mut Self::DecodedState) -> Result<()>; - fn push_null(&self, decoded: &mut Self::DecodedState); - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dictionary; -} - -/// The initial info of nested data types. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum InitNested { - /// Primitive data types - Primitive(bool), - /// List data types - List(bool), - /// Struct data types - Struct(bool), -} - -/// Initialize [`NestedState`] from `&[InitNested]`. -pub fn init_nested(init: &[InitNested], capacity: usize) -> NestedState { - let container = init - .iter() - .map(|init| match init { - InitNested::Primitive(is_nullable) => { - Box::new(NestedPrimitive::new(*is_nullable)) as Box - } - InitNested::List(is_nullable) => { - if *is_nullable { - Box::new(NestedOptional::with_capacity(capacity)) as Box - } else { - Box::new(NestedValid::with_capacity(capacity)) as Box - } - } - InitNested::Struct(is_nullable) => { - if *is_nullable { - Box::new(NestedStruct::with_capacity(capacity)) as Box - } else { - Box::new(NestedStructValid::new()) as Box - } - } - }) - .collect(); - NestedState::new(container) -} - -pub struct NestedPage<'a> { - iter: std::iter::Peekable, HybridRleDecoder<'a>>>, -} - -impl<'a> NestedPage<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (rep_levels, def_levels, _) = split_buffer(page)?; - - let max_rep_level = page.descriptor.max_rep_level; - let max_def_level = page.descriptor.max_def_level; - - let reps = - HybridRleDecoder::try_new(rep_levels, get_bit_width(max_rep_level), page.num_values())?; - let defs = - HybridRleDecoder::try_new(def_levels, get_bit_width(max_def_level), page.num_values())?; - - let iter = reps.zip(defs).peekable(); - - Ok(Self { iter }) - } - - // number of values (!= number of rows) - pub fn len(&self) -> usize { - self.iter.size_hint().0 - } -} - -/// The state of nested data types. -#[derive(Debug)] -pub struct NestedState { - /// The nesteds composing `NestedState`. - pub nested: Vec>, -} - -impl NestedState { - /// Creates a new [`NestedState`]. - pub fn new(nested: Vec>) -> Self { - Self { nested } - } - - /// The number of rows in this state - pub fn len(&self) -> usize { - // outermost is the number of rows - self.nested[0].len() - } - - /// Returns `true` if the number of rows is 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -/// Extends `items` by consuming `page`, first trying to complete the last `item` -/// and extending it if more are needed -pub(super) fn extend<'a, D: NestedDecoder<'a>>( - page: &'a DataPage, - init: &[InitNested], - items: &mut VecDeque<(NestedState, D::DecodedState)>, - dict: Option<&'a D::Dictionary>, - remaining: &mut usize, - decoder: &D, - chunk_size: Option, -) -> Result<()> { - let mut values_page = decoder.build_state(page, dict)?; - let mut page = NestedPage::try_new(page)?; - - let capacity = chunk_size.unwrap_or(0); - // chunk_size = None, remaining = 44 => chunk_size = 44 - let chunk_size = chunk_size.unwrap_or(usize::MAX); - - let (mut nested, mut decoded) = if let Some((nested, decoded)) = items.pop_back() { - (nested, decoded) - } else { - // there is no state => initialize it - (init_nested(init, capacity), decoder.with_capacity(0)) - }; - let existing = nested.len(); - - let additional = (chunk_size - existing).min(*remaining); - - // extend the current state - extend_offsets2( - &mut page, - &mut values_page, - &mut nested.nested, - &mut decoded, - decoder, - additional, - )?; - *remaining -= nested.len() - existing; - items.push_back((nested, decoded)); - - while page.len() > 0 && *remaining > 0 { - let additional = chunk_size.min(*remaining); - - let mut nested = init_nested(init, additional); - let mut decoded = decoder.with_capacity(0); - extend_offsets2( - &mut page, - &mut values_page, - &mut nested.nested, - &mut decoded, - decoder, - additional, - )?; - *remaining -= nested.len(); - items.push_back((nested, decoded)); - } - Ok(()) -} - -fn extend_offsets2<'a, D: NestedDecoder<'a>>( - page: &mut NestedPage<'a>, - values_state: &mut D::State, - nested: &mut [Box], - decoded: &mut D::DecodedState, - decoder: &D, - additional: usize, -) -> Result<()> { - let max_depth = nested.len(); - - let mut cum_sum = vec![0u32; max_depth + 1]; - for (i, nest) in nested.iter().enumerate() { - let delta = nest.is_nullable() as u32 + nest.is_repeated() as u32; - cum_sum[i + 1] = cum_sum[i] + delta; - } - - let mut cum_rep = vec![0u32; max_depth + 1]; - for (i, nest) in nested.iter().enumerate() { - let delta = nest.is_repeated() as u32; - cum_rep[i + 1] = cum_rep[i] + delta; - } - - let mut rows = 0; - while let Some((rep, def)) = page.iter.next() { - let rep = rep?; - let def = def?; - if rep == 0 { - rows += 1; - } - - let mut is_required = false; - for depth in 0..max_depth { - let right_level = rep <= cum_rep[depth] && def >= cum_sum[depth]; - if is_required || right_level { - let length = nested - .get(depth + 1) - .map(|x| x.len() as i64) - // the last depth is the leaf, which is always increased by 1 - .unwrap_or(1); - - let nest = &mut nested[depth]; - - let is_valid = nest.is_nullable() && def > cum_sum[depth]; - nest.push(length, is_valid); - is_required = nest.is_required() && !is_valid; - - if depth == max_depth - 1 { - // the leaf / primitive - let is_valid = (def != cum_sum[depth]) || !nest.is_nullable(); - if right_level && is_valid { - decoder.push_valid(values_state, decoded)?; - } else { - decoder.push_null(decoded); - } - } - } - } - - let next_rep = *page - .iter - .peek() - .map(|x| x.0.as_ref()) - .transpose() - .unwrap() // todo: fix this - .unwrap_or(&0); - - if next_rep == 0 && rows == additional { - break; - } - } - Ok(()) -} - -#[inline] -pub(super) fn next<'a, I, D>( - iter: &'a mut I, - items: &mut VecDeque<(NestedState, D::DecodedState)>, - dict: &'a mut Option, - remaining: &mut usize, - init: &[InitNested], - chunk_size: Option, - decoder: &D, -) -> MaybeNext> -where - I: Pages, - D: NestedDecoder<'a>, -{ - // front[a1, a2, a3, ...]back - if items.len() > 1 { - return MaybeNext::Some(Ok(items.pop_front().unwrap())); - } - if (items.len() == 1) && items.front().unwrap().0.len() == chunk_size.unwrap_or(usize::MAX) { - return MaybeNext::Some(Ok(items.pop_front().unwrap())); - } - if *remaining == 0 { - return match items.pop_front() { - Some(decoded) => MaybeNext::Some(Ok(decoded)), - None => MaybeNext::None, - }; - } - match iter.next() { - Err(e) => MaybeNext::Some(Err(e.into())), - Ok(None) => { - if let Some(decoded) = items.pop_front() { - MaybeNext::Some(Ok(decoded)) - } else { - MaybeNext::None - } - } - Ok(Some(page)) => { - let page = match page { - Page::Data(page) => page, - Page::Dict(dict_page) => { - *dict = Some(decoder.deserialize_dict(dict_page)); - return MaybeNext::More; - } - }; - - // there is a new page => consume the page from the start - let error = extend( - page, - init, - items, - dict.as_ref(), - remaining, - decoder, - chunk_size, - ); - match error { - Ok(_) => {} - Err(e) => return MaybeNext::Some(Err(e)), - }; - - if (items.len() == 1) - && items.front().unwrap().0.len() < chunk_size.unwrap_or(usize::MAX) - { - MaybeNext::More - } else { - MaybeNext::Some(Ok(items.pop_front().unwrap())) - } - } - } -} - -/// Type def for a sharable, boxed dyn [`Iterator`] of NestedStates and arrays -pub type NestedArrayIter<'a> = - Box)>> + Send + Sync + 'a>; diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/null/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/null/mod.rs deleted file mode 100644 index dc8c1dbfab1b..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/null/mod.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod nested; - -pub(super) use nested::NestedIter; -use parquet2::page::Page; - -use super::super::ArrayIter; -use super::super::Pages; -use crate::arrow::array::NullArray; -use crate::arrow::datatypes::DataType; - -/// Converts [`Pages`] to an [`ArrayIter`] -pub fn iter_to_arrays<'a, I>( - mut iter: I, - data_type: DataType, - chunk_size: Option, - num_rows: usize, -) -> ArrayIter<'a> -where - I: 'a + Pages, -{ - let mut len = 0usize; - - while let Ok(Some(page)) = iter.next() { - match page { - Page::Dict(_) => continue, - Page::Data(page) => { - let rows = page.num_values(); - len = (len + rows).min(num_rows); - if len == num_rows { - break; - } - } - } - } - - if len == 0 { - return Box::new(std::iter::empty()); - } - - let chunk_size = chunk_size.unwrap_or(len); - - let complete_chunks = len / chunk_size; - - let remainder = len - (complete_chunks * chunk_size); - let i_data_type = data_type.clone(); - let complete = (0..complete_chunks) - .map(move |_| Ok(NullArray::new(i_data_type.clone(), chunk_size).boxed())); - if len % chunk_size == 0 { - Box::new(complete) - } else { - let array = NullArray::new(data_type, remainder); - Box::new(complete.chain(std::iter::once(Ok(array.boxed())))) - } -} - -#[cfg(test)] -mod tests { - use parquet2::encoding::Encoding; - use parquet2::error::Error as ParquetError; - use parquet2::metadata::Descriptor; - use parquet2::page::DataPage; - use parquet2::page::DataPageHeader; - use parquet2::page::DataPageHeaderV1; - use parquet2::page::Page; - use parquet2::schema::types::PhysicalType; - use parquet2::schema::types::PrimitiveType; - - use super::iter_to_arrays; - use crate::arrow::array::NullArray; - use crate::arrow::datatypes::DataType; - use crate::arrow::error::Error; - - #[test] - fn limit() { - let new_page = |values: i32| { - Page::Data(DataPage::new( - DataPageHeader::V1(DataPageHeaderV1 { - num_values: values, - encoding: Encoding::Plain.into(), - definition_level_encoding: Encoding::Plain.into(), - repetition_level_encoding: Encoding::Plain.into(), - statistics: None, - }), - vec![], - Descriptor { - primitive_type: PrimitiveType::from_physical( - "a".to_string(), - PhysicalType::Int32, - ), - max_def_level: 0, - max_rep_level: 0, - }, - None, - )) - }; - - let p1 = new_page(100); - let p2 = new_page(100); - let pages = vec![Result::<_, ParquetError>::Ok(&p1), Ok(&p2)]; - let pages = fallible_streaming_iterator::convert(pages.into_iter()); - let arrays = iter_to_arrays(pages, DataType::Null, Some(10), 101); - - let arrays = arrays.collect::, Error>>().unwrap(); - let expected = std::iter::repeat(NullArray::new(DataType::Null, 10).boxed()) - .take(10) - .chain(std::iter::once(NullArray::new(DataType::Null, 1).boxed())); - assert_eq!(arrays, expected.collect::>()) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/null/nested.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/null/nested.rs deleted file mode 100644 index 0f9ce970d22c..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/null/nested.rs +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::page::DataPage; -use parquet2::page::DictPage; - -use super::super::nested_utils::*; -use super::super::utils; -use super::super::Pages; -use crate::arrow::array::NullArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::deserialize::utils::DecodedState; - -impl<'a> utils::PageState<'a> for usize { - fn len(&self) -> usize { - *self - } -} - -#[derive(Debug)] -struct NullDecoder {} - -impl DecodedState for usize { - fn len(&self) -> usize { - *self - } -} - -impl<'a> NestedDecoder<'a> for NullDecoder { - type State = usize; - type Dictionary = usize; - type DecodedState = usize; - - fn build_state( - &self, - _page: &'a DataPage, - dict: Option<&'a Self::Dictionary>, - ) -> Result { - if let Some(n) = dict { - return Ok(*n); - } - Ok(1) - } - - /// Initializes a new state - fn with_capacity(&self, _capacity: usize) -> Self::DecodedState { - 0 - } - - fn push_valid(&self, state: &mut Self::State, decoded: &mut Self::DecodedState) -> Result<()> { - *decoded += *state; - Ok(()) - } - - fn push_null(&self, decoded: &mut Self::DecodedState) { - let length = decoded; - *length += 1; - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dictionary { - page.num_values - } -} - -/// An iterator adapter over [`Pages`] assumed to be encoded as null arrays -#[derive(Debug)] -pub struct NestedIter -where I: Pages -{ - iter: I, - init: Vec, - data_type: DataType, - items: VecDeque<(NestedState, usize)>, - remaining: usize, - chunk_size: Option, - decoder: NullDecoder, -} - -impl NestedIter -where I: Pages -{ - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - ) -> Self { - Self { - iter, - init, - data_type, - items: VecDeque::new(), - chunk_size, - remaining: num_rows, - decoder: NullDecoder {}, - } - } -} - -impl Iterator for NestedIter -where I: Pages -{ - type Item = Result<(NestedState, NullArray)>; - - fn next(&mut self) -> Option { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut None, - &mut self.remaining, - &self.init, - self.chunk_size, - &self.decoder, - ); - match maybe_state { - utils::MaybeNext::Some(Ok((nested, state))) => { - Some(Ok((nested, NullArray::new(self.data_type.clone(), state)))) - } - utils::MaybeNext::Some(Err(e)) => Some(Err(e)), - utils::MaybeNext::None => None, - utils::MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/basic.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/basic.rs deleted file mode 100644 index b1631a13cc3d..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/basic.rs +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::deserialize::SliceFilteredIter; -use parquet2::encoding::hybrid_rle; -use parquet2::encoding::Encoding; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; -use parquet2::types::decode; -use parquet2::types::NativeType as ParquetNativeType; - -use super::super::utils; -use super::super::utils::get_selected_rows; -use super::super::utils::FilteredOptionalPageValidity; -use super::super::utils::OptionalPageValidity; -use super::super::Pages; -use crate::arrow::array::MutablePrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; - -#[derive(Debug)] -pub(super) struct FilteredRequiredValues<'a> { - values: SliceFilteredIter>, -} - -impl<'a> FilteredRequiredValues<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, _, values) = split_buffer(page)?; - assert_eq!(values.len() % std::mem::size_of::

(), 0); - - let values = values.chunks_exact(std::mem::size_of::

()); - - let rows = get_selected_rows(page); - let values = SliceFilteredIter::new(values, rows); - - Ok(Self { values }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(super) struct Values<'a> { - pub values: std::slice::ChunksExact<'a, u8>, -} - -impl<'a> Values<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, _, values) = split_buffer(page)?; - assert_eq!(values.len() % std::mem::size_of::

(), 0); - Ok(Self { - values: values.chunks_exact(std::mem::size_of::

()), - }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -#[derive(Debug)] -pub(super) struct ValuesDictionary<'a, T> -where T: NativeType -{ - pub values: hybrid_rle::HybridRleDecoder<'a>, - pub dict: &'a Vec, -} - -impl<'a, T> ValuesDictionary<'a, T> -where T: NativeType -{ - pub fn try_new(page: &'a DataPage, dict: &'a Vec) -> Result { - let values = utils::dict_indices_decoder(page)?; - - Ok(Self { dict, values }) - } - - #[inline] - pub fn len(&self) -> usize { - self.values.size_hint().0 - } -} - -// The state of a `DataPage` of `Primitive` parquet primitive type -#[derive(Debug)] -pub(super) enum State<'a, T> -where T: NativeType -{ - Optional(OptionalPageValidity<'a>, Values<'a>), - Required(Values<'a>), - RequiredDictionary(ValuesDictionary<'a, T>), - OptionalDictionary(OptionalPageValidity<'a>, ValuesDictionary<'a, T>), - FilteredRequired(FilteredRequiredValues<'a>), - FilteredOptional(FilteredOptionalPageValidity<'a>, Values<'a>), -} - -impl<'a, T> utils::PageState<'a> for State<'a, T> -where T: NativeType -{ - fn len(&self) -> usize { - match self { - State::Optional(optional, _) => optional.len(), - State::Required(values) => values.len(), - State::RequiredDictionary(values) => values.len(), - State::OptionalDictionary(optional, _) => optional.len(), - State::FilteredRequired(values) => values.len(), - State::FilteredOptional(optional, _) => optional.len(), - } - } -} - -#[derive(Debug)] -pub(super) struct PrimitiveDecoder -where - T: NativeType, - P: ParquetNativeType, - F: Fn(P) -> T, -{ - phantom: std::marker::PhantomData, - phantom_p: std::marker::PhantomData

, - pub op: F, -} - -impl PrimitiveDecoder -where - T: NativeType, - P: ParquetNativeType, - F: Fn(P) -> T, -{ - #[inline] - pub(super) fn new(op: F) -> Self { - Self { - phantom: std::marker::PhantomData, - phantom_p: std::marker::PhantomData, - op, - } - } -} - -impl utils::DecodedState for (Vec, MutableBitmap) { - fn len(&self) -> usize { - self.0.len() - } -} - -impl<'a, T, P, F> utils::Decoder<'a> for PrimitiveDecoder -where - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - type State = State<'a, T>; - type Dict = Vec; - type DecodedState = (Vec, MutableBitmap); - - fn build_state(&self, page: &'a DataPage, dict: Option<&'a Self::Dict>) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), dict, is_optional, is_filtered) { - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false, false) => { - ValuesDictionary::try_new(page, dict).map(State::RequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true, false) => { - Ok(State::OptionalDictionary( - OptionalPageValidity::try_new(page)?, - ValuesDictionary::try_new(page, dict)?, - )) - } - (Encoding::Plain, _, true, false) => { - let validity = OptionalPageValidity::try_new(page)?; - let values = Values::try_new::

(page)?; - - Ok(State::Optional(validity, values)) - } - (Encoding::Plain, _, false, false) => Ok(State::Required(Values::try_new::

(page)?)), - (Encoding::Plain, _, false, true) => { - FilteredRequiredValues::try_new::

(page).map(State::FilteredRequired) - } - (Encoding::Plain, _, true, true) => Ok(State::FilteredOptional( - FilteredOptionalPageValidity::try_new(page)?, - Values::try_new::

(page)?, - )), - _ => Err(utils::not_implemented(page)), - } - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - Vec::::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn extend_from_state( - &self, - state: &mut Self::State, - decoded: &mut Self::DecodedState, - remaining: usize, - ) { - let (values, validity) = decoded; - match state { - State::Optional(page_validity, page_values) => utils::extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - page_values.values.by_ref().map(decode).map(self.op), - ), - State::Required(page) => { - values.extend( - page.values - .by_ref() - .map(decode) - .map(self.op) - .take(remaining), - ); - } - State::OptionalDictionary(page_validity, page_values) => { - let op1 = |index: u32| page_values.dict[index as usize]; - utils::extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - &mut page_values.values.by_ref().map(|x| x.unwrap()).map(op1), - ) - } - State::RequiredDictionary(page) => { - let op1 = |index: u32| page.dict[index as usize]; - values.extend( - page.values - .by_ref() - .map(|x| x.unwrap()) - .map(op1) - .take(remaining), - ); - } - State::FilteredRequired(page) => { - values.extend( - page.values - .by_ref() - .map(decode) - .map(self.op) - .take(remaining), - ); - } - State::FilteredOptional(page_validity, page_values) => { - utils::extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - page_values.values.by_ref().map(decode).map(self.op), - ); - } - } - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dict { - deserialize_plain(&page.buffer, self.op) - } -} - -pub(super) fn finish( - data_type: &DataType, - values: Vec, - validity: MutableBitmap, -) -> MutablePrimitiveArray { - let validity = if validity.is_empty() { - None - } else { - Some(validity) - }; - MutablePrimitiveArray::try_new(data_type.clone(), values, validity).unwrap() -} - -/// An [`Iterator`] adapter over [`Pages`] assumed to be encoded as primitive arrays -#[derive(Debug)] -pub struct Iter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Fn(P) -> T, -{ - iter: I, - data_type: DataType, - items: VecDeque<(Vec, MutableBitmap)>, - remaining: usize, - chunk_size: Option, - dict: Option>, - op: F, - phantom: std::marker::PhantomData

, -} - -impl Iter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - pub fn new( - iter: I, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - op: F, - ) -> Self { - Self { - iter, - data_type, - items: VecDeque::new(), - dict: None, - remaining: num_rows, - chunk_size, - op, - phantom: Default::default(), - } - } -} - -impl Iterator for Iter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - type Item = Result>; - - fn next(&mut self) -> Option { - let maybe_state = utils::next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - self.chunk_size, - &PrimitiveDecoder::new(self.op), - ); - match maybe_state { - utils::MaybeNext::Some(Ok((values, validity))) => { - Some(Ok(finish(&self.data_type, values, validity))) - } - utils::MaybeNext::Some(Err(e)) => Some(Err(e)), - utils::MaybeNext::None => None, - utils::MaybeNext::More => self.next(), - } - } -} - -pub(super) fn deserialize_plain(values: &[u8], op: F) -> Vec -where - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - values - .chunks_exact(std::mem::size_of::

()) - .map(decode) - .map(op) - .collect::>() -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/dictionary.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/dictionary.rs deleted file mode 100644 index e055afb9579a..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/dictionary.rs +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::page::DictPage; -use parquet2::types::NativeType as ParquetNativeType; - -use super::super::dictionary::nested_next_dict; -use super::super::dictionary::*; -use super::super::nested_utils::InitNested; -use super::super::nested_utils::NestedState; -use super::super::utils::MaybeNext; -use super::super::Pages; -use super::basic::deserialize_plain; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; - -fn read_dict(data_type: DataType, op: F, dict: &DictPage) -> Box -where - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - let data_type = match data_type { - DataType::Dictionary(_, values, _) => *values, - _ => data_type, - }; - let values = deserialize_plain(&dict.buffer, op); - - Box::new(PrimitiveArray::new(data_type, values.into(), None)) -} - -/// An iterator adapter over [`Pages`] assumed to be encoded as boolean arrays -#[derive(Debug)] -pub struct DictIter -where - I: Pages, - T: NativeType, - K: DictionaryKey, - P: ParquetNativeType, - F: Fn(P) -> T, -{ - iter: I, - data_type: DataType, - values: Option>, - items: VecDeque<(Vec, MutableBitmap)>, - remaining: usize, - chunk_size: Option, - op: F, - phantom: std::marker::PhantomData

, -} - -impl DictIter -where - K: DictionaryKey, - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - pub fn new( - iter: I, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - op: F, - ) -> Self { - Self { - iter, - data_type, - values: None, - items: VecDeque::new(), - chunk_size, - remaining: num_rows, - op, - phantom: Default::default(), - } - } -} - -impl Iterator for DictIter -where - I: Pages, - T: NativeType, - K: DictionaryKey, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - type Item = Result>; - - fn next(&mut self) -> Option { - let maybe_state = next_dict( - &mut self.iter, - &mut self.items, - &mut self.values, - self.data_type.clone(), - &mut self.remaining, - self.chunk_size, - |dict| read_dict::(self.data_type.clone(), self.op, dict), - ); - match maybe_state { - MaybeNext::Some(Ok(dict)) => Some(Ok(dict)), - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} - -/// An iterator adapter that converts [`DataPages`] into an [`Iterator`] of [`DictionaryArray`] -#[derive(Debug)] -pub struct NestedDictIter -where - I: Pages, - T: NativeType, - K: DictionaryKey, - P: ParquetNativeType, - F: Fn(P) -> T, -{ - iter: I, - init: Vec, - data_type: DataType, - values: Option>, - items: VecDeque<(NestedState, (Vec, MutableBitmap))>, - remaining: usize, - chunk_size: Option, - op: F, - phantom: std::marker::PhantomData

, -} - -impl NestedDictIter -where - K: DictionaryKey, - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - op: F, - ) -> Self { - Self { - iter, - init, - data_type, - values: None, - items: VecDeque::new(), - remaining: num_rows, - chunk_size, - op, - phantom: Default::default(), - } - } -} - -impl Iterator for NestedDictIter -where - I: Pages, - T: NativeType, - K: DictionaryKey, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - type Item = Result<(NestedState, DictionaryArray)>; - - fn next(&mut self) -> Option { - let maybe_state = nested_next_dict( - &mut self.iter, - &mut self.items, - &mut self.remaining, - &self.init, - &mut self.values, - self.data_type.clone(), - self.chunk_size, - |dict| read_dict::(self.data_type.clone(), self.op, dict), - ); - match maybe_state { - MaybeNext::Some(Ok(dict)) => Some(Ok(dict)), - MaybeNext::Some(Err(e)) => Some(Err(e)), - MaybeNext::None => None, - MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/integer.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/integer.rs deleted file mode 100644 index fc529156e229..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/integer.rs +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use num_traits::AsPrimitive; -use parquet2::deserialize::SliceFilteredIter; -use parquet2::encoding::delta_bitpacked::Decoder; -use parquet2::encoding::Encoding; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; -use parquet2::types::NativeType as ParquetNativeType; - -use super::super::utils; -use super::super::Pages; -use super::basic::finish; -use super::basic::PrimitiveDecoder; -use super::basic::State as PrimitiveState; -use crate::arrow::array::MutablePrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::deserialize::utils::get_selected_rows; -use crate::arrow::io::parquet::read::deserialize::utils::FilteredOptionalPageValidity; -use crate::arrow::io::parquet::read::deserialize::utils::OptionalPageValidity; -use crate::arrow::types::NativeType; - -/// The state of a [`DataPage`] of an integer parquet type (i32 or i64) -#[derive(Debug)] -enum State<'a, T> -where T: NativeType -{ - Common(PrimitiveState<'a, T>), - DeltaBinaryPackedRequired(Decoder<'a>), - DeltaBinaryPackedOptional(OptionalPageValidity<'a>, Decoder<'a>), - FilteredDeltaBinaryPackedRequired(SliceFilteredIter>), - FilteredDeltaBinaryPackedOptional(FilteredOptionalPageValidity<'a>, Decoder<'a>), -} - -impl<'a, T> utils::PageState<'a> for State<'a, T> -where T: NativeType -{ - fn len(&self) -> usize { - match self { - State::Common(state) => state.len(), - State::DeltaBinaryPackedRequired(state) => state.size_hint().0, - State::DeltaBinaryPackedOptional(state, _) => state.len(), - State::FilteredDeltaBinaryPackedRequired(state) => state.size_hint().0, - State::FilteredDeltaBinaryPackedOptional(state, _) => state.len(), - } - } -} - -/// Decoder of integer parquet type -#[derive(Debug)] -struct IntDecoder(PrimitiveDecoder) -where - T: NativeType, - P: ParquetNativeType, - i64: num_traits::AsPrimitive

, - F: Fn(P) -> T; - -impl IntDecoder -where - T: NativeType, - P: ParquetNativeType, - i64: num_traits::AsPrimitive

, - F: Fn(P) -> T, -{ - #[inline] - fn new(op: F) -> Self { - Self(PrimitiveDecoder::new(op)) - } -} - -impl<'a, T, P, F> utils::Decoder<'a> for IntDecoder -where - T: NativeType, - P: ParquetNativeType, - i64: num_traits::AsPrimitive

, - F: Copy + Fn(P) -> T, -{ - type State = State<'a, T>; - type Dict = Vec; - type DecodedState = (Vec, MutableBitmap); - - fn build_state(&self, page: &'a DataPage, dict: Option<&'a Self::Dict>) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), dict, is_optional, is_filtered) { - (Encoding::DeltaBinaryPacked, _, false, false) => { - let (_, _, values) = split_buffer(page)?; - Decoder::try_new(values) - .map(State::DeltaBinaryPackedRequired) - .map_err(Error::from) - } - (Encoding::DeltaBinaryPacked, _, true, false) => { - let (_, _, values) = split_buffer(page)?; - Ok(State::DeltaBinaryPackedOptional( - OptionalPageValidity::try_new(page)?, - Decoder::try_new(values)?, - )) - } - (Encoding::DeltaBinaryPacked, _, false, true) => { - let (_, _, values) = split_buffer(page)?; - let values = Decoder::try_new(values)?; - - let rows = get_selected_rows(page); - let values = SliceFilteredIter::new(values, rows); - - Ok(State::FilteredDeltaBinaryPackedRequired(values)) - } - (Encoding::DeltaBinaryPacked, _, true, true) => { - let (_, _, values) = split_buffer(page)?; - let values = Decoder::try_new(values)?; - - Ok(State::FilteredDeltaBinaryPackedOptional( - FilteredOptionalPageValidity::try_new(page)?, - values, - )) - } - _ => self.0.build_state(page, dict).map(State::Common), - } - } - - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - self.0.with_capacity(capacity) - } - - fn extend_from_state( - &self, - state: &mut Self::State, - decoded: &mut Self::DecodedState, - remaining: usize, - ) { - let (values, validity) = decoded; - match state { - State::Common(state) => self.0.extend_from_state(state, decoded, remaining), - State::DeltaBinaryPackedRequired(state) => { - values.extend( - state - .by_ref() - .map(|x| x.unwrap().as_()) - .map(self.0.op) - .take(remaining), - ); - } - State::DeltaBinaryPackedOptional(page_validity, page_values) => { - utils::extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - page_values - .by_ref() - .map(|x| x.unwrap().as_()) - .map(self.0.op), - ) - } - State::FilteredDeltaBinaryPackedRequired(page) => { - values.extend( - page.by_ref() - .map(|x| x.unwrap().as_()) - .map(self.0.op) - .take(remaining), - ); - } - State::FilteredDeltaBinaryPackedOptional(page_validity, page_values) => { - utils::extend_from_decoder( - validity, - page_validity, - Some(remaining), - values, - page_values - .by_ref() - .map(|x| x.unwrap().as_()) - .map(self.0.op), - ); - } - } - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dict { - self.0.deserialize_dict(page) - } -} - -/// An [`Iterator`] adapter over [`Pages`] assumed to be encoded as primitive arrays -/// encoded as parquet integer types -#[derive(Debug)] -pub struct IntegerIter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Fn(P) -> T, -{ - iter: I, - data_type: DataType, - items: VecDeque<(Vec, MutableBitmap)>, - remaining: usize, - chunk_size: Option, - dict: Option>, - op: F, - phantom: std::marker::PhantomData

, -} - -impl IntegerIter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - pub fn new( - iter: I, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - op: F, - ) -> Self { - Self { - iter, - data_type, - items: VecDeque::new(), - dict: None, - remaining: num_rows, - chunk_size, - op, - phantom: Default::default(), - } - } -} - -impl Iterator for IntegerIter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - i64: num_traits::AsPrimitive

, - F: Copy + Fn(P) -> T, -{ - type Item = Result>; - - fn next(&mut self) -> Option { - let maybe_state = utils::next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - self.chunk_size, - &IntDecoder::new(self.op), - ); - match maybe_state { - utils::MaybeNext::Some(Ok((values, validity))) => { - Some(Ok(finish(&self.data_type, values, validity))) - } - utils::MaybeNext::Some(Err(e)) => Some(Err(e)), - utils::MaybeNext::None => None, - utils::MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/mod.rs deleted file mode 100644 index 6052b295b8d7..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod dictionary; -mod integer; -mod nested; - -pub use basic::Iter; -pub use dictionary::DictIter; -pub use dictionary::NestedDictIter; -pub use integer::IntegerIter; -pub use nested::NestedIter; diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/nested.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/nested.rs deleted file mode 100644 index 5323dbe6efda..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/primitive/nested.rs +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::schema::Repetition; -use parquet2::types::decode; -use parquet2::types::NativeType as ParquetNativeType; - -use super::super::nested_utils::*; -use super::super::utils; -use super::super::Pages; -use super::basic::deserialize_plain; -use super::basic::Values; -use super::basic::ValuesDictionary; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; - -// The state of a `DataPage` of `Primitive` parquet primitive type -#[allow(clippy::large_enum_variant)] -#[derive(Debug)] -enum State<'a, T> -where T: NativeType -{ - Optional(Values<'a>), - Required(Values<'a>), - RequiredDictionary(ValuesDictionary<'a, T>), - OptionalDictionary(ValuesDictionary<'a, T>), -} - -impl<'a, T> utils::PageState<'a> for State<'a, T> -where T: NativeType -{ - fn len(&self) -> usize { - match self { - State::Optional(values) => values.len(), - State::Required(values) => values.len(), - State::RequiredDictionary(values) => values.len(), - State::OptionalDictionary(values) => values.len(), - } - } -} - -#[derive(Debug)] -struct PrimitiveDecoder -where - T: NativeType, - P: ParquetNativeType, - F: Fn(P) -> T, -{ - phantom: std::marker::PhantomData, - phantom_p: std::marker::PhantomData

, - op: F, -} - -impl PrimitiveDecoder -where - T: NativeType, - P: ParquetNativeType, - F: Fn(P) -> T, -{ - #[inline] - fn new(op: F) -> Self { - Self { - phantom: std::marker::PhantomData, - phantom_p: std::marker::PhantomData, - op, - } - } -} - -impl<'a, T, P, F> NestedDecoder<'a> for PrimitiveDecoder -where - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - type State = State<'a, T>; - type Dictionary = Vec; - type DecodedState = (Vec, MutableBitmap); - - fn build_state( - &self, - page: &'a DataPage, - dict: Option<&'a Self::Dictionary>, - ) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - - match (page.encoding(), dict, is_optional, is_filtered) { - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false, false) => { - ValuesDictionary::try_new(page, dict).map(State::RequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true, false) => { - ValuesDictionary::try_new(page, dict).map(State::OptionalDictionary) - } - (Encoding::Plain, _, true, false) => Values::try_new::

(page).map(State::Optional), - (Encoding::Plain, _, false, false) => Values::try_new::

{ - /// Whether the page was sufficient to fill `chunk_size` - Some(P), - /// whether there are no more pages or intermediary decoded states - None, - /// Whether the page was insufficient to fill `chunk_size` and a new page is required - More, -} - -#[inline] -pub(super) fn next<'a, I: Pages, D: Decoder<'a>>( - iter: &'a mut I, - items: &'a mut VecDeque, - dict: &'a mut Option, - remaining: &'a mut usize, - chunk_size: Option, - decoder: &'a D, -) -> MaybeNext> { - // front[a1, a2, a3, ...]back - if items.len() > 1 { - return MaybeNext::Some(Ok(items.pop_front().unwrap())); - } - if (items.len() == 1) && items.front().unwrap().len() == chunk_size.unwrap_or(usize::MAX) { - return MaybeNext::Some(Ok(items.pop_front().unwrap())); - } - if *remaining == 0 { - return match items.pop_front() { - Some(decoded) => MaybeNext::Some(Ok(decoded)), - None => MaybeNext::None, - }; - } - - match iter.next() { - Err(e) => MaybeNext::Some(Err(e.into())), - Ok(Some(page)) => { - let page = match page { - Page::Data(page) => page, - Page::Dict(dict_page) => { - *dict = Some(decoder.deserialize_dict(dict_page)); - return MaybeNext::More; - } - }; - - // there is a new page => consume the page from the start - let maybe_page = decoder.build_state(page, dict.as_ref()); - let page = match maybe_page { - Ok(page) => page, - Err(e) => return MaybeNext::Some(Err(e)), - }; - - extend_from_new_page(page, chunk_size, items, remaining, decoder); - - if (items.len() == 1) && items.front().unwrap().len() < chunk_size.unwrap_or(usize::MAX) - { - MaybeNext::More - } else { - let decoded = items.pop_front().unwrap(); - MaybeNext::Some(Ok(decoded)) - } - } - Ok(None) => { - if let Some(decoded) = items.pop_front() { - // we have a populated item and no more pages - // the only case where an item's length may be smaller than chunk_size - debug_assert!(decoded.len() <= chunk_size.unwrap_or(usize::MAX)); - MaybeNext::Some(Ok(decoded)) - } else { - MaybeNext::None - } - } - } -} - -#[inline] -pub(super) fn dict_indices_decoder(page: &DataPage) -> Result { - let (_, _, indices_buffer) = split_buffer(page)?; - - // SPEC: Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32), - // SPEC: followed by the values encoded using RLE/Bit packed described above (with the given bit width). - let bit_width = indices_buffer[0]; - let indices_buffer = &indices_buffer[1..]; - - hybrid_rle::HybridRleDecoder::try_new(indices_buffer, bit_width as u32, page.num_values()) - .map_err(Error::from) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/file.rs b/src/common/arrow/src/arrow/io/parquet/read/file.rs deleted file mode 100644 index a12af00685c3..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/file.rs +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Read; -use std::io::Seek; - -use parquet2::indexes::FilteredPage; - -use super::RowGroupDeserializer; -use super::RowGroupMetaData; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::read_columns_many; - -/// An iterator of [`Chunk`]s coming from row groups of a parquet file. -/// -/// This can be thought of a flatten chain of [`Iterator`] - each row group is sequentially -/// mapped to an [`Iterator`] and each iterator is iterated upon until either the limit -/// or the last iterator ends. -/// # Implementation -/// This iterator is single threaded on both IO-bounded and CPU-bounded tasks, and mixes them. -pub struct FileReader { - row_groups: RowGroupReader, - remaining_rows: usize, - current_row_group: Option, -} - -impl FileReader { - /// Returns a new [`FileReader`]. - pub fn new( - reader: R, - row_groups: Vec, - schema: Schema, - chunk_size: Option, - limit: Option, - page_indexes: Option>>>>, - ) -> Self { - let row_groups = - RowGroupReader::new(reader, schema, row_groups, chunk_size, limit, page_indexes); - - Self { - row_groups, - remaining_rows: limit.unwrap_or(usize::MAX), - current_row_group: None, - } - } - - fn next_row_group(&mut self) -> Result> { - let result = self.row_groups.next().transpose()?; - - // If current_row_group is None, then there will be no elements to remove. - if self.current_row_group.is_some() { - self.remaining_rows = self.remaining_rows.saturating_sub( - result - .as_ref() - .map(|x| x.num_rows()) - .unwrap_or(self.remaining_rows), - ); - } - Ok(result) - } - - /// Returns the [`Schema`] associated to this file. - pub fn schema(&self) -> &Schema { - &self.row_groups.schema - } -} - -impl Iterator for FileReader { - type Item = Result>>; - - fn next(&mut self) -> Option { - if self.remaining_rows == 0 { - // reached the limit - return None; - } - - if let Some(row_group) = &mut self.current_row_group { - match row_group.next() { - // no more chunks in the current row group => try a new one - None => match self.next_row_group() { - Ok(Some(row_group)) => { - self.current_row_group = Some(row_group); - // new found => pull again - self.next() - } - Ok(None) => { - self.current_row_group = None; - None - } - Err(e) => Some(Err(e)), - }, - other => other, - } - } else { - match self.next_row_group() { - Ok(Some(row_group)) => { - self.current_row_group = Some(row_group); - self.next() - } - Ok(None) => { - self.current_row_group = None; - None - } - Err(e) => Some(Err(e)), - } - } - } -} - -/// An [`Iterator`] from row groups of a parquet file. -/// -/// # Implementation -/// Advancing this iterator is IO-bounded - each iteration reads all the column chunks from the file -/// to memory and attaches [`RowGroupDeserializer`] to them so that they can be iterated in chunks. -pub struct RowGroupReader { - reader: R, - schema: Schema, - row_groups: std::vec::IntoIter, - chunk_size: Option, - remaining_rows: usize, - page_indexes: Option>>>>, -} - -impl RowGroupReader { - /// Returns a new [`RowGroupReader`] - pub fn new( - reader: R, - schema: Schema, - row_groups: Vec, - chunk_size: Option, - limit: Option, - page_indexes: Option>>>>, - ) -> Self { - if let Some(pages) = &page_indexes { - assert_eq!(pages.len(), row_groups.len()) - } - Self { - reader, - schema, - row_groups: row_groups.into_iter(), - chunk_size, - remaining_rows: limit.unwrap_or(usize::MAX), - page_indexes: page_indexes.map(|pages| pages.into_iter()), - } - } - - #[inline] - fn _next(&mut self) -> Result> { - if self.schema.fields.is_empty() { - return Ok(None); - } - if self.remaining_rows == 0 { - // reached the limit - return Ok(None); - } - - let row_group = if let Some(row_group) = self.row_groups.next() { - row_group - } else { - return Ok(None); - }; - - let pages = self.page_indexes.as_mut().and_then(|iter| iter.next()); - - // the number of rows depends on whether indexes are selected or not. - let num_rows = pages - .as_ref() - .map(|x| { - // first field, first column within that field - x[0][0] - .iter() - .map(|page| { - page.selected_rows - .iter() - .map(|interval| interval.length) - .sum::() - }) - .sum() - }) - .unwrap_or_else(|| row_group.num_rows()); - - let column_chunks = read_columns_many( - &mut self.reader, - &row_group, - self.schema.fields.clone(), - self.chunk_size, - Some(self.remaining_rows), - pages, - )?; - - let result = RowGroupDeserializer::new(column_chunks, num_rows, Some(self.remaining_rows)); - self.remaining_rows = self.remaining_rows.saturating_sub(num_rows); - Ok(Some(result)) - } -} - -impl Iterator for RowGroupReader { - type Item = Result; - - fn next(&mut self) -> Option { - self._next().transpose() - } - - fn size_hint(&self) -> (usize, Option) { - self.row_groups.size_hint() - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/indexes/binary.rs b/src/common/arrow/src/arrow/io/parquet/read/indexes/binary.rs deleted file mode 100644 index bacfa6438a1e..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/indexes/binary.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::indexes::PageIndex; - -use super::ColumnPageStatistics; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; -use crate::arrow::trusted_len::TrustedLen; - -pub fn deserialize( - indexes: &[PageIndex>], - data_type: &DataType, -) -> Result { - Ok(ColumnPageStatistics { - min: deserialize_binary_iter(indexes.iter().map(|index| index.min.as_ref()), data_type)?, - max: deserialize_binary_iter(indexes.iter().map(|index| index.max.as_ref()), data_type)?, - null_count: PrimitiveArray::from_trusted_len_iter( - indexes - .iter() - .map(|index| index.null_count.map(|x| x as u64)), - ), - }) -} - -fn deserialize_binary_iter<'a, I: TrustedLen>>>( - iter: I, - data_type: &DataType, -) -> Result, Error> { - match data_type.to_physical_type() { - PhysicalType::LargeBinary => Ok(Box::new(BinaryArray::::from_iter(iter))), - PhysicalType::Utf8 => { - let iter = iter.map(|x| x.map(|x| std::str::from_utf8(x)).transpose()); - Ok(Box::new(Utf8Array::::try_from_trusted_len_iter(iter)?)) - } - PhysicalType::LargeUtf8 => { - let iter = iter.map(|x| x.map(|x| std::str::from_utf8(x)).transpose()); - Ok(Box::new(Utf8Array::::try_from_trusted_len_iter(iter)?)) - } - _ => Ok(Box::new(BinaryArray::::from_iter(iter))), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/indexes/boolean.rs b/src/common/arrow/src/arrow/io/parquet/read/indexes/boolean.rs deleted file mode 100644 index d9c8f4812d6d..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/indexes/boolean.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::indexes::PageIndex; - -use super::ColumnPageStatistics; -use crate::arrow::array::BooleanArray; -use crate::arrow::array::PrimitiveArray; - -pub fn deserialize(indexes: &[PageIndex]) -> ColumnPageStatistics { - ColumnPageStatistics { - min: Box::new(BooleanArray::from_trusted_len_iter( - indexes.iter().map(|index| index.min), - )), - max: Box::new(BooleanArray::from_trusted_len_iter( - indexes.iter().map(|index| index.max), - )), - null_count: PrimitiveArray::from_trusted_len_iter( - indexes - .iter() - .map(|index| index.null_count.map(|x| x as u64)), - ), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/indexes/fixed_len_binary.rs b/src/common/arrow/src/arrow/io/parquet/read/indexes/fixed_len_binary.rs deleted file mode 100644 index 5b00d00f032b..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/indexes/fixed_len_binary.rs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::indexes::PageIndex; - -use super::ColumnPageStatistics; -use crate::arrow::array::Array; -use crate::arrow::array::FixedSizeBinaryArray; -use crate::arrow::array::MutableFixedSizeBinaryArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::datatypes::PrimitiveType; -use crate::arrow::trusted_len::TrustedLen; -use crate::arrow::types::i256; -use crate::arrow::types::NativeType; - -pub fn deserialize(indexes: &[PageIndex>], data_type: DataType) -> ColumnPageStatistics { - ColumnPageStatistics { - min: deserialize_binary_iter( - indexes.iter().map(|index| index.min.as_ref()), - data_type.clone(), - ), - max: deserialize_binary_iter(indexes.iter().map(|index| index.max.as_ref()), data_type), - null_count: PrimitiveArray::from_trusted_len_iter( - indexes - .iter() - .map(|index| index.null_count.map(|x| x as u64)), - ), - } -} - -fn deserialize_binary_iter<'a, I: TrustedLen>>>( - iter: I, - data_type: DataType, -) -> Box { - match data_type.to_physical_type() { - PhysicalType::Primitive(PrimitiveType::Int128) => { - Box::new(PrimitiveArray::from_trusted_len_iter(iter.map(|v| { - v.map(|x| { - // Copy the fixed-size byte value to the start of a 16 byte stack - // allocated buffer, then use an arithmetic right shift to fill in - // MSBs, which accounts for leading 1's in negative (two's complement) - // values. - let n = x.len(); - let mut bytes = [0u8; 16]; - bytes[..n].copy_from_slice(x); - i128::from_be_bytes(bytes) >> (8 * (16 - n)) - }) - }))) - } - PhysicalType::Primitive(PrimitiveType::Int256) => { - Box::new(PrimitiveArray::from_trusted_len_iter(iter.map(|v| { - v.map(|x| { - let n = x.len(); - let mut bytes = [0u8; 32]; - bytes[..n].copy_from_slice(x); - i256::from_be_bytes(bytes) - }) - }))) - } - _ => { - let mut a = MutableFixedSizeBinaryArray::try_new( - data_type, - Vec::with_capacity(iter.size_hint().0), - None, - ) - .unwrap(); - for item in iter { - a.push(item); - } - let a: FixedSizeBinaryArray = a.into(); - Box::new(a) - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/indexes/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/indexes/mod.rs deleted file mode 100644 index 18fbb7685b29..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/indexes/mod.rs +++ /dev/null @@ -1,410 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! API to perform page-level filtering (also known as indexes) -use parquet2::error::Error as ParquetError; -use parquet2::indexes::select_pages; -use parquet2::indexes::BooleanIndex; -use parquet2::indexes::ByteIndex; -use parquet2::indexes::FixedLenByteIndex; -use parquet2::indexes::Index as ParquetIndex; -use parquet2::indexes::NativeIndex; -use parquet2::indexes::PageLocation; -use parquet2::metadata::ColumnChunkMetaData; -use parquet2::metadata::RowGroupMetaData; -use parquet2::read::read_columns_indexes as _read_columns_indexes; -use parquet2::read::read_pages_locations; -use parquet2::schema::types::PhysicalType as ParquetPhysicalType; - -mod binary; -mod boolean; -mod fixed_len_binary; -mod primitive; - -use std::collections::VecDeque; -use std::io::Read; -use std::io::Seek; - -pub use parquet2::indexes::FilteredPage; -pub use parquet2::indexes::Interval; - -use super::get_field_pages; -use crate::arrow::array::Array; -use crate::arrow::array::UInt64Array; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::datatypes::PrimitiveType; -use crate::arrow::error::Error; - -/// Page statistics of an Arrow field. -#[derive(Debug, PartialEq)] -pub enum FieldPageStatistics { - /// Variant used for fields with a single parquet column (e.g. primitives, dictionaries, list) - Single(ColumnPageStatistics), - /// Variant used for fields with multiple parquet columns (e.g. Struct, Map) - Multiple(Vec), -} - -impl From for FieldPageStatistics { - fn from(column: ColumnPageStatistics) -> Self { - Self::Single(column) - } -} - -/// [`ColumnPageStatistics`] contains the minimum, maximum, and null_count -/// of each page of a parquet column, as an [`Array`]. -/// This struct has the following invariants: -/// * `min`, `max` and `null_count` have the same length (equal to the number of pages in the column) -/// * `min`, `max` and `null_count` are guaranteed to be non-null -/// * `min` and `max` have the same logical type -#[derive(Debug, PartialEq)] -pub struct ColumnPageStatistics { - /// The minimum values in the pages - pub min: Box, - /// The maximum values in the pages - pub max: Box, - /// The number of null values in the pages. - pub null_count: UInt64Array, -} - -/// Given a sequence of [`ParquetIndex`] representing the page indexes of each column in the -/// parquet file, returns the page-level statistics as a [`FieldPageStatistics`]. -/// -/// This function maps timestamps, decimal types, etc. accordingly. -/// # Implementation -/// This function is CPU-bounded `O(P)` where `P` is the total number of pages on all columns. -/// # Error -/// This function errors iff the value is not deserializable to arrow (e.g. invalid utf-8) -#[allow(clippy::borrowed_box)] -fn deserialize( - indexes: &mut VecDeque<&Box>, - data_type: DataType, -) -> Result { - match data_type.to_physical_type() { - PhysicalType::Boolean => { - let index = indexes - .pop_front() - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - Ok(boolean::deserialize(&index.indexes).into()) - } - PhysicalType::Primitive(PrimitiveType::Int128) => { - let index = indexes.pop_front().unwrap(); - match index.physical_type() { - ParquetPhysicalType::Int32 => { - let index = index.as_any().downcast_ref::>().unwrap(); - Ok(primitive::deserialize_i32(&index.indexes, data_type).into()) - } - parquet2::schema::types::PhysicalType::Int64 => { - let index = index.as_any().downcast_ref::>().unwrap(); - Ok( - primitive::deserialize_i64( - &index.indexes, - &index.primitive_type, - data_type, - ) - .into(), - ) - } - parquet2::schema::types::PhysicalType::FixedLenByteArray(_) => { - let index = index.as_any().downcast_ref::().unwrap(); - Ok(fixed_len_binary::deserialize(&index.indexes, data_type).into()) - } - other => Err(Error::nyi(format!( - "Deserialize {other:?} to arrow's int64" - ))), - } - } - PhysicalType::Primitive(PrimitiveType::Int256) => { - let index = indexes.pop_front().unwrap(); - match index.physical_type() { - ParquetPhysicalType::Int32 => { - let index = index.as_any().downcast_ref::>().unwrap(); - Ok(primitive::deserialize_i32(&index.indexes, data_type).into()) - } - parquet2::schema::types::PhysicalType::Int64 => { - let index = index.as_any().downcast_ref::>().unwrap(); - Ok( - primitive::deserialize_i64( - &index.indexes, - &index.primitive_type, - data_type, - ) - .into(), - ) - } - parquet2::schema::types::PhysicalType::FixedLenByteArray(_) => { - let index = index.as_any().downcast_ref::().unwrap(); - Ok(fixed_len_binary::deserialize(&index.indexes, data_type).into()) - } - other => Err(Error::nyi(format!( - "Deserialize {other:?} to arrow's int64" - ))), - } - } - PhysicalType::Primitive(PrimitiveType::UInt8) - | PhysicalType::Primitive(PrimitiveType::UInt16) - | PhysicalType::Primitive(PrimitiveType::UInt32) - | PhysicalType::Primitive(PrimitiveType::Int32) => { - let index = indexes - .pop_front() - .unwrap() - .as_any() - .downcast_ref::>() - .unwrap(); - Ok(primitive::deserialize_i32(&index.indexes, data_type).into()) - } - PhysicalType::Primitive(PrimitiveType::UInt64) - | PhysicalType::Primitive(PrimitiveType::Int64) => { - let index = indexes.pop_front().unwrap(); - match index.physical_type() { - ParquetPhysicalType::Int64 => { - let index = index.as_any().downcast_ref::>().unwrap(); - Ok( - primitive::deserialize_i64( - &index.indexes, - &index.primitive_type, - data_type, - ) - .into(), - ) - } - parquet2::schema::types::PhysicalType::Int96 => { - let index = index - .as_any() - .downcast_ref::>() - .unwrap(); - Ok(primitive::deserialize_i96(&index.indexes, data_type).into()) - } - other => Err(Error::nyi(format!( - "Deserialize {other:?} to arrow's int64" - ))), - } - } - PhysicalType::Primitive(PrimitiveType::Float32) => { - let index = indexes - .pop_front() - .unwrap() - .as_any() - .downcast_ref::>() - .unwrap(); - Ok(primitive::deserialize_id(&index.indexes, data_type).into()) - } - PhysicalType::Primitive(PrimitiveType::Float64) => { - let index = indexes - .pop_front() - .unwrap() - .as_any() - .downcast_ref::>() - .unwrap(); - Ok(primitive::deserialize_id(&index.indexes, data_type).into()) - } - PhysicalType::Binary - | PhysicalType::LargeBinary - | PhysicalType::Utf8 - | PhysicalType::LargeUtf8 => { - let index = indexes - .pop_front() - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - binary::deserialize(&index.indexes, &data_type).map(|x| x.into()) - } - PhysicalType::FixedSizeBinary => { - let index = indexes - .pop_front() - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - Ok(fixed_len_binary::deserialize(&index.indexes, data_type).into()) - } - PhysicalType::Dictionary(_) => { - if let DataType::Dictionary(_, inner, _) = data_type.to_logical_type() { - deserialize(indexes, (**inner).clone()) - } else { - unreachable!() - } - } - PhysicalType::List => { - if let DataType::List(inner) = data_type.to_logical_type() { - deserialize(indexes, inner.data_type.clone()) - } else { - unreachable!() - } - } - PhysicalType::LargeList => { - if let DataType::LargeList(inner) = data_type.to_logical_type() { - deserialize(indexes, inner.data_type.clone()) - } else { - unreachable!() - } - } - PhysicalType::Map => { - if let DataType::Map(inner, _) = data_type.to_logical_type() { - deserialize(indexes, inner.data_type.clone()) - } else { - unreachable!() - } - } - PhysicalType::Struct => { - let children_fields = if let DataType::Struct(children) = data_type.to_logical_type() { - children - } else { - unreachable!() - }; - let children = children_fields - .iter() - .map(|child| deserialize(indexes, child.data_type.clone())) - .collect::, Error>>()?; - - Ok(FieldPageStatistics::Multiple(children)) - } - - other => Err(Error::nyi(format!( - "Deserialize into arrow's {other:?} page index" - ))), - } -} - -/// Checks whether the row group have page index information (page statistics) -pub fn has_indexes(row_group: &RowGroupMetaData) -> bool { - row_group - .columns() - .iter() - .all(|chunk| chunk.column_chunk().column_index_offset.is_some()) -} - -/// Reads the column indexes from the reader assuming a valid set of derived Arrow fields -/// for all parquet the columns in the file. -/// -/// It returns one [`FieldPageStatistics`] per field in `fields` -/// -/// This function is expected to be used to filter out parquet pages. -/// -/// # Implementation -/// This function is IO-bounded and calls `reader.read_exact` exactly once. -/// # Error -/// Errors iff the indexes can't be read or their deserialization to arrow is incorrect (e.g. invalid utf-8) -pub fn read_columns_indexes( - reader: &mut R, - chunks: &[ColumnChunkMetaData], - fields: &[Field], -) -> Result, Error> { - let indexes = _read_columns_indexes(reader, chunks)?; - - fields - .iter() - .map(|field| { - let indexes = get_field_pages(chunks, &indexes, &field.name); - let mut indexes = indexes.into_iter().collect(); - - deserialize(&mut indexes, field.data_type.clone()) - }) - .collect() -} - -/// Returns the set of (row) intervals of the pages. -pub fn compute_page_row_intervals( - locations: &[PageLocation], - num_rows: usize, -) -> Result, ParquetError> { - if locations.is_empty() { - return Ok(vec![]); - }; - - let last = (|| { - let start: usize = locations.last().unwrap().first_row_index.try_into()?; - let length = num_rows - start; - Result::<_, ParquetError>::Ok(Interval::new(start, length)) - })(); - - let pages_lengths = locations - .windows(2) - .map(|x| { - let start = usize::try_from(x[0].first_row_index)?; - let length = usize::try_from(x[1].first_row_index - x[0].first_row_index)?; - Ok(Interval::new(start, length)) - }) - .chain(std::iter::once(last)); - pages_lengths.collect() -} - -/// Reads all page locations and index locations (IO-bounded) and uses `predicate` to compute -/// the set of [`FilteredPage`] that fulfill the predicate. -/// -/// The non-trivial argument of this function is `predicate`, that controls which pages are selected. -/// -/// Its signature contains 2 arguments: -/// * 0th argument (indexes): contains one [`ColumnPageStatistics`] (page statistics) per field. -/// Use it to evaluate the predicate against -/// * 1th argument (intervals): contains one [`Vec>`] (row positions) per field. -/// For each field, the outermost vector corresponds to each parquet column: -/// a primitive field contains 1 column, a struct field with 2 primitive fields contain 2 columns. -/// The inner `Vec` contains one [`Interval`] per page: its length equals the length of [`ColumnPageStatistics`]. -/// -/// It returns a single [`Vec`] denoting the set of intervals that the predicate selects (over all columns). -/// -/// This returns one item per `field`. For each field, there is one item per column (for non-nested types it returns one column) -/// and finally [`Vec`], that corresponds to the set of selected pages. -pub fn read_filtered_pages< - R: Read + Seek, - F: Fn(&[FieldPageStatistics], &[Vec>]) -> Vec, ->( - reader: &mut R, - row_group: &RowGroupMetaData, - fields: &[Field], - predicate: F, - // is_intersection: bool, -) -> Result>>, Error> { - let num_rows = row_group.num_rows(); - - // one vec per column - let locations = read_pages_locations(reader, row_group.columns())?; - // one Vec> per field (non-nested contain a single entry on the first column) - let locations = fields - .iter() - .map(|field| get_field_pages(row_group.columns(), &locations, &field.name)) - .collect::>(); - - // one ColumnPageStatistics per field - let indexes = read_columns_indexes(reader, row_group.columns(), fields)?; - - let intervals = locations - .iter() - .map(|locations| { - locations - .iter() - .map(|locations| Ok(compute_page_row_intervals(locations, num_rows)?)) - .collect::, Error>>() - }) - .collect::, Error>>()?; - - let intervals = predicate(&indexes, &intervals); - - locations - .into_iter() - .map(|locations| { - locations - .into_iter() - .map(|locations| Ok(select_pages(&intervals, locations, num_rows)?)) - .collect::, Error>>() - }) - .collect() -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/indexes/primitive.rs b/src/common/arrow/src/arrow/io/parquet/read/indexes/primitive.rs deleted file mode 100644 index 268e20f1c421..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/indexes/primitive.rs +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use ethnum::I256; -use parquet2::indexes::PageIndex; -use parquet2::schema::types::PrimitiveLogicalType; -use parquet2::schema::types::PrimitiveType; -use parquet2::schema::types::TimeUnit as ParquetTimeUnit; -use parquet2::types::int96_to_i64_ns; - -use super::ColumnPageStatistics; -use crate::arrow::array::Array; -use crate::arrow::array::MutablePrimitiveArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::trusted_len::TrustedLen; -use crate::arrow::types::i256; -use crate::arrow::types::NativeType; - -#[inline] -fn deserialize_int32>>( - iter: I, - data_type: DataType, -) -> Box { - use DataType::*; - match data_type.to_logical_type() { - UInt8 => Box::new( - PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as u8))) - .to(data_type), - ) as _, - UInt16 => Box::new( - PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as u16))) - .to(data_type), - ), - UInt32 => Box::new( - PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as u32))) - .to(data_type), - ), - Decimal(_, _) => Box::new( - PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as i128))) - .to(data_type), - ), - Decimal256(_, _) => Box::new( - PrimitiveArray::::from_trusted_len_iter( - iter.map(|x| x.map(|x| i256(I256::new(x.into())))), - ) - .to(data_type), - ) as _, - _ => Box::new(PrimitiveArray::::from_trusted_len_iter(iter).to(data_type)), - } -} - -#[inline] -fn timestamp( - array: &mut MutablePrimitiveArray, - time_unit: TimeUnit, - logical_type: Option, -) { - let unit = if let Some(PrimitiveLogicalType::Timestamp { unit, .. }) = logical_type { - unit - } else { - return; - }; - - match (unit, time_unit) { - (ParquetTimeUnit::Milliseconds, TimeUnit::Second) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x /= 1_000), - (ParquetTimeUnit::Microseconds, TimeUnit::Second) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x /= 1_000_000), - (ParquetTimeUnit::Nanoseconds, TimeUnit::Second) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x /= 1_000_000_000), - - (ParquetTimeUnit::Milliseconds, TimeUnit::Millisecond) => {} - (ParquetTimeUnit::Microseconds, TimeUnit::Millisecond) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x /= 1_000), - (ParquetTimeUnit::Nanoseconds, TimeUnit::Millisecond) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x /= 1_000_000), - - (ParquetTimeUnit::Milliseconds, TimeUnit::Microsecond) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x *= 1_000), - (ParquetTimeUnit::Microseconds, TimeUnit::Microsecond) => {} - (ParquetTimeUnit::Nanoseconds, TimeUnit::Microsecond) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x /= 1_000), - - (ParquetTimeUnit::Milliseconds, TimeUnit::Nanosecond) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x *= 1_000_000), - (ParquetTimeUnit::Microseconds, TimeUnit::Nanosecond) => array - .values_mut_slice() - .iter_mut() - .for_each(|x| *x /= 1_000), - (ParquetTimeUnit::Nanoseconds, TimeUnit::Nanosecond) => {} - } -} - -#[inline] -fn deserialize_int64>>( - iter: I, - primitive_type: &PrimitiveType, - data_type: DataType, -) -> Box { - use DataType::*; - match data_type.to_logical_type() { - UInt64 => Box::new( - PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as u64))) - .to(data_type), - ) as _, - Decimal(_, _) => Box::new( - PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as i128))) - .to(data_type), - ) as _, - Decimal256(_, _) => Box::new( - PrimitiveArray::::from_trusted_len_iter( - iter.map(|x| x.map(|x| i256(I256::new(x.into())))), - ) - .to(data_type), - ) as _, - Timestamp(time_unit, _) => { - let mut array = - MutablePrimitiveArray::::from_trusted_len_iter(iter).to(data_type.clone()); - - timestamp(&mut array, *time_unit, primitive_type.logical_type); - - let array: PrimitiveArray = array.into(); - - Box::new(array) - } - _ => Box::new(PrimitiveArray::::from_trusted_len_iter(iter).to(data_type)), - } -} - -#[inline] -fn deserialize_int96>>( - iter: I, - data_type: DataType, -) -> Box { - Box::new( - PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(int96_to_i64_ns))) - .to(data_type), - ) -} - -#[inline] -fn deserialize_id_s>>( - iter: I, - data_type: DataType, -) -> Box { - Box::new(PrimitiveArray::::from_trusted_len_iter(iter).to(data_type)) -} - -pub fn deserialize_i32(indexes: &[PageIndex], data_type: DataType) -> ColumnPageStatistics { - ColumnPageStatistics { - min: deserialize_int32(indexes.iter().map(|index| index.min), data_type.clone()), - max: deserialize_int32(indexes.iter().map(|index| index.max), data_type), - null_count: PrimitiveArray::from_trusted_len_iter( - indexes - .iter() - .map(|index| index.null_count.map(|x| x as u64)), - ), - } -} - -pub fn deserialize_i64( - indexes: &[PageIndex], - primitive_type: &PrimitiveType, - data_type: DataType, -) -> ColumnPageStatistics { - ColumnPageStatistics { - min: deserialize_int64( - indexes.iter().map(|index| index.min), - primitive_type, - data_type.clone(), - ), - max: deserialize_int64( - indexes.iter().map(|index| index.max), - primitive_type, - data_type, - ), - null_count: PrimitiveArray::from_trusted_len_iter( - indexes - .iter() - .map(|index| index.null_count.map(|x| x as u64)), - ), - } -} - -pub fn deserialize_i96( - indexes: &[PageIndex<[u32; 3]>], - data_type: DataType, -) -> ColumnPageStatistics { - ColumnPageStatistics { - min: deserialize_int96(indexes.iter().map(|index| index.min), data_type.clone()), - max: deserialize_int96(indexes.iter().map(|index| index.max), data_type), - null_count: PrimitiveArray::from_trusted_len_iter( - indexes - .iter() - .map(|index| index.null_count.map(|x| x as u64)), - ), - } -} - -pub fn deserialize_id( - indexes: &[PageIndex], - data_type: DataType, -) -> ColumnPageStatistics { - ColumnPageStatistics { - min: deserialize_id_s(indexes.iter().map(|index| index.min), data_type.clone()), - max: deserialize_id_s(indexes.iter().map(|index| index.max), data_type), - null_count: PrimitiveArray::from_trusted_len_iter( - indexes - .iter() - .map(|index| index.null_count.map(|x| x as u64)), - ), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/mod.rs deleted file mode 100644 index 740042e33588..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/mod.rs +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to read from Parquet format. -#![allow(clippy::type_complexity)] - -mod deserialize; -mod file; -pub mod indexes; -mod row_group; -pub mod schema; -pub mod statistics; - -use std::io::Read; -use std::io::Seek; - -pub use deserialize::column_iter_to_arrays; -pub use deserialize::create_list; -pub use deserialize::create_map; -pub use deserialize::get_page_iterator; -pub use deserialize::init_nested; -pub use deserialize::n_columns; -pub use deserialize::nested_column_iter_to_arrays; -pub use deserialize::InitNested; -pub use deserialize::NestedArrayIter; -pub use deserialize::NestedState; -pub use deserialize::StructIterator; -pub use file::FileReader; -pub use file::RowGroupReader; -pub use parquet2::error::Error as ParquetError; -pub use parquet2::fallible_streaming_iterator; -pub use parquet2::metadata::ColumnChunkMetaData; -pub use parquet2::metadata::ColumnDescriptor; -pub use parquet2::metadata::RowGroupMetaData; -pub use parquet2::page::CompressedDataPage; -pub use parquet2::page::DataPageHeader; -pub use parquet2::page::Page; -pub use parquet2::read::decompress; -pub use parquet2::read::get_column_iterator; -pub use parquet2::read::read_columns_indexes as _read_columns_indexes; -pub use parquet2::read::read_metadata as _read_metadata; -pub use parquet2::read::read_pages_locations; -pub use parquet2::read::BasicDecompressor; -pub use parquet2::read::Decompressor; -pub use parquet2::read::MutStreamingIterator; -pub use parquet2::read::PageFilter; -pub use parquet2::read::PageReader; -pub use parquet2::read::ReadColumnIterator; -pub use parquet2::read::State; -// re-exports of parquet2's relevant APIs -#[cfg(feature = "io_parquet_async")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_parquet_async")))] -pub use parquet2::read::{get_page_stream, read_metadata_async as _read_metadata_async}; -pub use parquet2::schema::types::GroupLogicalType; -pub use parquet2::schema::types::ParquetType; -pub use parquet2::schema::types::PhysicalType; -pub use parquet2::schema::types::PrimitiveConvertedType; -pub use parquet2::schema::types::PrimitiveLogicalType; -pub use parquet2::schema::types::TimeUnit as ParquetTimeUnit; -pub use parquet2::types::int96_to_i64_ns; -pub use parquet2::FallibleStreamingIterator; -pub use row_group::*; -pub use schema::infer_schema; -pub use schema::FileMetaData; - -use crate::arrow::array::Array; -use crate::arrow::error::Result; -use crate::arrow::types::i256; -use crate::arrow::types::NativeType; - -/// Trait describing a [`FallibleStreamingIterator`] of [`Page`] -pub trait Pages: - FallibleStreamingIterator + Send + Sync -{ -} - -impl + Send + Sync> Pages for I {} - -/// Type def for a sharable, boxed dyn [`Iterator`] of arrays -pub type ArrayIter<'a> = Box>> + Send + Sync + 'a>; - -/// Reads parquets' metadata synchronously. -pub fn read_metadata(reader: &mut R) -> Result { - Ok(_read_metadata(reader)?) -} - -/// Reads parquets' metadata asynchronously. -#[cfg(feature = "io_parquet_async")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_parquet_async")))] -pub async fn read_metadata_async(reader: opendal::Reader, file_size: u64) -> Result { - Ok(_read_metadata_async(reader, file_size).await?) -} - -fn convert_days_ms(value: &[u8]) -> crate::arrow::types::days_ms { - crate::arrow::types::days_ms( - i32::from_le_bytes(value[4..8].try_into().unwrap()), - i32::from_le_bytes(value[8..12].try_into().unwrap()), - ) -} - -fn convert_i128(value: &[u8], n: usize) -> i128 { - // Copy the fixed-size byte value to the start of a 16 byte stack - // allocated buffer, then use an arithmetic right shift to fill in - // MSBs, which accounts for leading 1's in negative (two's complement) - // values. - let mut bytes = [0u8; 16]; - bytes[..n].copy_from_slice(value); - i128::from_be_bytes(bytes) >> (8 * (16 - n)) -} - -fn convert_i256(value: &[u8]) -> i256 { - if value[0] >= 128 { - let mut neg_bytes = [255u8; 32]; - neg_bytes[32 - value.len()..].copy_from_slice(value); - i256::from_be_bytes(neg_bytes) - } else { - let mut bytes = [0u8; 32]; - bytes[32 - value.len()..].copy_from_slice(value); - i256::from_be_bytes(bytes) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/row_group.rs b/src/common/arrow/src/arrow/io/parquet/read/row_group.rs deleted file mode 100644 index 0b3efb414d64..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/row_group.rs +++ /dev/null @@ -1,374 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Read; -use std::io::Seek; - -#[cfg(feature = "io_parquet_async")] -use futures::future::try_join_all; -#[cfg(feature = "io_parquet_async")] -use futures::future::BoxFuture; -#[cfg(feature = "io_parquet_async")] -use futures::AsyncRead; -#[cfg(feature = "io_parquet_async")] -use futures::AsyncReadExt; -#[cfg(feature = "io_parquet_async")] -use futures::AsyncSeek; -#[cfg(feature = "io_parquet_async")] -use futures::AsyncSeekExt; -use parquet2::indexes::FilteredPage; -use parquet2::metadata::ColumnChunkMetaData; -use parquet2::read::BasicDecompressor; -use parquet2::read::IndexedPageReader; -use parquet2::read::PageMetaData; -use parquet2::read::PageReader; - -use super::ArrayIter; -use super::RowGroupMetaData; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::column_iter_to_arrays; - -/// An [`Iterator`] of [`Chunk`] that (dynamically) adapts a vector of iterators of [`Array`] into -/// an iterator of [`Chunk`]. -/// -/// This struct tracks advances each of the iterators individually and combines the -/// result in a single [`Chunk`]. -/// -/// # Implementation -/// This iterator is single-threaded and advancing it is CPU-bounded. -pub struct RowGroupDeserializer { - num_rows: usize, - remaining_rows: usize, - column_chunks: Vec>, -} - -impl RowGroupDeserializer { - /// Creates a new [`RowGroupDeserializer`]. - /// - /// # Panic - /// This function panics iff any of the `column_chunks` - /// do not return an array with an equal length. - pub fn new( - column_chunks: Vec>, - num_rows: usize, - limit: Option, - ) -> Self { - Self { - num_rows, - remaining_rows: limit.unwrap_or(usize::MAX).min(num_rows), - column_chunks, - } - } - - /// Returns the number of rows on this row group - pub fn num_rows(&self) -> usize { - self.num_rows - } -} - -impl Iterator for RowGroupDeserializer { - type Item = Result>>; - - fn next(&mut self) -> Option { - if self.remaining_rows == 0 { - return None; - } - let chunk = self - .column_chunks - .iter_mut() - .map(|iter| iter.next().unwrap()) - .collect::>>() - .and_then(Chunk::try_new); - self.remaining_rows = self.remaining_rows.saturating_sub( - chunk - .as_ref() - .map(|x| x.len()) - .unwrap_or(self.remaining_rows), - ); - - Some(chunk) - } -} - -/// Returns all [`ColumnChunkMetaData`] associated to `field_name`. -/// For non-nested parquet types, this returns a single column -pub fn get_field_columns<'a>( - columns: &'a [ColumnChunkMetaData], - field_name: &str, -) -> Vec<&'a ColumnChunkMetaData> { - columns - .iter() - .filter(|x| x.descriptor().path_in_schema[0] == field_name) - .collect() -} - -/// Returns all [`ColumnChunkMetaData`] associated to `field_name`. -/// For non-nested parquet types, this returns a single column -pub fn get_field_pages<'a, T>( - columns: &'a [ColumnChunkMetaData], - items: &'a [T], - field_name: &str, -) -> Vec<&'a T> { - columns - .iter() - .zip(items) - .filter(|(metadata, _)| metadata.descriptor().path_in_schema[0] == field_name) - .map(|(_, item)| item) - .collect() -} - -/// Reads all columns that are part of the parquet field `field_name` -/// # Implementation -/// This operation is IO-bounded `O(C)` where C is the number of columns associated to -/// the field (one for non-nested types) -pub fn read_columns<'a, R: Read + Seek>( - reader: &mut R, - columns: &'a [ColumnChunkMetaData], - field_name: &str, -) -> Result)>> { - get_field_columns(columns, field_name) - .into_iter() - .map(|meta| _read_single_column(reader, meta)) - .collect() -} - -fn _read_single_column<'a, R>( - reader: &mut R, - meta: &'a ColumnChunkMetaData, -) -> Result<(&'a ColumnChunkMetaData, Vec)> -where - R: Read + Seek, -{ - let (start, length) = meta.byte_range(); - reader.seek(std::io::SeekFrom::Start(start))?; - - let mut chunk = vec![]; - chunk.try_reserve(length as usize)?; - reader.by_ref().take(length).read_to_end(&mut chunk)?; - Ok((meta, chunk)) -} - -#[cfg(feature = "io_parquet_async")] -async fn _read_single_column_async<'b, R, F>( - reader_factory: F, - meta: &ColumnChunkMetaData, -) -> Result<(&ColumnChunkMetaData, Vec)> -where - R: AsyncRead + AsyncSeek + Send + Unpin, - F: Fn() -> BoxFuture<'b, std::io::Result>, -{ - let mut reader = reader_factory().await?; - let (start, length) = meta.byte_range(); - reader.seek(std::io::SeekFrom::Start(start)).await?; - - let mut chunk = vec![]; - chunk.try_reserve(length as usize)?; - reader.take(length).read_to_end(&mut chunk).await?; - Result::Ok((meta, chunk)) -} - -/// Reads all columns that are part of the parquet field `field_name` -/// # Implementation -/// This operation is IO-bounded `O(C)` where C is the number of columns associated to -/// the field (one for non-nested types) -/// -/// It does so asynchronously via a single `join_all` over all the necessary columns for -/// `field_name`. -#[cfg(feature = "io_parquet_async")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_parquet_async")))] -pub async fn read_columns_async< - 'a, - 'b, - R: AsyncRead + AsyncSeek + Send + Unpin, - F: Fn() -> BoxFuture<'b, std::io::Result> + Clone, ->( - reader_factory: F, - columns: &'a [ColumnChunkMetaData], - field_name: &str, -) -> Result)>> { - let futures = get_field_columns(columns, field_name) - .into_iter() - .map(|meta| async { _read_single_column_async(reader_factory.clone(), meta).await }); - - try_join_all(futures).await -} - -type Pages = Box< - dyn Iterator> - + Sync - + Send, ->; - -/// Converts a vector of columns associated with the parquet field whose name is [`Field`] -/// to an iterator of [`Array`], [`ArrayIter`] of chunk size `chunk_size`. -pub fn to_deserializer<'a>( - columns: Vec<(&ColumnChunkMetaData, Vec)>, - field: Field, - num_rows: usize, - chunk_size: Option, - pages: Option>>, -) -> Result> { - let chunk_size = chunk_size.map(|c| c.min(num_rows)); - - let (columns, types) = if let Some(pages) = pages { - let (columns, types): (Vec<_>, Vec<_>) = columns - .into_iter() - .zip(pages) - .map(|((column_meta, chunk), mut pages)| { - // de-offset the start, since we read in chunks (and offset is from start of file) - let mut meta: PageMetaData = column_meta.into(); - pages - .iter_mut() - .for_each(|page| page.start -= meta.column_start); - meta.column_start = 0; - let pages = IndexedPageReader::new_with_page_meta( - std::io::Cursor::new(chunk), - meta, - pages, - vec![], - vec![], - ); - let pages = Box::new(pages) as Pages; - ( - BasicDecompressor::new(pages, vec![]), - &column_meta.descriptor().descriptor.primitive_type, - ) - }) - .unzip(); - - (columns, types) - } else { - let (columns, types): (Vec<_>, Vec<_>) = columns - .into_iter() - .map(|(column_meta, chunk)| { - let len = chunk.len(); - let pages = PageReader::new( - std::io::Cursor::new(chunk), - column_meta, - std::sync::Arc::new(|_, _| true), - vec![], - len * 2 + 1024, - ); - let pages = Box::new(pages) as Pages; - ( - BasicDecompressor::new(pages, vec![]), - &column_meta.descriptor().descriptor.primitive_type, - ) - }) - .unzip(); - - (columns, types) - }; - - column_iter_to_arrays(columns, types, field, chunk_size, num_rows) -} - -/// Returns a vector of iterators of [`Array`] ([`ArrayIter`]) corresponding to the top -/// level parquet fields whose name matches `fields`'s names. -/// -/// # Implementation -/// This operation is IO-bounded `O(C)` where C is the number of columns in the row group - -/// it reads all the columns to memory from the row group associated to the requested fields. -/// -/// This operation is single-threaded. For readers with stronger invariants -/// (e.g. implement [`Clone`]) you can use [`read_columns`] to read multiple columns at once -/// and convert them to [`ArrayIter`] via [`to_deserializer`]. -pub fn read_columns_many<'a, R: Read + Seek>( - reader: &mut R, - row_group: &RowGroupMetaData, - fields: Vec, - chunk_size: Option, - limit: Option, - pages: Option>>>, -) -> Result>> { - let num_rows = row_group.num_rows(); - let num_rows = limit.map(|limit| limit.min(num_rows)).unwrap_or(num_rows); - - // reads all the necessary columns for all fields from the row group - // This operation is IO-bounded `O(C)` where C is the number of columns in the row group - let field_columns = fields - .iter() - .map(|field| read_columns(reader, row_group.columns(), &field.name)) - .collect::>>()?; - - if let Some(pages) = pages { - field_columns - .into_iter() - .zip(fields) - .zip(pages) - .map(|((columns, field), pages)| { - to_deserializer(columns, field, num_rows, chunk_size, Some(pages)) - }) - .collect() - } else { - field_columns - .into_iter() - .zip(fields) - .map(|(columns, field)| to_deserializer(columns, field, num_rows, chunk_size, None)) - .collect() - } -} - -/// Returns a vector of iterators of [`Array`] corresponding to the top level parquet fields whose -/// name matches `fields`'s names. -/// -/// # Implementation -/// This operation is IO-bounded `O(C)` where C is the number of columns in the row group - -/// it reads all the columns to memory from the row group associated to the requested fields. -/// It does so asynchronously via `join_all` -#[cfg(feature = "io_parquet_async")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_parquet_async")))] -pub async fn read_columns_many_async< - 'a, - 'b, - R: AsyncRead + AsyncSeek + Send + Unpin, - F: Fn() -> BoxFuture<'b, std::io::Result> + Clone, ->( - reader_factory: F, - row_group: &RowGroupMetaData, - fields: Vec, - chunk_size: Option, - limit: Option, - pages: Option>>>, -) -> Result>> { - let num_rows = row_group.num_rows(); - let num_rows = limit.map(|limit| limit.min(num_rows)).unwrap_or(num_rows); - - let futures = fields - .iter() - .map(|field| read_columns_async(reader_factory.clone(), row_group.columns(), &field.name)); - - let field_columns = try_join_all(futures).await?; - - if let Some(pages) = pages { - field_columns - .into_iter() - .zip(fields) - .zip(pages) - .map(|((columns, field), pages)| { - to_deserializer(columns, field, num_rows, chunk_size, Some(pages)) - }) - .collect() - } else { - field_columns - .into_iter() - .zip(fields.into_iter()) - .map(|(columns, field)| to_deserializer(columns, field, num_rows, chunk_size, None)) - .collect() - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/schema/convert.rs b/src/common/arrow/src/arrow/io/parquet/read/schema/convert.rs deleted file mode 100644 index a0b582062c4a..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/schema/convert.rs +++ /dev/null @@ -1,1129 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! This module has entry points, [`parquet_to_arrow_schema`] and the more configurable [`parquet_to_arrow_schema_with_options`]. -use parquet2::schema::types::FieldInfo; -use parquet2::schema::types::GroupConvertedType; -use parquet2::schema::types::GroupLogicalType; -use parquet2::schema::types::IntegerType; -use parquet2::schema::types::ParquetType; -use parquet2::schema::types::PhysicalType; -use parquet2::schema::types::PrimitiveConvertedType; -use parquet2::schema::types::PrimitiveLogicalType; -use parquet2::schema::types::PrimitiveType; -use parquet2::schema::types::TimeUnit as ParquetTimeUnit; -use parquet2::schema::Repetition; - -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::io::parquet::read::schema::SchemaInferenceOptions; - -/// Converts [`ParquetType`]s to a [`Field`], ignoring parquet fields that do not contain -/// any physical column. -pub fn parquet_to_arrow_schema(fields: &[ParquetType]) -> Vec { - parquet_to_arrow_schema_with_options(fields, &None) -} - -/// Like [`parquet_to_arrow_schema`] but with configurable options which affect the behavior of schema inference -pub fn parquet_to_arrow_schema_with_options( - fields: &[ParquetType], - options: &Option, -) -> Vec { - fields - .iter() - .filter_map(|f| to_field(f, options.as_ref().unwrap_or(&Default::default()))) - .collect::>() -} - -fn from_int32( - logical_type: Option, - converted_type: Option, -) -> DataType { - use PrimitiveLogicalType::*; - match (logical_type, converted_type) { - // handle logical types first - (Some(Integer(t)), _) => match t { - IntegerType::Int8 => DataType::Int8, - IntegerType::Int16 => DataType::Int16, - IntegerType::Int32 => DataType::Int32, - IntegerType::UInt8 => DataType::UInt8, - IntegerType::UInt16 => DataType::UInt16, - IntegerType::UInt32 => DataType::UInt32, - // The above are the only possible annotations for parquet's int32. Anything else - // is a deviation to the parquet specification and we ignore - _ => DataType::Int32, - }, - (Some(Decimal(precision, scale)), _) => DataType::Decimal(precision, scale), - (Some(Date), _) => DataType::Date32, - (Some(Time { unit, .. }), _) => match unit { - ParquetTimeUnit::Milliseconds => DataType::Time32(TimeUnit::Millisecond), - // MILLIS is the only possible annotation for parquet's int32. Anything else - // is a deviation to the parquet specification and we ignore - _ => DataType::Int32, - }, - // handle converted types: - (_, Some(PrimitiveConvertedType::Uint8)) => DataType::UInt8, - (_, Some(PrimitiveConvertedType::Uint16)) => DataType::UInt16, - (_, Some(PrimitiveConvertedType::Uint32)) => DataType::UInt32, - (_, Some(PrimitiveConvertedType::Int8)) => DataType::Int8, - (_, Some(PrimitiveConvertedType::Int16)) => DataType::Int16, - (_, Some(PrimitiveConvertedType::Int32)) => DataType::Int32, - (_, Some(PrimitiveConvertedType::Date)) => DataType::Date32, - (_, Some(PrimitiveConvertedType::TimeMillis)) => DataType::Time32(TimeUnit::Millisecond), - (_, Some(PrimitiveConvertedType::Decimal(precision, scale))) => { - DataType::Decimal(precision, scale) - } - (_, _) => DataType::Int32, - } -} - -fn from_int64( - logical_type: Option, - converted_type: Option, -) -> DataType { - use PrimitiveLogicalType::*; - match (logical_type, converted_type) { - // handle logical types first - (Some(Integer(integer)), _) => match integer { - IntegerType::UInt64 => DataType::UInt64, - IntegerType::Int64 => DataType::Int64, - _ => DataType::Int64, - }, - ( - Some(Timestamp { - is_adjusted_to_utc, - unit, - }), - _, - ) => { - let timezone = if is_adjusted_to_utc { - // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md - // A TIMESTAMP with isAdjustedToUTC=true is defined as [...] elapsed since the Unix epoch - Some("+00:00".to_string()) - } else { - // PARQUET: - // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md - // A TIMESTAMP with isAdjustedToUTC=false represents [...] such - // timestamps should always be displayed the same way, regardless of the local time zone in effect - // ARROW: - // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md - // If the time zone is null or equal to an empty string, the data is "time - // zone naive" and shall be displayed *as is* to the user, not localized - // to the locale of the user. - None - }; - - match unit { - ParquetTimeUnit::Milliseconds => { - DataType::Timestamp(TimeUnit::Millisecond, timezone) - } - ParquetTimeUnit::Microseconds => { - DataType::Timestamp(TimeUnit::Microsecond, timezone) - } - ParquetTimeUnit::Nanoseconds => DataType::Timestamp(TimeUnit::Nanosecond, timezone), - } - } - (Some(Time { unit, .. }), _) => match unit { - ParquetTimeUnit::Microseconds => DataType::Time64(TimeUnit::Microsecond), - ParquetTimeUnit::Nanoseconds => DataType::Time64(TimeUnit::Nanosecond), - // MILLIS is only possible for int32. Appearing in int64 is a deviation - // to parquet's spec, which we ignore - _ => DataType::Int64, - }, - (Some(Decimal(precision, scale)), _) => DataType::Decimal(precision, scale), - // handle converted types: - (_, Some(PrimitiveConvertedType::TimeMicros)) => DataType::Time64(TimeUnit::Microsecond), - (_, Some(PrimitiveConvertedType::TimestampMillis)) => { - DataType::Timestamp(TimeUnit::Millisecond, None) - } - (_, Some(PrimitiveConvertedType::TimestampMicros)) => { - DataType::Timestamp(TimeUnit::Microsecond, None) - } - (_, Some(PrimitiveConvertedType::Int64)) => DataType::Int64, - (_, Some(PrimitiveConvertedType::Uint64)) => DataType::UInt64, - (_, Some(PrimitiveConvertedType::Decimal(precision, scale))) => { - DataType::Decimal(precision, scale) - } - - (_, _) => DataType::Int64, - } -} - -fn from_byte_array( - logical_type: &Option, - converted_type: &Option, -) -> DataType { - match (logical_type, converted_type) { - (Some(PrimitiveLogicalType::String), _) => DataType::Utf8, - (Some(PrimitiveLogicalType::Json), _) => DataType::Binary, - (Some(PrimitiveLogicalType::Bson), _) => DataType::Binary, - (Some(PrimitiveLogicalType::Enum), _) => DataType::Binary, - (_, Some(PrimitiveConvertedType::Json)) => DataType::Binary, - (_, Some(PrimitiveConvertedType::Bson)) => DataType::Binary, - (_, Some(PrimitiveConvertedType::Enum)) => DataType::Binary, - (_, Some(PrimitiveConvertedType::Utf8)) => DataType::Utf8, - (_, _) => DataType::Binary, - } -} - -fn from_fixed_len_byte_array( - length: usize, - logical_type: Option, - converted_type: Option, -) -> DataType { - match (logical_type, converted_type) { - (Some(PrimitiveLogicalType::Decimal(precision, scale)), _) => { - if length < 32 { - DataType::Decimal(precision, scale) - } else { - DataType::Decimal256(precision, scale) - } - } - (None, Some(PrimitiveConvertedType::Decimal(precision, scale))) => { - if length < 32 { - DataType::Decimal(precision, scale) - } else { - DataType::Decimal256(precision, scale) - } - } - (None, Some(PrimitiveConvertedType::Interval)) => { - // There is currently no reliable way of determining which IntervalUnit - // to return. Thus without the original Arrow schema, the results - // would be incorrect if all 12 bytes of the interval are populated - DataType::Interval(IntervalUnit::DayTime) - } - _ => DataType::FixedSizeBinary(length), - } -} - -/// Maps a [`PhysicalType`] with optional metadata to a [`DataType`] -fn to_primitive_type_inner( - primitive_type: &PrimitiveType, - options: &SchemaInferenceOptions, -) -> DataType { - match primitive_type.physical_type { - PhysicalType::Boolean => DataType::Boolean, - PhysicalType::Int32 => { - from_int32(primitive_type.logical_type, primitive_type.converted_type) - } - PhysicalType::Int64 => { - from_int64(primitive_type.logical_type, primitive_type.converted_type) - } - PhysicalType::Int96 => DataType::Timestamp(options.int96_coerce_to_timeunit, None), - PhysicalType::Float => DataType::Float32, - PhysicalType::Double => DataType::Float64, - PhysicalType::ByteArray => { - from_byte_array(&primitive_type.logical_type, &primitive_type.converted_type) - } - PhysicalType::FixedLenByteArray(length) => from_fixed_len_byte_array( - length, - primitive_type.logical_type, - primitive_type.converted_type, - ), - } -} - -/// Entry point for converting parquet primitive type to arrow type. -/// -/// This function takes care of repetition. -fn to_primitive_type(primitive_type: &PrimitiveType, options: &SchemaInferenceOptions) -> DataType { - let base_type = to_primitive_type_inner(primitive_type, options); - - if primitive_type.field_info.repetition == Repetition::Repeated { - DataType::List(Box::new(Field::new( - &primitive_type.field_info.name, - base_type, - is_nullable(&primitive_type.field_info), - ))) - } else { - base_type - } -} - -fn non_repeated_group( - logical_type: &Option, - converted_type: &Option, - fields: &[ParquetType], - parent_name: &str, - options: &SchemaInferenceOptions, -) -> Option { - debug_assert!(!fields.is_empty()); - match (logical_type, converted_type) { - (Some(GroupLogicalType::List), _) => to_list(fields, parent_name, options), - (None, Some(GroupConvertedType::List)) => to_list(fields, parent_name, options), - (Some(GroupLogicalType::Map), _) => to_list(fields, parent_name, options), - (None, Some(GroupConvertedType::Map) | Some(GroupConvertedType::MapKeyValue)) => { - to_map(fields, options) - } - _ => to_struct(fields, options), - } -} - -/// Converts a parquet group type to an arrow [`DataType::Struct`]. -/// Returns [`None`] if all its fields are empty -fn to_struct(fields: &[ParquetType], options: &SchemaInferenceOptions) -> Option { - let fields = fields - .iter() - .filter_map(|f| to_field(f, options)) - .collect::>(); - if fields.is_empty() { - None - } else { - Some(DataType::Struct(fields)) - } -} - -/// Converts a parquet group type to an arrow [`DataType::Struct`]. -/// Returns [`None`] if all its fields are empty -fn to_map(fields: &[ParquetType], options: &SchemaInferenceOptions) -> Option { - let inner = to_field(&fields[0], options)?; - Some(DataType::Map(Box::new(inner), false)) -} - -/// Entry point for converting parquet group type. -/// -/// This function takes care of logical type and repetition. -fn to_group_type( - field_info: &FieldInfo, - logical_type: &Option, - converted_type: &Option, - fields: &[ParquetType], - parent_name: &str, - options: &SchemaInferenceOptions, -) -> Option { - debug_assert!(!fields.is_empty()); - if field_info.repetition == Repetition::Repeated { - Some(DataType::List(Box::new(Field::new( - &field_info.name, - to_struct(fields, options)?, - is_nullable(field_info), - )))) - } else { - non_repeated_group(logical_type, converted_type, fields, parent_name, options) - } -} - -/// Checks whether this schema is nullable. -pub(crate) fn is_nullable(field_info: &FieldInfo) -> bool { - match field_info.repetition { - Repetition::Optional => true, - Repetition::Repeated => false, - Repetition::Required => false, - } -} - -/// Converts parquet schema to arrow field. -/// Returns `None` iff the parquet type has no associated primitive types, -/// i.e. if it is a column-less group type. -fn to_field(type_: &ParquetType, options: &SchemaInferenceOptions) -> Option { - Some(Field::new( - &type_.get_field_info().name, - to_data_type(type_, options)?, - is_nullable(type_.get_field_info()), - )) -} - -/// Converts a parquet list to arrow list. -/// -/// To fully understand this algorithm, please refer to -/// [parquet doc](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md). -fn to_list( - fields: &[ParquetType], - parent_name: &str, - options: &SchemaInferenceOptions, -) -> Option { - let item = fields.first().unwrap(); - - let item_type = match item { - ParquetType::PrimitiveType(primitive) => Some(to_primitive_type_inner(primitive, options)), - ParquetType::GroupType { fields, .. } => { - if fields.len() == 1 - && item.name() != "array" - && item.name() != format!("{parent_name}_tuple") - { - // extract the repetition field - let nested_item = fields.first().unwrap(); - to_data_type(nested_item, options) - } else { - to_struct(fields, options) - } - } - }?; - - // Check that the name of the list child is "list", in which case we - // get the child nullability and name (normally "element") from the nested - // group type. - // Without this step, the child incorrectly inherits the parent's optionality - let (list_item_name, item_is_optional) = match item { - ParquetType::GroupType { - field_info, fields, .. - } if field_info.name == "list" && fields.len() == 1 => { - let field = fields.first().unwrap(); - ( - &field.get_field_info().name, - field.get_field_info().repetition == Repetition::Optional, - ) - } - _ => ( - &item.get_field_info().name, - item.get_field_info().repetition == Repetition::Optional, - ), - }; - - Some(DataType::List(Box::new(Field::new( - list_item_name, - item_type, - item_is_optional, - )))) -} - -/// Converts parquet schema to arrow data type. -/// -/// This function discards schema name. -/// -/// If this schema is a primitive type and not included in the leaves, the result is -/// Ok(None). -/// -/// If this schema is a group type and none of its children is reserved in the -/// conversion, the result is Ok(None). -pub(crate) fn to_data_type( - type_: &ParquetType, - options: &SchemaInferenceOptions, -) -> Option { - match type_ { - ParquetType::PrimitiveType(primitive) => Some(to_primitive_type(primitive, options)), - ParquetType::GroupType { - field_info, - logical_type, - converted_type, - fields, - } => { - if fields.is_empty() { - None - } else { - to_group_type( - field_info, - logical_type, - converted_type, - fields, - &field_info.name, - options, - ) - } - } - } -} - -#[cfg(test)] -mod tests { - use parquet2::metadata::SchemaDescriptor; - - use super::*; - use crate::arrow::datatypes::DataType; - use crate::arrow::datatypes::Field; - use crate::arrow::datatypes::TimeUnit; - use crate::arrow::error::Result; - - #[test] - fn test_flat_primitives() -> Result<()> { - let message = " - message test_schema { - REQUIRED BOOLEAN boolean; - REQUIRED INT32 int8 (INT_8); - REQUIRED INT32 int16 (INT_16); - REQUIRED INT32 uint8 (INTEGER(8,false)); - REQUIRED INT32 uint16 (INTEGER(16,false)); - REQUIRED INT32 int32; - REQUIRED INT64 int64 ; - OPTIONAL DOUBLE double; - OPTIONAL FLOAT float; - OPTIONAL BINARY string (UTF8); - OPTIONAL BINARY string_2 (STRING); - } - "; - let expected = &[ - Field::new("boolean", DataType::Boolean, false), - Field::new("int8", DataType::Int8, false), - Field::new("int16", DataType::Int16, false), - Field::new("uint8", DataType::UInt8, false), - Field::new("uint16", DataType::UInt16, false), - Field::new("int32", DataType::Int32, false), - Field::new("int64", DataType::Int64, false), - Field::new("double", DataType::Float64, true), - Field::new("float", DataType::Float32, true), - Field::new("string", DataType::Utf8, true), - Field::new("string_2", DataType::Utf8, true), - ]; - - let parquet_schema = SchemaDescriptor::try_from_message(message)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(fields, expected); - Ok(()) - } - - #[test] - fn test_byte_array_fields() -> Result<()> { - let message = " - message test_schema { - REQUIRED BYTE_ARRAY binary; - REQUIRED FIXED_LEN_BYTE_ARRAY (20) fixed_binary; - REQUIRED FIXED_LEN_BYTE_ARRAY (7) decimal_128 (Decimal(16, 2)) ; - REQUIRED FIXED_LEN_BYTE_ARRAY (32) decimal_256 (Decimal(44, 2)) ; - } - "; - let expected = vec![ - Field::new("binary", DataType::Binary, false), - Field::new("fixed_binary", DataType::FixedSizeBinary(20), false), - Field::new("decimal_128", DataType::Decimal(16, 2), false), - Field::new("decimal_256", DataType::Decimal256(44, 2), false), - ]; - - let parquet_schema = SchemaDescriptor::try_from_message(message)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(fields, expected); - Ok(()) - } - - #[test] - fn test_duplicate_fields() -> Result<()> { - let message = " - message test_schema { - REQUIRED BOOLEAN boolean; - REQUIRED INT32 int8 (INT_8); - } - "; - let expected = &[ - Field::new("boolean", DataType::Boolean, false), - Field::new("int8", DataType::Int8, false), - ]; - - let parquet_schema = SchemaDescriptor::try_from_message(message)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(fields, expected); - Ok(()) - } - - #[test] - fn test_parquet_lists() -> Result<()> { - let mut arrow_fields = Vec::new(); - - // LIST encoding example taken from parquet-format/LogicalTypes.md - let message_type = " - message test_schema { - REQUIRED GROUP my_list (LIST) { - REPEATED GROUP list { - OPTIONAL BINARY element (UTF8); - } - } - OPTIONAL GROUP my_list (LIST) { - REPEATED GROUP list { - REQUIRED BINARY element (UTF8); - } - } - OPTIONAL GROUP array_of_arrays (LIST) { - REPEATED GROUP list { - REQUIRED GROUP element (LIST) { - REPEATED GROUP list { - REQUIRED INT32 element; - } - } - } - } - OPTIONAL GROUP my_list (LIST) { - REPEATED GROUP element { - REQUIRED BINARY str (UTF8); - } - } - OPTIONAL GROUP my_list (LIST) { - REPEATED INT32 element; - } - OPTIONAL GROUP my_list (LIST) { - REPEATED GROUP element { - REQUIRED BINARY str (UTF8); - REQUIRED INT32 num; - } - } - OPTIONAL GROUP my_list (LIST) { - REPEATED GROUP array { - REQUIRED BINARY str (UTF8); - } - - } - OPTIONAL GROUP my_list (LIST) { - REPEATED GROUP my_list_tuple { - REQUIRED BINARY str (UTF8); - } - } - REPEATED INT32 name; - } - "; - - // // List (list non-null, elements nullable) - // required group my_list (LIST) { - // repeated group list { - // optional binary element (UTF8); - // } - // } - { - arrow_fields.push(Field::new( - "my_list", - DataType::List(Box::new(Field::new("element", DataType::Utf8, true))), - false, - )); - } - - // // List (list nullable, elements non-null) - // optional group my_list (LIST) { - // repeated group list { - // required binary element (UTF8); - // } - // } - { - arrow_fields.push(Field::new( - "my_list", - DataType::List(Box::new(Field::new("element", DataType::Utf8, false))), - true, - )); - } - - // Element types can be nested structures. For example, a list of lists: - // - // // List> - // optional group array_of_arrays (LIST) { - // repeated group list { - // required group element (LIST) { - // repeated group list { - // required int32 element; - // } - // } - // } - // } - { - let arrow_inner_list = - DataType::List(Box::new(Field::new("element", DataType::Int32, false))); - arrow_fields.push(Field::new( - "array_of_arrays", - DataType::List(Box::new(Field::new("element", arrow_inner_list, false))), - true, - )); - } - - // // List (list nullable, elements non-null) - // optional group my_list (LIST) { - // repeated group element { - // required binary str (UTF8); - // }; - // } - { - arrow_fields.push(Field::new( - "my_list", - DataType::List(Box::new(Field::new("element", DataType::Utf8, false))), - true, - )); - } - - // // List (nullable list, non-null elements) - // optional group my_list (LIST) { - // repeated int32 element; - // } - { - arrow_fields.push(Field::new( - "my_list", - DataType::List(Box::new(Field::new("element", DataType::Int32, false))), - true, - )); - } - - // // List> (nullable list, non-null elements) - // optional group my_list (LIST) { - // repeated group element { - // required binary str (UTF8); - // required int32 num; - // }; - // } - { - let arrow_struct = DataType::Struct(vec![ - Field::new("str", DataType::Utf8, false), - Field::new("num", DataType::Int32, false), - ]); - arrow_fields.push(Field::new( - "my_list", - DataType::List(Box::new(Field::new("element", arrow_struct, false))), - true, - )); - } - - // // List> (nullable list, non-null elements) - // optional group my_list (LIST) { - // repeated group array { - // required binary str (UTF8); - // }; - // } - // Special case: group is named array - { - let arrow_struct = DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]); - arrow_fields.push(Field::new( - "my_list", - DataType::List(Box::new(Field::new("array", arrow_struct, false))), - true, - )); - } - - // // List> (nullable list, non-null elements) - // optional group my_list (LIST) { - // repeated group my_list_tuple { - // required binary str (UTF8); - // }; - // } - // Special case: group named ends in _tuple - { - let arrow_struct = DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]); - arrow_fields.push(Field::new( - "my_list", - DataType::List(Box::new(Field::new("my_list_tuple", arrow_struct, false))), - true, - )); - } - - // One-level encoding: Only allows required lists with required cells - // repeated value_type name - { - arrow_fields.push(Field::new( - "name", - DataType::List(Box::new(Field::new("name", DataType::Int32, false))), - false, - )); - } - - let parquet_schema = SchemaDescriptor::try_from_message(message_type)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(arrow_fields, fields); - Ok(()) - } - - #[test] - fn test_parquet_list_with_struct() -> Result<()> { - let mut arrow_fields = Vec::new(); - - let message_type = " - message eventlog { - REQUIRED group events (LIST) { - REPEATED group array { - REQUIRED BYTE_ARRAY event_name (STRING); - REQUIRED INT64 event_time (TIMESTAMP(MILLIS,true)); - } - } - } - "; - - { - let struct_fields = vec![ - Field::new("event_name", DataType::Utf8, false), - Field::new( - "event_time", - DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())), - false, - ), - ]; - arrow_fields.push(Field::new( - "events", - DataType::List(Box::new(Field::new( - "array", - DataType::Struct(struct_fields), - false, - ))), - false, - )); - } - - let parquet_schema = SchemaDescriptor::try_from_message(message_type)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(arrow_fields, fields); - Ok(()) - } - - #[test] - fn test_parquet_list_nullable() -> Result<()> { - let mut arrow_fields = Vec::new(); - - let message_type = " - message test_schema { - REQUIRED GROUP my_list1 (LIST) { - REPEATED GROUP list { - OPTIONAL BINARY element (UTF8); - } - } - OPTIONAL GROUP my_list2 (LIST) { - REPEATED GROUP list { - REQUIRED BINARY element (UTF8); - } - } - REQUIRED GROUP my_list3 (LIST) { - REPEATED GROUP list { - REQUIRED BINARY element (UTF8); - } - } - } - "; - - // // List (list non-null, elements nullable) - // required group my_list1 (LIST) { - // repeated group list { - // optional binary element (UTF8); - // } - // } - { - arrow_fields.push(Field::new( - "my_list1", - DataType::List(Box::new(Field::new("element", DataType::Utf8, true))), - false, - )); - } - - // // List (list nullable, elements non-null) - // optional group my_list2 (LIST) { - // repeated group list { - // required binary element (UTF8); - // } - // } - { - arrow_fields.push(Field::new( - "my_list2", - DataType::List(Box::new(Field::new("element", DataType::Utf8, false))), - true, - )); - } - - // // List (list non-null, elements non-null) - // repeated group my_list3 (LIST) { - // repeated group list { - // required binary element (UTF8); - // } - // } - { - arrow_fields.push(Field::new( - "my_list3", - DataType::List(Box::new(Field::new("element", DataType::Utf8, false))), - false, - )); - } - - let parquet_schema = SchemaDescriptor::try_from_message(message_type)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(arrow_fields, fields); - Ok(()) - } - - #[test] - fn test_nested_schema() -> Result<()> { - let mut arrow_fields = Vec::new(); - { - let group1_fields = vec![ - Field::new("leaf1", DataType::Boolean, false), - Field::new("leaf2", DataType::Int32, false), - ]; - let group1_struct = Field::new("group1", DataType::Struct(group1_fields), false); - arrow_fields.push(group1_struct); - - let leaf3_field = Field::new("leaf3", DataType::Int64, false); - arrow_fields.push(leaf3_field); - } - - let message_type = " - message test_schema { - REQUIRED GROUP group1 { - REQUIRED BOOLEAN leaf1; - REQUIRED INT32 leaf2; - } - REQUIRED INT64 leaf3; - } - "; - - let parquet_schema = SchemaDescriptor::try_from_message(message_type)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(arrow_fields, fields); - Ok(()) - } - - #[test] - fn test_repeated_nested_schema() -> Result<()> { - let mut arrow_fields = Vec::new(); - { - arrow_fields.push(Field::new("leaf1", DataType::Int32, true)); - - let inner_group_list = Field::new( - "innerGroup", - DataType::List(Box::new(Field::new( - "innerGroup", - DataType::Struct(vec![Field::new("leaf3", DataType::Int32, true)]), - false, - ))), - false, - ); - - let outer_group_list = Field::new( - "outerGroup", - DataType::List(Box::new(Field::new( - "outerGroup", - DataType::Struct(vec![ - Field::new("leaf2", DataType::Int32, true), - inner_group_list, - ]), - false, - ))), - false, - ); - arrow_fields.push(outer_group_list); - } - - let message_type = " - message test_schema { - OPTIONAL INT32 leaf1; - REPEATED GROUP outerGroup { - OPTIONAL INT32 leaf2; - REPEATED GROUP innerGroup { - OPTIONAL INT32 leaf3; - } - } - } - "; - - let parquet_schema = SchemaDescriptor::try_from_message(message_type)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(arrow_fields, fields); - Ok(()) - } - - #[test] - fn test_column_desc_to_field() -> Result<()> { - let message_type = " - message test_schema { - REQUIRED BOOLEAN boolean; - REQUIRED INT32 int8 (INT_8); - REQUIRED INT32 uint8 (INTEGER(8,false)); - REQUIRED INT32 int16 (INT_16); - REQUIRED INT32 uint16 (INTEGER(16,false)); - REQUIRED INT32 int32; - REQUIRED INT64 int64; - OPTIONAL DOUBLE double; - OPTIONAL FLOAT float; - OPTIONAL BINARY string (UTF8); - REPEATED BOOLEAN bools; - OPTIONAL INT32 date (DATE); - OPTIONAL INT32 time_milli (TIME_MILLIS); - OPTIONAL INT64 time_micro (TIME_MICROS); - OPTIONAL INT64 time_nano (TIME(NANOS,false)); - OPTIONAL INT64 ts_milli (TIMESTAMP_MILLIS); - REQUIRED INT64 ts_micro (TIMESTAMP_MICROS); - REQUIRED INT64 ts_nano (TIMESTAMP(NANOS,true)); - } - "; - let arrow_fields = vec![ - Field::new("boolean", DataType::Boolean, false), - Field::new("int8", DataType::Int8, false), - Field::new("uint8", DataType::UInt8, false), - Field::new("int16", DataType::Int16, false), - Field::new("uint16", DataType::UInt16, false), - Field::new("int32", DataType::Int32, false), - Field::new("int64", DataType::Int64, false), - Field::new("double", DataType::Float64, true), - Field::new("float", DataType::Float32, true), - Field::new("string", DataType::Utf8, true), - Field::new( - "bools", - DataType::List(Box::new(Field::new("bools", DataType::Boolean, false))), - false, - ), - Field::new("date", DataType::Date32, true), - Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true), - Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true), - Field::new("time_nano", DataType::Time64(TimeUnit::Nanosecond), true), - Field::new( - "ts_milli", - DataType::Timestamp(TimeUnit::Millisecond, None), - true, - ), - Field::new( - "ts_micro", - DataType::Timestamp(TimeUnit::Microsecond, None), - false, - ), - Field::new( - "ts_nano", - DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".to_string())), - false, - ), - ]; - - let parquet_schema = SchemaDescriptor::try_from_message(message_type)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(arrow_fields, fields); - Ok(()) - } - - #[test] - fn test_field_to_column_desc() -> Result<()> { - let message_type = " - message arrow_schema { - REQUIRED BOOLEAN boolean; - REQUIRED INT32 int8 (INT_8); - REQUIRED INT32 int16 (INTEGER(16,true)); - REQUIRED INT32 int32; - REQUIRED INT64 int64; - OPTIONAL DOUBLE double; - OPTIONAL FLOAT float; - OPTIONAL BINARY string (STRING); - OPTIONAL GROUP bools (LIST) { - REPEATED GROUP list { - OPTIONAL BOOLEAN element; - } - } - REQUIRED GROUP bools_non_null (LIST) { - REPEATED GROUP list { - REQUIRED BOOLEAN element; - } - } - OPTIONAL INT32 date (DATE); - OPTIONAL INT32 time_milli (TIME(MILLIS,false)); - OPTIONAL INT64 time_micro (TIME_MICROS); - OPTIONAL INT64 ts_milli (TIMESTAMP_MILLIS); - REQUIRED INT64 ts_micro (TIMESTAMP(MICROS,false)); - REQUIRED GROUP struct { - REQUIRED BOOLEAN bools; - REQUIRED INT32 uint32 (INTEGER(32,false)); - REQUIRED GROUP int32 (LIST) { - REPEATED GROUP list { - OPTIONAL INT32 element; - } - } - } - REQUIRED BINARY dictionary_strings (STRING); - } - "; - - let arrow_fields = vec![ - Field::new("boolean", DataType::Boolean, false), - Field::new("int8", DataType::Int8, false), - Field::new("int16", DataType::Int16, false), - Field::new("int32", DataType::Int32, false), - Field::new("int64", DataType::Int64, false), - Field::new("double", DataType::Float64, true), - Field::new("float", DataType::Float32, true), - Field::new("string", DataType::Utf8, true), - Field::new( - "bools", - DataType::List(Box::new(Field::new("element", DataType::Boolean, true))), - true, - ), - Field::new( - "bools_non_null", - DataType::List(Box::new(Field::new("element", DataType::Boolean, false))), - false, - ), - Field::new("date", DataType::Date32, true), - Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true), - Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true), - Field::new( - "ts_milli", - DataType::Timestamp(TimeUnit::Millisecond, None), - true, - ), - Field::new( - "ts_micro", - DataType::Timestamp(TimeUnit::Microsecond, None), - false, - ), - Field::new( - "struct", - DataType::Struct(vec![ - Field::new("bools", DataType::Boolean, false), - Field::new("uint32", DataType::UInt32, false), - Field::new( - "int32", - DataType::List(Box::new(Field::new("element", DataType::Int32, true))), - false, - ), - ]), - false, - ), - Field::new("dictionary_strings", DataType::Utf8, false), - ]; - - let parquet_schema = SchemaDescriptor::try_from_message(message_type)?; - let fields = parquet_to_arrow_schema(parquet_schema.fields()); - - assert_eq!(arrow_fields, fields); - Ok(()) - } - - #[test] - fn test_int96_options() -> Result<()> { - for tu in [ - TimeUnit::Second, - TimeUnit::Microsecond, - TimeUnit::Millisecond, - TimeUnit::Nanosecond, - ] { - let message_type = " - message arrow_schema { - REQUIRED INT96 int96_field; - OPTIONAL GROUP int96_list (LIST) { - REPEATED GROUP list { - OPTIONAL INT96 element; - } - } - REQUIRED GROUP int96_struct { - REQUIRED INT96 int96_field; - } - } - "; - let coerced_to = DataType::Timestamp(tu, None); - let arrow_fields = vec![ - Field::new("int96_field", coerced_to.clone(), false), - Field::new( - "int96_list", - DataType::List(Box::new(Field::new("element", coerced_to.clone(), true))), - true, - ), - Field::new( - "int96_struct", - DataType::Struct(vec![Field::new("int96_field", coerced_to.clone(), false)]), - false, - ), - ]; - - let parquet_schema = SchemaDescriptor::try_from_message(message_type)?; - let fields = parquet_to_arrow_schema_with_options( - parquet_schema.fields(), - &Some(SchemaInferenceOptions { - int96_coerce_to_timeunit: tu, - }), - ); - assert_eq!(arrow_fields, fields); - } - Ok(()) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/schema/metadata.rs b/src/common/arrow/src/arrow/io/parquet/read/schema/metadata.rs deleted file mode 100644 index 19229b9d49d8..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/schema/metadata.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use base64::engine::general_purpose; -use base64::Engine as _; -pub use parquet2::metadata::KeyValue; - -use super::super::super::ARROW_SCHEMA_META_KEY; -use crate::arrow::datatypes::Metadata; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::read::deserialize_schema; - -/// Reads an arrow schema from Parquet's file metadata. Returns `None` if no schema was found. -/// # Errors -/// Errors iff the schema cannot be correctly parsed. -pub fn read_schema_from_metadata(metadata: &mut Metadata) -> Result> { - metadata - .remove(ARROW_SCHEMA_META_KEY) - .map(|encoded| get_arrow_schema_from_metadata(&encoded)) - .transpose() -} - -/// Try to convert Arrow schema metadata into a schema -fn get_arrow_schema_from_metadata(encoded_meta: &str) -> Result { - let decoded = general_purpose::STANDARD.decode(encoded_meta); - match decoded { - Ok(bytes) => { - let slice = if bytes[0..4] == [255u8; 4] { - &bytes[8..] - } else { - bytes.as_slice() - }; - deserialize_schema(slice).map(|x| x.0) - } - Err(err) => { - // The C++ implementation returns an error if the schema can't be parsed. - Err(Error::InvalidArgumentError(format!( - "Unable to decode the encoded schema stored in {ARROW_SCHEMA_META_KEY}, {err:?}" - ))) - } - } -} - -pub(super) fn parse_key_value_metadata(key_value_metadata: &Option>) -> Metadata { - key_value_metadata - .as_ref() - .map(|key_values| { - key_values - .iter() - .filter_map(|kv| { - kv.value - .as_ref() - .map(|value| (kv.key.clone(), value.clone())) - }) - .collect() - }) - .unwrap_or_default() -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/schema/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/schema/mod.rs deleted file mode 100644 index 7b8692b34417..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/schema/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to handle Parquet <-> Arrow schemas. -use crate::arrow::datatypes::Schema; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::error::Result; - -mod convert; -mod metadata; - -pub use convert::parquet_to_arrow_schema; -pub use convert::parquet_to_arrow_schema_with_options; -pub(crate) use convert::*; -pub use metadata::read_schema_from_metadata; -pub use parquet2::metadata::FileMetaData; -pub use parquet2::metadata::KeyValue; -pub use parquet2::metadata::SchemaDescriptor; -pub use parquet2::schema::types::ParquetType; - -use self::metadata::parse_key_value_metadata; - -/// Options when inferring schemas from Parquet -pub struct SchemaInferenceOptions { - /// When inferring schemas from the Parquet INT96 timestamp type, this is the corresponding TimeUnit - /// in the inferred Arrow Timestamp type. - /// - /// This defaults to `TimeUnit::Nanosecond`, but INT96 timestamps outside of the range of years 1678-2262, - /// will overflow when parsed as `Timestamp(TimeUnit::Nanosecond)`. Setting this to a lower resolution - /// (e.g. TimeUnit::Milliseconds) will result in loss of precision, but support a larger range of dates - /// without overflowing when parsing the data. - pub int96_coerce_to_timeunit: TimeUnit, -} - -impl Default for SchemaInferenceOptions { - fn default() -> Self { - SchemaInferenceOptions { - int96_coerce_to_timeunit: TimeUnit::Nanosecond, - } - } -} - -/// Infers a [`Schema`] from parquet's [`FileMetaData`]. This first looks for the metadata key -/// `"ARROW:schema"`; if it does not exist, it converts the parquet types declared in the -/// file's parquet schema to Arrow's equivalent. -/// # Error -/// This function errors iff the key `"ARROW:schema"` exists but is not correctly encoded, -/// indicating that that the file's arrow metadata was incorrectly written. -pub fn infer_schema(file_metadata: &FileMetaData) -> Result { - infer_schema_with_options(file_metadata, &None) -} - -/// Like [`infer_schema`] but with configurable options which affects the behavior of inference -pub fn infer_schema_with_options( - file_metadata: &FileMetaData, - options: &Option, -) -> Result { - let mut metadata = parse_key_value_metadata(file_metadata.key_value_metadata()); - - let schema = read_schema_from_metadata(&mut metadata)?; - Ok(schema.unwrap_or_else(|| { - let fields = parquet_to_arrow_schema_with_options(file_metadata.schema().fields(), options); - Schema { fields, metadata } - })) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/binary.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/binary.rs deleted file mode 100644 index c3c0e056f7cc..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/binary.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::statistics::BinaryStatistics; -use parquet2::statistics::Statistics as ParquetStatistics; - -use crate::arrow::array::MutableArray; -use crate::arrow::array::MutableBinaryArray; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; - -pub(super) fn push( - from: Option<&dyn ParquetStatistics>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - min.push(from.and_then(|s| s.min_value.as_ref())); - max.push(from.and_then(|s| s.max_value.as_ref())); - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/boolean.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/boolean.rs deleted file mode 100644 index 3dd9d5ea85fd..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/boolean.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::statistics::BooleanStatistics; -use parquet2::statistics::Statistics as ParquetStatistics; - -use crate::arrow::array::MutableArray; -use crate::arrow::array::MutableBooleanArray; -use crate::arrow::error::Result; - -pub(super) fn push( - from: Option<&dyn ParquetStatistics>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - min.push(from.and_then(|s| s.min_value)); - max.push(from.and_then(|s| s.max_value)); - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/dictionary.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/dictionary.rs deleted file mode 100644 index e212eb40c8ff..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/dictionary.rs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::make_mutable; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; - -#[derive(Debug)] -pub struct DynMutableDictionary { - data_type: DataType, - pub inner: Box, -} - -impl DynMutableDictionary { - pub fn try_with_capacity(data_type: DataType, capacity: usize) -> Result { - let inner = if let DataType::Dictionary(_, inner, _) = &data_type { - inner.as_ref() - } else { - unreachable!() - }; - let inner = make_mutable(inner, capacity)?; - - Ok(Self { data_type, inner }) - } -} - -impl MutableArray for DynMutableDictionary { - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn len(&self) -> usize { - self.inner.len() - } - - fn validity(&self) -> Option<&crate::arrow::bitmap::MutableBitmap> { - self.inner.validity() - } - - fn as_box(&mut self) -> Box { - let inner = self.inner.as_box(); - match self.data_type.to_physical_type() { - PhysicalType::Dictionary(key) => match_integer_type!(key, |$T| { - let keys: Vec<$T> = (0..inner.len() as $T).collect(); - let keys = PrimitiveArray::<$T>::from_vec(keys); - Box::new(DictionaryArray::<$T>::try_new(self.data_type.clone(), keys, inner).unwrap()) - }), - _ => todo!(), - } - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn push_null(&mut self) { - todo!() - } - - fn reserve(&mut self, _: usize) { - todo!(); - } - - fn shrink_to_fit(&mut self) { - todo!() - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/fixlen.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/fixlen.rs deleted file mode 100644 index ea7339ad38ac..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/fixlen.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use ethnum::I256; -use parquet2::statistics::FixedLenStatistics; -use parquet2::statistics::Statistics as ParquetStatistics; - -use super::super::convert_days_ms; -use super::super::convert_i128; -use crate::arrow::array::*; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::convert_i256; -use crate::arrow::types::days_ms; -use crate::arrow::types::i256; - -pub(super) fn push_i128( - from: Option<&dyn ParquetStatistics>, - n: usize, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - - min.push(from.and_then(|s| s.min_value.as_deref().map(|x| convert_i128(x, n)))); - max.push(from.and_then(|s| s.max_value.as_deref().map(|x| convert_i128(x, n)))); - - Ok(()) -} - -pub(super) fn push_i256_with_i128( - from: Option<&dyn ParquetStatistics>, - n: usize, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - - min.push(from.and_then(|s| { - s.min_value - .as_deref() - .map(|x| i256(I256::new(convert_i128(x, n)))) - })); - max.push(from.and_then(|s| { - s.max_value - .as_deref() - .map(|x| i256(I256::new(convert_i128(x, n)))) - })); - - Ok(()) -} - -pub(super) fn push_i256( - from: Option<&dyn ParquetStatistics>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - - min.push(from.and_then(|s| s.min_value.as_deref().map(convert_i256))); - max.push(from.and_then(|s| s.max_value.as_deref().map(convert_i256))); - - Ok(()) -} - -pub(super) fn push( - from: Option<&dyn ParquetStatistics>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - min.push(from.and_then(|s| s.min_value.as_ref())); - max.push(from.and_then(|s| s.max_value.as_ref())); - Ok(()) -} - -fn convert_year_month(value: &[u8]) -> i32 { - i32::from_le_bytes(value[..4].try_into().unwrap()) -} - -pub(super) fn push_year_month( - from: Option<&dyn ParquetStatistics>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - - min.push(from.and_then(|s| s.min_value.as_deref().map(convert_year_month))); - max.push(from.and_then(|s| s.max_value.as_deref().map(convert_year_month))); - - Ok(()) -} - -pub(super) fn push_days_ms( - from: Option<&dyn ParquetStatistics>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - - min.push(from.and_then(|s| s.min_value.as_deref().map(convert_days_ms))); - max.push(from.and_then(|s| s.max_value.as_deref().map(convert_days_ms))); - - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/list.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/list.rs deleted file mode 100644 index 85b1e4b6d1c1..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/list.rs +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::make_mutable; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::offset::Offsets; - -#[derive(Debug)] -pub struct DynMutableListArray { - data_type: DataType, - pub inner: Box, -} - -impl DynMutableListArray { - pub fn try_with_capacity(data_type: DataType, capacity: usize) -> Result { - let inner = match data_type.to_logical_type() { - DataType::List(inner) | DataType::LargeList(inner) => inner.data_type(), - _ => unreachable!(), - }; - let inner = make_mutable(inner, capacity)?; - - Ok(Self { data_type, inner }) - } -} - -impl MutableArray for DynMutableListArray { - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn len(&self) -> usize { - self.inner.len() - } - - fn validity(&self) -> Option<&crate::arrow::bitmap::MutableBitmap> { - self.inner.validity() - } - - fn as_box(&mut self) -> Box { - let inner = self.inner.as_box(); - - match self.data_type.to_logical_type() { - DataType::List(_) => { - let offsets = - Offsets::try_from_lengths(std::iter::repeat(1).take(inner.len())).unwrap(); - Box::new(ListArray::::new( - self.data_type.clone(), - offsets.into(), - inner, - None, - )) - } - DataType::LargeList(_) => { - let offsets = - Offsets::try_from_lengths(std::iter::repeat(1).take(inner.len())).unwrap(); - Box::new(ListArray::::new( - self.data_type.clone(), - offsets.into(), - inner, - None, - )) - } - _ => unreachable!(), - } - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn push_null(&mut self) { - todo!() - } - - fn reserve(&mut self, _: usize) { - todo!(); - } - - fn shrink_to_fit(&mut self) { - todo!() - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/map.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/map.rs deleted file mode 100644 index b96a67492e83..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/map.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::make_mutable; -use crate::arrow::array::Array; -use crate::arrow::array::MapArray; -use crate::arrow::array::MutableArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; - -#[derive(Debug)] -pub struct DynMutableMapArray { - data_type: DataType, - pub inner: Box, -} - -impl DynMutableMapArray { - pub fn try_with_capacity(data_type: DataType, capacity: usize) -> Result { - let inner = match data_type.to_logical_type() { - DataType::Map(inner, _) => inner, - _ => unreachable!(), - }; - let inner = make_mutable(inner.data_type(), capacity)?; - - Ok(Self { data_type, inner }) - } -} - -impl MutableArray for DynMutableMapArray { - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn len(&self) -> usize { - self.inner.len() - } - - fn validity(&self) -> Option<&crate::arrow::bitmap::MutableBitmap> { - None - } - - fn as_box(&mut self) -> Box { - Box::new(MapArray::new( - self.data_type.clone(), - vec![0, self.inner.len() as i32].try_into().unwrap(), - self.inner.as_box(), - None, - )) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn push_null(&mut self) { - todo!() - } - - fn reserve(&mut self, _: usize) { - todo!(); - } - - fn shrink_to_fit(&mut self) { - todo!() - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/mod.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/mod.rs deleted file mode 100644 index a06fc16fe124..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/mod.rs +++ /dev/null @@ -1,596 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs exposing `parquet2`'s statistics as arrow's statistics. -use std::collections::VecDeque; -use std::sync::Arc; - -use ethnum::I256; -use parquet2::metadata::RowGroupMetaData; -use parquet2::schema::types::PhysicalType as ParquetPhysicalType; -use parquet2::schema::types::PrimitiveType as ParquetPrimitiveType; -use parquet2::statistics::BinaryStatistics; -use parquet2::statistics::BooleanStatistics; -use parquet2::statistics::FixedLenStatistics; -use parquet2::statistics::PrimitiveStatistics; -use parquet2::statistics::Statistics as ParquetStatistics; -use parquet2::types::int96_to_i64_ns; - -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::i256; - -mod binary; -mod boolean; -mod dictionary; -mod fixlen; -mod list; -mod map; -mod null; -mod primitive; -mod struct_; -mod utf8; - -use self::list::DynMutableListArray; -use super::get_field_columns; - -/// Arrow-deserialized parquet Statistics of a file -#[derive(Debug, PartialEq)] -pub struct Statistics { - /// number of nulls. This is a [`UInt64Array`] for non-nested types - pub null_count: Box, - /// number of dictinct values. This is a [`UInt64Array`] for non-nested types - pub distinct_count: Box, - /// Minimum - pub min_value: Box, - /// Maximum - pub max_value: Box, -} - -/// Arrow-deserialized parquet Statistics of a file -#[derive(Debug)] -struct MutableStatistics { - /// number of nulls - pub null_count: Box, - /// number of dictinct values - pub distinct_count: Box, - /// Minimum - pub min_value: Box, - /// Maximum - pub max_value: Box, -} - -impl From for Statistics { - fn from(mut s: MutableStatistics) -> Self { - let null_count = if let PhysicalType::Struct = s.null_count.data_type().to_physical_type() { - s.null_count - .as_box() - .as_any() - .downcast_ref::() - .unwrap() - .clone() - .boxed() - } else if let PhysicalType::Map = s.null_count.data_type().to_physical_type() { - s.null_count - .as_box() - .as_any() - .downcast_ref::() - .unwrap() - .clone() - .boxed() - } else if let PhysicalType::List = s.null_count.data_type().to_physical_type() { - s.null_count - .as_box() - .as_any() - .downcast_ref::>() - .unwrap() - .clone() - .boxed() - } else if let PhysicalType::LargeList = s.null_count.data_type().to_physical_type() { - s.null_count - .as_box() - .as_any() - .downcast_ref::>() - .unwrap() - .clone() - .boxed() - } else { - s.null_count - .as_box() - .as_any() - .downcast_ref::() - .unwrap() - .clone() - .boxed() - }; - let distinct_count = if let PhysicalType::Struct = - s.distinct_count.data_type().to_physical_type() - { - s.distinct_count - .as_box() - .as_any() - .downcast_ref::() - .unwrap() - .clone() - .boxed() - } else if let PhysicalType::Map = s.distinct_count.data_type().to_physical_type() { - s.distinct_count - .as_box() - .as_any() - .downcast_ref::() - .unwrap() - .clone() - .boxed() - } else if let PhysicalType::List = s.distinct_count.data_type().to_physical_type() { - s.distinct_count - .as_box() - .as_any() - .downcast_ref::>() - .unwrap() - .clone() - .boxed() - } else if let PhysicalType::LargeList = s.distinct_count.data_type().to_physical_type() { - s.distinct_count - .as_box() - .as_any() - .downcast_ref::>() - .unwrap() - .clone() - .boxed() - } else { - s.distinct_count - .as_box() - .as_any() - .downcast_ref::() - .unwrap() - .clone() - .boxed() - }; - Self { - null_count, - distinct_count, - min_value: s.min_value.as_box(), - max_value: s.max_value.as_box(), - } - } -} - -fn make_mutable(data_type: &DataType, capacity: usize) -> Result> { - Ok(match data_type.to_physical_type() { - PhysicalType::Boolean => { - Box::new(MutableBooleanArray::with_capacity(capacity)) as Box - } - PhysicalType::Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - Box::new(MutablePrimitiveArray::<$T>::with_capacity(capacity).to(data_type.clone())) - as Box - }), - PhysicalType::Binary => { - Box::new(MutableBinaryArray::::with_capacity(capacity)) as Box - } - PhysicalType::LargeBinary => { - Box::new(MutableBinaryArray::::with_capacity(capacity)) as Box - } - PhysicalType::Utf8 => { - Box::new(MutableUtf8Array::::with_capacity(capacity)) as Box - } - PhysicalType::LargeUtf8 => { - Box::new(MutableUtf8Array::::with_capacity(capacity)) as Box - } - PhysicalType::FixedSizeBinary => { - Box::new(MutableFixedSizeBinaryArray::try_new(data_type.clone(), vec![], None).unwrap()) - as _ - } - PhysicalType::LargeList | PhysicalType::List => Box::new( - DynMutableListArray::try_with_capacity(data_type.clone(), capacity)?, - ) as Box, - PhysicalType::Dictionary(_) => Box::new( - dictionary::DynMutableDictionary::try_with_capacity(data_type.clone(), capacity)?, - ), - PhysicalType::Struct => Box::new(struct_::DynMutableStructArray::try_with_capacity( - data_type.clone(), - capacity, - )?), - PhysicalType::Map => Box::new(map::DynMutableMapArray::try_with_capacity( - data_type.clone(), - capacity, - )?), - PhysicalType::Null => { - Box::new(MutableNullArray::new(DataType::Null, 0)) as Box - } - other => { - return Err(Error::NotYetImplemented(format!( - "Deserializing parquet stats from {other:?} is still not implemented" - ))); - } - }) -} - -fn create_dt(data_type: &DataType) -> DataType { - if let DataType::Struct(fields) = data_type.to_logical_type() { - DataType::Struct( - fields - .iter() - .map(|f| Field::new(&f.name, create_dt(&f.data_type), f.is_nullable)) - .collect(), - ) - } else if let DataType::Map(f, ordered) = data_type.to_logical_type() { - DataType::Map( - Box::new(Field::new(&f.name, create_dt(&f.data_type), f.is_nullable)), - *ordered, - ) - } else if let DataType::List(f) = data_type.to_logical_type() { - DataType::List(Box::new(Field::new( - &f.name, - create_dt(&f.data_type), - f.is_nullable, - ))) - } else if let DataType::LargeList(f) = data_type.to_logical_type() { - DataType::LargeList(Box::new(Field::new( - &f.name, - create_dt(&f.data_type), - f.is_nullable, - ))) - } else { - DataType::UInt64 - } -} - -impl MutableStatistics { - fn try_new(field: &Field) -> Result { - let min_value = make_mutable(&field.data_type, 0)?; - let max_value = make_mutable(&field.data_type, 0)?; - - let dt = create_dt(&field.data_type); - Ok(Self { - null_count: make_mutable(&dt, 0)?, - distinct_count: make_mutable(&dt, 0)?, - min_value, - max_value, - }) - } -} - -fn push_others( - from: Option<&dyn ParquetStatistics>, - distinct_count: &mut UInt64Vec, - null_count: &mut UInt64Vec, -) { - let from = if let Some(from) = from { - from - } else { - distinct_count.push(None); - null_count.push(None); - return; - }; - let (distinct, null_count1) = match from.physical_type() { - ParquetPhysicalType::Boolean => { - let from = from.as_any().downcast_ref::().unwrap(); - (from.distinct_count, from.null_count) - } - ParquetPhysicalType::Int32 => { - let from = from - .as_any() - .downcast_ref::>() - .unwrap(); - (from.distinct_count, from.null_count) - } - ParquetPhysicalType::Int64 => { - let from = from - .as_any() - .downcast_ref::>() - .unwrap(); - (from.distinct_count, from.null_count) - } - ParquetPhysicalType::Int96 => { - let from = from - .as_any() - .downcast_ref::>() - .unwrap(); - (from.distinct_count, from.null_count) - } - ParquetPhysicalType::Float => { - let from = from - .as_any() - .downcast_ref::>() - .unwrap(); - (from.distinct_count, from.null_count) - } - ParquetPhysicalType::Double => { - let from = from - .as_any() - .downcast_ref::>() - .unwrap(); - (from.distinct_count, from.null_count) - } - ParquetPhysicalType::ByteArray => { - let from = from.as_any().downcast_ref::().unwrap(); - (from.distinct_count, from.null_count) - } - ParquetPhysicalType::FixedLenByteArray(_) => { - let from = from.as_any().downcast_ref::().unwrap(); - (from.distinct_count, from.null_count) - } - }; - - distinct_count.push(distinct.map(|x| x as u64)); - null_count.push(null_count1.map(|x| x as u64)); -} - -fn push( - stats: &mut VecDeque<(Option>, ParquetPrimitiveType)>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, - distinct_count: &mut dyn MutableArray, - null_count: &mut dyn MutableArray, -) -> Result<()> { - match min.data_type().to_logical_type() { - List(_) | LargeList(_) => { - let min = min - .as_mut_any() - .downcast_mut::() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::() - .unwrap(); - let distinct_count = distinct_count - .as_mut_any() - .downcast_mut::() - .unwrap(); - let null_count = null_count - .as_mut_any() - .downcast_mut::() - .unwrap(); - return push( - stats, - min.inner.as_mut(), - max.inner.as_mut(), - distinct_count.inner.as_mut(), - null_count.inner.as_mut(), - ); - } - Dictionary(_, _, _) => { - let min = min - .as_mut_any() - .downcast_mut::() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::() - .unwrap(); - return push( - stats, - min.inner.as_mut(), - max.inner.as_mut(), - distinct_count, - null_count, - ); - } - Struct(_) => { - let min = min - .as_mut_any() - .downcast_mut::() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::() - .unwrap(); - let distinct_count = distinct_count - .as_mut_any() - .downcast_mut::() - .unwrap(); - let null_count = null_count - .as_mut_any() - .downcast_mut::() - .unwrap(); - return min - .inner - .iter_mut() - .zip(max.inner.iter_mut()) - .zip(distinct_count.inner.iter_mut()) - .zip(null_count.inner.iter_mut()) - .try_for_each(|(((min, max), distinct_count), null_count)| { - push( - stats, - min.as_mut(), - max.as_mut(), - distinct_count.as_mut(), - null_count.as_mut(), - ) - }); - } - Map(_, _) => { - let min = min - .as_mut_any() - .downcast_mut::() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::() - .unwrap(); - let distinct_count = distinct_count - .as_mut_any() - .downcast_mut::() - .unwrap(); - let null_count = null_count - .as_mut_any() - .downcast_mut::() - .unwrap(); - return push( - stats, - min.inner.as_mut(), - max.inner.as_mut(), - distinct_count.inner.as_mut(), - null_count.inner.as_mut(), - ); - } - _ => {} - } - - let (from, type_) = stats.pop_front().unwrap(); - let from = from.as_deref(); - - let distinct_count = distinct_count - .as_mut_any() - .downcast_mut::() - .unwrap(); - let null_count = null_count.as_mut_any().downcast_mut::().unwrap(); - - push_others(from, distinct_count, null_count); - - let physical_type = &type_.physical_type; - - use DataType::*; - match min.data_type().to_logical_type() { - Boolean => boolean::push(from, min, max), - Int8 => primitive::push(from, min, max, |x: i32| Ok(x as i8)), - Int16 => primitive::push(from, min, max, |x: i32| Ok(x as i16)), - Date32 | Time32(_) => primitive::push::(from, min, max, Ok), - Interval(IntervalUnit::YearMonth) => fixlen::push_year_month(from, min, max), - Interval(IntervalUnit::DayTime) => fixlen::push_days_ms(from, min, max), - UInt8 => primitive::push(from, min, max, |x: i32| Ok(x as u8)), - UInt16 => primitive::push(from, min, max, |x: i32| Ok(x as u16)), - UInt32 => match physical_type { - // some implementations of parquet write arrow's u32 into i64. - ParquetPhysicalType::Int64 => primitive::push(from, min, max, |x: i64| Ok(x as u32)), - ParquetPhysicalType::Int32 => primitive::push(from, min, max, |x: i32| Ok(x as u32)), - other => Err(Error::NotYetImplemented(format!( - "Can't decode UInt32 type from parquet type {other:?}" - ))), - }, - Int32 => primitive::push::(from, min, max, Ok), - Date64 => match physical_type { - ParquetPhysicalType::Int64 => primitive::push::(from, min, max, Ok), - // some implementations of parquet write arrow's date64 into i32. - ParquetPhysicalType::Int32 => { - primitive::push(from, min, max, |x: i32| Ok(x as i64 * 86400000)) - } - other => Err(Error::NotYetImplemented(format!( - "Can't decode Date64 type from parquet type {other:?}" - ))), - }, - Int64 | Time64(_) | Duration(_) => primitive::push::(from, min, max, Ok), - UInt64 => primitive::push(from, min, max, |x: i64| Ok(x as u64)), - Timestamp(time_unit, _) => { - let time_unit = *time_unit; - if physical_type == &ParquetPhysicalType::Int96 { - let from = from.map(|from| { - let from = from - .as_any() - .downcast_ref::>() - .unwrap(); - PrimitiveStatistics:: { - primitive_type: from.primitive_type.clone(), - null_count: from.null_count, - distinct_count: from.distinct_count, - min_value: from.min_value.map(int96_to_i64_ns), - max_value: from.max_value.map(int96_to_i64_ns), - } - }); - primitive::push( - from.as_ref().map(|x| x as &dyn ParquetStatistics), - min, - max, - |x: i64| { - Ok(primitive::timestamp( - type_.logical_type.as_ref(), - time_unit, - x, - )) - }, - ) - } else { - primitive::push(from, min, max, |x: i64| { - Ok(primitive::timestamp( - type_.logical_type.as_ref(), - time_unit, - x, - )) - }) - } - } - Float32 => primitive::push::(from, min, max, Ok), - Float64 => primitive::push::(from, min, max, Ok), - Decimal(_, _) => match physical_type { - ParquetPhysicalType::Int32 => primitive::push(from, min, max, |x: i32| Ok(x as i128)), - ParquetPhysicalType::Int64 => primitive::push(from, min, max, |x: i64| Ok(x as i128)), - ParquetPhysicalType::FixedLenByteArray(n) if *n > 16 => Err(Error::NotYetImplemented( - format!("Can't decode Decimal128 type from Fixed Size Byte Array of len {n:?}"), - )), - ParquetPhysicalType::FixedLenByteArray(n) => fixlen::push_i128(from, *n, min, max), - _ => unreachable!(), - }, - Decimal256(_, _) => match physical_type { - ParquetPhysicalType::Int32 => { - primitive::push(from, min, max, |x: i32| Ok(i256(I256::new(x.into())))) - } - ParquetPhysicalType::Int64 => { - primitive::push(from, min, max, |x: i64| Ok(i256(I256::new(x.into())))) - } - ParquetPhysicalType::FixedLenByteArray(n) if *n <= 16 => { - fixlen::push_i256_with_i128(from, *n, min, max) - } - ParquetPhysicalType::FixedLenByteArray(n) if *n > 32 => Err(Error::NotYetImplemented( - format!("Can't decode Decimal256 type from Fixed Size Byte Array of len {n:?}"), - )), - ParquetPhysicalType::FixedLenByteArray(_) => fixlen::push_i256(from, min, max), - _ => unreachable!(), - }, - Binary => binary::push::(from, min, max), - LargeBinary => binary::push::(from, min, max), - Utf8 => utf8::push::(from, min, max), - LargeUtf8 => utf8::push::(from, min, max), - FixedSizeBinary(_) => fixlen::push(from, min, max), - Null => null::push(min, max), - other => todo!("{:?}", other), - } -} - -/// Deserializes the statistics in the column chunks from all `row_groups` -/// into [`Statistics`] associated from `field`'s name. -/// -/// # Errors -/// This function errors if the deserialization of the statistics fails (e.g. invalid utf8) -pub fn deserialize(field: &Field, row_groups: &[RowGroupMetaData]) -> Result { - let mut statistics = MutableStatistics::try_new(field)?; - - // transpose - row_groups.iter().try_for_each(|group| { - let columns = get_field_columns(group.columns(), field.name.as_ref()); - let mut stats = columns - .into_iter() - .map(|column| { - Ok(( - column.statistics().transpose()?, - column.descriptor().descriptor.primitive_type.clone(), - )) - }) - .collect::, ParquetPrimitiveType)>>>()?; - push( - &mut stats, - statistics.min_value.as_mut(), - statistics.max_value.as_mut(), - statistics.distinct_count.as_mut(), - statistics.null_count.as_mut(), - ) - })?; - - Ok(statistics.into()) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/null.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/null.rs deleted file mode 100644 index f11fb971100c..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/null.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::*; -use crate::arrow::error::Result; - -pub(super) fn push(min: &mut dyn MutableArray, max: &mut dyn MutableArray) -> Result<()> { - let min = min.as_mut_any().downcast_mut::().unwrap(); - let max = max.as_mut_any().downcast_mut::().unwrap(); - min.push_null(); - max.push_null(); - - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/primitive.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/primitive.rs deleted file mode 100644 index 3d785d56bfab..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/primitive.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::schema::types::PrimitiveLogicalType; -use parquet2::schema::types::TimeUnit as ParquetTimeUnit; -use parquet2::statistics::PrimitiveStatistics; -use parquet2::statistics::Statistics as ParquetStatistics; -use parquet2::types::NativeType as ParquetNativeType; - -use crate::arrow::array::*; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; - -pub fn timestamp(logical_type: Option<&PrimitiveLogicalType>, time_unit: TimeUnit, x: i64) -> i64 { - let unit = if let Some(PrimitiveLogicalType::Timestamp { unit, .. }) = logical_type { - unit - } else { - return x; - }; - - match (unit, time_unit) { - (ParquetTimeUnit::Milliseconds, TimeUnit::Second) => x / 1_000, - (ParquetTimeUnit::Microseconds, TimeUnit::Second) => x / 1_000_000, - (ParquetTimeUnit::Nanoseconds, TimeUnit::Second) => x * 1_000_000_000, - - (ParquetTimeUnit::Milliseconds, TimeUnit::Millisecond) => x, - (ParquetTimeUnit::Microseconds, TimeUnit::Millisecond) => x / 1_000, - (ParquetTimeUnit::Nanoseconds, TimeUnit::Millisecond) => x / 1_000_000, - - (ParquetTimeUnit::Milliseconds, TimeUnit::Microsecond) => x * 1_000, - (ParquetTimeUnit::Microseconds, TimeUnit::Microsecond) => x, - (ParquetTimeUnit::Nanoseconds, TimeUnit::Microsecond) => x / 1_000, - - (ParquetTimeUnit::Milliseconds, TimeUnit::Nanosecond) => x * 1_000_000, - (ParquetTimeUnit::Microseconds, TimeUnit::Nanosecond) => x * 1_000, - (ParquetTimeUnit::Nanoseconds, TimeUnit::Nanosecond) => x, - } -} - -pub(super) fn push Result + Copy>( - from: Option<&dyn ParquetStatistics>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, - map: F, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::>().unwrap()); - min.push(from.and_then(|s| s.min_value.map(map)).transpose()?); - max.push(from.and_then(|s| s.max_value.map(map)).transpose()?); - - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/struct_.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/struct_.rs deleted file mode 100644 index 9a942a05e20e..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/struct_.rs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::make_mutable; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::StructArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; - -#[derive(Debug)] -pub struct DynMutableStructArray { - data_type: DataType, - pub inner: Vec>, -} - -impl DynMutableStructArray { - pub fn try_with_capacity(data_type: DataType, capacity: usize) -> Result { - let inners = match data_type.to_logical_type() { - DataType::Struct(inner) => inner, - _ => unreachable!(), - }; - let inner = inners - .iter() - .map(|f| make_mutable(f.data_type(), capacity)) - .collect::>>()?; - - Ok(Self { data_type, inner }) - } -} -impl MutableArray for DynMutableStructArray { - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn len(&self) -> usize { - self.inner[0].len() - } - - fn validity(&self) -> Option<&crate::arrow::bitmap::MutableBitmap> { - None - } - - fn as_box(&mut self) -> Box { - let inner = self.inner.iter_mut().map(|x| x.as_box()).collect(); - - Box::new(StructArray::new(self.data_type.clone(), inner, None)) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn push_null(&mut self) { - todo!() - } - - fn reserve(&mut self, _: usize) { - todo!(); - } - - fn shrink_to_fit(&mut self) { - todo!() - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/statistics/utf8.rs b/src/common/arrow/src/arrow/io/parquet/read/statistics/utf8.rs deleted file mode 100644 index c6b6e098c456..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/statistics/utf8.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::statistics::BinaryStatistics; -use parquet2::statistics::Statistics as ParquetStatistics; - -use crate::arrow::array::MutableArray; -use crate::arrow::array::MutableUtf8Array; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; - -pub(super) fn push( - from: Option<&dyn ParquetStatistics>, - min: &mut dyn MutableArray, - max: &mut dyn MutableArray, -) -> Result<()> { - let min = min - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let max = max - .as_mut_any() - .downcast_mut::>() - .unwrap(); - let from = from.map(|s| s.as_any().downcast_ref::().unwrap()); - - min.push( - from.and_then(|s| s.min_value.as_deref().map(simdutf8::basic::from_utf8)) - .transpose()?, - ); - max.push( - from.and_then(|s| s.max_value.as_deref().map(simdutf8::basic::from_utf8)) - .transpose()?, - ); - Ok(()) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/binary/basic.rs b/src/common/arrow/src/arrow/io/parquet/write/binary/basic.rs deleted file mode 100644 index fcf79c82aa4e..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/binary/basic.rs +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::delta_bitpacked; -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::serialize_statistics; -use parquet2::statistics::BinaryStatistics; -use parquet2::statistics::ParquetStatistics; -use parquet2::statistics::Statistics; - -use super::super::utils; -use super::super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::offset::Offset; - -pub(crate) fn encode_plain( - array: &BinaryArray, - is_optional: bool, - buffer: &mut Vec, -) { - // append the non-null values - if is_optional { - array.iter().for_each(|x| { - if let Some(x) = x { - // BYTE_ARRAY: first 4 bytes denote length in littleendian. - let len = (x.len() as u32).to_le_bytes(); - buffer.extend_from_slice(&len); - buffer.extend_from_slice(x); - } - }) - } else { - array.values_iter().for_each(|x| { - // BYTE_ARRAY: first 4 bytes denote length in littleendian. - let len = (x.len() as u32).to_le_bytes(); - buffer.extend_from_slice(&len); - buffer.extend_from_slice(x); - }) - } -} - -pub fn array_to_page( - array: &BinaryArray, - options: WriteOptions, - type_: PrimitiveType, - encoding: Encoding, -) -> Result { - let validity = array.validity(); - let is_optional = is_nullable(&type_.field_info); - - let mut buffer = vec![]; - utils::write_def_levels( - &mut buffer, - is_optional, - validity, - array.len(), - options.version, - )?; - - let definition_levels_byte_length = buffer.len(); - - match encoding { - Encoding::Plain => encode_plain(array, is_optional, &mut buffer), - Encoding::DeltaLengthByteArray => encode_delta( - array.values(), - array.offsets().buffer(), - array.validity(), - is_optional, - &mut buffer, - ), - _ => { - return Err(Error::InvalidArgumentError(format!( - "Datatype {:?} cannot be encoded by {:?} encoding", - array.data_type(), - encoding - ))); - } - } - - let statistics = if options.write_statistics { - Some(build_statistics(array, type_.clone())) - } else { - None - }; - - utils::build_plain_page( - buffer, - array.len(), - array.len(), - array.null_count(), - 0, - definition_levels_byte_length, - statistics, - type_, - options, - encoding, - ) -} - -pub(crate) fn build_statistics( - array: &BinaryArray, - primitive_type: PrimitiveType, -) -> ParquetStatistics { - let statistics = &BinaryStatistics { - primitive_type, - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array - .iter() - .flatten() - .max_by(|x, y| ord_binary(x, y)) - .map(|x| x.to_vec()), - min_value: array - .iter() - .flatten() - .min_by(|x, y| ord_binary(x, y)) - .map(|x| x.to_vec()), - } as &dyn Statistics; - serialize_statistics(statistics) -} - -pub(crate) fn encode_delta( - values: &[u8], - offsets: &[O], - validity: Option<&Bitmap>, - is_optional: bool, - buffer: &mut Vec, -) { - if is_optional { - if let Some(validity) = validity { - let lengths = offsets - .windows(2) - .map(|w| (w[1] - w[0]).to_usize() as i64) - .zip(validity.iter()) - .flat_map(|(x, is_valid)| if is_valid { Some(x) } else { None }); - let length = offsets.len() - 1 - validity.unset_bits(); - let lengths = utils::ExactSizedIter::new(lengths, length); - - delta_bitpacked::encode(lengths, buffer); - } else { - let lengths = offsets.windows(2).map(|w| (w[1] - w[0]).to_usize() as i64); - delta_bitpacked::encode(lengths, buffer); - } - } else { - let lengths = offsets.windows(2).map(|w| (w[1] - w[0]).to_usize() as i64); - delta_bitpacked::encode(lengths, buffer); - } - - buffer.extend_from_slice( - &values[offsets.first().unwrap().to_usize()..offsets.last().unwrap().to_usize()], - ) -} - -/// Returns the ordering of two binary values. This corresponds to pyarrows' ordering -/// of statistics. -pub(crate) fn ord_binary<'a>(a: &'a [u8], b: &'a [u8]) -> std::cmp::Ordering { - use std::cmp::Ordering::*; - match (a.is_empty(), b.is_empty()) { - (true, true) => return Equal, - (true, false) => return Less, - (false, true) => return Greater, - (false, false) => {} - } - - for (v1, v2) in a.iter().zip(b.iter()) { - match v1.cmp(v2) { - Equal => continue, - other => return other, - } - } - Equal -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/binary/mod.rs b/src/common/arrow/src/arrow/io/parquet/write/binary/mod.rs deleted file mode 100644 index 865f80c5466a..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/binary/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod nested; - -pub use basic::array_to_page; -pub(crate) use basic::build_statistics; -pub(super) use basic::encode_delta; -pub(crate) use basic::encode_plain; -pub(super) use basic::ord_binary; -pub use nested::array_to_page as nested_array_to_page; diff --git a/src/common/arrow/src/arrow/io/parquet/write/binary/nested.rs b/src/common/arrow/src/arrow/io/parquet/write/binary/nested.rs deleted file mode 100644 index 1fce75327abc..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/binary/nested.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; - -use super::super::nested; -use super::super::utils; -use super::super::WriteOptions; -use super::basic::build_statistics; -use super::basic::encode_plain; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::io::parquet::write::Nested; -use crate::arrow::offset::Offset; - -pub fn array_to_page( - array: &BinaryArray, - options: WriteOptions, - type_: PrimitiveType, - nested: &[Nested], -) -> Result -where - O: Offset, -{ - let is_optional = is_nullable(&type_.field_info); - - let mut buffer = vec![]; - let (repetition_levels_byte_length, definition_levels_byte_length) = - nested::write_rep_and_def(options.version, nested, &mut buffer)?; - - encode_plain(array, is_optional, &mut buffer); - - let statistics = if options.write_statistics { - Some(build_statistics(array, type_.clone())) - } else { - None - }; - - utils::build_plain_page( - buffer, - nested::num_values(nested), - nested[0].len(), - array.null_count(), - repetition_levels_byte_length, - definition_levels_byte_length, - statistics, - type_, - options, - Encoding::Plain, - ) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/binview/basic.rs b/src/common/arrow/src/arrow/io/parquet/write/binview/basic.rs deleted file mode 100644 index 57a697d1557c..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/binview/basic.rs +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::delta_bitpacked; -use parquet2::encoding::Encoding; -use parquet2::page::Page; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::serialize_statistics; -use parquet2::statistics::BinaryStatistics; -use parquet2::statistics::ParquetStatistics; -use parquet2::statistics::Statistics; - -use crate::arrow::array::Array; -use crate::arrow::array::BinaryViewArray; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::io::parquet::write::binary::ord_binary; -use crate::arrow::io::parquet::write::utils; -use crate::arrow::io::parquet::write::WriteOptions; - -pub(crate) fn encode_non_null_values<'a, I: Iterator>( - iter: I, - buffer: &mut Vec, -) { - iter.for_each(|x| { - // BYTE_ARRAY: first 4 bytes denote length in littleendian. - let len = (x.len() as u32).to_le_bytes(); - buffer.extend_from_slice(&len); - buffer.extend_from_slice(x); - }) -} - -pub(crate) fn encode_plain(array: &BinaryViewArray, buffer: &mut Vec) { - let capacity = - array.total_bytes_len() + (array.len() - array.null_count()) * std::mem::size_of::(); - - let len_before = buffer.len(); - buffer.reserve(capacity); - - encode_non_null_values(array.non_null_values_iter(), buffer); - // Append the non-null values. - debug_assert_eq!(buffer.len() - len_before, capacity); -} - -pub(crate) fn encode_delta(array: &BinaryViewArray, buffer: &mut Vec) { - let lengths = array.non_null_views_iter().map(|v| v.length as i64); - delta_bitpacked::encode(lengths, buffer); - - for slice in array.non_null_values_iter() { - buffer.extend_from_slice(slice) - } -} - -pub fn array_to_page( - array: &BinaryViewArray, - options: WriteOptions, - type_: PrimitiveType, - encoding: Encoding, -) -> Result { - let validity = array.validity(); - let is_optional = is_nullable(&type_.field_info); - - let mut buffer = vec![]; - utils::write_def_levels( - &mut buffer, - is_optional, - validity, - array.len(), - options.version, - )?; - - let definition_levels_byte_length = buffer.len(); - - match encoding { - Encoding::Plain => encode_plain(array, &mut buffer), - Encoding::DeltaLengthByteArray => encode_delta(array, &mut buffer), - _ => { - return Err(Error::oos(format!( - "Datatype {:?} cannot be encoded by {:?} encoding", - array.data_type(), - encoding - ))); - } - } - - let statistics = if options.write_statistics { - Some(build_statistics(array, type_.clone())) - } else { - None - }; - - utils::build_plain_page( - buffer, - array.len(), - array.len(), - array.null_count(), - 0, - definition_levels_byte_length, - statistics, - type_, - options, - encoding, - ) - .map(Page::Data) -} - -pub(crate) fn build_statistics( - array: &BinaryViewArray, - primitive_type: PrimitiveType, -) -> ParquetStatistics { - let statistics = &BinaryStatistics { - primitive_type, - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array - .iter() - .flatten() - .max_by(|x, y| ord_binary(x, y)) - .map(|x| x.to_vec()), - min_value: array - .iter() - .flatten() - .min_by(|x, y| ord_binary(x, y)) - .map(|x| x.to_vec()), - } as &dyn Statistics; - serialize_statistics(statistics) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/binview/mod.rs b/src/common/arrow/src/arrow/io/parquet/write/binview/mod.rs deleted file mode 100644 index a2acb98474d3..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/binview/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod nested; - -pub(crate) use basic::array_to_page; -pub(crate) use basic::build_statistics; -pub(crate) use basic::encode_plain; -pub use nested::array_to_page as nested_array_to_page; diff --git a/src/common/arrow/src/arrow/io/parquet/write/binview/nested.rs b/src/common/arrow/src/arrow/io/parquet/write/binview/nested.rs deleted file mode 100644 index ee855a4f9387..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/binview/nested.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; - -use super::super::nested; -use super::super::utils; -use super::super::WriteOptions; -use super::basic::build_statistics; -use super::basic::encode_plain; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryViewArray; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::write::Nested; - -pub fn array_to_page( - array: &BinaryViewArray, - options: WriteOptions, - type_: PrimitiveType, - nested: &[Nested], -) -> Result { - let mut buffer = vec![]; - let (repetition_levels_byte_length, definition_levels_byte_length) = - nested::write_rep_and_def(options.version, nested, &mut buffer)?; - - encode_plain(array, &mut buffer); - - let statistics = if options.write_statistics { - Some(build_statistics(array, type_.clone())) - } else { - None - }; - - utils::build_plain_page( - buffer, - nested::num_values(nested), - nested[0].len(), - array.null_count(), - repetition_levels_byte_length, - definition_levels_byte_length, - statistics, - type_, - options, - Encoding::Plain, - ) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/boolean/basic.rs b/src/common/arrow/src/arrow/io/parquet/write/boolean/basic.rs deleted file mode 100644 index 4cb26b44f641..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/boolean/basic.rs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::hybrid_rle::bitpacked_encode; -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::serialize_statistics; -use parquet2::statistics::BooleanStatistics; -use parquet2::statistics::ParquetStatistics; -use parquet2::statistics::Statistics; - -use super::super::utils; -use super::super::WriteOptions; -use crate::arrow::array::*; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; - -fn encode(iterator: impl Iterator, buffer: &mut Vec) -> Result<()> { - // encode values using bitpacking - let len = buffer.len(); - let mut buffer = std::io::Cursor::new(buffer); - buffer.set_position(len as u64); - Ok(bitpacked_encode(&mut buffer, iterator)?) -} - -pub(super) fn encode_plain( - array: &BooleanArray, - is_optional: bool, - buffer: &mut Vec, -) -> Result<()> { - if is_optional { - let iter = array.iter().flatten().take( - array - .validity() - .as_ref() - .map(|x| x.len() - x.unset_bits()) - .unwrap_or_else(|| array.len()), - ); - encode(iter, buffer) - } else { - let iter = array.values().iter(); - encode(iter, buffer) - } -} - -pub fn array_to_page( - array: &BooleanArray, - options: WriteOptions, - type_: PrimitiveType, -) -> Result { - let is_optional = is_nullable(&type_.field_info); - - let validity = array.validity(); - - let mut buffer = vec![]; - utils::write_def_levels( - &mut buffer, - is_optional, - validity, - array.len(), - options.version, - )?; - - let definition_levels_byte_length = buffer.len(); - - encode_plain(array, is_optional, &mut buffer)?; - - let statistics = if options.write_statistics { - Some(build_statistics(array)) - } else { - None - }; - - utils::build_plain_page( - buffer, - array.len(), - array.len(), - array.null_count(), - 0, - definition_levels_byte_length, - statistics, - type_, - options, - Encoding::Plain, - ) -} - -pub(super) fn build_statistics(array: &BooleanArray) -> ParquetStatistics { - let statistics = &BooleanStatistics { - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array.iter().flatten().max(), - min_value: array.iter().flatten().min(), - } as &dyn Statistics; - serialize_statistics(statistics) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/boolean/mod.rs b/src/common/arrow/src/arrow/io/parquet/write/boolean/mod.rs deleted file mode 100644 index 1d8773fe5a1b..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/boolean/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod nested; - -pub use basic::array_to_page; -pub use nested::array_to_page as nested_array_to_page; diff --git a/src/common/arrow/src/arrow/io/parquet/write/boolean/nested.rs b/src/common/arrow/src/arrow/io/parquet/write/boolean/nested.rs deleted file mode 100644 index 73cf57fe1cd7..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/boolean/nested.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; - -use super::super::nested; -use super::super::utils; -use super::super::WriteOptions; -use super::basic::build_statistics; -use super::basic::encode_plain; -use crate::arrow::array::Array; -use crate::arrow::array::BooleanArray; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::io::parquet::write::Nested; - -pub fn array_to_page( - array: &BooleanArray, - options: WriteOptions, - type_: PrimitiveType, - nested: &[Nested], -) -> Result { - let is_optional = is_nullable(&type_.field_info); - - let mut buffer = vec![]; - let (repetition_levels_byte_length, definition_levels_byte_length) = - nested::write_rep_and_def(options.version, nested, &mut buffer)?; - - encode_plain(array, is_optional, &mut buffer)?; - - let statistics = if options.write_statistics { - Some(build_statistics(array)) - } else { - None - }; - - utils::build_plain_page( - buffer, - nested::num_values(nested), - nested[0].len(), - array.null_count(), - repetition_levels_byte_length, - definition_levels_byte_length, - statistics, - type_, - options, - Encoding::Plain, - ) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/dictionary.rs b/src/common/arrow/src/arrow/io/parquet/write/dictionary.rs deleted file mode 100644 index 9054e8399c6d..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/dictionary.rs +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::hybrid_rle::encode_u32; -use parquet2::encoding::Encoding; -use parquet2::page::DictPage; -use parquet2::page::Page; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::serialize_statistics; -use parquet2::statistics::ParquetStatistics; -use parquet2::write::DynIter; - -use super::binary::build_statistics as binary_build_statistics; -use super::binary::encode_plain as binary_encode_plain; -use super::binview; -use super::fixed_len_bytes::build_statistics as fixed_binary_build_statistics; -use super::fixed_len_bytes::encode_plain as fixed_binary_encode_plain; -use super::nested; -use super::primitive::build_statistics as primitive_build_statistics; -use super::primitive::encode_plain as primitive_encode_plain; -use super::utf8::build_statistics as utf8_build_statistics; -use super::utf8::encode_plain as utf8_encode_plain; -use super::Nested; -use super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::Utf8ViewArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::io::parquet::write::slice_nested_leaf; -use crate::arrow::io::parquet::write::utils; - -fn serialize_def_levels_simple( - validity: Option<&Bitmap>, - length: usize, - is_optional: bool, - options: WriteOptions, - buffer: &mut Vec, -) -> Result<()> { - utils::write_def_levels(buffer, is_optional, validity, length, options.version) -} - -fn serialize_keys_values( - array: &DictionaryArray, - validity: Option<&Bitmap>, - buffer: &mut Vec, -) -> Result<()> { - let keys = array.keys_values_iter().map(|x| x as u32); - if let Some(validity) = validity { - // discard indices whose values are null. - let keys = keys - .zip(validity.iter()) - .filter_map(|(key, is_valid)| is_valid.then_some(key)); - let num_bits = utils::get_bit_width(keys.clone().max().unwrap_or(0) as u64); - - let keys = utils::ExactSizedIter::new(keys, array.len() - validity.unset_bits()); - - // num_bits as a single byte - buffer.push(num_bits as u8); - - // followed by the encoded indices. - Ok(encode_u32(buffer, keys, num_bits)?) - } else { - let num_bits = utils::get_bit_width(keys.clone().max().unwrap_or(0) as u64); - - // num_bits as a single byte - buffer.push(num_bits as u8); - - // followed by the encoded indices. - Ok(encode_u32(buffer, keys, num_bits)?) - } -} - -fn serialize_levels( - validity: Option<&Bitmap>, - length: usize, - type_: &PrimitiveType, - nested: &[Nested], - options: WriteOptions, - buffer: &mut Vec, -) -> Result<(usize, usize)> { - if nested.len() == 1 { - let is_optional = is_nullable(&type_.field_info); - serialize_def_levels_simple(validity, length, is_optional, options, buffer)?; - let definition_levels_byte_length = buffer.len(); - Ok((0, definition_levels_byte_length)) - } else { - nested::write_rep_and_def(options.version, nested, buffer) - } -} - -fn normalized_validity(array: &DictionaryArray) -> Option { - match (array.keys().validity(), array.values().validity()) { - (None, None) => None, - (None, rhs) => rhs.cloned(), - (lhs, None) => lhs.cloned(), - (Some(_), Some(rhs)) => { - let projected_validity = array - .keys_iter() - .map(|x| x.map(|x| rhs.get_bit(x)).unwrap_or(false)); - MutableBitmap::from_trusted_len_iter(projected_validity).into() - } - } -} - -fn serialize_keys( - array: &DictionaryArray, - type_: PrimitiveType, - nested: &[Nested], - statistics: Option, - options: WriteOptions, -) -> Result { - let mut buffer = vec![]; - - // parquet only accepts a single validity - we "&" the validities into a single one - // and ignore keys whole _value_ is null. - let validity = normalized_validity(array); - let (start, len) = slice_nested_leaf(nested); - - let mut nested = nested.to_vec(); - let array = array.clone().sliced(start, len); - if let Some(Nested::Primitive(_, _, c)) = nested.last_mut() { - *c = len; - } else { - unreachable!("") - } - - let (repetition_levels_byte_length, definition_levels_byte_length) = serialize_levels( - validity.as_ref(), - array.len(), - &type_, - &nested, - options, - &mut buffer, - )?; - - serialize_keys_values(&array, validity.as_ref(), &mut buffer)?; - - let (num_values, num_rows) = if nested.len() == 1 { - (array.len(), array.len()) - } else { - (nested::num_values(&nested), nested[0].len()) - }; - - utils::build_plain_page( - buffer, - num_values, - num_rows, - array.null_count(), - repetition_levels_byte_length, - definition_levels_byte_length, - statistics, - type_, - options, - Encoding::RleDictionary, - ) - .map(Page::Data) -} - -macro_rules! dyn_prim { - ($from:ty, $to:ty, $array:expr, $options:expr, $type_:expr) => {{ - let values = $array.values().as_any().downcast_ref().unwrap(); - - let buffer = primitive_encode_plain::<$from, $to>(values, false, vec![]); - - let stats: Option = if $options.write_statistics { - let mut stats = primitive_build_statistics::<$from, $to>(values, $type_.clone()); - stats.null_count = Some($array.null_count() as i64); - let stats = serialize_statistics(&stats); - Some(stats) - } else { - None - }; - (DictPage::new(buffer, values.len(), false), stats) - }}; -} - -pub fn array_to_pages( - array: &DictionaryArray, - type_: PrimitiveType, - nested: &[Nested], - options: WriteOptions, - encoding: Encoding, -) -> Result>> { - match encoding { - Encoding::PlainDictionary | Encoding::RleDictionary => { - // write DictPage - let (dict_page, statistics): (_, Option) = - match array.values().data_type().to_logical_type() { - DataType::Int8 => dyn_prim!(i8, i32, array, options, type_), - DataType::Int16 => dyn_prim!(i16, i32, array, options, type_), - DataType::Int32 | DataType::Date32 | DataType::Time32(_) => { - dyn_prim!(i32, i32, array, options, type_) - } - DataType::Int64 - | DataType::Date64 - | DataType::Time64(_) - | DataType::Timestamp(_, _) - | DataType::Duration(_) => dyn_prim!(i64, i64, array, options, type_), - DataType::UInt8 => dyn_prim!(u8, i32, array, options, type_), - DataType::UInt16 => dyn_prim!(u16, i32, array, options, type_), - DataType::UInt32 => dyn_prim!(u32, i32, array, options, type_), - DataType::UInt64 => dyn_prim!(u64, i64, array, options, type_), - DataType::Float32 => dyn_prim!(f32, f32, array, options, type_), - DataType::Float64 => dyn_prim!(f64, f64, array, options, type_), - DataType::Utf8 => { - let array = array.values().as_any().downcast_ref().unwrap(); - - let mut buffer = vec![]; - utf8_encode_plain::(array, false, &mut buffer); - let stats = if options.write_statistics { - Some(utf8_build_statistics(array, type_.clone())) - } else { - None - }; - (DictPage::new(buffer, array.len(), false), stats) - } - DataType::LargeUtf8 => { - let array = array.values().as_any().downcast_ref().unwrap(); - - let mut buffer = vec![]; - utf8_encode_plain::(array, false, &mut buffer); - let stats = if options.write_statistics { - Some(utf8_build_statistics(array, type_.clone())) - } else { - None - }; - (DictPage::new(buffer, array.len(), false), stats) - } - DataType::Binary => { - let array = array.values().as_any().downcast_ref().unwrap(); - - let mut buffer = vec![]; - binary_encode_plain::(array, false, &mut buffer); - let stats = if options.write_statistics { - Some(binary_build_statistics(array, type_.clone())) - } else { - None - }; - (DictPage::new(buffer, array.len(), false), stats) - } - DataType::LargeBinary => { - let values = array.values().as_any().downcast_ref().unwrap(); - - let mut buffer = vec![]; - binary_encode_plain::(values, false, &mut buffer); - let stats = if options.write_statistics { - let mut stats = binary_build_statistics(values, type_.clone()); - stats.null_count = Some(array.null_count() as i64); - Some(stats) - } else { - None - }; - (DictPage::new(buffer, values.len(), false), stats) - } - DataType::Utf8View => { - let array = array - .values() - .as_any() - .downcast_ref::() - .unwrap() - .to_binview(); - let mut buffer = vec![]; - binview::encode_plain(&array, &mut buffer); - - let stats = if options.write_statistics { - Some(binview::build_statistics(&array, type_.clone())) - } else { - None - }; - (DictPage::new(buffer, array.len(), false), stats) - } - DataType::FixedSizeBinary(_) => { - let mut buffer = vec![]; - let array = array.values().as_any().downcast_ref().unwrap(); - fixed_binary_encode_plain(array, false, &mut buffer); - let stats = if options.write_statistics { - let mut stats = fixed_binary_build_statistics(array, type_.clone()); - stats.null_count = Some(array.null_count() as i64); - Some(serialize_statistics(&stats)) - } else { - None - }; - (DictPage::new(buffer, array.len(), false), stats) - } - other => { - return Err(Error::NotYetImplemented(format!( - "Writing dictionary arrays to parquet only support data type {other:?}" - ))); - } - }; - let dict_page = Page::Dict(dict_page); - - // write DataPage pointing to DictPage - let data_page = serialize_keys(array, type_, nested, statistics, options)?; - - let iter = std::iter::once(Ok(dict_page)).chain(std::iter::once(Ok(data_page))); - Ok(DynIter::new(Box::new(iter))) - } - _ => Err(Error::NotYetImplemented( - "Dictionary arrays only support dictionary encoding".to_string(), - )), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/file.rs b/src/common/arrow/src/arrow/io/parquet/write/file.rs deleted file mode 100644 index d04cbbe74506..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/file.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -use parquet2::metadata::KeyValue; -use parquet2::metadata::SchemaDescriptor; -use parquet2::write::RowGroupIter; -use parquet2::write::WriteOptions as FileWriteOptions; - -use super::schema::schema_to_metadata_key; -use super::to_parquet_schema; -use super::ThriftFileMetaData; -use super::WriteOptions; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// Attaches [`Schema`] to `key_value_metadata` -pub fn add_arrow_schema( - schema: &Schema, - key_value_metadata: Option>, -) -> Option> { - key_value_metadata - .map(|mut x| { - x.push(schema_to_metadata_key(schema)); - x - }) - .or_else(|| Some(vec![schema_to_metadata_key(schema)])) -} - -/// An interface to write a parquet to a [`Write`] -pub struct FileWriter { - writer: parquet2::write::FileWriter, - schema: Schema, - options: WriteOptions, -} - -// Accessors -impl FileWriter { - /// The options assigned to the file - pub fn options(&self) -> WriteOptions { - self.options - } - - /// The [`SchemaDescriptor`] assigned to this file - pub fn parquet_schema(&self) -> &SchemaDescriptor { - self.writer.schema() - } - - /// The [`Schema`] assigned to this file - pub fn schema(&self) -> &Schema { - &self.schema - } -} - -impl FileWriter { - /// Returns a new [`FileWriter`]. - /// # Error - /// If it is unable to derive a parquet schema from [`Schema`]. - pub fn try_new(writer: W, schema: Schema, options: WriteOptions) -> Result { - let parquet_schema = to_parquet_schema(&schema)?; - - let created_by = Some("Arrow2 - Native Rust implementation of Arrow".to_string()); - - Ok(Self { - writer: parquet2::write::FileWriter::new( - writer, - parquet_schema, - FileWriteOptions { - version: options.version, - write_statistics: options.write_statistics, - }, - created_by, - ), - schema, - options, - }) - } - - /// Writes a row group to the file. - pub fn write(&mut self, row_group: RowGroupIter<'_, Error>) -> Result<()> { - Ok(self.writer.write(row_group)?) - } - - /// Writes the footer of the parquet file. Returns the total size of the file. - pub fn end(&mut self, key_value_metadata: Option>) -> Result { - let key_value_metadata = add_arrow_schema(&self.schema, key_value_metadata); - Ok(self.writer.end(key_value_metadata)?) - } - - /// Consumes this writer and returns the inner writer - pub fn into_inner(self) -> W { - self.writer.into_inner() - } - - /// Returns the underlying writer and [`ThriftFileMetaData`] - /// # Panics - /// This function panics if [`Self::end`] has not yet been called - pub fn into_inner_and_metadata(self) -> (W, ThriftFileMetaData) { - self.writer.into_inner_and_metadata() - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/fixed_len_bytes.rs b/src/common/arrow/src/arrow/io/parquet/write/fixed_len_bytes.rs deleted file mode 100644 index 864f90b38b4e..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/fixed_len_bytes.rs +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::serialize_statistics; -use parquet2::statistics::FixedLenStatistics; - -use super::binary::ord_binary; -use super::utils; -use super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::array::FixedSizeBinaryArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::io::parquet::write::nested; -use crate::arrow::io::parquet::write::Nested; -use crate::arrow::types::i256; - -pub(crate) fn encode_plain(array: &FixedSizeBinaryArray, is_optional: bool, buffer: &mut Vec) { - // append the non-null values - if is_optional { - array.iter().for_each(|x| { - if let Some(x) = x { - buffer.extend_from_slice(x); - } - }) - } else { - buffer.extend_from_slice(array.values()); - } -} - -pub fn array_to_page( - array: &FixedSizeBinaryArray, - options: WriteOptions, - type_: PrimitiveType, - statistics: Option, -) -> Result { - let is_optional = is_nullable(&type_.field_info); - let validity = array.validity(); - - let mut buffer = vec![]; - utils::write_def_levels( - &mut buffer, - is_optional, - validity, - array.len(), - options.version, - )?; - - let definition_levels_byte_length = buffer.len(); - - encode_plain(array, is_optional, &mut buffer); - - utils::build_plain_page( - buffer, - array.len(), - array.len(), - array.null_count(), - 0, - definition_levels_byte_length, - statistics.map(|x| serialize_statistics(&x)), - type_, - options, - Encoding::Plain, - ) -} - -pub fn nested_array_to_page( - array: &FixedSizeBinaryArray, - options: WriteOptions, - type_: PrimitiveType, - statistics: Option, - nested: &[Nested], -) -> Result { - let is_optional = is_nullable(&type_.field_info); - - let mut buffer = vec![]; - let (repetition_levels_byte_length, definition_levels_byte_length) = - nested::write_rep_and_def(options.version, nested, &mut buffer)?; - - encode_plain(array, is_optional, &mut buffer); - - utils::build_plain_page( - buffer, - nested::num_values(nested), - nested[0].len(), - array.null_count(), - repetition_levels_byte_length, - definition_levels_byte_length, - statistics.map(|x| serialize_statistics(&x)), - type_, - options, - Encoding::Plain, - ) -} - -pub fn build_statistics( - array: &FixedSizeBinaryArray, - primitive_type: PrimitiveType, -) -> FixedLenStatistics { - FixedLenStatistics { - primitive_type, - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array - .iter() - .flatten() - .max_by(|x, y| ord_binary(x, y)) - .map(|x| x.to_vec()), - min_value: array - .iter() - .flatten() - .min_by(|x, y| ord_binary(x, y)) - .map(|x| x.to_vec()), - } -} - -pub(super) fn build_statistics_decimal( - array: &PrimitiveArray, - primitive_type: PrimitiveType, - size: usize, -) -> FixedLenStatistics { - FixedLenStatistics { - primitive_type, - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array - .iter() - .flatten() - .max() - .map(|x| x.to_be_bytes()[16 - size..].to_vec()), - min_value: array - .iter() - .flatten() - .min() - .map(|x| x.to_be_bytes()[16 - size..].to_vec()), - } -} - -pub(super) fn build_statistics_decimal256_with_i128( - array: &PrimitiveArray, - primitive_type: PrimitiveType, - size: usize, -) -> FixedLenStatistics { - FixedLenStatistics { - primitive_type, - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array - .iter() - .flatten() - .max() - .map(|x| x.0.low().to_be_bytes()[16 - size..].to_vec()), - min_value: array - .iter() - .flatten() - .min() - .map(|x| x.0.low().to_be_bytes()[16 - size..].to_vec()), - } -} - -pub(super) fn build_statistics_decimal256( - array: &PrimitiveArray, - primitive_type: PrimitiveType, - size: usize, -) -> FixedLenStatistics { - FixedLenStatistics { - primitive_type, - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array - .iter() - .flatten() - .max() - .map(|x| x.0.to_be_bytes()[32 - size..].to_vec()), - min_value: array - .iter() - .flatten() - .min() - .map(|x| x.0.to_be_bytes()[32 - size..].to_vec()), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/mod.rs b/src/common/arrow/src/arrow/io/parquet/write/mod.rs deleted file mode 100644 index 8cb70eedf3d1..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/mod.rs +++ /dev/null @@ -1,951 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! APIs to write to Parquet format. -//! -//! # Arrow/Parquet Interoperability -//! As of [parquet-format v2.9](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md) -//! there are Arrow [DataTypes](crate::datatypes::DataType) which do not have a parquet -//! representation. These include but are not limited to: -//! * `DataType::Timestamp(TimeUnit::Second, _)` -//! * `DataType::Int64` -//! * `DataType::Duration` -//! * `DataType::Date64` -//! * `DataType::Time32(TimeUnit::Second)` -//! -//! The use of these arrow types will result in no logical type being stored within a parquet file. - -mod binary; -mod binview; -mod boolean; -mod dictionary; -mod file; -mod fixed_len_bytes; -mod nested; -mod pages; -mod primitive; -mod row_group; -mod schema; -#[cfg(feature = "io_parquet_async")] -mod sink; -mod utf8; -mod utils; - -pub use nested::num_values; -pub use nested::write_rep_and_def; -pub use pages::to_leaves; -pub use pages::to_nested; -pub use pages::to_parquet_leaves; -pub use parquet2::compression::BrotliLevel; -pub use parquet2::compression::CompressionOptions; -pub use parquet2::compression::GzipLevel; -pub use parquet2::compression::ZstdLevel; -pub use parquet2::encoding::Encoding; -pub use parquet2::fallible_streaming_iterator; -pub use parquet2::metadata::Descriptor; -pub use parquet2::metadata::FileMetaData; -pub use parquet2::metadata::KeyValue; -pub use parquet2::metadata::SchemaDescriptor; -pub use parquet2::metadata::ThriftFileMetaData; -pub use parquet2::page::CompressedDataPage; -pub use parquet2::page::CompressedPage; -pub use parquet2::page::Page; -pub use parquet2::schema::types::FieldInfo; -pub use parquet2::schema::types::ParquetType; -pub use parquet2::schema::types::PhysicalType as ParquetPhysicalType; -use parquet2::schema::types::PrimitiveType as ParquetPrimitiveType; -pub use parquet2::write::compress; -pub use parquet2::write::write_metadata_sidecar; -pub use parquet2::write::Compressor; -pub use parquet2::write::DynIter; -pub use parquet2::write::DynStreamingIterator; -pub use parquet2::write::RowGroupIter; -pub use parquet2::write::Version; -pub use parquet2::FallibleStreamingIterator; -pub use utils::write_def_levels; - -use crate::arrow::array::*; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::days_ms; -use crate::arrow::types::i256; -use crate::arrow::types::NativeType; - -/// Currently supported options to write to parquet -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct WriteOptions { - /// Whether to write statistics - pub write_statistics: bool, - /// The page and file version to use - pub version: Version, - /// The compression to apply to every page - pub compression: CompressionOptions, - /// The size to flush a page, defaults to 1024 * 1024 if None - pub data_pagesize_limit: Option, -} - -pub use file::FileWriter; -pub use pages::array_to_columns; -pub use pages::Nested; -pub use row_group::row_group_iter; -pub use row_group::RowGroupIterator; -pub use schema::to_parquet_type; -#[cfg(feature = "io_parquet_async")] -#[cfg_attr(docsrs, doc(cfg(feature = "io_parquet_async")))] -pub use sink::FileSink; - -use crate::arrow; -use crate::arrow::compute::aggregate::estimated_bytes_size; - -/// returns offset and length to slice the leaf values -pub fn slice_nested_leaf(nested: &[Nested]) -> (usize, usize) { - // find the deepest recursive dremel structure as that one determines how many values we must - // take - let mut out = (0, 0); - for nested in nested.iter().rev() { - match nested { - Nested::LargeList(l_nested) => { - let start = *l_nested.offsets.first(); - let end = *l_nested.offsets.last(); - return (start as usize, (end - start) as usize); - } - Nested::List(l_nested) => { - let start = *l_nested.offsets.first(); - let end = *l_nested.offsets.last(); - return (start as usize, (end - start) as usize); - } - Nested::Primitive(_, _, len) => out = (0, *len), - _ => {} - } - } - out -} - -fn decimal_length_from_precision(precision: usize) -> usize { - // digits = floor(log_10(2^(8*n - 1) - 1)) - // ceil(digits) = log10(2^(8*n - 1) - 1) - // 10^ceil(digits) = 2^(8*n - 1) - 1 - // 10^ceil(digits) + 1 = 2^(8*n - 1) - // log2(10^ceil(digits) + 1) = (8*n - 1) - // log2(10^ceil(digits) + 1) + 1 = 8*n - // (log2(10^ceil(a) + 1) + 1) / 8 = n - (((10.0_f64.powi(precision as i32) + 1.0).log2() + 1.0) / 8.0).ceil() as usize -} - -/// Creates a parquet [`SchemaDescriptor`] from a [`Schema`]. -pub fn to_parquet_schema(schema: &Schema) -> Result { - let parquet_types = schema - .fields - .iter() - .map(to_parquet_type) - .collect::>>()?; - Ok(SchemaDescriptor::new("root".to_string(), parquet_types)) -} - -/// Checks whether the `data_type` can be encoded as `encoding`. -/// Note that this is whether this implementation supports it, which is a subset of -/// what the parquet spec allows. -pub fn can_encode(data_type: &DataType, encoding: Encoding) -> bool { - if let (Encoding::DeltaBinaryPacked, DataType::Decimal(p, _)) = - (encoding, data_type.to_logical_type()) - { - return *p <= 18; - }; - - matches!( - (encoding, data_type.to_logical_type()), - (Encoding::Plain, _) - | ( - Encoding::DeltaLengthByteArray, - DataType::Binary | DataType::LargeBinary | DataType::Utf8 | DataType::LargeUtf8, - ) - | (Encoding::RleDictionary, DataType::Dictionary(_, _, _)) - | (Encoding::PlainDictionary, DataType::Dictionary(_, _, _)) - | ( - Encoding::DeltaBinaryPacked, - DataType::Null - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::Date32 - | DataType::Time32(_) - | DataType::Int64 - | DataType::Date64 - | DataType::Time64(_) - | DataType::Timestamp(_, _) - | DataType::Duration(_) - ) - ) -} - -/// Slices the [`Array`] to `Box` and `Vec`. -pub fn slice_parquet_array( - primitive_array: &mut dyn Array, - nested: &mut [Nested], - mut current_offset: usize, - mut current_length: usize, -) { - for nested in nested.iter_mut() { - match nested { - Nested::LargeList(l_nested) => { - l_nested.offsets.slice(current_offset, current_length + 1); - if let Some(validity) = l_nested.validity.as_mut() { - validity.slice(current_offset, current_length) - }; - - current_length = l_nested.offsets.range() as usize; - current_offset = *l_nested.offsets.first() as usize; - } - Nested::List(l_nested) => { - l_nested.offsets.slice(current_offset, current_length + 1); - if let Some(validity) = l_nested.validity.as_mut() { - validity.slice(current_offset, current_length) - }; - - current_length = l_nested.offsets.range() as usize; - current_offset = *l_nested.offsets.first() as usize; - } - Nested::Struct(validity, _, length) => { - *length = current_length; - if let Some(validity) = validity.as_mut() { - validity.slice(current_offset, current_length) - }; - } - Nested::Primitive(validity, _, length) => { - *length = current_length; - if let Some(validity) = validity.as_mut() { - validity.slice(current_offset, current_length) - }; - primitive_array.slice(current_offset, current_length); - } - } - } -} - -/// Get the length of [`Array`] that should be sliced. -pub fn get_max_length(nested: &[Nested]) -> usize { - let mut length = 0; - for nested in nested.iter() { - match nested { - Nested::LargeList(l_nested) => length += l_nested.offsets.range() as usize, - Nested::List(l_nested) => length += l_nested.offsets.range() as usize, - _ => {} - } - } - length -} - -/// Returns an iterator of [`Page`]. -pub fn array_to_pages( - primitive_array: &dyn Array, - type_: ParquetPrimitiveType, - nested: &[Nested], - options: WriteOptions, - encoding: Encoding, -) -> Result>> { - if let DataType::Dictionary(key_type, _, _) = primitive_array.data_type().to_logical_type() { - return match_integer_type!(key_type, |$T| { - dictionary::array_to_pages::<$T>( - primitive_array.as_any().downcast_ref().unwrap(), - type_, - &nested, - options, - encoding, - ) - }); - }; - - let nested = nested.to_vec(); - let primitive_array = primitive_array.to_boxed(); - - let number_of_rows = nested[0].len(); - - // note: this is not correct if the array is sliced - the estimation should happen on the - // primitive after sliced for parquet - let byte_size = estimated_bytes_size(primitive_array.as_ref()); - - const DEFAULT_PAGE_SIZE: usize = 1024 * 1024; - let max_page_size = options.data_pagesize_limit.unwrap_or(DEFAULT_PAGE_SIZE); - let max_page_size = max_page_size.min(2usize.pow(31) - 2usize.pow(25)); // allowed maximum page size - let bytes_per_row = if number_of_rows == 0 { - 0 - } else { - ((byte_size as f64) / (number_of_rows as f64)) as usize - }; - let rows_per_page = (max_page_size / (bytes_per_row + 1)).max(1); - - let pages = (0..number_of_rows) - .step_by(rows_per_page) - .map(move |offset| { - let length = if offset + rows_per_page > number_of_rows { - number_of_rows - offset - } else { - rows_per_page - }; - - let mut right_array = primitive_array.clone(); - let mut right_nested = nested.clone(); - slice_parquet_array(right_array.as_mut(), &mut right_nested, offset, length); - - array_to_page( - right_array.as_ref(), - type_.clone(), - &right_nested, - options, - encoding, - ) - }); - - Ok(DynIter::new(pages)) -} - -/// Converts an [`Array`] to a [`CompressedPage`] based on options, descriptor and `encoding`. -pub fn array_to_page( - array: &dyn Array, - type_: ParquetPrimitiveType, - nested: &[Nested], - options: WriteOptions, - encoding: Encoding, -) -> Result { - if nested.len() == 1 { - // special case where validity == def levels - return array_to_page_simple(array, type_, options, encoding); - } - array_to_page_nested(array, type_, nested, options, encoding) -} - -/// Converts an [`Array`] to a [`CompressedPage`] based on options, descriptor and `encoding`. -pub fn array_to_page_simple( - array: &dyn Array, - type_: ParquetPrimitiveType, - options: WriteOptions, - encoding: Encoding, -) -> Result { - let data_type = array.data_type(); - if !can_encode(data_type, encoding) { - return Err(Error::InvalidArgumentError(format!( - "The datatype {data_type:?} cannot be encoded by {encoding:?}" - ))); - } - - match data_type.to_logical_type() { - DataType::Boolean => { - boolean::array_to_page(array.as_any().downcast_ref().unwrap(), options, type_) - } - // casts below MUST match the casts done at the metadata (field -> parquet type). - DataType::UInt8 => primitive::array_to_page_integer::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::UInt16 => primitive::array_to_page_integer::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::UInt32 => primitive::array_to_page_integer::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::UInt64 => primitive::array_to_page_integer::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::Int8 => primitive::array_to_page_integer::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::Int16 => primitive::array_to_page_integer::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::Int32 | DataType::Date32 | DataType::Time32(_) => { - primitive::array_to_page_integer::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ) - } - DataType::Int64 - | DataType::Date64 - | DataType::Time64(_) - | DataType::Timestamp(_, _) - | DataType::Duration(_) => primitive::array_to_page_integer::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::Float32 => primitive::array_to_page_plain::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - ), - DataType::Float64 => primitive::array_to_page_plain::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - ), - DataType::Utf8 => utf8::array_to_page::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::LargeUtf8 => utf8::array_to_page::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::Binary => binary::array_to_page::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::LargeBinary => binary::array_to_page::( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ), - DataType::BinaryView => { - return binview::array_to_page( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ); - } - DataType::Utf8View => { - let array = - arrow::compute::cast::cast(array, &DataType::BinaryView, Default::default()) - .unwrap(); - return binview::array_to_page( - array.as_any().downcast_ref().unwrap(), - options, - type_, - encoding, - ); - } - DataType::Null => { - let array = Int32Array::new_null(DataType::Int32, array.len()); - primitive::array_to_page_plain::(&array, options, type_) - } - DataType::Interval(IntervalUnit::YearMonth) => { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let mut values = Vec::::with_capacity(12 * array.len()); - array.values().iter().for_each(|x| { - let bytes = &x.to_le_bytes(); - values.extend_from_slice(bytes); - values.extend_from_slice(&[0; 8]); - }); - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(12), - values.into(), - array.validity().cloned(), - ); - let statistics = if options.write_statistics { - Some(fixed_len_bytes::build_statistics(&array, type_.clone())) - } else { - None - }; - fixed_len_bytes::array_to_page(&array, options, type_, statistics) - } - DataType::Interval(IntervalUnit::DayTime) => { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let mut values = Vec::::with_capacity(12 * array.len()); - array.values().iter().for_each(|x| { - let bytes = &x.to_le_bytes(); - values.extend_from_slice(&[0; 4]); // months - values.extend_from_slice(bytes); // days and seconds - }); - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(12), - values.into(), - array.validity().cloned(), - ); - let statistics = if options.write_statistics { - Some(fixed_len_bytes::build_statistics(&array, type_.clone())) - } else { - None - }; - fixed_len_bytes::array_to_page(&array, options, type_, statistics) - } - DataType::FixedSizeBinary(_) => { - let array = array.as_any().downcast_ref().unwrap(); - let statistics = if options.write_statistics { - Some(fixed_len_bytes::build_statistics(array, type_.clone())) - } else { - None - }; - - fixed_len_bytes::array_to_page(array, options, type_, statistics) - } - DataType::Decimal256(precision, _) => { - let precision = *precision; - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - if precision <= 9 { - let values = array - .values() - .iter() - .map(|x| x.0.as_i32()) - .collect::>() - .into(); - - let array = - PrimitiveArray::::new(DataType::Int32, values, array.validity().cloned()); - primitive::array_to_page_integer::(&array, options, type_, encoding) - } else if precision <= 18 { - let values = array - .values() - .iter() - .map(|x| x.0.as_i64()) - .collect::>() - .into(); - - let array = - PrimitiveArray::::new(DataType::Int64, values, array.validity().cloned()); - primitive::array_to_page_integer::(&array, options, type_, encoding) - } else if precision <= 38 { - let size = decimal_length_from_precision(precision); - let statistics = if options.write_statistics { - let stats = fixed_len_bytes::build_statistics_decimal256_with_i128( - array, - type_.clone(), - size, - ); - Some(stats) - } else { - None - }; - - let mut values = Vec::::with_capacity(size * array.len()); - array.values().iter().for_each(|x| { - let bytes = &x.0.low().to_be_bytes()[16 - size..]; - values.extend_from_slice(bytes) - }); - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(size), - values.into(), - array.validity().cloned(), - ); - fixed_len_bytes::array_to_page(&array, options, type_, statistics) - } else { - let size = 32; - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let statistics = if options.write_statistics { - let stats = - fixed_len_bytes::build_statistics_decimal256(array, type_.clone(), size); - Some(stats) - } else { - None - }; - let mut values = Vec::::with_capacity(size * array.len()); - array.values().iter().for_each(|x| { - let bytes = &x.to_be_bytes(); - values.extend_from_slice(bytes) - }); - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(size), - values.into(), - array.validity().cloned(), - ); - - fixed_len_bytes::array_to_page(&array, options, type_, statistics) - } - } - DataType::Decimal(precision, _) => { - let precision = *precision; - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - if precision <= 9 { - let values = array - .values() - .iter() - .map(|x| *x as i32) - .collect::>() - .into(); - - let array = - PrimitiveArray::::new(DataType::Int32, values, array.validity().cloned()); - primitive::array_to_page_integer::(&array, options, type_, encoding) - } else if precision <= 18 { - let values = array - .values() - .iter() - .map(|x| *x as i64) - .collect::>() - .into(); - - let array = - PrimitiveArray::::new(DataType::Int64, values, array.validity().cloned()); - primitive::array_to_page_integer::(&array, options, type_, encoding) - } else { - let size = decimal_length_from_precision(precision); - - let statistics = if options.write_statistics { - let stats = - fixed_len_bytes::build_statistics_decimal(array, type_.clone(), size); - Some(stats) - } else { - None - }; - - let mut values = Vec::::with_capacity(size * array.len()); - array.values().iter().for_each(|x| { - let bytes = &x.to_be_bytes()[16 - size..]; - values.extend_from_slice(bytes) - }); - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(size), - values.into(), - array.validity().cloned(), - ); - fixed_len_bytes::array_to_page(&array, options, type_, statistics) - } - } - other => Err(Error::NotYetImplemented(format!( - "Writing parquet pages for data type {other:?}" - ))), - } - .map(Page::Data) -} - -fn array_to_page_nested( - array: &dyn Array, - type_: ParquetPrimitiveType, - nested: &[Nested], - options: WriteOptions, - _encoding: Encoding, -) -> Result { - use DataType::*; - match array.data_type().to_logical_type() { - Null => { - let array = Int32Array::new_null(DataType::Int32, array.len()); - primitive::nested_array_to_page::(&array, options, type_, nested) - } - Boolean => { - let array = array.as_any().downcast_ref().unwrap(); - boolean::nested_array_to_page(array, options, type_, nested) - } - Utf8 => { - let array = array.as_any().downcast_ref().unwrap(); - utf8::nested_array_to_page::(array, options, type_, nested) - } - LargeUtf8 => { - let array = array.as_any().downcast_ref().unwrap(); - utf8::nested_array_to_page::(array, options, type_, nested) - } - Binary => { - let array = array.as_any().downcast_ref().unwrap(); - binary::nested_array_to_page::(array, options, type_, nested) - } - LargeBinary => { - let array = array.as_any().downcast_ref().unwrap(); - binary::nested_array_to_page::(array, options, type_, nested) - } - BinaryView => { - let array = array.as_any().downcast_ref().unwrap(); - binview::nested_array_to_page(array, options, type_, nested) - } - Utf8View => { - let array = arrow::compute::cast::cast(array, &BinaryView, Default::default()).unwrap(); - let array = array.as_any().downcast_ref().unwrap(); - binview::nested_array_to_page(array, options, type_, nested) - } - UInt8 => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - UInt16 => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - UInt32 => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - UInt64 => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - Int8 => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - Int16 => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - Int32 | Date32 | Time32(_) => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - Int64 | Date64 | Time64(_) | Timestamp(_, _) | Duration(_) => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - Float32 => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - Float64 => { - let array = array.as_any().downcast_ref().unwrap(); - primitive::nested_array_to_page::(array, options, type_, nested) - } - Decimal(precision, _) => { - let precision = *precision; - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - if precision <= 9 { - let values = array - .values() - .iter() - .map(|x| *x as i32) - .collect::>() - .into(); - - let array = - PrimitiveArray::::new(DataType::Int32, values, array.validity().cloned()); - primitive::nested_array_to_page::(&array, options, type_, nested) - } else if precision <= 18 { - let values = array - .values() - .iter() - .map(|x| *x as i64) - .collect::>() - .into(); - - let array = - PrimitiveArray::::new(DataType::Int64, values, array.validity().cloned()); - primitive::nested_array_to_page::(&array, options, type_, nested) - } else { - let size = decimal_length_from_precision(precision); - - let statistics = if options.write_statistics { - let stats = - fixed_len_bytes::build_statistics_decimal(array, type_.clone(), size); - Some(stats) - } else { - None - }; - - let mut values = Vec::::with_capacity(size * array.len()); - array.values().iter().for_each(|x| { - let bytes = &x.to_be_bytes()[16 - size..]; - values.extend_from_slice(bytes) - }); - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(size), - values.into(), - array.validity().cloned(), - ); - fixed_len_bytes::nested_array_to_page(&array, options, type_, statistics, nested) - } - } - Decimal256(precision, _) => { - let precision = *precision; - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - if precision <= 9 { - let values = array - .values() - .iter() - .map(|x| x.0.as_i32()) - .collect::>() - .into(); - - let array = - PrimitiveArray::::new(DataType::Int32, values, array.validity().cloned()); - primitive::nested_array_to_page::(&array, options, type_, nested) - } else if precision <= 18 { - let values = array - .values() - .iter() - .map(|x| x.0.as_i64()) - .collect::>() - .into(); - - let array = - PrimitiveArray::::new(DataType::Int64, values, array.validity().cloned()); - primitive::nested_array_to_page::(&array, options, type_, nested) - } else if precision <= 38 { - let size = decimal_length_from_precision(precision); - let statistics = if options.write_statistics { - let stats = fixed_len_bytes::build_statistics_decimal256_with_i128( - array, - type_.clone(), - size, - ); - Some(stats) - } else { - None - }; - - let mut values = Vec::::with_capacity(size * array.len()); - array.values().iter().for_each(|x| { - let bytes = &x.0.low().to_be_bytes()[16 - size..]; - values.extend_from_slice(bytes) - }); - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(size), - values.into(), - array.validity().cloned(), - ); - fixed_len_bytes::nested_array_to_page(&array, options, type_, statistics, nested) - } else { - let size = 32; - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let statistics = if options.write_statistics { - let stats = - fixed_len_bytes::build_statistics_decimal256(array, type_.clone(), size); - Some(stats) - } else { - None - }; - let mut values = Vec::::with_capacity(size * array.len()); - array.values().iter().for_each(|x| { - let bytes = &x.to_be_bytes(); - values.extend_from_slice(bytes) - }); - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(size), - values.into(), - array.validity().cloned(), - ); - - fixed_len_bytes::nested_array_to_page(&array, options, type_, statistics, nested) - } - } - other => Err(Error::NotYetImplemented(format!( - "Writing nested parquet pages for data type {other:?}" - ))), - } - .map(Page::Data) -} - -fn transverse_recursive T + Clone>( - data_type: &DataType, - map: F, - encodings: &mut Vec, -) { - use crate::arrow::datatypes::PhysicalType::*; - match data_type.to_physical_type() { - Null | Boolean | Primitive(_) | Binary | FixedSizeBinary | LargeBinary | Utf8 - | Dictionary(_) | LargeUtf8 | BinaryView | Utf8View => encodings.push(map(data_type)), - List | FixedSizeList | LargeList => { - let a = data_type.to_logical_type(); - if let DataType::List(inner) = a { - transverse_recursive(&inner.data_type, map, encodings) - } else if let DataType::LargeList(inner) = a { - transverse_recursive(&inner.data_type, map, encodings) - } else if let DataType::FixedSizeList(inner, _) = a { - transverse_recursive(&inner.data_type, map, encodings) - } else { - unreachable!() - } - } - Struct => { - if let DataType::Struct(fields) = data_type.to_logical_type() { - for field in fields { - transverse_recursive(&field.data_type, map.clone(), encodings) - } - } else { - unreachable!() - } - } - Map => { - if let DataType::Map(field, _) = data_type.to_logical_type() { - if let DataType::Struct(fields) = field.data_type.to_logical_type() { - for field in fields { - transverse_recursive(&field.data_type, map.clone(), encodings) - } - } else { - unreachable!() - } - } else { - unreachable!() - } - } - Union => todo!(), - } -} - -/// Transverses the `data_type` up to its (parquet) columns and returns a vector of -/// items based on `map`. -/// This is used to assign an [`Encoding`] to every parquet column based on the columns' type (see example) -/// # Example -/// ``` -/// use arrow2::datatypes::DataType; -/// use arrow2::datatypes::Field; -/// use arrow2::io::parquet::write::transverse; -/// use arrow2::io::parquet::write::Encoding; -/// -/// let dt = DataType::Struct(vec![ -/// Field::new("a", DataType::Int64, true), -/// Field::new( -/// "b", -/// DataType::List(Box::new(Field::new("item", DataType::Int32, true))), -/// true, -/// ), -/// ]); -/// -/// let encodings = transverse(&dt, |dt| Encoding::Plain); -/// assert_eq!(encodings, vec![Encoding::Plain, Encoding::Plain]); -/// ``` -pub fn transverse T + Clone>(data_type: &DataType, map: F) -> Vec { - let mut encodings = vec![]; - transverse_recursive(data_type, map, &mut encodings); - encodings -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/nested/def.rs b/src/common/arrow/src/arrow/io/parquet/write/nested/def.rs deleted file mode 100644 index 1159fc7da149..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/nested/def.rs +++ /dev/null @@ -1,584 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::pages::ListNested; -use super::super::pages::Nested; -use super::rep::num_values; -use super::to_length; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::offset::Offset; - -trait DebugIter: Iterator + std::fmt::Debug {} - -impl + std::fmt::Debug> DebugIter for A {} - -fn single_iter<'a>( - validity: &'a Option, - is_optional: bool, - length: usize, -) -> Box { - match (is_optional, validity) { - (false, _) => { - Box::new(std::iter::repeat((0u32, 1usize)).take(length)) as Box - } - (true, None) => { - Box::new(std::iter::repeat((1u32, 1usize)).take(length)) as Box - } - (true, Some(validity)) => { - Box::new(validity.iter().map(|v| (v as u32, 1usize)).take(length)) as Box - } - } -} - -fn single_list_iter<'a, O: Offset>(nested: &'a ListNested) -> Box { - match (nested.is_optional, &nested.validity) { - (false, _) => Box::new( - std::iter::repeat(0u32) - .zip(to_length(&nested.offsets)) - .map(|(a, b)| (a + (b != 0) as u32, b)), - ) as Box, - (true, None) => Box::new( - std::iter::repeat(1u32) - .zip(to_length(&nested.offsets)) - .map(|(a, b)| (a + (b != 0) as u32, b)), - ) as Box, - (true, Some(validity)) => Box::new( - validity - .iter() - .map(|x| (x as u32)) - .zip(to_length(&nested.offsets)) - .map(|(a, b)| (a + (b != 0) as u32, b)), - ) as Box, - } -} - -fn iter<'a>(nested: &'a [Nested]) -> Vec> { - nested - .iter() - .map(|nested| match nested { - Nested::Primitive(validity, is_optional, length) => { - single_iter(validity, *is_optional, *length) - } - Nested::List(nested) => single_list_iter(nested), - Nested::LargeList(nested) => single_list_iter(nested), - Nested::Struct(validity, is_optional, length) => { - single_iter(validity, *is_optional, *length) - } - }) - .collect() -} - -/// Iterator adapter of parquet / dremel definition levels -#[derive(Debug)] -pub struct DefLevelsIter<'a> { - // iterators of validities and lengths. E.g. [[[None,b,c], None], None] -> [[(true, 2), (false, 0)], [(true, 3), (false, 0)], [(false, 1), (true, 1), (true, 1)]] - iter: Vec>, - // vector containing the remaining number of values of each iterator. - // e.g. the iters [[2, 2], [3, 4, 1, 2]] after the first iteration will return [2, 3], - // and remaining will be [2, 3]. - // on the second iteration, it will be `[2, 2]` (since iterations consume the last items) - remaining: Vec, // < remaining.len() == iter.len() - validity: Vec, - // cache of the first `remaining` that is non-zero. Examples: - // * `remaining = [2, 2] => current_level = 2` - // * `remaining = [2, 0] => current_level = 1` - // * `remaining = [0, 0] => current_level = 0` - current_level: usize, // < iter.len() - // the total definition level at any given point during the iteration - total: u32, // < iter.len() - // the total number of items that this iterator will return - remaining_values: usize, -} - -impl<'a> DefLevelsIter<'a> { - pub fn new(nested: &'a [Nested]) -> Self { - let remaining_values = num_values(nested); - - let iter = iter(nested); - let remaining = vec![0; iter.len()]; - let validity = vec![0; iter.len()]; - - Self { - iter, - remaining, - validity, - total: 0, - current_level: 0, - remaining_values, - } - } -} - -impl<'a> Iterator for DefLevelsIter<'a> { - type Item = u32; - - fn next(&mut self) -> Option { - if self.remaining_values == 0 { - return None; - } - - if self.remaining.is_empty() { - self.remaining_values -= 1; - return Some(0); - } - - let mut empty_contrib = 0u32; - for ((iter, remaining), validity) in self - .iter - .iter_mut() - .zip(self.remaining.iter_mut()) - .zip(self.validity.iter_mut()) - .skip(self.current_level) - { - let (is_valid, length): (u32, usize) = iter.next()?; - *validity = is_valid; - self.total += is_valid; - - *remaining = length; - if length == 0 { - *validity = 0; - self.total -= is_valid; - empty_contrib = is_valid; - break; - } - self.current_level += 1; - } - - // track - if let Some(x) = self.remaining.get_mut(self.current_level.saturating_sub(1)) { - *x = x.saturating_sub(1) - } - - let r = Some(self.total + empty_contrib); - - for index in (1..self.current_level).rev() { - if self.remaining[index] == 0 { - self.current_level -= 1; - self.remaining[index - 1] -= 1; - self.total -= self.validity[index]; - } - } - if self.remaining[0] == 0 { - self.current_level = self.current_level.saturating_sub(1); - self.total -= self.validity[0]; - } - self.remaining_values -= 1; - r - } - - fn size_hint(&self) -> (usize, Option) { - let length = self.remaining_values; - (length, Some(length)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn test(nested: Vec, expected: Vec) { - let mut iter = DefLevelsIter::new(&nested); - assert_eq!(iter.size_hint().0, expected.len()); - let result = iter.by_ref().collect::>(); - assert_eq!(result, expected); - assert_eq!(iter.size_hint().0, 0); - } - - #[test] - fn struct_optional() { - let b = [ - true, false, true, true, false, true, false, false, true, true, - ]; - let nested = vec![ - Nested::Struct(None, true, 10), - Nested::Primitive(Some(b.into()), true, 10), - ]; - let expected = vec![2, 1, 2, 2, 1, 2, 1, 1, 2, 2]; - - test(nested, expected) - } - - #[test] - fn nested_edge_simple() { - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, true, 2), - ]; - let expected = vec![3, 3]; - - test(nested, expected) - } - - #[test] - fn struct_optional_1() { - let b = [ - true, false, true, true, false, true, false, false, true, true, - ]; - let nested = vec![ - Nested::Struct(None, true, 10), - Nested::Primitive(Some(b.into()), true, 10), - ]; - let expected = vec![2, 1, 2, 2, 1, 2, 1, 1, 2, 2]; - - test(nested, expected) - } - - #[test] - fn struct_optional_optional() { - let nested = vec![ - Nested::Struct(None, true, 10), - Nested::Primitive(None, true, 10), - ]; - let expected = vec![2, 2, 2, 2, 2, 2, 2, 2, 2, 2]; - - test(nested, expected) - } - - #[test] - fn l1_required_required() { - let nested = vec![ - // [[0, 1], [], [2, 0, 3], [4, 5, 6], [], [7, 8, 9], [], [10]] - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 2, 2, 5, 8, 8, 11, 11, 12].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, false, 12), - ]; - let expected = vec![1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1]; - - test(nested, expected) - } - - #[test] - fn l1_optional_optional() { - // [[0, 1], None, [2, None, 3], [4, 5, 6], [], [7, 8, 9], None, [10]] - - let v0 = [true, false, true, true, true, true, false, true]; - let v1 = [ - true, true, //[0, 1] - true, false, true, //[2, None, 3] - true, true, true, //[4, 5, 6] - true, true, true, //[7, 8, 9] - true, //[10] - ]; - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 2, 5, 8, 8, 11, 11, 12].try_into().unwrap(), - validity: Some(v0.into()), - }), - Nested::Primitive(Some(v1.into()), true, 12), - ]; - let expected = vec![3u32, 3, 0, 3, 2, 3, 3, 3, 3, 1, 3, 3, 3, 0, 3]; - - test(nested, expected) - } - - #[test] - fn l2_required_required_required() { - // [ - // [ - // [1,2,3], - // [4,5,6,7], - // ], - // [ - // [8], - // [9, 10] - // ] - // ] - let nested = vec![ - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 2, 4].try_into().unwrap(), - validity: None, - }), - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 3, 7, 8, 10].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, false, 10), - ]; - let expected = vec![2, 2, 2, 2, 2, 2, 2, 2, 2, 2]; - - test(nested, expected) - } - - #[test] - fn l2_optional_required_required() { - let a = [true, false, true, true]; - // [ - // [ - // [1,2,3], - // [4,5,6,7], - // ], - // None, - // [ - // [8], - // [], - // [9, 10] - // ] - // ] - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 2, 2, 5].try_into().unwrap(), - validity: Some(a.into()), - }), - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 3, 7, 8, 8, 10].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, false, 10), - ]; - let expected = vec![3, 3, 3, 3, 3, 3, 3, 0, 1, 3, 2, 3, 3]; - - test(nested, expected) - } - - #[test] - fn l2_optional_optional_required() { - let a = [true, false, true]; - let b = [true, true, true, true, false]; - // [ - // [ - // [1,2,3], - // [4,5,6,7], - // ], - // None, - // [ - // [8], - // [], - // None, - // ], - // ] - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 2, 5].try_into().unwrap(), - validity: Some(a.into()), - }), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 3, 7, 8, 8, 8].try_into().unwrap(), - validity: Some(b.into()), - }), - Nested::Primitive(None, false, 8), - ]; - let expected = vec![4, 4, 4, 4, 4, 4, 4, 0, 4, 3, 2]; - - test(nested, expected) - } - - #[test] - fn l2_optional_optional_optional() { - let a = [true, false, true]; - let b = [true, true, true, false]; - let c = [true, true, true, true, false, true, true, true]; - // [ - // [ - // [1,2,3], - // [4,None,6,7], - // ], - // None, - // [ - // [8], - // None, - // ], - // ] - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 2, 4].try_into().unwrap(), - validity: Some(a.into()), - }), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 3, 7, 8, 8].try_into().unwrap(), - validity: Some(b.into()), - }), - Nested::Primitive(Some(c.into()), true, 8), - ]; - let expected = vec![5, 5, 5, 5, 4, 5, 5, 0, 5, 2]; - - test(nested, expected) - } - - // [{"a": "a"}, {"a": "b"}], - // None, - // [{"a": "b"}, None, {"a": "b"}], - // [{"a": None}, {"a": None}, {"a": None}], - // [], - // [{"a": "d"}, {"a": "d"}, {"a": "d"}], - // None, - // [{"a": "e"}], - #[test] - fn nested_list_struct_nullable() { - let a = [ - true, true, true, false, true, false, false, false, true, true, true, true, - ]; - let b = [ - true, true, true, false, true, true, true, true, true, true, true, true, - ]; - let c = [true, false, true, true, true, true, false, true]; - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 2, 5, 8, 8, 11, 11, 12].try_into().unwrap(), - validity: Some(c.into()), - }), - Nested::Struct(Some(b.into()), true, 12), - Nested::Primitive(Some(a.into()), true, 12), - ]; - let expected = vec![4, 4, 0, 4, 2, 4, 3, 3, 3, 1, 4, 4, 4, 0, 4]; - - test(nested, expected) - } - - #[test] - fn nested_list_struct_nullable1() { - let c = [true, false]; - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 1, 1].try_into().unwrap(), - validity: Some(c.into()), - }), - Nested::Struct(None, true, 1), - Nested::Primitive(None, true, 1), - ]; - let expected = vec![4, 0]; - - test(nested, expected) - } - - #[test] - fn nested_struct_list_nullable() { - let a = [true, false, true, true, true, true, false, true]; - let b = [ - true, true, true, false, true, true, true, true, true, true, true, true, - ]; - let nested = vec![ - Nested::Struct(None, true, 12), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 2, 5, 8, 8, 11, 11, 12].try_into().unwrap(), - validity: Some(a.into()), - }), - Nested::Primitive(Some(b.into()), true, 12), - ]; - let expected = vec![4, 4, 1, 4, 3, 4, 4, 4, 4, 2, 4, 4, 4, 1, 4]; - - test(nested, expected) - } - - #[test] - fn nested_struct_list_nullable1() { - let a = [true, true, false]; - let nested = vec![ - Nested::Struct(None, true, 3), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 1, 1, 1].try_into().unwrap(), - validity: Some(a.into()), - }), - Nested::Primitive(None, true, 1), - ]; - let expected = vec![4, 2, 1]; - - test(nested, expected) - } - - #[test] - fn nested_list_struct_list_nullable1() { - // [ - // [{"a": ["b"]}, None], - // ] - - let a = [true]; - let b = [true, false]; - let c = [true, false]; - let d = [true]; - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2].try_into().unwrap(), - validity: Some(a.into()), - }), - Nested::Struct(Some(b.into()), true, 2), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 1, 1].try_into().unwrap(), - validity: Some(c.into()), - }), - Nested::Primitive(Some(d.into()), true, 1), - ]; - // 0 6 - // 1 6 - // 0 0 - // 0 6 - // 1 2 - let expected = vec![6, 2]; - - test(nested, expected) - } - - #[test] - fn nested_list_struct_list_nullable() { - // [ - // [{"a": ["a"]}, {"a": ["b"]}], - // None, - // [{"a": ["b"]}, None, {"a": ["b"]}], - // [{"a": None}, {"a": None}, {"a": None}], - // [], - // [{"a": ["d"]}, {"a": [None]}, {"a": ["c", "d"]}], - // None, - // [{"a": []}], - // ] - let a = [true, false, true, true, true, true, false, true]; - let b = [ - true, true, true, false, true, true, true, true, true, true, true, true, - ]; - let c = [ - true, true, true, false, true, false, false, false, true, true, true, true, - ]; - let d = [true, true, true, true, true, false, true, true]; - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 2, 5, 8, 8, 11, 11, 12].try_into().unwrap(), - validity: Some(a.into()), - }), - Nested::Struct(Some(b.into()), true, 12), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 1, 2, 3, 3, 4, 4, 4, 4, 5, 6, 8, 8] - .try_into() - .unwrap(), - validity: Some(c.into()), - }), - Nested::Primitive(Some(d.into()), true, 8), - ]; - let expected = vec![6, 6, 0, 6, 2, 6, 3, 3, 3, 1, 6, 5, 6, 6, 0, 4]; - - test(nested, expected) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/nested/mod.rs b/src/common/arrow/src/arrow/io/parquet/write/nested/mod.rs deleted file mode 100644 index 560f098bbc8b..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/nested/mod.rs +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod def; -mod rep; - -use parquet2::encoding::hybrid_rle::encode_u32; -use parquet2::read::levels::get_bit_width; -use parquet2::write::Version; -pub use rep::num_values; - -use super::Nested; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; - -fn write_levels_v1) -> Result<()>>( - buffer: &mut Vec, - encode: F, -) -> Result<()> { - buffer.extend_from_slice(&[0; 4]); - let start = buffer.len(); - - encode(buffer)?; - - let end = buffer.len(); - let length = end - start; - - // write the first 4 bytes as length - let length = (length as i32).to_le_bytes(); - (0..4).for_each(|i| buffer[start - 4 + i] = length[i]); - Ok(()) -} - -/// writes the rep levels to a `Vec`. -fn write_rep_levels(buffer: &mut Vec, nested: &[Nested], version: Version) -> Result<()> { - let max_level = max_rep_level(nested) as i16; - if max_level == 0 { - return Ok(()); - } - let num_bits = get_bit_width(max_level); - - let levels = rep::RepLevelsIter::new(nested); - - match version { - Version::V1 => { - write_levels_v1(buffer, |buffer: &mut Vec| { - encode_u32(buffer, levels, num_bits)?; - Ok(()) - })?; - } - Version::V2 => { - encode_u32(buffer, levels, num_bits)?; - } - } - - Ok(()) -} - -/// writes the rep levels to a `Vec`. -fn write_def_levels(buffer: &mut Vec, nested: &[Nested], version: Version) -> Result<()> { - let max_level = max_def_level(nested) as i16; - if max_level == 0 { - return Ok(()); - } - let num_bits = get_bit_width(max_level); - - let levels = def::DefLevelsIter::new(nested); - - match version { - Version::V1 => write_levels_v1(buffer, move |buffer: &mut Vec| { - encode_u32(buffer, levels, num_bits)?; - Ok(()) - }), - Version::V2 => Ok(encode_u32(buffer, levels, num_bits)?), - } -} - -fn max_def_level(nested: &[Nested]) -> usize { - nested - .iter() - .map(|nested| match nested { - Nested::Primitive(_, is_optional, _) => *is_optional as usize, - Nested::List(nested) => 1 + (nested.is_optional as usize), - Nested::LargeList(nested) => 1 + (nested.is_optional as usize), - Nested::Struct(_, is_optional, _) => *is_optional as usize, - }) - .sum() -} - -fn max_rep_level(nested: &[Nested]) -> usize { - nested - .iter() - .map(|nested| match nested { - Nested::LargeList(_) | Nested::List(_) => 1, - Nested::Primitive(_, _, _) | Nested::Struct(_, _, _) => 0, - }) - .sum() -} - -fn to_length( - offsets: &[O], -) -> impl Iterator + std::fmt::Debug + Clone + '_ { - offsets - .windows(2) - .map(|w| w[1].to_usize() - w[0].to_usize()) -} - -/// Write `repetition_levels` and `definition_levels` to buffer. -pub fn write_rep_and_def( - page_version: Version, - nested: &[Nested], - buffer: &mut Vec, -) -> Result<(usize, usize)> { - write_rep_levels(buffer, nested, page_version)?; - let repetition_levels_byte_length = buffer.len(); - - write_def_levels(buffer, nested, page_version)?; - let definition_levels_byte_length = buffer.len() - repetition_levels_byte_length; - - Ok((repetition_levels_byte_length, definition_levels_byte_length)) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/nested/rep.rs b/src/common/arrow/src/arrow/io/parquet/write/nested/rep.rs deleted file mode 100644 index d4f9fcad7acd..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/nested/rep.rs +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::pages::Nested; -use super::to_length; - -trait DebugIter: Iterator + std::fmt::Debug {} - -impl + std::fmt::Debug> DebugIter for A {} - -fn iter<'a>(nested: &'a [Nested]) -> Vec> { - nested - .iter() - .filter_map(|nested| match nested { - Nested::Primitive(_, _, _) => None, - Nested::List(nested) => { - Some(Box::new(to_length(&nested.offsets)) as Box) - } - Nested::LargeList(nested) => { - Some(Box::new(to_length(&nested.offsets)) as Box) - } - Nested::Struct(_, _, _) => None, - }) - .collect() -} - -/// return number values of the nested -pub fn num_values(nested: &[Nested]) -> usize { - let pr = match nested.last().unwrap() { - Nested::Primitive(_, _, len) => *len, - _ => todo!(), - }; - - iter(nested) - .into_iter() - .map(|lengths| { - lengths - .map(|length| if length == 0 { 1 } else { 0 }) - .sum::() - }) - .sum::() - + pr -} - -/// Iterator adapter of parquet / dremel repetition levels -#[derive(Debug)] -pub struct RepLevelsIter<'a> { - // iterators of lengths. E.g. [[[a,b,c], [d,e,f,g]], [[h], [i,j]]] -> [[2, 2], [3, 4, 1, 2]] - iter: Vec>, - // vector containing the remaining number of values of each iterator. - // e.g. the iters [[2, 2], [3, 4, 1, 2]] after the first iteration will return [2, 3], - // and remaining will be [2, 3]. - // on the second iteration, it will be `[2, 2]` (since iterations consume the last items) - remaining: Vec, // < remaining.len() == iter.len() - // cache of the first `remaining` that is non-zero. Examples: - // * `remaining = [2, 2] => current_level = 2` - // * `remaining = [2, 0] => current_level = 1` - // * `remaining = [0, 0] => current_level = 0` - current_level: usize, // < iter.len() - // the number to discount due to being the first element of the iterators. - total: usize, // < iter.len() - - // the total number of items that this iterator will return - remaining_values: usize, -} - -impl<'a> RepLevelsIter<'a> { - pub fn new(nested: &'a [Nested]) -> Self { - let remaining_values = num_values(nested); - - let iter = iter(nested); - let remaining = vec![0; iter.len()]; - - Self { - iter, - remaining, - total: 0, - current_level: 0, - remaining_values, - } - } -} - -impl<'a> Iterator for RepLevelsIter<'a> { - type Item = u32; - - fn next(&mut self) -> Option { - if self.remaining_values == 0 { - return None; - } - if self.remaining.is_empty() { - self.remaining_values -= 1; - return Some(0); - } - - for (iter, remaining) in self - .iter - .iter_mut() - .zip(self.remaining.iter_mut()) - .skip(self.current_level) - { - let length: usize = iter.next()?; - *remaining = length; - if length == 0 { - break; - } - self.current_level += 1; - self.total += 1; - } - - // track - if let Some(x) = self.remaining.get_mut(self.current_level.saturating_sub(1)) { - *x = x.saturating_sub(1) - } - let r = Some((self.current_level - self.total) as u32); - - // update - for index in (1..self.current_level).rev() { - if self.remaining[index] == 0 { - self.current_level -= 1; - self.remaining[index - 1] -= 1; - } - } - if self.remaining[0] == 0 { - self.current_level = self.current_level.saturating_sub(1); - } - self.total = 0; - self.remaining_values -= 1; - - r - } - - fn size_hint(&self) -> (usize, Option) { - let length = self.remaining_values; - (length, Some(length)) - } -} - -#[cfg(test)] -mod tests { - use super::super::super::pages::ListNested; - use super::*; - - fn test(nested: Vec, expected: Vec) { - let mut iter = RepLevelsIter::new(&nested); - assert_eq!(iter.size_hint().0, expected.len()); - assert_eq!(iter.by_ref().collect::>(), expected); - assert_eq!(iter.size_hint().0, 0); - } - - #[test] - fn struct_required() { - let nested = vec![ - Nested::Struct(None, false, 10), - Nested::Primitive(None, true, 10), - ]; - let expected = vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; - - test(nested, expected) - } - - #[test] - fn struct_optional() { - let nested = vec![ - Nested::Struct(None, true, 10), - Nested::Primitive(None, true, 10), - ]; - let expected = vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; - - test(nested, expected) - } - - #[test] - fn l1() { - let nested = vec![ - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 2, 2, 5, 8, 8, 11, 11, 12].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, false, 12), - ]; - let expected = vec![0u32, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0]; - - test(nested, expected) - } - - #[test] - fn l2() { - let nested = vec![ - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 2, 2, 4].try_into().unwrap(), - validity: None, - }), - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 3, 7, 8, 10].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, false, 10), - ]; - let expected = vec![0, 2, 2, 1, 2, 2, 2, 0, 0, 1, 2]; - - test(nested, expected) - } - - #[test] - fn list_of_struct() { - // [ - // [{"a": "b"}],[{"a": "c"}] - // ] - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 1, 2].try_into().unwrap(), - validity: None, - }), - Nested::Struct(None, true, 2), - Nested::Primitive(None, true, 2), - ]; - let expected = vec![0, 0]; - - test(nested, expected) - } - - #[test] - fn list_struct_list() { - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 3].try_into().unwrap(), - validity: None, - }), - Nested::Struct(None, true, 3), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 3, 6, 7].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, true, 7), - ]; - let expected = vec![0, 2, 2, 1, 2, 2, 0]; - - test(nested, expected) - } - - #[test] - fn struct_list_optional() { - // {"f1": ["a", "b", None, "c"]} - let nested = vec![ - Nested::Struct(None, true, 1), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 4].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, true, 4), - ]; - let expected = vec![0, 1, 1, 1]; - - test(nested, expected) - } - - #[test] - fn l2_other() { - let nested = vec![ - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 1, 1, 3, 5, 5, 8, 8, 9].try_into().unwrap(), - validity: None, - }), - Nested::List(ListNested { - is_optional: false, - offsets: vec![0, 2, 4, 5, 7, 8, 9, 10, 11, 12].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, false, 12), - ]; - let expected = vec![0, 2, 0, 0, 2, 1, 0, 2, 1, 0, 0, 1, 1, 0, 0]; - - test(nested, expected) - } - - #[test] - fn list_struct_list_1() { - // [ - // [{"a": ["a"]}, {"a": ["b"]}], - // [], - // [{"a": ["b"]}, None, {"a": ["b"]}], - // [{"a": []}, {"a": []}, {"a": []}], - // [], - // [{"a": ["d"]}, {"a": ["a"]}, {"a": ["c", "d"]}], - // [], - // [{"a": []}], - // ] - // reps: [0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 2, 0, 0] - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 2, 2, 5, 8, 8, 11, 11, 12].try_into().unwrap(), - validity: None, - }), - Nested::Struct(None, true, 12), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 1, 2, 3, 3, 4, 4, 4, 4, 5, 6, 8].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, true, 8), - ]; - let expected = vec![0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 2, 0]; - - test(nested, expected) - } - - #[test] - fn list_struct_list_2() { - // [ - // [{"a": []}], - // ] - // reps: [0] - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 1].try_into().unwrap(), - validity: None, - }), - Nested::Struct(None, true, 12), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 0].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, true, 0), - ]; - let expected = vec![0]; - - test(nested, expected) - } - - #[test] - fn list_struct_list_3() { - let nested = vec![ - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 1, 1].try_into().unwrap(), - validity: None, - }), - Nested::Struct(None, true, 12), - Nested::List(ListNested { - is_optional: true, - offsets: vec![0, 0].try_into().unwrap(), - validity: None, - }), - Nested::Primitive(None, true, 0), - ]; - let expected = vec![0, 0]; - // [1, 0], [0] - // pick last - - test(nested, expected) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/pages.rs b/src/common/arrow/src/arrow/io/parquet/write/pages.rs deleted file mode 100644 index c8325bc54a7f..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/pages.rs +++ /dev/null @@ -1,653 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; - -use parquet2::page::Page; -use parquet2::schema::types::ParquetType; -use parquet2::schema::types::PrimitiveType as ParquetPrimitiveType; -use parquet2::write::DynIter; - -use super::array_to_pages; -use super::Encoding; -use super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::array::ListArray; -use crate::arrow::array::MapArray; -use crate::arrow::array::StructArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::offset::Offset; -use crate::arrow::offset::OffsetsBuffer; - -#[derive(Debug, Clone, PartialEq)] -pub struct ListNested { - pub is_optional: bool, - pub offsets: OffsetsBuffer, - pub validity: Option, -} - -impl ListNested { - pub fn new(offsets: OffsetsBuffer, validity: Option, is_optional: bool) -> Self { - Self { - is_optional, - offsets, - validity, - } - } -} - -/// Descriptor of nested information of a field -#[derive(Debug, Clone, PartialEq)] -pub enum Nested { - /// a primitive (leaf or parquet column) - /// bitmap, _, length - Primitive(Option, bool, usize), - /// a list - List(ListNested), - /// a list - LargeList(ListNested), - /// a struct - Struct(Option, bool, usize), -} - -impl Nested { - /// Returns the length (number of rows) of the element - pub fn len(&self) -> usize { - match self { - Nested::Primitive(_, _, length) => *length, - Nested::List(nested) => nested.offsets.len_proxy(), - Nested::LargeList(nested) => nested.offsets.len_proxy(), - Nested::Struct(_, _, len) => *len, - } - } - - /// Returns `true` if the length of the element is 0. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -/// Constructs the necessary `Vec>` to write the rep and def levels of `array` to parquet -pub fn to_nested(array: &dyn Array, type_: &ParquetType) -> Result>> { - let mut nested = vec![]; - - to_nested_recursive(array, type_, &mut nested, vec![])?; - Ok(nested) -} - -fn to_nested_recursive( - array: &dyn Array, - type_: &ParquetType, - nested: &mut Vec>, - mut parents: Vec, -) -> Result<()> { - let is_optional = is_nullable(type_.get_field_info()); - - use PhysicalType::*; - match array.data_type().to_physical_type() { - Struct => { - let array = array.as_any().downcast_ref::().unwrap(); - let fields = if let ParquetType::GroupType { fields, .. } = type_ { - fields - } else { - return Err(Error::InvalidArgumentError( - "Parquet type must be a group for a struct array".to_string(), - )); - }; - - parents.push(Nested::Struct( - array.validity().cloned(), - is_optional, - array.len(), - )); - - for (type_, array) in fields.iter().zip(array.values()) { - to_nested_recursive(array.as_ref(), type_, nested, parents.clone())?; - } - } - List => { - let array = array.as_any().downcast_ref::>().unwrap(); - let type_ = if let ParquetType::GroupType { fields, .. } = type_ { - if let ParquetType::GroupType { fields, .. } = &fields[0] { - &fields[0] - } else { - return Err(Error::InvalidArgumentError( - "Parquet type must be a group for a list array".to_string(), - )); - } - } else { - return Err(Error::InvalidArgumentError( - "Parquet type must be a group for a list array".to_string(), - )); - }; - - parents.push(Nested::List(ListNested::new( - array.offsets().clone(), - array.validity().cloned(), - is_optional, - ))); - to_nested_recursive(array.values().as_ref(), type_, nested, parents)?; - } - LargeList => { - let array = array.as_any().downcast_ref::>().unwrap(); - let type_ = if let ParquetType::GroupType { fields, .. } = type_ { - if let ParquetType::GroupType { fields, .. } = &fields[0] { - &fields[0] - } else { - return Err(Error::InvalidArgumentError( - "Parquet type must be a group for a list array".to_string(), - )); - } - } else { - return Err(Error::InvalidArgumentError( - "Parquet type must be a group for a list array".to_string(), - )); - }; - - parents.push(Nested::LargeList(ListNested::new( - array.offsets().clone(), - array.validity().cloned(), - is_optional, - ))); - to_nested_recursive(array.values().as_ref(), type_, nested, parents)?; - } - Map => { - let array = array.as_any().downcast_ref::().unwrap(); - let type_ = if let ParquetType::GroupType { fields, .. } = type_ { - if let ParquetType::GroupType { fields, .. } = &fields[0] { - &fields[0] - } else { - return Err(Error::InvalidArgumentError( - "Parquet type must be a group for a map array".to_string(), - )); - } - } else { - return Err(Error::InvalidArgumentError( - "Parquet type must be a group for a map array".to_string(), - )); - }; - - parents.push(Nested::List(ListNested::new( - array.offsets().clone(), - array.validity().cloned(), - is_optional, - ))); - to_nested_recursive(array.field().as_ref(), type_, nested, parents)?; - } - _ => { - parents.push(Nested::Primitive( - array.validity().cloned(), - is_optional, - array.len(), - )); - nested.push(parents) - } - } - Ok(()) -} - -/// Convert [`Array`] to `Vec<&dyn Array>` leaves in DFS order. -pub fn to_leaves(array: &dyn Array) -> Vec<&dyn Array> { - let mut leaves = vec![]; - to_leaves_recursive(array, &mut leaves); - leaves -} - -fn to_leaves_recursive<'a>(array: &'a dyn Array, leaves: &mut Vec<&'a dyn Array>) { - use PhysicalType::*; - match array.data_type().to_physical_type() { - Struct => { - let array = array.as_any().downcast_ref::().unwrap(); - array - .values() - .iter() - .for_each(|a| to_leaves_recursive(a.as_ref(), leaves)); - } - List => { - let array = array.as_any().downcast_ref::>().unwrap(); - to_leaves_recursive(array.values().as_ref(), leaves); - } - LargeList => { - let array = array.as_any().downcast_ref::>().unwrap(); - to_leaves_recursive(array.values().as_ref(), leaves); - } - Map => { - let array = array.as_any().downcast_ref::().unwrap(); - to_leaves_recursive(array.field().as_ref(), leaves); - } - Null | Boolean | Primitive(_) | Binary | FixedSizeBinary | LargeBinary | Utf8 - | LargeUtf8 | Dictionary(_) | BinaryView | Utf8View => leaves.push(array), - other => todo!("Writing {:?} to parquet not yet implemented", other), - } -} - -/// Convert `ParquetType` to `Vec` leaves in DFS order. -pub fn to_parquet_leaves(type_: ParquetType) -> Vec { - let mut leaves = vec![]; - to_parquet_leaves_recursive(type_, &mut leaves); - leaves -} - -fn to_parquet_leaves_recursive(type_: ParquetType, leaves: &mut Vec) { - match type_ { - ParquetType::PrimitiveType(primitive) => leaves.push(primitive), - ParquetType::GroupType { fields, .. } => { - fields - .into_iter() - .for_each(|type_| to_parquet_leaves_recursive(type_, leaves)); - } - } -} - -/// Returns a vector of iterators of [`Page`], one per leaf column in the array -pub fn array_to_columns + Send + Sync>( - array: A, - type_: ParquetType, - options: WriteOptions, - encoding: &[Encoding], -) -> Result>>> { - let array = array.as_ref(); - let nested = to_nested(array, &type_)?; - - let types = to_parquet_leaves(type_); - - let values = to_leaves(array); - - assert_eq!(encoding.len(), types.len()); - - values - .iter() - .zip(nested) - .zip(types) - .zip(encoding.iter()) - .map(|(((values, nested), type_), encoding)| { - array_to_pages(*values, type_, &nested, options, *encoding) - }) - .collect() -} - -#[cfg(test)] -mod tests { - use parquet2::schema::types::GroupLogicalType; - use parquet2::schema::types::PrimitiveConvertedType; - use parquet2::schema::types::PrimitiveLogicalType; - use parquet2::schema::Repetition; - - use super::super::FieldInfo; - use super::super::ParquetPhysicalType; - use super::super::ParquetPrimitiveType; - use super::*; - use crate::arrow::array::*; - use crate::arrow::bitmap::Bitmap; - use crate::arrow::datatypes::*; - - #[test] - fn test_struct() { - let boolean = BooleanArray::from_slice([false, false, true, true]).boxed(); - let int = Int32Array::from_slice([42, 28, 19, 31]).boxed(); - - let fields = vec![ - Field::new("b", DataType::Boolean, false), - Field::new("c", DataType::Int32, false), - ]; - - let array = StructArray::new( - DataType::Struct(fields), - vec![boolean.clone(), int.clone()], - Some(Bitmap::from([true, true, false, true])), - ); - - let type_ = ParquetType::GroupType { - field_info: FieldInfo { - name: "a".to_string(), - repetition: Repetition::Optional, - id: None, - }, - logical_type: None, - converted_type: None, - fields: vec![ - ParquetType::PrimitiveType(ParquetPrimitiveType { - field_info: FieldInfo { - name: "b".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - physical_type: ParquetPhysicalType::Boolean, - }), - ParquetType::PrimitiveType(ParquetPrimitiveType { - field_info: FieldInfo { - name: "c".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - physical_type: ParquetPhysicalType::Int32, - }), - ], - }; - let a = to_nested(&array, &type_).unwrap(); - - assert_eq!(a, vec![ - vec![ - Nested::Struct(Some(Bitmap::from([true, true, false, true])), true, 4), - Nested::Primitive(None, false, 4), - ], - vec![ - Nested::Struct(Some(Bitmap::from([true, true, false, true])), true, 4), - Nested::Primitive(None, false, 4), - ], - ]); - } - - #[test] - fn test_struct_struct() { - let boolean = BooleanArray::from_slice([false, false, true, true]).boxed(); - let int = Int32Array::from_slice([42, 28, 19, 31]).boxed(); - - let fields = vec![ - Field::new("b", DataType::Boolean, false), - Field::new("c", DataType::Int32, false), - ]; - - let array = StructArray::new( - DataType::Struct(fields), - vec![boolean.clone(), int.clone()], - Some(Bitmap::from([true, true, false, true])), - ); - - let fields = vec![ - Field::new("b", array.data_type().clone(), true), - Field::new("c", array.data_type().clone(), true), - ]; - - let array = StructArray::new( - DataType::Struct(fields), - vec![Box::new(array.clone()), Box::new(array)], - None, - ); - - let type_ = ParquetType::GroupType { - field_info: FieldInfo { - name: "a".to_string(), - repetition: Repetition::Optional, - id: None, - }, - logical_type: None, - converted_type: None, - fields: vec![ - ParquetType::PrimitiveType(ParquetPrimitiveType { - field_info: FieldInfo { - name: "b".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - physical_type: ParquetPhysicalType::Boolean, - }), - ParquetType::PrimitiveType(ParquetPrimitiveType { - field_info: FieldInfo { - name: "c".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - physical_type: ParquetPhysicalType::Int32, - }), - ], - }; - - let type_ = ParquetType::GroupType { - field_info: FieldInfo { - name: "a".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - fields: vec![type_.clone(), type_], - }; - - let a = to_nested(&array, &type_).unwrap(); - - assert_eq!(a, vec![ - // a.b.b - vec![ - Nested::Struct(None, false, 4), - Nested::Struct(Some(Bitmap::from([true, true, false, true])), true, 4), - Nested::Primitive(None, false, 4), - ], - // a.b.c - vec![ - Nested::Struct(None, false, 4), - Nested::Struct(Some(Bitmap::from([true, true, false, true])), true, 4), - Nested::Primitive(None, false, 4), - ], - // a.c.b - vec![ - Nested::Struct(None, false, 4), - Nested::Struct(Some(Bitmap::from([true, true, false, true])), true, 4), - Nested::Primitive(None, false, 4), - ], - // a.c.c - vec![ - Nested::Struct(None, false, 4), - Nested::Struct(Some(Bitmap::from([true, true, false, true])), true, 4), - Nested::Primitive(None, false, 4), - ], - ]); - } - - #[test] - fn test_list_struct() { - let boolean = BooleanArray::from_slice([false, false, true, true]).boxed(); - let int = Int32Array::from_slice([42, 28, 19, 31]).boxed(); - - let fields = vec![ - Field::new("b", DataType::Boolean, false), - Field::new("c", DataType::Int32, false), - ]; - - let array = StructArray::new( - DataType::Struct(fields), - vec![boolean.clone(), int.clone()], - Some(Bitmap::from([true, true, false, true])), - ); - - let array = ListArray::new( - DataType::List(Box::new(Field::new("l", array.data_type().clone(), true))), - vec![0i32, 2, 4].try_into().unwrap(), - Box::new(array), - None, - ); - - let type_ = ParquetType::GroupType { - field_info: FieldInfo { - name: "a".to_string(), - repetition: Repetition::Optional, - id: None, - }, - logical_type: None, - converted_type: None, - fields: vec![ - ParquetType::PrimitiveType(ParquetPrimitiveType { - field_info: FieldInfo { - name: "b".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - physical_type: ParquetPhysicalType::Boolean, - }), - ParquetType::PrimitiveType(ParquetPrimitiveType { - field_info: FieldInfo { - name: "c".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - physical_type: ParquetPhysicalType::Int32, - }), - ], - }; - - let type_ = ParquetType::GroupType { - field_info: FieldInfo { - name: "l".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - fields: vec![ParquetType::GroupType { - field_info: FieldInfo { - name: "list".to_string(), - repetition: Repetition::Repeated, - id: None, - }, - logical_type: None, - converted_type: None, - fields: vec![type_], - }], - }; - - let a = to_nested(&array, &type_).unwrap(); - - assert_eq!(a, vec![ - vec![ - Nested::List(ListNested:: { - is_optional: false, - offsets: vec![0, 2, 4].try_into().unwrap(), - validity: None, - }), - Nested::Struct(Some(Bitmap::from([true, true, false, true])), true, 4), - Nested::Primitive(None, false, 4), - ], - vec![ - Nested::List(ListNested:: { - is_optional: false, - offsets: vec![0, 2, 4].try_into().unwrap(), - validity: None, - }), - Nested::Struct(Some(Bitmap::from([true, true, false, true])), true, 4), - Nested::Primitive(None, false, 4), - ], - ]); - } - - #[test] - fn test_map() { - let kv_type = DataType::Struct(vec![ - Field::new("k", DataType::Utf8, false), - Field::new("v", DataType::Int32, false), - ]); - let kv_field = Field::new("kv", kv_type.clone(), false); - let map_type = DataType::Map(Box::new(kv_field), false); - - let key_array = Utf8Array::::from_slice(["k1", "k2", "k3", "k4", "k5", "k6"]).boxed(); - let val_array = Int32Array::from_slice([42, 28, 19, 31, 21, 17]).boxed(); - let kv_array = StructArray::try_new(kv_type, vec![key_array, val_array], None) - .unwrap() - .boxed(); - let offsets = OffsetsBuffer::try_from(vec![0, 2, 3, 4, 6]).unwrap(); - - let array = MapArray::try_new(map_type, offsets, kv_array, None).unwrap(); - - let type_ = ParquetType::GroupType { - field_info: FieldInfo { - name: "kv".to_string(), - repetition: Repetition::Optional, - id: None, - }, - logical_type: None, - converted_type: None, - fields: vec![ - ParquetType::PrimitiveType(ParquetPrimitiveType { - field_info: FieldInfo { - name: "k".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: Some(PrimitiveLogicalType::String), - converted_type: Some(PrimitiveConvertedType::Utf8), - physical_type: ParquetPhysicalType::ByteArray, - }), - ParquetType::PrimitiveType(ParquetPrimitiveType { - field_info: FieldInfo { - name: "v".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: None, - converted_type: None, - physical_type: ParquetPhysicalType::Int32, - }), - ], - }; - - let type_ = ParquetType::GroupType { - field_info: FieldInfo { - name: "m".to_string(), - repetition: Repetition::Required, - id: None, - }, - logical_type: Some(GroupLogicalType::Map), - converted_type: None, - fields: vec![ParquetType::GroupType { - field_info: FieldInfo { - name: "map".to_string(), - repetition: Repetition::Repeated, - id: None, - }, - logical_type: None, - converted_type: None, - fields: vec![type_], - }], - }; - - let a = to_nested(&array, &type_).unwrap(); - - assert_eq!(a, vec![ - vec![ - Nested::List(ListNested:: { - is_optional: false, - offsets: vec![0, 2, 3, 4, 6].try_into().unwrap(), - validity: None, - }), - Nested::Struct(None, true, 6), - Nested::Primitive(None, false, 6), - ], - vec![ - Nested::List(ListNested:: { - is_optional: false, - offsets: vec![0, 2, 3, 4, 6].try_into().unwrap(), - validity: None, - }), - Nested::Struct(None, true, 6), - Nested::Primitive(None, false, 6), - ], - ]); - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/primitive/basic.rs b/src/common/arrow/src/arrow/io/parquet/write/primitive/basic.rs deleted file mode 100644 index 64db2c72468c..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/primitive/basic.rs +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::delta_bitpacked::encode; -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::serialize_statistics; -use parquet2::statistics::PrimitiveStatistics; -use parquet2::types::NativeType as ParquetNativeType; - -use super::super::utils; -use super::super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Error; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::io::parquet::write::utils::ExactSizedIter; -use crate::arrow::types::NativeType; - -pub(crate) fn encode_plain( - array: &PrimitiveArray, - is_optional: bool, - mut buffer: Vec, -) -> Vec -where - T: NativeType, - P: ParquetNativeType, - T: num_traits::AsPrimitive

, -{ - if is_optional { - buffer.reserve(std::mem::size_of::

() * (array.len() - array.null_count())); - // append the non-null values - array.iter().for_each(|x| { - if let Some(x) = x { - let parquet_native: P = x.as_(); - buffer.extend_from_slice(parquet_native.to_le_bytes().as_ref()) - } - }); - } else { - buffer.reserve(std::mem::size_of::

() * array.len()); - // append all values - array.values().iter().for_each(|x| { - let parquet_native: P = x.as_(); - buffer.extend_from_slice(parquet_native.to_le_bytes().as_ref()) - }); - } - buffer -} - -pub(crate) fn encode_delta( - array: &PrimitiveArray, - is_optional: bool, - mut buffer: Vec, -) -> Vec -where - T: NativeType, - P: ParquetNativeType, - T: num_traits::AsPrimitive

, - P: num_traits::AsPrimitive, -{ - if is_optional { - // append the non-null values - let iterator = array.iter().flatten().map(|x| { - let parquet_native: P = x.as_(); - let integer: i64 = parquet_native.as_(); - integer - }); - let iterator = ExactSizedIter::new(iterator, array.len() - array.null_count()); - encode(iterator, &mut buffer) - } else { - // append all values - let iterator = array.values().iter().map(|x| { - let parquet_native: P = x.as_(); - let integer: i64 = parquet_native.as_(); - integer - }); - encode(iterator, &mut buffer) - } - buffer -} - -pub fn array_to_page_plain( - array: &PrimitiveArray, - options: WriteOptions, - type_: PrimitiveType, -) -> Result -where - T: NativeType, - P: ParquetNativeType, - T: num_traits::AsPrimitive

, -{ - array_to_page(array, options, type_, Encoding::Plain, encode_plain) -} - -pub fn array_to_page_integer( - array: &PrimitiveArray, - options: WriteOptions, - type_: PrimitiveType, - encoding: Encoding, -) -> Result -where - T: NativeType, - P: ParquetNativeType, - T: num_traits::AsPrimitive

, - P: num_traits::AsPrimitive, -{ - match encoding { - Encoding::DeltaBinaryPacked => array_to_page(array, options, type_, encoding, encode_delta), - Encoding::Plain => array_to_page(array, options, type_, encoding, encode_plain), - other => Err(Error::nyi(format!("Encoding integer as {other:?}"))), - } -} - -pub fn array_to_page, bool, Vec) -> Vec>( - array: &PrimitiveArray, - options: WriteOptions, - type_: PrimitiveType, - encoding: Encoding, - encode: F, -) -> Result -where - T: NativeType, - P: ParquetNativeType, - // constraint required to build statistics - T: num_traits::AsPrimitive

, -{ - let is_optional = is_nullable(&type_.field_info); - - let validity = array.validity(); - - let mut buffer = vec![]; - utils::write_def_levels( - &mut buffer, - is_optional, - validity, - array.len(), - options.version, - )?; - - let definition_levels_byte_length = buffer.len(); - - let buffer = encode(array, is_optional, buffer); - - let statistics = if options.write_statistics { - Some(serialize_statistics(&build_statistics( - array, - type_.clone(), - ))) - } else { - None - }; - - utils::build_plain_page( - buffer, - array.len(), - array.len(), - array.null_count(), - 0, - definition_levels_byte_length, - statistics, - type_, - options, - encoding, - ) -} - -pub fn build_statistics( - array: &PrimitiveArray, - primitive_type: PrimitiveType, -) -> PrimitiveStatistics

-where - T: NativeType, - P: ParquetNativeType, - T: num_traits::AsPrimitive

, -{ - PrimitiveStatistics::

{ - primitive_type, - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array - .iter() - .flatten() - .map(|x| { - let x: P = x.as_(); - x - }) - .max_by(|x, y| x.ord(y)), - min_value: array - .iter() - .flatten() - .map(|x| { - let x: P = x.as_(); - x - }) - .min_by(|x, y| x.ord(y)), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/primitive/mod.rs b/src/common/arrow/src/arrow/io/parquet/write/primitive/mod.rs deleted file mode 100644 index 97297e2b4203..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/primitive/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod nested; - -pub use basic::array_to_page_integer; -pub use basic::array_to_page_plain; -pub(crate) use basic::build_statistics; -pub(crate) use basic::encode_plain; -pub use nested::array_to_page as nested_array_to_page; diff --git a/src/common/arrow/src/arrow/io/parquet/write/primitive/nested.rs b/src/common/arrow/src/arrow/io/parquet/write/primitive/nested.rs deleted file mode 100644 index 2c93058b08c6..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/primitive/nested.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::serialize_statistics; -use parquet2::types::NativeType; - -use super::super::nested; -use super::super::utils; -use super::super::WriteOptions; -use super::basic::build_statistics; -use super::basic::encode_plain; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::io::parquet::write::Nested; -use crate::arrow::types::NativeType as ArrowNativeType; - -pub fn array_to_page( - array: &PrimitiveArray, - options: WriteOptions, - type_: PrimitiveType, - nested: &[Nested], -) -> Result -where - T: ArrowNativeType, - R: NativeType, - T: num_traits::AsPrimitive, -{ - let is_optional = is_nullable(&type_.field_info); - - let mut buffer = vec![]; - - let (repetition_levels_byte_length, definition_levels_byte_length) = - nested::write_rep_and_def(options.version, nested, &mut buffer)?; - - let buffer = encode_plain(array, is_optional, buffer); - - let statistics = if options.write_statistics { - Some(serialize_statistics(&build_statistics( - array, - type_.clone(), - ))) - } else { - None - }; - - utils::build_plain_page( - buffer, - nested::num_values(nested), - nested[0].len(), - array.null_count(), - repetition_levels_byte_length, - definition_levels_byte_length, - statistics, - type_, - options, - Encoding::Plain, - ) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/row_group.rs b/src/common/arrow/src/arrow/io/parquet/write/row_group.rs deleted file mode 100644 index 30a3bfbe8665..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/row_group.rs +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::error::Error as ParquetError; -use parquet2::schema::types::ParquetType; -use parquet2::write::Compressor; -use parquet2::FallibleStreamingIterator; - -use super::array_to_columns; -use super::to_parquet_schema; -use super::DynIter; -use super::DynStreamingIterator; -use super::Encoding; -use super::RowGroupIter; -use super::SchemaDescriptor; -use super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// Maps a [`Chunk`] and parquet-specific options to an [`RowGroupIter`] used to -/// write to parquet -/// # Panics -/// Iff -/// * `encodings.len() != fields.len()` or -/// * `encodings.len() != chunk.arrays().len()` -pub fn row_group_iter + 'static + Send + Sync>( - chunk: Chunk, - encodings: Vec>, - fields: Vec, - options: WriteOptions, -) -> RowGroupIter<'static, Error> { - assert_eq!(encodings.len(), fields.len()); - assert_eq!(encodings.len(), chunk.arrays().len()); - DynIter::new( - chunk - .into_arrays() - .into_iter() - .zip(fields) - .zip(encodings) - .flat_map(move |((array, type_), encoding)| { - let encoded_columns = array_to_columns(array, type_, options, &encoding).unwrap(); - encoded_columns - .into_iter() - .map(|encoded_pages| { - let pages = encoded_pages; - - let pages = DynIter::new( - pages - .into_iter() - .map(|x| x.map_err(|e| ParquetError::OutOfSpec(e.to_string()))), - ); - - let compressed_pages = Compressor::new(pages, options.compression, vec![]) - .map_err(Error::from); - Ok(DynStreamingIterator::new(compressed_pages)) - }) - .collect::>() - }), - ) -} - -/// An iterator adapter that converts an iterator over [`Chunk`] into an iterator -/// of row groups. -/// Use it to create an iterator consumable by the parquet's API. -pub struct RowGroupIterator + 'static, I: Iterator>>> { - iter: I, - options: WriteOptions, - parquet_schema: SchemaDescriptor, - encodings: Vec>, -} - -impl + 'static, I: Iterator>>> RowGroupIterator { - /// Creates a new [`RowGroupIterator`] from an iterator over [`Chunk`]. - /// - /// # Errors - /// Iff - /// * the Arrow schema can't be converted to a valid Parquet schema. - /// * the length of the encodings is different from the number of fields in schema - pub fn try_new( - iter: I, - schema: &Schema, - options: WriteOptions, - encodings: Vec>, - ) -> Result { - if encodings.len() != schema.fields.len() { - return Err(Error::InvalidArgumentError( - "The number of encodings must equal the number of fields".to_string(), - )); - } - let parquet_schema = to_parquet_schema(schema)?; - - Ok(Self { - iter, - options, - parquet_schema, - encodings, - }) - } - - /// Returns the [`SchemaDescriptor`] of the [`RowGroupIterator`]. - pub fn parquet_schema(&self) -> &SchemaDescriptor { - &self.parquet_schema - } -} - -impl + 'static + Send + Sync, I: Iterator>>> Iterator - for RowGroupIterator -{ - type Item = Result>; - - fn next(&mut self) -> Option { - let options = self.options; - - self.iter.next().map(|maybe_chunk| { - let chunk = maybe_chunk?; - if self.encodings.len() != chunk.arrays().len() { - return Err(Error::InvalidArgumentError( - "The number of arrays in the chunk must equal the number of fields in the schema" - .to_string(), - )); - }; - let encodings = self.encodings.clone(); - Ok(row_group_iter( - chunk, - encodings, - self.parquet_schema.fields().to_vec(), - options, - )) - }) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/schema.rs b/src/common/arrow/src/arrow/io/parquet/write/schema.rs deleted file mode 100644 index b7888bc7b834..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/schema.rs +++ /dev/null @@ -1,407 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use base64::engine::general_purpose; -use base64::Engine as _; -use parquet2::metadata::KeyValue; -use parquet2::schema::types::GroupConvertedType; -use parquet2::schema::types::GroupLogicalType; -use parquet2::schema::types::IntegerType; -use parquet2::schema::types::ParquetType; -use parquet2::schema::types::PhysicalType; -use parquet2::schema::types::PrimitiveConvertedType; -use parquet2::schema::types::PrimitiveLogicalType; -use parquet2::schema::types::TimeUnit as ParquetTimeUnit; -use parquet2::schema::Repetition; - -use super::super::ARROW_SCHEMA_META_KEY; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::Schema; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::ipc::write::default_ipc_fields; -use crate::arrow::io::ipc::write::schema_to_bytes; -use crate::arrow::io::parquet::write::decimal_length_from_precision; - -pub fn schema_to_metadata_key(schema: &Schema) -> KeyValue { - let serialized_schema = schema_to_bytes(schema, &default_ipc_fields(&schema.fields)); - - // manually prepending the length to the schema as arrow uses the legacy IPC format - // TODO: change after addressing ARROW-9777 - let schema_len = serialized_schema.len(); - let mut len_prefix_schema = Vec::with_capacity(schema_len + 8); - len_prefix_schema.extend_from_slice(&[255u8, 255, 255, 255]); - len_prefix_schema.extend_from_slice(&(schema_len as u32).to_le_bytes()); - len_prefix_schema.extend_from_slice(&serialized_schema); - - let encoded = general_purpose::STANDARD.encode(&len_prefix_schema); - - KeyValue { - key: ARROW_SCHEMA_META_KEY.to_string(), - value: Some(encoded), - } -} - -/// Creates a [`ParquetType`] from a [`Field`]. -pub fn to_parquet_type(field: &Field) -> Result { - let name = field.name.clone(); - let repetition = if field.is_nullable { - Repetition::Optional - } else { - Repetition::Required - }; - // create type from field - match field.data_type().to_logical_type() { - DataType::Null => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - None, - Some(PrimitiveLogicalType::Unknown), - None, - )?), - DataType::Boolean => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Boolean, - repetition, - None, - None, - None, - )?), - DataType::Int32 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - None, - None, - None, - )?), - // DataType::Duration(_) has no parquet representation => do not apply any logical type - DataType::Int64 | DataType::Duration(_) => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int64, - repetition, - None, - None, - None, - )?), - // no natural representation in parquet; leave it as is. - // arrow consumers MAY use the arrow schema in the metadata to parse them. - DataType::Date64 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int64, - repetition, - None, - None, - None, - )?), - DataType::Float32 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Float, - repetition, - None, - None, - None, - )?), - DataType::Float64 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Double, - repetition, - None, - None, - None, - )?), - DataType::Binary | DataType::LargeBinary | DataType::BinaryView => { - Ok(ParquetType::try_from_primitive( - name, - PhysicalType::ByteArray, - repetition, - None, - None, - None, - )?) - } - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => { - Ok(ParquetType::try_from_primitive( - name, - PhysicalType::ByteArray, - repetition, - Some(PrimitiveConvertedType::Utf8), - Some(PrimitiveLogicalType::String), - None, - )?) - } - DataType::Date32 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - Some(PrimitiveConvertedType::Date), - Some(PrimitiveLogicalType::Date), - None, - )?), - DataType::Int8 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - Some(PrimitiveConvertedType::Int8), - Some(PrimitiveLogicalType::Integer(IntegerType::Int8)), - None, - )?), - DataType::Int16 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - Some(PrimitiveConvertedType::Int16), - Some(PrimitiveLogicalType::Integer(IntegerType::Int16)), - None, - )?), - DataType::UInt8 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - Some(PrimitiveConvertedType::Uint8), - Some(PrimitiveLogicalType::Integer(IntegerType::UInt8)), - None, - )?), - DataType::UInt16 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - Some(PrimitiveConvertedType::Uint16), - Some(PrimitiveLogicalType::Integer(IntegerType::UInt16)), - None, - )?), - DataType::UInt32 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - Some(PrimitiveConvertedType::Uint32), - Some(PrimitiveLogicalType::Integer(IntegerType::UInt32)), - None, - )?), - DataType::UInt64 => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int64, - repetition, - Some(PrimitiveConvertedType::Uint64), - Some(PrimitiveLogicalType::Integer(IntegerType::UInt64)), - None, - )?), - // no natural representation in parquet; leave it as is. - // arrow consumers MAY use the arrow schema in the metadata to parse them. - DataType::Timestamp(TimeUnit::Second, _) => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int64, - repetition, - None, - None, - None, - )?), - DataType::Timestamp(time_unit, zone) => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int64, - repetition, - None, - Some(PrimitiveLogicalType::Timestamp { - is_adjusted_to_utc: matches!(zone, Some(z) if !z.as_str().is_empty()), - unit: match time_unit { - TimeUnit::Second => unreachable!(), - TimeUnit::Millisecond => ParquetTimeUnit::Milliseconds, - TimeUnit::Microsecond => ParquetTimeUnit::Microseconds, - TimeUnit::Nanosecond => ParquetTimeUnit::Nanoseconds, - }, - }), - None, - )?), - // no natural representation in parquet; leave it as is. - // arrow consumers MAY use the arrow schema in the metadata to parse them. - DataType::Time32(TimeUnit::Second) => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - None, - None, - None, - )?), - DataType::Time32(TimeUnit::Millisecond) => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - Some(PrimitiveConvertedType::TimeMillis), - Some(PrimitiveLogicalType::Time { - is_adjusted_to_utc: false, - unit: ParquetTimeUnit::Milliseconds, - }), - None, - )?), - DataType::Time64(time_unit) => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int64, - repetition, - match time_unit { - TimeUnit::Microsecond => Some(PrimitiveConvertedType::TimeMicros), - TimeUnit::Nanosecond => None, - _ => unreachable!(), - }, - Some(PrimitiveLogicalType::Time { - is_adjusted_to_utc: false, - unit: match time_unit { - TimeUnit::Microsecond => ParquetTimeUnit::Microseconds, - TimeUnit::Nanosecond => ParquetTimeUnit::Nanoseconds, - _ => unreachable!(), - }, - }), - None, - )?), - DataType::Struct(fields) => { - if fields.is_empty() { - return Err(Error::InvalidArgumentError( - "Parquet does not support writing empty structs".to_string(), - )); - } - // recursively convert children to types/nodes - let fields = fields - .iter() - .map(to_parquet_type) - .collect::>>()?; - Ok(ParquetType::from_group( - name, repetition, None, None, fields, None, - )) - } - DataType::Dictionary(_, value, _) => { - let dict_field = Field::new(name.as_str(), value.as_ref().clone(), field.is_nullable); - to_parquet_type(&dict_field) - } - DataType::FixedSizeBinary(size) => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::FixedLenByteArray(*size), - repetition, - None, - None, - None, - )?), - DataType::Decimal(precision, scale) => { - let precision = *precision; - let scale = *scale; - let logical_type = Some(PrimitiveLogicalType::Decimal(precision, scale)); - - let physical_type = if precision <= 9 { - PhysicalType::Int32 - } else if precision <= 18 { - PhysicalType::Int64 - } else { - let len = decimal_length_from_precision(precision); - PhysicalType::FixedLenByteArray(len) - }; - Ok(ParquetType::try_from_primitive( - name, - physical_type, - repetition, - Some(PrimitiveConvertedType::Decimal(precision, scale)), - logical_type, - None, - )?) - } - DataType::Decimal256(precision, scale) => { - let precision = *precision; - let scale = *scale; - let logical_type = Some(PrimitiveLogicalType::Decimal(precision, scale)); - - if precision <= 9 { - Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int32, - repetition, - Some(PrimitiveConvertedType::Decimal(precision, scale)), - logical_type, - None, - )?) - } else if precision <= 18 { - Ok(ParquetType::try_from_primitive( - name, - PhysicalType::Int64, - repetition, - Some(PrimitiveConvertedType::Decimal(precision, scale)), - logical_type, - None, - )?) - } else if precision <= 38 { - let len = decimal_length_from_precision(precision); - Ok(ParquetType::try_from_primitive( - name, - PhysicalType::FixedLenByteArray(len), - repetition, - Some(PrimitiveConvertedType::Decimal(precision, scale)), - logical_type, - None, - )?) - } else { - Ok(ParquetType::try_from_primitive( - name, - PhysicalType::FixedLenByteArray(32), - repetition, - Some(PrimitiveConvertedType::Decimal(precision, scale)), - logical_type, - None, - )?) - } - } - DataType::Interval(_) => Ok(ParquetType::try_from_primitive( - name, - PhysicalType::FixedLenByteArray(12), - repetition, - Some(PrimitiveConvertedType::Interval), - None, - None, - )?), - DataType::List(f) | DataType::FixedSizeList(f, _) | DataType::LargeList(f) => { - Ok(ParquetType::from_group( - name, - repetition, - Some(GroupConvertedType::List), - Some(GroupLogicalType::List), - vec![ParquetType::from_group( - "list".to_string(), - Repetition::Repeated, - None, - None, - vec![to_parquet_type(f)?], - None, - )], - None, - )) - } - DataType::Map(f, _) => Ok(ParquetType::from_group( - name, - repetition, - Some(GroupConvertedType::Map), - Some(GroupLogicalType::Map), - vec![ParquetType::from_group( - "map".to_string(), - Repetition::Repeated, - None, - None, - vec![to_parquet_type(f)?], - None, - )], - None, - )), - other => Err(Error::NotYetImplemented(format!( - "Writing the data type {other:?} is not yet implemented" - ))), - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/sink.rs b/src/common/arrow/src/arrow/io/parquet/write/sink.rs deleted file mode 100644 index 4b92216f60d5..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/sink.rs +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::pin::Pin; -use std::task::Poll; - -use ahash::AHashMap; -use futures::future::BoxFuture; -use futures::AsyncWrite; -use futures::AsyncWriteExt; -use futures::FutureExt; -use futures::Sink; -use futures::TryFutureExt; -use parquet2::metadata::KeyValue; -use parquet2::write::FileStreamer; -use parquet2::write::WriteOptions as ParquetWriteOptions; - -use super::file::add_arrow_schema; -use super::Encoding; -use super::SchemaDescriptor; -use super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; - -/// Sink that writes array [`chunks`](Chunk) as a Parquet file. -/// -/// Any values in the sink's `metadata` field will be written to the file's footer -/// when the sink is closed. -/// -/// # Examples -/// -/// ``` -/// use arrow2::array::Array; -/// use arrow2::array::Int32Array; -/// use arrow2::chunk::Chunk; -/// use arrow2::datatypes::DataType; -/// use arrow2::datatypes::Field; -/// use arrow2::datatypes::Schema; -/// use arrow2::io::parquet::write::CompressionOptions; -/// use arrow2::io::parquet::write::Encoding; -/// use arrow2::io::parquet::write::Version; -/// use arrow2::io::parquet::write::WriteOptions; -/// use futures::SinkExt; -/// # use arrow2::io::parquet::write::FileSink; -/// # futures::executor::block_on(async move { -/// -/// let schema = Schema::from(vec![Field::new("values", DataType::Int32, true)]); -/// let encoding = vec![vec![Encoding::Plain]]; -/// let options = WriteOptions { -/// write_statistics: true, -/// compression: CompressionOptions::Uncompressed, -/// version: Version::V2, -/// data_pagesize_limit: None, -/// }; -/// -/// let mut buffer = vec![]; -/// let mut sink = FileSink::try_new(&mut buffer, schema, encoding, options)?; -/// -/// for i in 0..3 { -/// let values = Int32Array::from(&[Some(i), None]); -/// let chunk = Chunk::new(vec![values.boxed()]); -/// sink.feed(chunk).await?; -/// } -/// sink.metadata -/// .insert(String::from("key"), Some(String::from("value"))); -/// sink.close().await?; -/// # arrow2::error::Result::Ok(()) -/// # }).unwrap(); -/// ``` -pub struct FileSink<'a, W: AsyncWrite + Send + Unpin> { - writer: Option>, - task: Option>, Error>>>, - options: WriteOptions, - encodings: Vec>, - schema: Schema, - parquet_schema: SchemaDescriptor, - /// Key-value metadata that will be written to the file on close. - pub metadata: AHashMap>, -} - -impl<'a, W> FileSink<'a, W> -where W: AsyncWrite + Send + Unpin + 'a -{ - /// Create a new sink that writes arrays to the provided `writer`. - /// - /// # Error - /// Iff - /// * the Arrow schema can't be converted to a valid Parquet schema. - /// * the length of the encodings is different from the number of fields in schema - pub fn try_new( - writer: W, - schema: Schema, - encodings: Vec>, - options: WriteOptions, - ) -> Result { - if encodings.len() != schema.fields.len() { - return Err(Error::InvalidArgumentError( - "The number of encodings must equal the number of fields".to_string(), - )); - } - - let parquet_schema = crate::arrow::io::parquet::write::to_parquet_schema(&schema)?; - let created_by = Some("Arrow2 - Native Rust implementation of Arrow".to_string()); - let writer = FileStreamer::new( - writer, - parquet_schema.clone(), - ParquetWriteOptions { - version: options.version, - write_statistics: options.write_statistics, - }, - created_by, - ); - Ok(Self { - writer: Some(writer), - task: None, - options, - schema, - encodings, - parquet_schema, - metadata: AHashMap::default(), - }) - } - - /// The Arrow [`Schema`] for the file. - pub fn schema(&self) -> &Schema { - &self.schema - } - - /// The Parquet [`SchemaDescriptor`] for the file. - pub fn parquet_schema(&self) -> &SchemaDescriptor { - &self.parquet_schema - } - - /// The write options for the file. - pub fn options(&self) -> &WriteOptions { - &self.options - } - - fn poll_complete( - &mut self, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - if let Some(task) = &mut self.task { - match futures::ready!(task.poll_unpin(cx)) { - Ok(writer) => { - self.task = None; - self.writer = writer; - Poll::Ready(Ok(())) - } - Err(error) => { - self.task = None; - Poll::Ready(Err(error)) - } - } - } else { - Poll::Ready(Ok(())) - } - } -} - -impl<'a, W> Sink>> for FileSink<'a, W> -where W: AsyncWrite + Send + Unpin + 'a -{ - type Error = Error; - - fn start_send(self: Pin<&mut Self>, item: Chunk>) -> Result<(), Self::Error> { - if self.schema.fields.len() != item.arrays().len() { - return Err(Error::InvalidArgumentError( - "The number of arrays in the chunk must equal the number of fields in the schema" - .to_string(), - )); - } - let this = self.get_mut(); - if let Some(mut writer) = this.writer.take() { - let rows = crate::arrow::io::parquet::write::row_group_iter( - item, - this.encodings.clone(), - this.parquet_schema.fields().to_vec(), - this.options, - ); - this.task = Some(Box::pin(async move { - writer.write(rows).await?; - Ok(Some(writer)) - })); - Ok(()) - } else { - Err(Error::Io(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "writer closed".to_string(), - ))) - } - } - - fn poll_ready( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.get_mut().poll_complete(cx) - } - - fn poll_flush( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.get_mut().poll_complete(cx) - } - - fn poll_close( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - let this = self.get_mut(); - match futures::ready!(this.poll_complete(cx)) { - Ok(()) => { - let writer = this.writer.take(); - if let Some(mut writer) = writer { - let meta = std::mem::take(&mut this.metadata); - let metadata = if meta.is_empty() { - None - } else { - Some( - meta.into_iter() - .map(|(k, v)| KeyValue::new(k, v)) - .collect::>(), - ) - }; - let kv_meta = add_arrow_schema(&this.schema, metadata); - - this.task = Some(Box::pin(async move { - writer.end(kv_meta).map_err(Error::from).await?; - writer.into_inner().close().map_err(Error::from).await?; - Ok(None) - })); - this.poll_complete(cx) - } else { - Poll::Ready(Ok(())) - } - } - Err(error) => Poll::Ready(Err(error)), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/utf8/basic.rs b/src/common/arrow/src/arrow/io/parquet/write/utf8/basic.rs deleted file mode 100644 index fd83bb43793a..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/utf8/basic.rs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::serialize_statistics; -use parquet2::statistics::BinaryStatistics; -use parquet2::statistics::ParquetStatistics; -use parquet2::statistics::Statistics; - -use super::super::binary::encode_delta; -use super::super::binary::ord_binary; -use super::super::utils; -use super::super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::array::Utf8Array; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::offset::Offset; - -pub(crate) fn encode_plain( - array: &Utf8Array, - is_optional: bool, - buffer: &mut Vec, -) { - if is_optional { - array.iter().for_each(|x| { - if let Some(x) = x { - // BYTE_ARRAY: first 4 bytes denote length in littleendian. - let len = (x.len() as u32).to_le_bytes(); - buffer.extend_from_slice(&len); - buffer.extend_from_slice(x.as_bytes()); - } - }) - } else { - array.values_iter().for_each(|x| { - // BYTE_ARRAY: first 4 bytes denote length in littleendian. - let len = (x.len() as u32).to_le_bytes(); - buffer.extend_from_slice(&len); - buffer.extend_from_slice(x.as_bytes()); - }) - } -} - -pub fn array_to_page( - array: &Utf8Array, - options: WriteOptions, - type_: PrimitiveType, - encoding: Encoding, -) -> Result { - let validity = array.validity(); - let is_optional = is_nullable(&type_.field_info); - - let mut buffer = vec![]; - utils::write_def_levels( - &mut buffer, - is_optional, - validity, - array.len(), - options.version, - )?; - - let definition_levels_byte_length = buffer.len(); - - match encoding { - Encoding::Plain => encode_plain(array, is_optional, &mut buffer), - Encoding::DeltaLengthByteArray => encode_delta( - array.values(), - array.offsets().buffer(), - array.validity(), - is_optional, - &mut buffer, - ), - _ => { - return Err(Error::InvalidArgumentError(format!( - "Datatype {:?} cannot be encoded by {:?} encoding", - array.data_type(), - encoding - ))); - } - } - - let statistics = if options.write_statistics { - Some(build_statistics(array, type_.clone())) - } else { - None - }; - - utils::build_plain_page( - buffer, - array.len(), - array.len(), - array.null_count(), - 0, - definition_levels_byte_length, - statistics, - type_, - options, - encoding, - ) -} - -pub(crate) fn build_statistics( - array: &Utf8Array, - primitive_type: PrimitiveType, -) -> ParquetStatistics { - let statistics = &BinaryStatistics { - primitive_type, - null_count: Some(array.null_count() as i64), - distinct_count: None, - max_value: array - .iter() - .flatten() - .map(|x| x.as_bytes()) - .max_by(|x, y| ord_binary(x, y)) - .map(|x| x.to_vec()), - min_value: array - .iter() - .flatten() - .map(|x| x.as_bytes()) - .min_by(|x, y| ord_binary(x, y)) - .map(|x| x.to_vec()), - } as &dyn Statistics; - serialize_statistics(statistics) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/utf8/mod.rs b/src/common/arrow/src/arrow/io/parquet/write/utf8/mod.rs deleted file mode 100644 index b9a3791ab139..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/utf8/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod basic; -mod nested; - -pub use basic::array_to_page; -pub(crate) use basic::build_statistics; -pub(crate) use basic::encode_plain; -pub use nested::array_to_page as nested_array_to_page; diff --git a/src/common/arrow/src/arrow/io/parquet/write/utf8/nested.rs b/src/common/arrow/src/arrow/io/parquet/write/utf8/nested.rs deleted file mode 100644 index 0046aa627c11..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/utf8/nested.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::encoding::Encoding; -use parquet2::page::DataPage; -use parquet2::schema::types::PrimitiveType; - -use super::super::nested; -use super::super::utils; -use super::super::WriteOptions; -use super::basic::build_statistics; -use super::basic::encode_plain; -use crate::arrow::array::Array; -use crate::arrow::array::Utf8Array; -use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::schema::is_nullable; -use crate::arrow::io::parquet::write::Nested; -use crate::arrow::offset::Offset; - -pub fn array_to_page( - array: &Utf8Array, - options: WriteOptions, - type_: PrimitiveType, - nested: &[Nested], -) -> Result -where - O: Offset, -{ - let is_optional = is_nullable(&type_.field_info); - - let mut buffer = vec![]; - let (repetition_levels_byte_length, definition_levels_byte_length) = - nested::write_rep_and_def(options.version, nested, &mut buffer)?; - - encode_plain(array, is_optional, &mut buffer); - - let statistics = if options.write_statistics { - Some(build_statistics(array, type_.clone())) - } else { - None - }; - - utils::build_plain_page( - buffer, - nested::num_values(nested), - nested[0].len(), - array.null_count(), - repetition_levels_byte_length, - definition_levels_byte_length, - statistics, - type_, - options, - Encoding::Plain, - ) -} diff --git a/src/common/arrow/src/arrow/io/parquet/write/utils.rs b/src/common/arrow/src/arrow/io/parquet/write/utils.rs deleted file mode 100644 index aef07138cf52..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/write/utils.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet2::compression::CompressionOptions; -use parquet2::encoding::hybrid_rle::encode_bool; -use parquet2::encoding::Encoding; -use parquet2::metadata::Descriptor; -use parquet2::page::DataPage; -use parquet2::page::DataPageHeader; -use parquet2::page::DataPageHeaderV1; -use parquet2::page::DataPageHeaderV2; -use parquet2::schema::types::PrimitiveType; -use parquet2::statistics::ParquetStatistics; - -use super::Version; -use super::WriteOptions; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::error::Result; - -fn encode_iter_v1>(buffer: &mut Vec, iter: I) -> Result<()> { - buffer.extend_from_slice(&[0; 4]); - let start = buffer.len(); - encode_bool(buffer, iter)?; - let end = buffer.len(); - let length = end - start; - - // write the first 4 bytes as length - let length = (length as i32).to_le_bytes(); - (0..4).for_each(|i| buffer[start - 4 + i] = length[i]); - Ok(()) -} - -fn encode_iter_v2>(writer: &mut Vec, iter: I) -> Result<()> { - Ok(encode_bool(writer, iter)?) -} - -fn encode_iter>( - writer: &mut Vec, - iter: I, - version: Version, -) -> Result<()> { - match version { - Version::V1 => encode_iter_v1(writer, iter), - Version::V2 => encode_iter_v2(writer, iter), - } -} - -/// writes the def levels to a `Vec` and returns it. -pub fn write_def_levels( - writer: &mut Vec, - is_optional: bool, - validity: Option<&Bitmap>, - len: usize, - version: Version, -) -> Result<()> { - // encode def levels - match (is_optional, validity) { - (true, Some(validity)) => encode_iter(writer, validity.iter(), version), - (true, None) => encode_iter(writer, std::iter::repeat(true).take(len), version), - _ => Ok(()), // is required => no def levels - } -} - -#[allow(clippy::too_many_arguments)] -pub fn build_plain_page( - buffer: Vec, - num_values: usize, - num_rows: usize, - null_count: usize, - repetition_levels_byte_length: usize, - definition_levels_byte_length: usize, - statistics: Option, - type_: PrimitiveType, - options: WriteOptions, - encoding: Encoding, -) -> Result { - let header = match options.version { - Version::V1 => DataPageHeader::V1(DataPageHeaderV1 { - num_values: num_values as i32, - encoding: encoding.into(), - definition_level_encoding: Encoding::Rle.into(), - repetition_level_encoding: Encoding::Rle.into(), - statistics, - }), - Version::V2 => DataPageHeader::V2(DataPageHeaderV2 { - num_values: num_values as i32, - encoding: encoding.into(), - num_nulls: null_count as i32, - num_rows: num_rows as i32, - definition_levels_byte_length: definition_levels_byte_length as i32, - repetition_levels_byte_length: repetition_levels_byte_length as i32, - is_compressed: Some(options.compression != CompressionOptions::Uncompressed), - statistics, - }), - }; - Ok(DataPage::new( - header, - buffer, - Descriptor { - primitive_type: type_, - max_def_level: 0, - max_rep_level: 0, - }, - Some(num_rows), - )) -} - -/// Auxiliary iterator adapter to declare the size hint of an iterator. -pub(super) struct ExactSizedIter> { - iter: I, - remaining: usize, -} - -impl + Clone> Clone for ExactSizedIter { - fn clone(&self) -> Self { - Self { - iter: self.iter.clone(), - remaining: self.remaining, - } - } -} - -impl> ExactSizedIter { - pub fn new(iter: I, length: usize) -> Self { - Self { - iter, - remaining: length, - } - } -} - -impl> Iterator for ExactSizedIter { - type Item = T; - - #[inline] - fn next(&mut self) -> Option { - self.iter.next().inspect(|_| { - self.remaining -= 1; - }) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.remaining, Some(self.remaining)) - } -} - -/// Returns the number of bits needed to bitpack `max` -#[inline] -pub fn get_bit_width(max: u64) -> u32 { - 64 - max.leading_zeros() -} diff --git a/src/common/arrow/src/arrow/mod.rs b/src/common/arrow/src/arrow/mod.rs index cbd0881d8636..a7e59cdd435e 100644 --- a/src/common/arrow/src/arrow/mod.rs +++ b/src/common/arrow/src/arrow/mod.rs @@ -17,7 +17,6 @@ pub mod bitmap; pub mod buffer; pub mod datatypes; pub mod error; -pub mod ffi; pub mod offset; pub mod trusted_len; pub mod types; @@ -25,7 +24,6 @@ pub mod types; pub mod array; pub mod chunk; pub mod compute; -pub mod io; pub mod scalar; pub mod temporal_conversions; pub mod util; diff --git a/src/common/arrow/src/arrow/util/lexical.rs b/src/common/arrow/src/arrow/util/lexical.rs deleted file mode 100644 index d286b0b7b77a..000000000000 --- a/src/common/arrow/src/arrow/util/lexical.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/// Converts numeric type to a `String` -#[inline] -pub fn lexical_to_bytes(n: N) -> Vec { - let mut buf = Vec::::with_capacity(N::FORMATTED_SIZE_DECIMAL); - lexical_to_bytes_mut(n, &mut buf); - buf -} - -/// Converts numeric type to a `String` -#[inline] -pub fn lexical_to_bytes_mut(n: N, buf: &mut Vec) { - buf.clear(); - buf.reserve(N::FORMATTED_SIZE_DECIMAL); - unsafe { - // JUSTIFICATION - // Benefit - // Allows using the faster serializer lexical core and convert to string - // Soundness - // Length of buf is set as written length afterwards. lexical_core - // creates a valid string, so doesn't need to be checked. - let slice = std::slice::from_raw_parts_mut(buf.as_mut_ptr(), buf.capacity()); - - // Safety: - // Omits an unneeded bound check as we just ensured that we reserved `N::FORMATTED_SIZE_DECIMAL` - #[cfg(debug_assertions)] - { - let len = lexical_core::write(n, slice).len(); - buf.set_len(len); - } - #[cfg(not(debug_assertions))] - { - let len = lexical_core::write(n, slice).len(); - buf.set_len(len); - } - } -} - -/// Converts numeric type to a `String` -#[inline] -pub fn lexical_to_string(n: N) -> String { - unsafe { String::from_utf8_unchecked(lexical_to_bytes(n)) } -} diff --git a/src/common/arrow/src/arrow/util/mod.rs b/src/common/arrow/src/arrow/util/mod.rs index 1cfc9452bd5e..2e6c349ef88e 100644 --- a/src/common/arrow/src/arrow/util/mod.rs +++ b/src/common/arrow/src/arrow/util/mod.rs @@ -15,6 +15,4 @@ //! Misc utilities used in different places in the crate. -mod lexical; -pub use lexical::*; pub mod bench_util; diff --git a/src/common/arrow/src/lib.rs b/src/common/arrow/src/lib.rs index d0068d3448f2..ab58813bfb7a 100644 --- a/src/common/arrow/src/lib.rs +++ b/src/common/arrow/src/lib.rs @@ -26,7 +26,4 @@ pub mod arrow; pub mod native; pub mod schema_projection; -pub use arrow_format; -pub use parquet2 as parquet; - pub type ArrayRef = Box; diff --git a/src/common/arrow/src/native/mod.rs b/src/common/arrow/src/native/mod.rs index 93153e3432bf..ee65c1b91973 100644 --- a/src/common/arrow/src/native/mod.rs +++ b/src/common/arrow/src/native/mod.rs @@ -18,15 +18,18 @@ mod errors; mod util; mod compression; +pub mod nested; pub use compression::CommonCompression; pub use compression::Compression; -pub type SchemaDescriptor = parquet2::metadata::SchemaDescriptor; pub mod read; pub mod stat; pub mod write; +pub use util::*; -const ARROW_MAGIC: [u8; 6] = [b'A', b'R', b'R', b'O', b'W', b'2']; -pub(crate) const CONTINUATION_MARKER: [u8; 4] = [0xff; 4]; +// StrawBoat = SB, haha +const STRAWBOAT_MAGIC: [u8; 2] = [b'S', b'B']; +const STRAWBOAT_VERSION: u16 = 2; +pub(crate) const EOF_MARKER: [u8; 4] = [0xff; 4]; #[derive( Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, serde::Serialize, serde::Deserialize, diff --git a/src/common/arrow/src/native/nested.rs b/src/common/arrow/src/native/nested.rs new file mode 100644 index 000000000000..87c7dd49ffda --- /dev/null +++ b/src/common/arrow/src/native/nested.rs @@ -0,0 +1,403 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::arrow::array::Array; +use crate::arrow::array::FixedSizeListArray; +use crate::arrow::array::ListArray; +use crate::arrow::array::MapArray; +use crate::arrow::array::StructArray; +use crate::arrow::bitmap::Bitmap; +use crate::arrow::buffer::Buffer; +use crate::arrow::datatypes::DataType; +use crate::arrow::datatypes::Field; +use crate::arrow::datatypes::PhysicalType; +use crate::arrow::error::Error; +use crate::arrow::error::Result; +use crate::arrow::offset::Offset; +use crate::arrow::offset::Offsets; +use crate::arrow::offset::OffsetsBuffer; + +/// Descriptor of nested information of a field +#[derive(Debug, Clone, PartialEq)] +pub enum Nested { + /// A primitive array + Primitive(usize, bool, Option), + /// a list + List(ListNested), + /// a list + LargeList(ListNested), + /// A struct array + Struct(usize, bool, Option), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ListNested { + pub is_nullable: bool, + pub offsets: OffsetsBuffer, + pub validity: Option, +} + +impl ListNested { + pub fn new(offsets: OffsetsBuffer, validity: Option, is_nullable: bool) -> Self { + Self { + is_nullable, + offsets, + validity, + } + } +} + +pub type NestedState = Vec; + +impl Nested { + pub fn length(&self) -> usize { + match self { + Nested::Primitive(len, _, _) => *len, + Nested::List(l) => l.offsets.len_proxy(), + Nested::LargeList(l) => l.offsets.len_proxy(), + Nested::Struct(len, _, _) => *len, + } + } + + pub fn is_nullable(&self) -> bool { + match self { + Nested::Primitive(_, b, _) => *b, + Nested::List(l) => l.is_nullable, + Nested::LargeList(l) => l.is_nullable, + Nested::Struct(_, b, _) => *b, + } + } + + pub fn inner(&self) -> (Buffer, &Option) { + match self { + Nested::Primitive(_, _, v) => (Buffer::new(), v), + Nested::List(l) => { + let start = l.offsets.first(); + let buffer = l + .offsets + .buffer() + .iter() + .map(|x| (*x - start) as i64) + .collect(); + (buffer, &l.validity) + } + Nested::LargeList(l) => { + let start = l.offsets.first(); + let buffer = if *start == 0 { + l.offsets.buffer().clone() + } else { + l.offsets.buffer().iter().map(|x| *x - start).collect() + }; + (buffer, &l.validity) + } + Nested::Struct(_, _, v) => (Buffer::new(), v), + } + } + + pub fn validity(&self) -> &Option { + match self { + Nested::Primitive(_, _, v) => v, + Nested::List(l) => &l.validity, + Nested::LargeList(l) => &l.validity, + Nested::Struct(_, _, v) => v, + } + } + + pub fn is_list(&self) -> bool { + matches!(self, Nested::List(_) | Nested::LargeList(_)) + } +} + +/// Constructs the necessary `Vec>` to write the rep and def levels of `array` to parquet +pub fn to_nested(array: &dyn Array, f: &Field) -> Result>> { + let mut nested = vec![]; + + to_nested_recursive(array, f, &mut nested, vec![])?; + Ok(nested) +} + +pub fn is_nested_type(t: &DataType) -> bool { + matches!( + t, + DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_) | DataType::Map(_, _) + ) +} + +/// Slices the [`Array`] to `Box` and `Vec`. +pub fn slice_nest_array( + primitive_array: &mut dyn Array, + nested: &mut [Nested], + mut current_offset: usize, + mut current_length: usize, +) { + for nested in nested.iter_mut() { + match nested { + Nested::LargeList(l_nested) => { + l_nested.offsets.slice(current_offset, current_length + 1); + if let Some(validity) = l_nested.validity.as_mut() { + validity.slice(current_offset, current_length) + }; + + current_length = l_nested.offsets.range() as usize; + current_offset = *l_nested.offsets.first() as usize; + } + Nested::List(l_nested) => { + l_nested.offsets.slice(current_offset, current_length + 1); + if let Some(validity) = l_nested.validity.as_mut() { + validity.slice(current_offset, current_length) + }; + + current_length = l_nested.offsets.range() as usize; + current_offset = *l_nested.offsets.first() as usize; + } + Nested::Struct(length, _, validity) => { + *length = current_length; + if let Some(validity) = validity.as_mut() { + validity.slice(current_offset, current_length) + }; + } + Nested::Primitive(length, _, validity) => { + *length = current_length; + if let Some(validity) = validity.as_mut() { + validity.slice(current_offset, current_length) + }; + primitive_array.slice(current_offset, current_length); + } + } + } +} + +fn to_nested_recursive( + array: &dyn Array, + f: &Field, + nested: &mut Vec>, + mut parents: Vec, +) -> Result<()> { + use PhysicalType::*; + let lt = f.data_type.to_logical_type(); + let nullable = f.is_nullable; + match array.data_type().to_physical_type() { + Struct => { + let array = array.as_any().downcast_ref::().unwrap(); + parents.push(Nested::Struct( + array.len(), + nullable, + array.validity().cloned(), + )); + + if let DataType::Struct(fs) = lt { + for (array, f) in array.values().iter().zip(fs.iter()) { + to_nested_recursive(array.as_ref(), f, nested, parents.clone())?; + } + } else { + return Err(Error::InvalidArgumentError( + "DataType type must be a group for a struct array".to_string(), + )); + } + } + List => { + let array = array.as_any().downcast_ref::>().unwrap(); + + if let DataType::List(fs) = lt { + parents.push(Nested::List(ListNested::new( + array.offsets().clone(), + array.validity().cloned(), + nullable, + ))); + to_nested_recursive(array.values().as_ref(), fs.as_ref(), nested, parents)?; + } else { + return Err(Error::InvalidArgumentError( + "DataType type must be a group for a List array".to_string(), + )); + } + } + LargeList => { + let array = array.as_any().downcast_ref::>().unwrap(); + if let DataType::LargeList(fs) = lt { + parents.push(Nested::LargeList(ListNested::::new( + array.offsets().clone(), + array.validity().cloned(), + nullable, + ))); + to_nested_recursive(array.values().as_ref(), fs.as_ref(), nested, parents)?; + } else { + return Err(Error::InvalidArgumentError( + "DataType type must be a group for a LargeList array".to_string(), + )); + } + } + Map => { + let array = array.as_any().downcast_ref::().unwrap(); + if let DataType::Map(fs, _) = lt { + parents.push(Nested::List(ListNested::new( + array.offsets().clone(), + array.validity().cloned(), + nullable, + ))); + to_nested_recursive(array.field().as_ref(), fs.as_ref(), nested, parents)?; + } else { + return Err(Error::InvalidArgumentError( + "DataType type must be a group for a LargeList array".to_string(), + )); + } + } + _ => { + parents.push(Nested::Primitive( + array.len(), + nullable, + array.validity().cloned(), + )); + nested.push(parents); + } + } + Ok(()) +} + +/// Convert [`Array`] to `Vec<&dyn Array>` leaves in DFS order. +pub fn to_leaves(array: &dyn Array) -> Vec<&dyn Array> { + let mut leaves = vec![]; + to_leaves_recursive(array, &mut leaves); + leaves +} + +fn to_leaves_recursive<'a>(array: &'a dyn Array, leaves: &mut Vec<&'a dyn Array>) { + use PhysicalType::*; + match array.data_type().to_physical_type() { + Struct => { + let array = array.as_any().downcast_ref::().unwrap(); + array + .values() + .iter() + .for_each(|a| to_leaves_recursive(a.as_ref(), leaves)); + } + List => { + let array = array.as_any().downcast_ref::>().unwrap(); + to_leaves_recursive(array.values().as_ref(), leaves); + } + LargeList => { + let array = array.as_any().downcast_ref::>().unwrap(); + to_leaves_recursive(array.values().as_ref(), leaves); + } + Map => { + let array = array.as_any().downcast_ref::().unwrap(); + to_leaves_recursive(array.field().as_ref(), leaves); + } + Null | Boolean | Primitive(_) | Binary | FixedSizeBinary | LargeBinary | Utf8 + | LargeUtf8 | Dictionary(_) | BinaryView | Utf8View => leaves.push(array), + other => todo!("Writing {:?} to native not yet implemented", other), + } +} + +/// The initial info of nested data types. +/// The initial info of nested data types. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InitNested { + /// Primitive data types + Primitive(bool), + /// List data types + List(bool), + /// Struct data types + Struct(bool), +} + +impl InitNested { + pub fn is_nullable(&self) -> bool { + match self { + InitNested::Primitive(b) => *b, + InitNested::List(b) => *b, + InitNested::Struct(b) => *b, + } + } +} + +/// Creates a new [`ListArray`] or [`FixedSizeListArray`]. +pub fn create_list( + data_type: DataType, + nested: &mut NestedState, + values: Box, +) -> Box { + let n = nested.pop().unwrap(); + let (offsets, validity) = n.inner(); + match data_type.to_logical_type() { + DataType::List(_) => { + let offsets = offsets.iter().map(|x| *x as i32).collect::>(); + let offsets: Offsets = offsets + .try_into() + .expect("i64 offsets do not fit in i32 offsets"); + + Box::new(ListArray::::new( + data_type, + OffsetsBuffer::from(offsets), + values, + validity.clone(), + )) + } + DataType::LargeList(_) => Box::new(ListArray::::new( + data_type, + unsafe { OffsetsBuffer::new_unchecked(offsets) }, + values, + validity.clone(), + )), + DataType::FixedSizeList(_, _) => { + Box::new(FixedSizeListArray::new(data_type, values, validity.clone())) + } + _ => unreachable!(), + } +} + +/// Creates a new [`MapArray`]. +pub fn create_map( + data_type: DataType, + nested: &mut NestedState, + values: Box, +) -> Box { + let n = nested.pop().unwrap(); + let (offsets, validity) = n.inner(); + match data_type.to_logical_type() { + DataType::Map(_, _) => { + let offsets = offsets.iter().map(|x| *x as i32).collect::>(); + + let offsets: Offsets = offsets + .try_into() + .expect("i64 offsets do not fit in i32 offsets"); + + Box::new(MapArray::new( + data_type, + offsets.into(), + values, + validity.clone(), + )) + } + _ => unreachable!(), + } +} + +pub fn create_struct( + fields: Vec, + nested: &mut Vec, + values: Vec>, +) -> (NestedState, Box) { + let mut nest = nested.pop().unwrap(); + let n = nest.pop().unwrap(); + let (_, validity) = n.inner(); + + ( + nest, + Box::new(StructArray::new( + DataType::Struct(fields), + values, + validity.clone(), + )), + ) +} diff --git a/src/common/arrow/src/native/read/array/binary.rs b/src/common/arrow/src/native/read/array/binary.rs index ac9284e5b5ae..31fc3981a460 100644 --- a/src/common/arrow/src/native/read/array/binary.rs +++ b/src/common/arrow/src/native/read/array/binary.rs @@ -15,117 +15,25 @@ use std::io::Cursor; use std::marker::PhantomData; -use parquet2::metadata::ColumnDescriptor; - use crate::arrow::array::Array; use crate::arrow::array::BinaryArray; use crate::arrow::array::Utf8Array; use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; use crate::arrow::buffer::Buffer; use crate::arrow::datatypes::DataType; use crate::arrow::error::Error; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; use crate::arrow::offset::OffsetsBuffer; use crate::arrow::types::Offset; use crate::native::compression::binary::decompress_binary; +use crate::native::nested::InitNested; +use crate::native::nested::NestedState; use crate::native::read::read_basic::*; use crate::native::read::BufReader; use crate::native::read::NativeReadBuf; use crate::native::read::PageIterator; use crate::native::PageMeta; -#[derive(Debug)] -pub struct BinaryIter -where - I: Iterator)>> + PageIterator + Send + Sync, - O: Offset, -{ - iter: I, - is_nullable: bool, - data_type: DataType, - scratch: Vec, - _phantom: PhantomData, -} - -impl BinaryIter -where - I: Iterator)>> + PageIterator + Send + Sync, - O: Offset, -{ - pub fn new(iter: I, is_nullable: bool, data_type: DataType) -> Self { - Self { - iter, - is_nullable, - data_type, - scratch: vec![], - _phantom: PhantomData, - } - } -} - -impl BinaryIter -where - I: Iterator)>> + PageIterator + Send + Sync, - O: Offset, -{ - fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result> { - let length = num_values as usize; - let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let validity = if self.is_nullable { - let mut validity_builder = MutableBitmap::with_capacity(length); - read_validity(&mut reader, length, &mut validity_builder)?; - Some(std::mem::take(&mut validity_builder).into()) - } else { - None - }; - - let mut offsets: Vec = Vec::with_capacity(length + 1); - let mut values = Vec::with_capacity(0); - - decompress_binary( - &mut reader, - length, - &mut offsets, - &mut values, - &mut self.scratch, - )?; - - try_new_binary_array( - self.data_type.clone(), - unsafe { OffsetsBuffer::new_unchecked(offsets.into()) }, - values.into(), - validity, - ) - } -} - -impl Iterator for BinaryIter -where - I: Iterator)>> + PageIterator + Send + Sync, - O: Offset, -{ - type Item = Result>; - - fn nth(&mut self, n: usize) -> Option { - match self.iter.nth(n) { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } -} - #[derive(Debug)] pub struct BinaryNestedIter where @@ -134,7 +42,6 @@ where { iter: I, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, scratch: Vec, _phantom: PhantomData, @@ -145,16 +52,10 @@ where I: Iterator)>> + PageIterator + Send + Sync, O: Offset, { - pub fn new( - iter: I, - data_type: DataType, - leaf: ColumnDescriptor, - init: Vec, - ) -> Self { + pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { Self { iter, data_type, - leaf, init, scratch: vec![], _phantom: PhantomData, @@ -173,14 +74,8 @@ where buffer: Vec, ) -> Result<(NestedState, Box)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let (mut nested, validity) = read_validity_nested( - &mut reader, - num_values as usize, - &self.leaf, - self.init.clone(), - )?; - let length = nested.nested.pop().unwrap().len(); - + let length = num_values as usize; + let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; let mut offsets: Vec = Vec::with_capacity(length + 1); let mut values = Vec::with_capacity(0); @@ -226,54 +121,9 @@ where } } -pub fn read_binary( - reader: &mut R, - is_nullable: bool, - data_type: DataType, - page_metas: Vec, -) -> Result> { - let num_values = page_metas.iter().map(|p| p.num_values as usize).sum(); - - let total_length: usize = page_metas.iter().map(|p| p.length as usize).sum(); - - let mut validity_builder = if is_nullable { - Some(MutableBitmap::with_capacity(num_values)) - } else { - None - }; - let mut scratch = vec![]; - let out_off_len = num_values + 2; - // don't know how much space is needed for the buffer, - // if not enough, it may need to be reallocated several times. - let out_buf_len = total_length * 4; - let mut offsets: Vec = Vec::with_capacity(out_off_len); - let mut values: Vec = Vec::with_capacity(out_buf_len); - - for page_meta in page_metas { - let length = page_meta.num_values as usize; - if let Some(ref mut validity_builder) = validity_builder { - read_validity(reader, length, validity_builder)?; - } - - decompress_binary(reader, length, &mut offsets, &mut values, &mut scratch)?; - } - let validity = - validity_builder.map(|mut validity_builder| std::mem::take(&mut validity_builder).into()); - let offsets: Buffer = offsets.into(); - let values: Buffer = values.into(); - - try_new_binary_array( - data_type, - unsafe { OffsetsBuffer::new_unchecked(offsets) }, - values, - validity, - ) -} - pub fn read_nested_binary( reader: &mut R, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, page_metas: Vec, ) -> Result)>> { @@ -283,13 +133,11 @@ pub fn read_nested_binary( for page_meta in page_metas { let num_values = page_meta.num_values as usize; - let (mut nested, validity) = read_validity_nested(reader, num_values, &leaf, init.clone())?; - let length = nested.nested.pop().unwrap().len(); - - let mut offsets: Vec = Vec::with_capacity(length + 1); + let (nested, validity) = read_nested(reader, &init, num_values)?; + let mut offsets: Vec = Vec::with_capacity(num_values + 1); let mut values = Vec::with_capacity(0); - decompress_binary(reader, length, &mut offsets, &mut values, &mut scratch)?; + decompress_binary(reader, num_values, &mut offsets, &mut values, &mut scratch)?; let array = try_new_binary_array( data_type.clone(), diff --git a/src/common/arrow/src/native/read/array/boolean.rs b/src/common/arrow/src/native/read/array/boolean.rs index b4a42311bde0..c507bc8b9dfa 100644 --- a/src/common/arrow/src/native/read/array/boolean.rs +++ b/src/common/arrow/src/native/read/array/boolean.rs @@ -14,100 +14,26 @@ use std::io::Cursor; -use parquet2::metadata::ColumnDescriptor; - use crate::arrow::array::Array; use crate::arrow::array::BooleanArray; use crate::arrow::bitmap::MutableBitmap; use crate::arrow::datatypes::DataType; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; use crate::native::compression::boolean::decompress_boolean; +use crate::native::nested::InitNested; +use crate::native::nested::NestedState; use crate::native::read::read_basic::*; use crate::native::read::BufReader; use crate::native::read::NativeReadBuf; use crate::native::read::PageIterator; use crate::native::PageMeta; -#[derive(Debug)] -pub struct BooleanIter -where I: Iterator)>> + PageIterator + Send + Sync -{ - iter: I, - is_nullable: bool, - data_type: DataType, - scratch: Vec, -} - -impl BooleanIter -where I: Iterator)>> + PageIterator + Send + Sync -{ - pub fn new(iter: I, is_nullable: bool, data_type: DataType) -> Self { - Self { - iter, - is_nullable, - data_type, - scratch: vec![], - } - } -} - -impl BooleanIter -where I: Iterator)>> + PageIterator + Send + Sync -{ - fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result> { - let length = num_values as usize; - let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let validity = if self.is_nullable { - let mut validity_builder = MutableBitmap::with_capacity(length); - read_validity(&mut reader, length, &mut validity_builder)?; - Some(std::mem::take(&mut validity_builder).into()) - } else { - None - }; - let mut bitmap_builder = MutableBitmap::with_capacity(length); - - decompress_boolean(&mut reader, length, &mut bitmap_builder, &mut self.scratch)?; - - let values = std::mem::take(&mut bitmap_builder).into(); - let mut buffer = reader.into_inner().into_inner(); - self.iter.swap_buffer(&mut buffer); - - let array = BooleanArray::try_new(self.data_type.clone(), values, validity)?; - Ok(Box::new(array) as Box) - } -} - -impl Iterator for BooleanIter -where I: Iterator)>> + PageIterator + Send + Sync -{ - type Item = Result>; - - fn nth(&mut self, n: usize) -> Option { - match self.iter.nth(n) { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } -} - #[derive(Debug)] pub struct BooleanNestedIter where I: Iterator)>> + PageIterator + Send + Sync { iter: I, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, scratch: Vec, } @@ -115,16 +41,10 @@ where I: Iterator)>> + PageIterator + Send + Sync impl BooleanNestedIter where I: Iterator)>> + PageIterator + Send + Sync { - pub fn new( - iter: I, - data_type: DataType, - leaf: ColumnDescriptor, - init: Vec, - ) -> Self { + pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { Self { iter, data_type, - leaf, init, scratch: vec![], } @@ -136,23 +56,17 @@ where I: Iterator)>> + PageIterator + Send + Sync { fn deserialize( &mut self, - num_values: u64, + length: u64, buffer: Vec, ) -> Result<(NestedState, Box)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let (mut nested, validity) = read_validity_nested( - &mut reader, - num_values as usize, - &self.leaf, - self.init.clone(), - )?; - let length = nested.nested.pop().unwrap().len(); - let mut bitmap_builder = MutableBitmap::with_capacity(length); + let length = length as usize; + let (nested, validity) = read_nested(&mut reader, &self.init, length)?; + let mut bitmap_builder = MutableBitmap::with_capacity(length); decompress_boolean(&mut reader, length, &mut bitmap_builder, &mut self.scratch)?; let values = std::mem::take(&mut bitmap_builder).into(); - let mut buffer = reader.into_inner().into_inner(); self.iter.swap_buffer(&mut buffer); @@ -183,40 +97,9 @@ where I: Iterator)>> + PageIterator + Send + Sync } } -pub fn read_boolean( - reader: &mut R, - is_nullable: bool, - data_type: DataType, - page_metas: Vec, -) -> Result> { - let num_values = page_metas.iter().map(|p| p.num_values as usize).sum(); - let mut scratch = vec![]; - let mut validity_builder = if is_nullable { - Some(MutableBitmap::with_capacity(num_values)) - } else { - None - }; - let mut bitmap_builder = MutableBitmap::with_capacity(num_values); - for page_meta in page_metas { - let length = page_meta.num_values as usize; - if let Some(ref mut validity_builder) = validity_builder { - read_validity(reader, length, validity_builder)?; - } - - decompress_boolean(reader, length, &mut bitmap_builder, &mut scratch)?; - } - let validity = - validity_builder.map(|mut validity_builder| std::mem::take(&mut validity_builder).into()); - let values = std::mem::take(&mut bitmap_builder).into(); - - let array = BooleanArray::try_new(data_type, values, validity)?; - Ok(Box::new(array) as Box) -} - pub fn read_nested_boolean( reader: &mut R, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, page_metas: Vec, ) -> Result)>> { @@ -225,11 +108,10 @@ pub fn read_nested_boolean( let mut results = Vec::with_capacity(page_metas.len()); for page_meta in page_metas { let num_values = page_meta.num_values as usize; - let (mut nested, validity) = read_validity_nested(reader, num_values, &leaf, init.clone())?; - let length = nested.nested.pop().unwrap().len(); - let mut bitmap_builder = MutableBitmap::with_capacity(length); + let (nested, validity) = read_nested(reader, &init, num_values)?; + let mut bitmap_builder = MutableBitmap::with_capacity(num_values); - decompress_boolean(reader, length, &mut bitmap_builder, &mut scratch)?; + decompress_boolean(reader, num_values, &mut bitmap_builder, &mut scratch)?; let values = std::mem::take(&mut bitmap_builder).into(); let array = BooleanArray::try_new(data_type.clone(), values, validity)?; diff --git a/src/common/arrow/src/native/read/array/double.rs b/src/common/arrow/src/native/read/array/double.rs index abeca49b83c2..6be5f30f1f46 100644 --- a/src/common/arrow/src/native/read/array/double.rs +++ b/src/common/arrow/src/native/read/array/double.rs @@ -16,106 +16,20 @@ use std::convert::TryInto; use std::io::Cursor; use std::marker::PhantomData; -use parquet2::metadata::ColumnDescriptor; - use crate::arrow::array::Array; use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::buffer::Buffer; use crate::arrow::datatypes::DataType; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; use crate::native::compression::double::decompress_double; use crate::native::compression::double::DoubleType; +use crate::native::nested::InitNested; +use crate::native::nested::NestedState; use crate::native::read::read_basic::*; use crate::native::read::BufReader; use crate::native::read::NativeReadBuf; use crate::native::read::PageIterator; use crate::native::PageMeta; -pub struct DoubleIter -where - I: Iterator)>> + PageIterator + Send + Sync, - T: DoubleType, -{ - iter: I, - is_nullable: bool, - data_type: DataType, - scratch: Vec, - _phantom: PhantomData, -} - -impl DoubleIter -where - I: Iterator)>> + PageIterator + Send + Sync, - T: DoubleType, -{ - pub fn new(iter: I, is_nullable: bool, data_type: DataType) -> Self { - Self { - iter, - is_nullable, - data_type, - scratch: vec![], - _phantom: PhantomData, - } - } -} - -impl DoubleIter -where - I: Iterator)>> + PageIterator + Send + Sync, - T: DoubleType, - Vec: TryInto, -{ - fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result> { - let length = num_values as usize; - let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let validity = if self.is_nullable { - let mut validity_builder = MutableBitmap::with_capacity(length); - read_validity(&mut reader, length, &mut validity_builder)?; - Some(std::mem::take(&mut validity_builder).into()) - } else { - None - }; - let mut values: Vec = Vec::with_capacity(length); - - decompress_double(&mut reader, length, &mut values, &mut self.scratch)?; - assert_eq!(values.len(), length); - - let mut buffer = reader.into_inner().into_inner(); - self.iter.swap_buffer(&mut buffer); - - let array = PrimitiveArray::::try_new(self.data_type.clone(), values.into(), validity)?; - Ok(Box::new(array) as Box) - } -} - -impl Iterator for DoubleIter -where - I: Iterator)>> + PageIterator + Send + Sync, - T: DoubleType, - Vec: TryInto, -{ - type Item = Result>; - - fn nth(&mut self, n: usize) -> Option { - match self.iter.nth(n) { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } -} - #[derive(Debug)] pub struct DoubleNestedIter where @@ -124,7 +38,6 @@ where { iter: I, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, scratch: Vec, _phantom: PhantomData, @@ -135,16 +48,10 @@ where I: Iterator)>> + PageIterator + Send + Sync, T: DoubleType, { - pub fn new( - iter: I, - data_type: DataType, - leaf: ColumnDescriptor, - init: Vec, - ) -> Self { + pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { Self { iter, data_type, - leaf, init, scratch: vec![], _phantom: PhantomData, @@ -164,13 +71,8 @@ where buffer: Vec, ) -> Result<(NestedState, Box)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let (mut nested, validity) = read_validity_nested( - &mut reader, - num_values as usize, - &self.leaf, - self.init.clone(), - )?; - let length = nested.nested.pop().unwrap().len(); + let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; + let length = num_values as usize; let mut values = Vec::with_capacity(length); decompress_double(&mut reader, length, &mut values, &mut self.scratch)?; @@ -210,40 +112,9 @@ where } } -pub fn read_double( - reader: &mut R, - is_nullable: bool, - data_type: DataType, - page_metas: Vec, -) -> Result> { - let num_values = page_metas.iter().map(|p| p.num_values as usize).sum(); - - let mut scratch = vec![]; - let mut validity_builder = if is_nullable { - Some(MutableBitmap::with_capacity(num_values)) - } else { - None - }; - let mut out_buffer: Vec = Vec::with_capacity(num_values); - for page_meta in page_metas { - let length = page_meta.num_values as usize; - if let Some(ref mut validity_builder) = validity_builder { - read_validity(reader, length, validity_builder)?; - } - decompress_double(reader, length, &mut out_buffer, &mut scratch)?; - } - let validity = - validity_builder.map(|mut validity_builder| std::mem::take(&mut validity_builder).into()); - let values: Buffer = std::mem::take(&mut out_buffer).into(); - - let array = PrimitiveArray::::try_new(data_type, values, validity)?; - Ok(Box::new(array) as Box) -} - pub fn read_nested_primitive( reader: &mut R, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, page_metas: Vec, ) -> Result)>> { @@ -251,11 +122,10 @@ pub fn read_nested_primitive( let mut results = Vec::with_capacity(page_metas.len()); for page_meta in page_metas { let num_values = page_meta.num_values as usize; - let (mut nested, validity) = read_validity_nested(reader, num_values, &leaf, init.clone())?; - let length = nested.nested.pop().unwrap().len(); + let (nested, validity) = read_nested(reader, &init, num_values)?; - let mut values = Vec::with_capacity(length); - decompress_double(reader, length, &mut values, &mut scratch)?; + let mut values = Vec::with_capacity(num_values); + decompress_double(reader, num_values, &mut values, &mut scratch)?; let array = PrimitiveArray::::try_new(data_type.clone(), values.into(), validity)?; results.push((nested, Box::new(array) as Box)); diff --git a/src/common/arrow/src/native/read/array/integer.rs b/src/common/arrow/src/native/read/array/integer.rs index bb1b4636d7b7..f89c2e79db81 100644 --- a/src/common/arrow/src/native/read/array/integer.rs +++ b/src/common/arrow/src/native/read/array/integer.rs @@ -16,106 +16,20 @@ use std::convert::TryInto; use std::io::Cursor; use std::marker::PhantomData; -use parquet2::metadata::ColumnDescriptor; - use crate::arrow::array::Array; use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::buffer::Buffer; use crate::arrow::datatypes::DataType; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; use crate::native::compression::integer::decompress_integer; use crate::native::compression::integer::IntegerType; +use crate::native::nested::InitNested; +use crate::native::nested::NestedState; use crate::native::read::read_basic::*; use crate::native::read::BufReader; use crate::native::read::NativeReadBuf; use crate::native::read::PageIterator; use crate::native::PageMeta; -pub struct IntegerIter -where - I: Iterator)>> + PageIterator + Send + Sync, - T: IntegerType, -{ - iter: I, - is_nullable: bool, - data_type: DataType, - scratch: Vec, - _phantom: PhantomData, -} - -impl IntegerIter -where - I: Iterator)>> + PageIterator + Send + Sync, - T: IntegerType, -{ - pub fn new(iter: I, is_nullable: bool, data_type: DataType) -> Self { - Self { - iter, - is_nullable, - data_type, - scratch: vec![], - _phantom: PhantomData, - } - } -} - -impl IntegerIter -where - I: Iterator)>> + PageIterator + Send + Sync, - T: IntegerType, - Vec: TryInto, -{ - fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result> { - let length = num_values as usize; - let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let validity = if self.is_nullable { - let mut validity_builder = MutableBitmap::with_capacity(length); - read_validity(&mut reader, length, &mut validity_builder)?; - Some(std::mem::take(&mut validity_builder).into()) - } else { - None - }; - let mut values: Vec = Vec::with_capacity(length); - - decompress_integer(&mut reader, length, &mut values, &mut self.scratch)?; - assert_eq!(values.len(), length); - - let mut buffer = reader.into_inner().into_inner(); - self.iter.swap_buffer(&mut buffer); - - let array = PrimitiveArray::::try_new(self.data_type.clone(), values.into(), validity)?; - Ok(Box::new(array) as Box) - } -} - -impl Iterator for IntegerIter -where - I: Iterator)>> + PageIterator + Send + Sync, - T: IntegerType, - Vec: TryInto, -{ - type Item = Result>; - - fn nth(&mut self, n: usize) -> Option { - match self.iter.nth(n) { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } -} - #[derive(Debug)] pub struct IntegerNestedIter where @@ -124,7 +38,6 @@ where { iter: I, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, scratch: Vec, _phantom: PhantomData, @@ -135,16 +48,10 @@ where I: Iterator)>> + PageIterator + Send + Sync, T: IntegerType, { - pub fn new( - iter: I, - data_type: DataType, - leaf: ColumnDescriptor, - init: Vec, - ) -> Self { + pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { Self { iter, data_type, - leaf, init, scratch: vec![], _phantom: PhantomData, @@ -164,13 +71,8 @@ where buffer: Vec, ) -> Result<(NestedState, Box)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let (mut nested, validity) = read_validity_nested( - &mut reader, - num_values as usize, - &self.leaf, - self.init.clone(), - )?; - let length = nested.nested.pop().unwrap().len(); + let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; + let length = num_values as usize; let mut values = Vec::with_capacity(length); decompress_integer(&mut reader, length, &mut values, &mut self.scratch)?; @@ -210,40 +112,9 @@ where } } -pub fn read_integer( - reader: &mut R, - is_nullable: bool, - data_type: DataType, - page_metas: Vec, -) -> Result> { - let num_values = page_metas.iter().map(|p| p.num_values as usize).sum(); - - let mut scratch = vec![]; - let mut validity_builder = if is_nullable { - Some(MutableBitmap::with_capacity(num_values)) - } else { - None - }; - let mut out_buffer: Vec = Vec::with_capacity(num_values); - for page_meta in page_metas { - let length = page_meta.num_values as usize; - if let Some(ref mut validity_builder) = validity_builder { - read_validity(reader, length, validity_builder)?; - } - decompress_integer(reader, length, &mut out_buffer, &mut scratch)?; - } - let validity = - validity_builder.map(|mut validity_builder| std::mem::take(&mut validity_builder).into()); - let values: Buffer = std::mem::take(&mut out_buffer).into(); - - let array = PrimitiveArray::::try_new(data_type, values, validity)?; - Ok(Box::new(array) as Box) -} - pub fn read_nested_integer( reader: &mut R, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, page_metas: Vec, ) -> Result)>> { @@ -251,11 +122,10 @@ pub fn read_nested_integer( let mut results = Vec::with_capacity(page_metas.len()); for page_meta in page_metas { let num_values = page_meta.num_values as usize; - let (mut nested, validity) = read_validity_nested(reader, num_values, &leaf, init.clone())?; - let length = nested.nested.pop().unwrap().len(); + let (nested, validity) = read_nested(reader, &init, num_values)?; - let mut values = Vec::with_capacity(length); - decompress_integer(reader, length, &mut values, &mut scratch)?; + let mut values = Vec::with_capacity(num_values); + decompress_integer(reader, num_values, &mut values, &mut scratch)?; let array = PrimitiveArray::::try_new(data_type.clone(), values.into(), validity)?; results.push((nested, Box::new(array) as Box)); diff --git a/src/common/arrow/src/native/read/array/list.rs b/src/common/arrow/src/native/read/array/list.rs index 0c901fdfdd22..c478bb110dcb 100644 --- a/src/common/arrow/src/native/read/array/list.rs +++ b/src/common/arrow/src/native/read/array/list.rs @@ -15,8 +15,8 @@ use crate::arrow::array::Array; use crate::arrow::datatypes::Field; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::create_list; -use crate::arrow::io::parquet::read::NestedState; +use crate::native::nested::create_list; +use crate::native::nested::NestedState; use crate::native::read::deserialize::DynIter; /// An iterator adapter over [`DynIter`] assumed to be encoded as List arrays diff --git a/src/common/arrow/src/native/read/array/map.rs b/src/common/arrow/src/native/read/array/map.rs index b275d8d02979..f8de5824169c 100644 --- a/src/common/arrow/src/native/read/array/map.rs +++ b/src/common/arrow/src/native/read/array/map.rs @@ -15,8 +15,8 @@ use crate::arrow::array::Array; use crate::arrow::datatypes::Field; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::create_map; -use crate::arrow::io::parquet::read::NestedState; +use crate::native::nested::create_map; +use crate::native::nested::NestedState; use crate::native::read::deserialize::DynIter; /// An iterator adapter over [`DynIter`] assumed to be encoded as Map arrays diff --git a/src/common/arrow/src/native/read/array/mod.rs b/src/common/arrow/src/native/read/array/mod.rs index a1443034ea1a..e3da3130c23a 100644 --- a/src/common/arrow/src/native/read/array/mod.rs +++ b/src/common/arrow/src/native/read/array/mod.rs @@ -26,7 +26,6 @@ mod view; pub use view::*; mod null; -pub use null::*; mod struct_; pub use struct_::*; mod list; diff --git a/src/common/arrow/src/native/read/array/struct_.rs b/src/common/arrow/src/native/read/array/struct_.rs index 1a52ef77b2e3..320166b12e16 100644 --- a/src/common/arrow/src/native/read/array/struct_.rs +++ b/src/common/arrow/src/native/read/array/struct_.rs @@ -13,11 +13,10 @@ // limitations under the License. use crate::arrow::array::Array; -use crate::arrow::array::StructArray; -use crate::arrow::datatypes::DataType; use crate::arrow::datatypes::Field; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::NestedState; +use crate::native::nested::create_struct; +use crate::native::nested::NestedState; use crate::native::read::deserialize::NestedIters; type StructValues = Vec)>>>; @@ -59,11 +58,9 @@ impl<'a> StructIterator<'a> { Err(e) => return Some(Err(e)), } } - Some(Ok(create_struct( - self.fields.clone(), - &mut nested, - new_values, - ))) + + let array = create_struct(self.fields.clone(), &mut nested, new_values); + Some(Ok(array)) } } @@ -90,20 +87,3 @@ impl<'a> Iterator for StructIterator<'a> { self.deserialize(values) } } - -pub fn create_struct( - fields: Vec, - nested: &mut Vec, - values: Vec>, -) -> (NestedState, Box) { - let mut nested = nested.pop().unwrap(); - let (_, validity) = nested.nested.pop().unwrap().inner(); - ( - nested, - Box::new(StructArray::new( - DataType::Struct(fields), - values, - validity.and_then(|x| x.into()), - )), - ) -} diff --git a/src/common/arrow/src/native/read/array/view.rs b/src/common/arrow/src/native/read/array/view.rs index 2d577ebdcf3b..e129d4146d87 100644 --- a/src/common/arrow/src/native/read/array/view.rs +++ b/src/common/arrow/src/native/read/array/view.rs @@ -16,19 +16,16 @@ use std::io::Cursor; use byteorder::LittleEndian; use byteorder::ReadBytesExt; -use parquet2::metadata::ColumnDescriptor; use crate::arrow::array::Array; use crate::arrow::array::BinaryViewArray; use crate::arrow::array::View; use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; use crate::arrow::buffer::Buffer; -use crate::arrow::compute::concatenate::concatenate; use crate::arrow::datatypes::DataType; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; +use crate::native::nested::InitNested; +use crate::native::nested::NestedState; use crate::native::read::read_basic::*; use crate::native::read::BufReader; use crate::native::read::NativeReadBuf; @@ -36,76 +33,12 @@ use crate::native::read::PageIterator; use crate::native::CommonCompression; use crate::native::PageMeta; -#[derive(Debug)] -pub struct ViewArrayIter -where I: Iterator)>> + PageIterator + Send + Sync -{ - iter: I, - is_nullable: bool, - data_type: DataType, - scratch: Vec, -} - -impl ViewArrayIter -where I: Iterator)>> + PageIterator + Send + Sync -{ - pub fn new(iter: I, is_nullable: bool, data_type: DataType) -> Self { - Self { - iter, - is_nullable, - data_type, - scratch: vec![], - } - } -} - -impl ViewArrayIter -where I: Iterator)>> + PageIterator + Send + Sync -{ - fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result> { - let length = num_values as usize; - let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let validity = if self.is_nullable { - let mut validity_builder = MutableBitmap::with_capacity(length); - read_validity(&mut reader, length, &mut validity_builder)?; - Some(std::mem::take(&mut validity_builder).into()) - } else { - None - }; - - read_view_array(&mut reader, length, self.data_type.clone(), validity) - } -} - -impl Iterator for ViewArrayIter -where I: Iterator)>> + PageIterator + Send + Sync -{ - type Item = Result>; - - fn nth(&mut self, n: usize) -> Option { - match self.iter.nth(n) { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } -} - #[derive(Debug)] pub struct ViewArrayNestedIter where I: Iterator)>> + PageIterator + Send + Sync { iter: I, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, scratch: Vec, } @@ -113,16 +46,10 @@ where I: Iterator)>> + PageIterator + Send + Sync impl ViewArrayNestedIter where I: Iterator)>> + PageIterator + Send + Sync { - pub fn new( - iter: I, - data_type: DataType, - leaf: ColumnDescriptor, - init: Vec, - ) -> Self { + pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { Self { iter, data_type, - leaf, init, scratch: vec![], } @@ -138,13 +65,9 @@ where I: Iterator)>> + PageIterator + Send + Sync buffer: Vec, ) -> Result<(NestedState, Box)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let (mut nested, validity) = read_validity_nested( - &mut reader, - num_values as usize, - &self.leaf, - self.init.clone(), - )?; - let length = nested.nested.pop().unwrap().len(); + let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; + let length = num_values as usize; + let array = read_view_array(&mut reader, length, self.data_type.clone(), validity)?; Ok((nested, array)) } @@ -172,40 +95,9 @@ where I: Iterator)>> + PageIterator + Send + Sync } } -pub fn read_view( - reader: &mut R, - is_nullable: bool, - data_type: DataType, - page_metas: Vec, -) -> Result> { - let num_values = page_metas.iter().map(|p| p.num_values as usize).sum(); - let mut validity_builder = if is_nullable { - Some(MutableBitmap::with_capacity(num_values)) - } else { - None - }; - let mut arrays = vec![]; - for page_meta in page_metas { - let length = page_meta.num_values as usize; - if let Some(ref mut validity_builder) = validity_builder { - read_validity(reader, length, validity_builder)?; - } - let array = read_view_array(reader, length, data_type.clone(), None)?; - arrays.push(array); - } - - let validity = - validity_builder.map(|mut validity_builder| std::mem::take(&mut validity_builder).into()); - let arrays = arrays.iter().map(|x| x.as_ref()).collect::>(); - let array = concatenate(&arrays)?; - let array = array.with_validity(validity); - Ok(array) -} - pub fn read_nested_view_array( reader: &mut R, data_type: DataType, - leaf: ColumnDescriptor, init: Vec, page_metas: Vec, ) -> Result)>> { @@ -213,10 +105,8 @@ pub fn read_nested_view_array( for page_meta in page_metas { let num_values = page_meta.num_values as usize; - let (mut nested, validity) = read_validity_nested(reader, num_values, &leaf, init.clone())?; - let length = nested.nested.pop().unwrap().len(); - - let array = read_view_array(reader, length, data_type.clone(), validity)?; + let (nested, validity) = read_nested(reader, &init, num_values)?; + let array = read_view_array(reader, num_values, data_type.clone(), validity)?; results.push((nested, array)); } Ok(results) diff --git a/src/common/arrow/src/native/read/batch_read.rs b/src/common/arrow/src/native/read/batch_read.rs index 7c9ed0c1d7a2..c8d70e33a8b9 100644 --- a/src/common/arrow/src/native/read/batch_read.rs +++ b/src/common/arrow/src/native/read/batch_read.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use parquet2::metadata::ColumnDescriptor; - use super::array::*; use super::NativeReadBuf; use crate::arrow::array::*; @@ -22,57 +20,17 @@ use crate::arrow::datatypes::DataType; use crate::arrow::datatypes::Field; use crate::arrow::datatypes::PhysicalType; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::create_list; -use crate::arrow::io::parquet::read::create_map; -use crate::arrow::io::parquet::read::n_columns; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; +use crate::native::nested::create_list; +use crate::native::nested::create_map; +use crate::native::nested::create_struct; +use crate::native::nested::InitNested; +use crate::native::nested::NestedState; +use crate::native::util::n_columns; use crate::native::PageMeta; -pub fn read_simple( - reader: &mut R, - field: Field, - page_metas: Vec, -) -> Result> { - use PhysicalType::*; - - let is_nullable = field.is_nullable; - let data_type = field.data_type().clone(); - - match data_type.to_physical_type() { - Null => read_null(data_type, page_metas), - Boolean => read_boolean(reader, is_nullable, data_type, page_metas), - Primitive(primitive) => with_match_integer_double_type!(primitive, - |$T| { - read_integer::<$T, _>( - reader, - is_nullable, - data_type, - page_metas, - ) - }, - |$T| { - read_double::<$T, _>( - reader, - is_nullable, - data_type, - page_metas, - ) - }), - Binary | Utf8 => read_binary::(reader, is_nullable, data_type, page_metas), - BinaryView | Utf8View => read_view::<_>(reader, is_nullable, data_type, page_metas), - LargeBinary | LargeUtf8 => { - read_binary::(reader, is_nullable, data_type, page_metas) - } - FixedSizeBinary => unimplemented!(), - _ => unreachable!(), - } -} - pub fn read_nested( mut readers: Vec, field: Field, - mut leaves: Vec, mut init: Vec, mut page_metas: Vec>, ) -> Result)>> { @@ -85,7 +43,6 @@ pub fn read_nested( read_nested_boolean( &mut readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, page_metas.pop().unwrap(), )? @@ -96,7 +53,6 @@ pub fn read_nested( read_nested_integer::<$T, _>( &mut readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, page_metas.pop().unwrap(), )? @@ -106,7 +62,6 @@ pub fn read_nested( read_nested_primitive::<$T, _>( &mut readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, page_metas.pop().unwrap(), )? @@ -117,7 +72,6 @@ pub fn read_nested( read_nested_binary::( &mut readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, page_metas.pop().unwrap(), )? @@ -128,7 +82,6 @@ pub fn read_nested( read_nested_view_array::<_>( &mut readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, page_metas.pop().unwrap(), )? @@ -139,7 +92,6 @@ pub fn read_nested( read_nested_binary::( &mut readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, page_metas.pop().unwrap(), )? @@ -151,8 +103,7 @@ pub fn read_nested( | DataType::LargeList(inner) | DataType::FixedSizeList(inner, _) => { init.push(InitNested::List(field.is_nullable)); - let results = - read_nested(readers, inner.as_ref().clone(), leaves, init, page_metas)?; + let results = read_nested(readers, inner.as_ref().clone(), init, page_metas)?; let mut arrays = Vec::with_capacity(results.len()); for (mut nested, values) in results { let array = create_list(field.data_type().clone(), &mut nested, values); @@ -162,8 +113,7 @@ pub fn read_nested( } DataType::Map(inner, _) => { init.push(InitNested::List(field.is_nullable)); - let results = - read_nested(readers, inner.as_ref().clone(), leaves, init, page_metas)?; + let results = read_nested(readers, inner.as_ref().clone(), init, page_metas)?; let mut arrays = Vec::with_capacity(results.len()); for (mut nested, values) in results { let array = create_map(field.data_type().clone(), &mut nested, values); @@ -179,9 +129,8 @@ pub fn read_nested( init.push(InitNested::Struct(field.is_nullable)); let n = n_columns(&f.data_type); let readers = readers.drain(..n).collect(); - let leaves = leaves.drain(..n).collect(); let page_metas = page_metas.drain(..n).collect(); - read_nested(readers, f.clone(), leaves, init, page_metas) + read_nested(readers, f.clone(), init, page_metas) }) .collect::>>()?; let mut arrays = Vec::with_capacity(results[0].len()); @@ -206,22 +155,12 @@ pub fn read_nested( /// Read all pages of column at once. pub fn batch_read_array( - mut readers: Vec, - leaves: Vec, + readers: Vec, field: Field, - is_nested: bool, - mut page_metas: Vec>, + page_metas: Vec>, ) -> Result> { - if is_nested { - let results = read_nested(readers, field, leaves, vec![], page_metas)?; - let arrays: Vec<&dyn Array> = results.iter().map(|(_, v)| v.as_ref()).collect(); - let array = concatenate(&arrays).unwrap(); - Ok(array) - } else { - read_simple( - &mut readers.pop().unwrap(), - field, - page_metas.pop().unwrap(), - ) - } + let results = read_nested(readers, field, vec![], page_metas)?; + let arrays: Vec<&dyn Array> = results.iter().map(|(_, v)| v.as_ref()).collect(); + let array = concatenate(&arrays).unwrap(); + Ok(array) } diff --git a/src/common/arrow/src/native/read/deserialize.rs b/src/common/arrow/src/native/read/deserialize.rs index 8ed459e8d201..094daa18a307 100644 --- a/src/common/arrow/src/native/read/deserialize.rs +++ b/src/common/arrow/src/native/read/deserialize.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use parquet2::metadata::ColumnDescriptor; - use super::array::*; use super::PageIterator; use crate::arrow::array::*; @@ -21,9 +19,9 @@ use crate::arrow::datatypes::DataType; use crate::arrow::datatypes::Field; use crate::arrow::datatypes::PhysicalType; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::n_columns; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; +use crate::native::nested::InitNested; +use crate::native::nested::NestedState; +use crate::native::util::n_columns; /// [`DynIter`] is an iterator adapter adds a custom `nth` method implementation. pub struct DynIter<'a, V> { @@ -94,52 +92,8 @@ where I: Iterator)>> + Send + Sync pub type NestedIters<'a> = DynIter<'a, Result<(NestedState, Box)>>; -fn deserialize_simple<'a, I>( - reader: I, - field: Field, -) -> Result>>> -where - I: Iterator)>> + PageIterator + Send + Sync + 'a, -{ - use PhysicalType::*; - - let is_nullable = field.is_nullable; - let data_type = field.data_type().clone(); - - Ok(match data_type.to_physical_type() { - Null => DynIter::new(NullIter::new(reader, data_type)), - Boolean => DynIter::new(BooleanIter::new(reader, is_nullable, data_type)), - Primitive(primitive) => with_match_integer_double_type!(primitive, - |$I| { - DynIter::new(IntegerIter::<_, $I>::new( - reader, - is_nullable, - data_type, - )) - }, - |$T| { - DynIter::new(DoubleIter::<_, $T>::new( - reader, - is_nullable, - data_type, - )) - } - ), - Binary | Utf8 => DynIter::new(BinaryIter::<_, i32>::new(reader, is_nullable, data_type)), - BinaryView | Utf8View => { - DynIter::new(ViewArrayIter::<_>::new(reader, is_nullable, data_type)) - } - LargeBinary | LargeUtf8 => { - DynIter::new(BinaryIter::<_, i64>::new(reader, is_nullable, data_type)) - } - FixedSizeBinary => unimplemented!(), - _ => unreachable!(), - }) -} - fn deserialize_nested<'a, I>( mut readers: Vec, - mut leaves: Vec, field: Field, mut init: Vec, ) -> Result> @@ -155,7 +109,6 @@ where DynIter::new(BooleanNestedIter::new( readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, )) } @@ -165,7 +118,6 @@ where DynIter::new(IntegerNestedIter::<_, $I>::new( readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, )) }, @@ -174,7 +126,6 @@ where DynIter::new(DoubleNestedIter::<_, $T>::new( readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, )) } @@ -184,7 +135,6 @@ where DynIter::new(BinaryNestedIter::<_, i32>::new( readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, )) } @@ -193,7 +143,6 @@ where DynIter::new(ViewArrayNestedIter::<_>::new( readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, )) } @@ -202,7 +151,6 @@ where DynIter::new(BinaryNestedIter::<_, i64>::new( readers.pop().unwrap(), field.data_type().clone(), - leaves.pop().unwrap(), init, )) } @@ -213,12 +161,12 @@ where | DataType::LargeList(inner) | DataType::FixedSizeList(inner, _) => { init.push(InitNested::List(field.is_nullable)); - let iter = deserialize_nested(readers, leaves, inner.as_ref().clone(), init)?; + let iter = deserialize_nested(readers, inner.as_ref().clone(), init)?; DynIter::new(ListIterator::new(iter, field.clone())) } DataType::Map(inner, _) => { init.push(InitNested::List(field.is_nullable)); - let iter = deserialize_nested(readers, leaves, inner.as_ref().clone(), init)?; + let iter = deserialize_nested(readers, inner.as_ref().clone(), init)?; DynIter::new(MapIterator::new(iter, field.clone())) } DataType::Struct(fields) => { @@ -230,8 +178,7 @@ where init.push(InitNested::Struct(field.is_nullable)); let n = n_columns(&f.data_type); let readers = readers.drain(readers.len() - n..).collect(); - let leaves = leaves.drain(leaves.len() - n..).collect(); - deserialize_nested(readers, leaves, f.clone(), init) + deserialize_nested(readers, f.clone(), init) }) .collect::>>()?; let columns = columns.into_iter().rev().collect(); @@ -244,19 +191,14 @@ where /// An iterator adapter that maps [`PageIterator`]s into an iterator of [`Array`]s. pub fn column_iter_to_arrays<'a, I>( - mut readers: Vec, - leaves: Vec, + readers: Vec, field: Field, - is_nested: bool, + init: Vec, ) -> Result> where I: Iterator)>> + PageIterator + Send + Sync + 'a, { - if is_nested { - let iter = deserialize_nested(readers, leaves, field, vec![])?; - let nested_iter = NestedIter::new(iter); - Ok(DynIter::new(nested_iter)) - } else { - deserialize_simple(readers.pop().unwrap(), field) - } + let iter = deserialize_nested(readers, field, init)?; + let nested_iter = NestedIter::new(iter); + Ok(DynIter::new(nested_iter)) } diff --git a/src/common/arrow/src/native/read/mod.rs b/src/common/arrow/src/native/read/mod.rs index 0c5c4e7456b9..a00bc1f0af02 100644 --- a/src/common/arrow/src/native/read/mod.rs +++ b/src/common/arrow/src/native/read/mod.rs @@ -25,10 +25,9 @@ use crate::arrow::error::Result; pub(crate) mod read_basic; use std::io::BufReader; +use super::nested::InitNested; use super::PageMeta; -use super::SchemaDescriptor; use crate::arrow::datatypes::Schema; -use crate::arrow::io::parquet::write::to_parquet_schema; pub mod reader; pub trait NativeReadBuf: std::io::BufRead { @@ -73,51 +72,33 @@ pub trait PageIterator { #[derive(Clone)] pub struct NativeColumnsReader { schema: Schema, - schema_desc: SchemaDescriptor, } impl NativeColumnsReader { pub fn new(schema: Schema) -> Result { - let schema_desc = to_parquet_schema(&schema)?; - Ok(Self { - schema, - schema_desc, - }) + Ok(Self { schema }) } /// An iterator adapter that maps [`PageIterator`]s into an iterator of [`Array`]s. pub fn column_iter_to_arrays<'a, I>( &self, readers: Vec, - leaf_indexes: &[usize], field: Field, - is_nested: bool, + init: Vec, ) -> Result> where I: Iterator)>> + PageIterator + Send + Sync + 'a, { - let leaves = leaf_indexes - .iter() - .map(|i| self.schema_desc.columns()[*i].clone()) - .collect(); - - column_iter_to_arrays(readers, leaves, field, is_nested) + column_iter_to_arrays(readers, field, init) } /// Read all pages of column at once. pub fn batch_read_array( &self, readers: Vec, - leaf_indexes: &[usize], field: Field, - is_nested: bool, page_metas: Vec>, ) -> Result> { - let leaves = leaf_indexes - .iter() - .map(|i| self.schema_desc.columns()[*i].clone()) - .collect(); - - batch_read_array(readers, leaves, field, is_nested, page_metas) + batch_read_array(readers, field, page_metas) } } diff --git a/src/common/arrow/src/native/read/read_basic.rs b/src/common/arrow/src/native/read/read_basic.rs index 8a4d6c1fec25..17a1e355eb31 100644 --- a/src/common/arrow/src/native/read/read_basic.rs +++ b/src/common/arrow/src/native/read/read_basic.rs @@ -15,155 +15,81 @@ use std::convert::TryInto; use std::io::Read; -use parquet2::encoding::hybrid_rle::BitmapIter; -use parquet2::encoding::hybrid_rle::Decoder; -use parquet2::encoding::hybrid_rle::HybridEncoded; -use parquet2::encoding::hybrid_rle::HybridRleDecoder; -use parquet2::metadata::ColumnDescriptor; -use parquet2::read::levels::get_bit_width; - use super::NativeReadBuf; use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; use crate::arrow::error::Result; -use crate::arrow::io::parquet::read::init_nested; -use crate::arrow::io::parquet::read::InitNested; -use crate::arrow::io::parquet::read::NestedState; +use crate::arrow::offset::Offsets; +use crate::arrow::offset::OffsetsBuffer; use crate::native::compression::Compression; +use crate::native::nested::InitNested; +use crate::native::nested::ListNested; +use crate::native::nested::Nested; -pub fn read_validity( - reader: &mut R, - length: usize, - builder: &mut MutableBitmap, -) -> Result<()> { +pub fn read_validity(reader: &mut R) -> Result> { let mut buf = vec![0u8; 4]; - let def_levels_len = read_u32(reader, buf.as_mut_slice())?; - if def_levels_len == 0 { - return Ok(()); - } - let mut def_levels = vec![0u8; def_levels_len as usize]; - reader.read_exact(def_levels.as_mut_slice())?; - - let decoder = Decoder::new(def_levels.as_slice(), 1); - for encoded in decoder { - let encoded = encoded.unwrap(); - match encoded { - HybridEncoded::Bitpacked(r) => { - let bitmap_iter = BitmapIter::new(r, 0, length); - for v in bitmap_iter { - unsafe { builder.push_unchecked(v) }; - } - } - HybridEncoded::Rle(_, _) => unreachable!(), - } + let length = read_u32(reader, &mut buf)? as usize; + if length > 0 { + buf.resize((length + 7) / 8, 0); + reader.read_exact(&mut buf)?; + Ok(Some(Bitmap::try_new(buf, length)?)) + } else { + Ok(None) } - Ok(()) } -pub fn read_validity_nested( +// Read nested from reader and pop the leaf nested +pub fn read_nested( reader: &mut R, - num_values: usize, - leaf: &ColumnDescriptor, - init: Vec, -) -> Result<(NestedState, Option)> { - let mut buf = vec![0u8; 4]; - let additional = read_u32(reader, buf.as_mut_slice())?; - let rep_levels_len = read_u32(reader, buf.as_mut_slice())?; - let def_levels_len = read_u32(reader, buf.as_mut_slice())?; - let max_rep_level = leaf.descriptor.max_rep_level; - let max_def_level = leaf.descriptor.max_def_level; - - let mut rep_levels = vec![0u8; rep_levels_len as usize]; - reader.read_exact(rep_levels.as_mut_slice())?; - let mut def_levels = vec![0u8; def_levels_len as usize]; - reader.read_exact(def_levels.as_mut_slice())?; - - let reps = HybridRleDecoder::try_new(&rep_levels, get_bit_width(max_rep_level), num_values)?; - let defs = HybridRleDecoder::try_new(&def_levels, get_bit_width(max_def_level), num_values)?; - let mut page_iter = reps.zip(defs).peekable(); - - let mut nested = init_nested(&init, num_values); - - // The following code is copied from arrow2 `extend_offsets2` function. - // https://github.com/jorgecarleitao/arrow2/blob/main/src/io/parquet/read/deserialize/nested_utils.rs#L403 - // The main purpose of this code is to calculate the `NestedState` and `Bitmap` - // of the nested information by decode `rep_levels` and `def_levels`. - let max_depth = nested.nested.len(); - - let mut cum_sum = vec![0u32; max_depth + 1]; - for (i, nest) in nested.nested.iter().enumerate() { - let delta = nest.is_nullable() as u32 + nest.is_repeated() as u32; - cum_sum[i + 1] = cum_sum[i] + delta; - } - - let mut cum_rep = vec![0u32; max_depth + 1]; - for (i, nest) in nested.nested.iter().enumerate() { - let delta = nest.is_repeated() as u32; - cum_rep[i + 1] = cum_rep[i] + delta; - } - - let mut is_nullable = false; - let mut builder = MutableBitmap::with_capacity(num_values); - - let mut rows = 0; - while let Some((rep, def)) = page_iter.next() { - let rep = rep?; - let def = def?; - if rep == 0 { - rows += 1; - } - - let mut is_required = false; - for depth in 0..max_depth { - let right_level = rep <= cum_rep[depth] && def >= cum_sum[depth]; - if is_required || right_level { - let length = nested - .nested - .get(depth + 1) - .map(|x| x.len() as i64) - // the last depth is the leaf, which is always increased by 1 - .unwrap_or(1); - - let nest = &mut nested.nested[depth]; - - let is_valid = nest.is_nullable() && def > cum_sum[depth]; - nest.push(length, is_valid); - is_required = nest.is_required() && !is_valid; - - if depth == max_depth - 1 { - // the leaf / primitive - is_nullable = nest.is_nullable(); - if is_nullable { - let is_valid = (def != cum_sum[depth]) || !nest.is_nullable(); - if right_level && is_valid { - unsafe { builder.push_unchecked(true) }; - } else { - unsafe { builder.push_unchecked(false) }; - } - } + init: &[InitNested], + leaf_length: usize, +) -> Result<(Vec, Option)> { + assert!(!init.is_empty()); + let is_simple_nested = init.len() == 1; + + if is_simple_nested { + let n = init[0]; + let bitmap = if n.is_nullable() { + read_validity(reader)? + } else { + None + }; + + Ok((vec![], bitmap)) + } else { + let mut results = Vec::with_capacity(init.len()); + for n in init { + let bitmap = if n.is_nullable() { + read_validity(reader)? + } else { + None + }; + + match n { + InitNested::Primitive(_) => { + results.push(Nested::Primitive(leaf_length, n.is_nullable(), bitmap)) + } + InitNested::List(_) => { + let mut buf = vec![0u8; 4]; + let length = read_u32(reader, &mut buf)?; + let mut values = vec![0i64; length as usize]; + let bytes: &mut [u8] = bytemuck::cast_slice_mut(values.as_mut()); + reader.read_exact(bytes)?; + + let offsets = Offsets::try_from(values).unwrap(); + results.push(Nested::LargeList(ListNested::new( + OffsetsBuffer::from(offsets), + bitmap, + n.is_nullable(), + ))) + } + InitNested::Struct(_) => { + results.push(Nested::Struct(leaf_length, n.is_nullable(), bitmap)) } } } - - let next_rep = *page_iter - .peek() - .map(|x| x.0.as_ref()) - .transpose() - .unwrap() // todo: fix this - .unwrap_or(&0); - - if next_rep == 0 && rows == additional { - break; - } + let bitmap = results.pop().unwrap().validity().clone(); + Ok((results, bitmap)) } - - let validity = if is_nullable { - Some(std::mem::take(&mut builder).into()) - } else { - None - }; - - Ok((nested, validity)) } #[inline(always)] diff --git a/src/common/arrow/src/native/read/reader.rs b/src/common/arrow/src/native/read/reader.rs index a9912b274e22..640b6c30732f 100644 --- a/src/common/arrow/src/native/read/reader.rs +++ b/src/common/arrow/src/native/read/reader.rs @@ -27,7 +27,6 @@ use crate::arrow::datatypes::PhysicalType; use crate::arrow::datatypes::Schema; use crate::arrow::error::Error; use crate::arrow::error::Result; -use crate::arrow::io::ipc::read::deserialize_schema; use crate::native::ColumnMeta; use crate::native::PageMeta; @@ -132,6 +131,11 @@ impl Iterator for NativeReader { self.current_page += 1; Some(Ok((page_meta.num_values, buffer))) } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.page_metas.len() - self.current_page; + (remaining, Some(remaining)) + } } impl NativeReader { @@ -190,9 +194,9 @@ pub fn infer_schema(reader: &mut Reader) -> Result reader.seek(SeekFrom::Current( -(column_meta_size as i64) - (schema_size as i64) - 8, ))?; - let mut schema_bytes = vec![0u8; schema_size]; - reader.read_exact(&mut schema_bytes)?; - let (schema, _) = deserialize_schema(&schema_bytes).expect("deserialize schema error"); + let mut schema_buf = vec![0u8; schema_size]; + reader.read_exact(&mut schema_buf)?; + let schema = serde_json::from_slice(&schema_buf).expect("deserialize schema error"); Ok(schema) } @@ -237,7 +241,7 @@ pub async fn read_meta_async( let mut schema_buf = vec![0u8; schema_size as usize]; footer_reader.read_exact(&mut schema_buf)?; - let (schema, _) = deserialize_schema(&schema_buf)?; + let schema = serde_json::from_slice(&schema_buf).expect("deserialize schema error"); footer_reader.seek(SeekFrom::End(-footer_size - meta_size))?; let mut meta_buf = vec![0u8; meta_size as usize]; diff --git a/src/common/arrow/src/native/stat.rs b/src/common/arrow/src/native/stat.rs index 62e1aee7c6fe..da8207bba62b 100644 --- a/src/common/arrow/src/native/stat.rs +++ b/src/common/arrow/src/native/stat.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::io::BufRead; + use crate::arrow::datatypes::Field; use crate::arrow::datatypes::PhysicalType; use crate::arrow::error::Result; @@ -61,15 +63,20 @@ pub struct DictPageBody { pub fn stat_simple<'a, I>(reader: I, field: Field) -> Result where I: Iterator)>> + PageIterator + Send + Sync + 'a { let mut pages = vec![]; + for compressed in reader { - let (_, buffer) = compressed?; + let (num_values, buffer) = compressed?; let mut buffer = buffer.as_slice(); let mut opt_validity_size = None; if field.is_nullable { let validity_size = u32::from_le_bytes(buffer[0..4].try_into().unwrap()); - buffer = &buffer[4 + validity_size as usize..]; - opt_validity_size = Some(u32::from_le_bytes(buffer[0..4].try_into().unwrap())); + debug_assert!(validity_size == 0 || validity_size as u64 == num_values); + let consume_validity_size = 4 + ((validity_size + 7) / 8) as usize; + buffer.consume(consume_validity_size); + if validity_size > 0 { + opt_validity_size = Some(validity_size); + } }; let physical_type = field.data_type.to_physical_type(); @@ -174,6 +181,7 @@ mod test { use super::ColumnInfo; use crate::arrow::array::Array; use crate::arrow::array::BinaryArray; + use crate::arrow::array::PrimitiveArray; use crate::arrow::chunk::Chunk; use crate::arrow::datatypes::Field; use crate::arrow::datatypes::Schema; @@ -225,6 +233,18 @@ mod test { #[test] fn test_stat_simple() { remove_all_env(); + + let values: Vec> = (0..COLUMN_SIZE) + .map(|d| if d % 3 == 0 { None } else { Some(d as i64) }) + .collect(); + let array = Box::new(PrimitiveArray::::from_iter(values)); + let column_info = write_and_stat_simple_column(array.clone()); + + assert_eq!(column_info.pages.len(), 10); + for p in column_info.pages { + assert_eq!(p.validity_size, Some(PAGE_SIZE as u32)); + } + let array = Box::new(BinaryArray::::from_iter_values( ["a"; COLUMN_SIZE].iter(), )); diff --git a/src/common/arrow/src/native/util/bit_util.rs b/src/common/arrow/src/native/util/bit_util.rs index f399c47dddec..ebadee3c6bf7 100644 --- a/src/common/arrow/src/native/util/bit_util.rs +++ b/src/common/arrow/src/native/util/bit_util.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::io::Write; use std::mem::size_of; use crate::arrow::buffer::Buffer; @@ -37,6 +38,46 @@ fn array_from_slice(bs: &[u8]) -> Result<[u8; N]> { } } +/// Sets bit at position `i` in `byte` +#[inline] +pub fn set(byte: u8, i: usize) -> u8 { + byte | BIT_MASK[i] +} + +/// Writes an iterator of bools into writer, with LSB first. +pub fn encode_bool>( + writer: &mut W, + mut iterator: I, +) -> std::io::Result<()> { + // the length of the iterator. + let length = iterator.size_hint().1.unwrap(); + + let chunks = length / 8; + let reminder = length % 8; + + (0..chunks).try_for_each(|_| { + let mut byte = 0u8; + (0..8).for_each(|i| { + if iterator.next().unwrap() { + byte = set(byte, i) + } + }); + writer.write_all(&[byte]) + })?; + + if reminder != 0 { + let mut last = 0u8; + iterator.enumerate().for_each(|(i, value)| { + if value { + last = set(last, i) + } + }); + writer.write_all(&[last]) + } else { + Ok(()) + } +} + pub trait FromBytes: Sized { type Buffer: AsMut<[u8]> + Default; fn try_from_le_slice(b: &[u8]) -> Result; diff --git a/src/common/arrow/src/native/util/mod.rs b/src/common/arrow/src/native/util/mod.rs index d0a8905ff576..a6953a428635 100644 --- a/src/common/arrow/src/native/util/mod.rs +++ b/src/common/arrow/src/native/util/mod.rs @@ -22,6 +22,8 @@ pub mod memory; pub use bit_util::*; pub use byte_writer::ByteWriter; +use crate::arrow::datatypes::DataType; + #[macro_export] macro_rules! with_match_integer_double_type { ( @@ -60,3 +62,40 @@ macro_rules! with_match_integer_double_type { } }}; } + +/// Returns the number of (parquet) columns that a [`DataType`] contains. +pub fn n_columns(data_type: &DataType) -> usize { + use crate::arrow::datatypes::PhysicalType::*; + match data_type.to_physical_type() { + Null | Boolean | Primitive(_) | Binary | FixedSizeBinary | LargeBinary | Utf8 + | Dictionary(_) | LargeUtf8 | BinaryView | Utf8View => 1, + List | FixedSizeList | LargeList => { + let a = data_type.to_logical_type(); + if let DataType::List(inner) = a { + n_columns(&inner.data_type) + } else if let DataType::LargeList(inner) = a { + n_columns(&inner.data_type) + } else if let DataType::FixedSizeList(inner, _) = a { + n_columns(&inner.data_type) + } else { + unreachable!() + } + } + Map => { + let a = data_type.to_logical_type(); + if let DataType::Map(inner, _) = a { + n_columns(&inner.data_type) + } else { + unreachable!() + } + } + Struct => { + if let DataType::Struct(fields) = data_type.to_logical_type() { + fields.iter().map(|inner| n_columns(&inner.data_type)).sum() + } else { + unreachable!() + } + } + _ => todo!(), + } +} diff --git a/src/common/arrow/src/native/write/common.rs b/src/common/arrow/src/native/write/common.rs index 72fb0273b69e..d3c0fef93f4e 100644 --- a/src/common/arrow/src/native/write/common.rs +++ b/src/common/arrow/src/native/write/common.rs @@ -19,16 +19,14 @@ use super::NativeWriter; use crate::arrow::array::*; use crate::arrow::chunk::Chunk; use crate::arrow::error::Result; -use crate::arrow::io::parquet::write::num_values; -use crate::arrow::io::parquet::write::slice_parquet_array; -use crate::arrow::io::parquet::write::to_leaves; -use crate::arrow::io::parquet::write::to_nested; -use crate::arrow::io::parquet::write::to_parquet_leaves; use crate::native::compression::CommonCompression; use crate::native::compression::Compression; +use crate::native::nested::slice_nest_array; +use crate::native::nested::to_leaves; +use crate::native::nested::to_nested; use crate::native::ColumnMeta; use crate::native::PageMeta; -use crate::native::CONTINUATION_MARKER; +use crate::native::EOF_MARKER; /// Options declaring the behaviour of writing to IPC #[derive(Debug, Clone, PartialEq, Default)] @@ -51,56 +49,46 @@ impl NativeWriter { .unwrap_or(chunk.len()) .min(chunk.len()); - for (array, type_) in chunk - .arrays() - .iter() - .zip(self.schema_descriptor.fields().to_vec()) - { - let array = array.as_ref(); - let nested = to_nested(array, &type_)?; - let types: Vec = to_parquet_leaves(type_); - let leaf_arrays = to_leaves(array); + for (array, field) in chunk.arrays().iter().zip(self.schema.fields.iter()) { let length = array.len(); - for ((leaf_array, nested), type_) in leaf_arrays - .iter() - .zip(nested.into_iter()) - .zip(types.into_iter()) - { - let start = self.writer.offset; + let nested = to_nested(array.as_ref(), field)?; + let leaf_arrays = to_leaves(array.as_ref()); + + for (leaf_array, nested) in leaf_arrays.iter().zip(nested.into_iter()) { let leaf_array = leaf_array.to_boxed(); + let mut page_metas = Vec::with_capacity((length + 1) / page_size + 1); + let start = self.writer.offset; - let page_metas: Vec = (0..length) - .step_by(page_size) - .map(|offset| { - let length = if offset + page_size > length { - length - offset - } else { - page_size - }; - let mut sub_array = leaf_array.clone(); - let mut sub_nested = nested.clone(); - slice_parquet_array(sub_array.as_mut(), &mut sub_nested, offset, length); + for offset in (0..length).step_by(page_size) { + let length = if offset + page_size > length { + length - offset + } else { + page_size + }; + + let mut sub_array = leaf_array.clone(); + let mut sub_nested = nested.clone(); + slice_nest_array(sub_array.as_mut(), &mut sub_nested, offset, length); + + { let page_start = self.writer.offset; write( &mut self.writer, sub_array.as_ref(), &sub_nested, - type_.clone(), - length, self.options.clone(), &mut self.scratch, ) .unwrap(); let page_end = self.writer.offset; - let num_values = num_values(&sub_nested); - PageMeta { + page_metas.push(PageMeta { length: (page_end - page_start), - num_values: num_values as u64, - } - }) - .collect(); + num_values: sub_array.len() as u64, + }); + } + } self.metas.push(ColumnMeta { offset: start, @@ -108,15 +96,14 @@ impl NativeWriter { }) } } - Ok(()) } } /// Write a record batch to the writer, writing the message size before the message /// if the record batch is being written to a stream -pub fn write_continuation(writer: &mut W, total_len: i32) -> Result { - writer.write_all(&CONTINUATION_MARKER)?; +pub fn write_eof(writer: &mut W, total_len: i32) -> Result { + writer.write_all(&EOF_MARKER)?; writer.write_all(&total_len.to_le_bytes()[..])?; Ok(8) } diff --git a/src/common/arrow/src/native/write/serialize.rs b/src/common/arrow/src/native/write/serialize.rs index c285065b9dc2..35bae7636280 100644 --- a/src/common/arrow/src/native/write/serialize.rs +++ b/src/common/arrow/src/native/write/serialize.rs @@ -14,23 +14,16 @@ use std::io::Write; -use parquet2::schema::types::FieldInfo; -use parquet2::schema::types::PrimitiveType; -use parquet2::schema::Repetition; - use super::boolean::write_bitmap; -use super::primitive::write_primitive; use super::WriteOptions; use crate::arrow::array::*; -use crate::arrow::bitmap::Bitmap; use crate::arrow::datatypes::DataType; use crate::arrow::datatypes::PhysicalType; use crate::arrow::error::Result; -use crate::arrow::io::parquet::write::write_def_levels; -use crate::arrow::io::parquet::write::write_rep_and_def; -use crate::arrow::io::parquet::write::Nested; -use crate::arrow::io::parquet::write::Version; +use crate::native::nested::Nested; +use crate::native::util::encode_bool; use crate::native::write::binary::write_binary; +use crate::native::write::primitive::write_primitive; use crate::native::write::view::write_view; use crate::with_match_primitive_type; @@ -39,65 +32,31 @@ pub fn write( w: &mut W, array: &dyn Array, nested: &[Nested], - type_: PrimitiveType, - length: usize, - write_options: WriteOptions, - scratch: &mut Vec, -) -> Result<()> { - if nested.len() == 1 { - return write_simple(w, array, type_, write_options, scratch); - } - write_nested(w, array, nested, length, write_options, scratch) -} - -/// Writes an [`Array`] to `arrow_data` -pub fn write_simple( - w: &mut W, - array: &dyn Array, - type_: PrimitiveType, write_options: WriteOptions, scratch: &mut Vec, ) -> Result<()> { use PhysicalType::*; - - let is_optional = is_nullable(&type_.field_info); + write_nest_info::(w, nested)?; match array.data_type().to_physical_type() { Null => {} Boolean => { let array: &BooleanArray = array.as_any().downcast_ref().unwrap(); - if is_optional { - write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; - } write_bitmap::(w, array, write_options, scratch)? } Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { let array: &PrimitiveArray<$T> = array.as_any().downcast_ref().unwrap(); - if is_optional { - write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; - } write_primitive::<$T, W>(w, array, write_options, scratch)?; }), Binary => { let array: &BinaryArray = array.as_any().downcast_ref().unwrap(); - if is_optional { - write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; - } write_binary::(w, array, write_options, scratch)?; } LargeBinary => { let array: &BinaryArray = array.as_any().downcast_ref().unwrap(); - if is_optional { - write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; - } write_binary::(w, array, write_options, scratch)?; } Utf8 => { let binary_array: &Utf8Array = array.as_any().downcast_ref().unwrap(); - - if is_optional { - write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; - } - let binary_array = BinaryArray::new( DataType::Binary, binary_array.offsets().clone(), @@ -109,10 +68,6 @@ pub fn write_simple( LargeUtf8 => { let binary_array: &Utf8Array = array.as_any().downcast_ref().unwrap(); - if is_optional { - write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; - } - let binary_array = BinaryArray::new( DataType::LargeBinary, binary_array.offsets().clone(), @@ -123,20 +78,11 @@ pub fn write_simple( } BinaryView => { let array: &BinaryViewArray = array.as_any().downcast_ref().unwrap(); - if is_optional { - write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; - } - write_view::(w, array, write_options, scratch)?; } Utf8View => { let array: &Utf8ViewArray = array.as_any().downcast_ref().unwrap(); let array = array.clone().to_binview(); - - if is_optional { - write_validity::(w, is_optional, array.validity(), array.len(), scratch)?; - } - write_view::(w, &array, write_options, scratch)?; } Struct => unreachable!(), @@ -151,119 +97,51 @@ pub fn write_simple( Ok(()) } -/// Writes a nested [`Array`] to `arrow_data` -pub fn write_nested( - w: &mut W, - array: &dyn Array, - nested: &[Nested], - length: usize, - write_options: WriteOptions, - scratch: &mut Vec, -) -> Result<()> { - write_nested_validity::(w, nested, length, scratch)?; - - scratch.clear(); - - use PhysicalType::*; - match array.data_type().to_physical_type() { - Null => {} - Boolean => { - let array: &BooleanArray = array.as_any().downcast_ref().unwrap(); - write_bitmap::(w, array, write_options, scratch)? - } - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - let array = array.as_any().downcast_ref().unwrap(); - write_primitive::<$T, W>(w, array, write_options, scratch)?; - }), - Binary => { - let binary_array: &BinaryArray = array.as_any().downcast_ref().unwrap(); - write_binary::(w, binary_array, write_options, scratch)?; - } - LargeBinary => { - let binary_array: &BinaryArray = array.as_any().downcast_ref().unwrap(); - write_binary::(w, binary_array, write_options, scratch)?; - } - Utf8 => { - let binary_array: &Utf8Array = array.as_any().downcast_ref().unwrap(); - let binary_array = BinaryArray::new( - DataType::Binary, - binary_array.offsets().clone(), - binary_array.values().clone(), - binary_array.validity().cloned(), - ); - - write_binary::(w, &binary_array, write_options, scratch)?; +fn write_nest_info(w: &mut W, nesteds: &[Nested]) -> Result<()> { + let is_simple = nesteds.len() == 1; + + if is_simple { + let nest = nesteds.last().unwrap(); + + if nest.is_nullable() { + let (_, validity) = nest.inner(); + if let Some(bitmap) = validity { + w.write_all(&(bitmap.len() as u32).to_le_bytes())?; + let (s, offset, _) = bitmap.as_slice(); + if offset == 0 { + w.write_all(s)?; + } else { + encode_bool(w, bitmap.iter())?; + } + } else { + w.write_all(&0u32.to_le_bytes())?; + } } - LargeUtf8 => { - let binary_array: &Utf8Array = array.as_any().downcast_ref().unwrap(); - let binary_array = BinaryArray::new( - DataType::LargeBinary, - binary_array.offsets().clone(), - binary_array.values().clone(), - binary_array.validity().cloned(), - ); + } else { + for nested in nesteds { + let (values, validity) = nested.inner(); + + if nested.is_nullable() { + if let Some(bitmap) = validity { + w.write_all(&(bitmap.len() as u32).to_le_bytes())?; + let (s, offset, _) = bitmap.as_slice(); + if offset == 0 { + w.write_all(s)?; + } else { + encode_bool(w, bitmap.iter())?; + } + } else { + w.write_all(&0u32.to_le_bytes())?; + } + } - write_binary::(w, &binary_array, write_options, scratch)?; - } - BinaryView => { - let array: &BinaryViewArray = array.as_any().downcast_ref().unwrap(); - write_view::(w, array, write_options, scratch)?; - } - Utf8View => { - let array: &Utf8ViewArray = array.as_any().downcast_ref().unwrap(); - let array = array.clone().to_binview(); - write_view::(w, &array, write_options, scratch)?; + if nested.is_list() { + w.write_all(&(values.len() as u32).to_le_bytes())?; + let input_buf: &[u8] = bytemuck::cast_slice(&values); + w.write_all(input_buf)?; + } } - Struct => unreachable!(), - List => unreachable!(), - FixedSizeList => unreachable!(), - Dictionary(_key_type) => unreachable!(), - Union => unreachable!(), - Map => unreachable!(), - _ => todo!(), } Ok(()) } - -fn write_validity( - w: &mut W, - is_optional: bool, - validity: Option<&Bitmap>, - length: usize, - scratch: &mut Vec, -) -> Result<()> { - scratch.clear(); - - write_def_levels(scratch, is_optional, validity, length, Version::V2)?; - let def_levels_len = scratch.len(); - w.write_all(&(def_levels_len as u32).to_le_bytes())?; - w.write_all(&scratch[..def_levels_len])?; - - Ok(()) -} - -fn write_nested_validity( - w: &mut W, - nested: &[Nested], - length: usize, - scratch: &mut Vec, -) -> Result<()> { - scratch.clear(); - - let (rep_levels_len, def_levels_len) = write_rep_and_def(Version::V2, nested, scratch)?; - w.write_all(&(length as u32).to_le_bytes())?; - w.write_all(&(rep_levels_len as u32).to_le_bytes())?; - w.write_all(&(def_levels_len as u32).to_le_bytes())?; - w.write_all(&scratch[..scratch.len()])?; - - Ok(()) -} - -fn is_nullable(field_info: &FieldInfo) -> bool { - match field_info.repetition { - Repetition::Optional => true, - Repetition::Repeated => true, - Repetition::Required => false, - } -} diff --git a/src/common/arrow/src/native/write/writer.rs b/src/common/arrow/src/native/write/writer.rs index cf661790b358..37be134448fc 100644 --- a/src/common/arrow/src/native/write/writer.rs +++ b/src/common/arrow/src/native/write/writer.rs @@ -14,19 +14,16 @@ use std::io::Write; -use super::super::ARROW_MAGIC; -use super::common::write_continuation; +use super::common::write_eof; use super::common::WriteOptions; use crate::arrow::array::Array; use crate::arrow::chunk::Chunk; use crate::arrow::datatypes::Schema; use crate::arrow::error::Error; use crate::arrow::error::Result; -use crate::arrow::io::ipc::write::default_ipc_fields; -use crate::arrow::io::ipc::write::schema_to_bytes; -use crate::arrow::io::parquet::write::to_parquet_schema; use crate::native::ColumnMeta; -use crate::native::SchemaDescriptor; +use crate::native::STRAWBOAT_MAGIC; +use crate::native::STRAWBOAT_VERSION; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum State { @@ -44,7 +41,6 @@ pub struct NativeWriter { pub(crate) options: WriteOptions, /// A reference to the schema, used in validating record batches pub(crate) schema: Schema, - pub(crate) schema_descriptor: SchemaDescriptor, /// Record blocks that will be written as part of the strawboat footer pub metas: Vec, @@ -66,7 +62,6 @@ impl NativeWriter { /// Creates a new [`NativeWriter`]. pub fn new(writer: W, schema: Schema, options: WriteOptions) -> Result { let num_cols = schema.fields.len(); - let schema_descriptor = to_parquet_schema(&schema)?; Ok(Self { writer: OffsetWriter { w: writer, @@ -74,7 +69,6 @@ impl NativeWriter { }, options, schema, - schema_descriptor, metas: Vec::with_capacity(num_cols), scratch: Vec::with_capacity(0), state: State::None, @@ -95,10 +89,10 @@ impl NativeWriter { "The strawboat file can only be started once".to_string(), )); } - // write magic to header - self.writer.write_all(&ARROW_MAGIC[..])?; - // create an 8-byte boundary after the header - self.writer.write_all(&[0, 0])?; + // write magic to header, 2 + 2 + 4 = 8 bytes + self.writer.write_all(&STRAWBOAT_MAGIC[..])?; + self.writer.write_all(&STRAWBOAT_VERSION.to_le_bytes())?; + self.writer.write_all(&[0, 0, 0, 0])?; self.state = State::Started; Ok(()) @@ -133,7 +127,7 @@ impl NativeWriter { // write footer // footer = schema(variable bytes) + column_meta(variable bytes) // + schema size(4 bytes) + column_meta size(4bytes) + EOS(8 bytes) - let schema_bytes = schema_to_bytes(&self.schema, &default_ipc_fields(&self.schema.fields)); + let schema_bytes = serde_json::to_vec(&self.schema)?; // write the schema, set the written bytes to the schema self.writer.write_all(&schema_bytes)?; @@ -159,7 +153,7 @@ impl NativeWriter { self.writer .write_all(&((meta_end - meta_start) as u32).to_le_bytes())?; // write EOS - write_continuation(&mut self.writer, 0)?; + write_eof(&mut self.writer, 0)?; self.writer.flush()?; self.state = State::Finished; Ok(()) diff --git a/src/common/arrow/tests/it/arrow/compute/aggregate/memory.rs b/src/common/arrow/tests/it/arrow/compute/aggregate/memory.rs deleted file mode 100644 index 066e2e2923ee..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/aggregate/memory.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::compute::aggregate::estimated_bytes_size; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; - -#[test] -fn primitive() { - let a = Int32Array::from_slice([1, 2, 3, 4, 5]); - assert_eq!(5 * std::mem::size_of::(), estimated_bytes_size(&a)); -} - -#[test] -fn boolean() { - let a = BooleanArray::from_slice([true]); - assert_eq!(1, estimated_bytes_size(&a)); -} - -#[test] -fn utf8() { - let a = Utf8Array::::from_slice(["aaa"]); - assert_eq!(3 + 2 * std::mem::size_of::(), estimated_bytes_size(&a)); -} - -#[test] -fn fixed_size_list() { - let data_type = - DataType::FixedSizeList(Box::new(Field::new("elem", DataType::Float32, false)), 3); - let values = Box::new(Float32Array::from_slice([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])); - let a = FixedSizeListArray::new(data_type, values, None); - assert_eq!(6 * std::mem::size_of::(), estimated_bytes_size(&a)); -} diff --git a/src/common/arrow/tests/it/arrow/compute/aggregate/mod.rs b/src/common/arrow/tests/it/arrow/compute/aggregate/mod.rs deleted file mode 100644 index 50ce01f697e6..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/aggregate/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod memory; diff --git a/src/common/arrow/tests/it/arrow/compute/cast.rs b/src/common/arrow/tests/it/arrow/compute/cast.rs deleted file mode 100644 index cba4f4e2cea2..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/cast.rs +++ /dev/null @@ -1,903 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::compute::cast::can_cast_types; -use databend_common_arrow::arrow::compute::cast::cast; -use databend_common_arrow::arrow::compute::cast::CastOptions; -use databend_common_arrow::arrow::datatypes::DataType::LargeList; -use databend_common_arrow::arrow::datatypes::*; -use databend_common_arrow::arrow::types::days_ms; -use databend_common_arrow::arrow::types::months_days_ns; -use databend_common_arrow::arrow::types::NativeType; - -#[test] -fn i32_to_f64() { - let array = Int32Array::from_slice([5, 6, 7, 8, 9]); - let b = cast(&array, &DataType::Float64, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - assert!((5.0 - c.value(0)).abs() < f64::EPSILON); - assert!((6.0 - c.value(1)).abs() < f64::EPSILON); - assert!((7.0 - c.value(2)).abs() < f64::EPSILON); - assert!((8.0 - c.value(3)).abs() < f64::EPSILON); - assert!((9.0 - c.value(4)).abs() < f64::EPSILON); -} - -#[test] -fn i32_as_f64_no_overflow() { - let array = Int32Array::from_slice([5, 6, 7, 8, 9]); - let b = cast(&array, &DataType::Float64, CastOptions { - wrapped: true, - ..Default::default() - }) - .unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - assert!((5.0 - c.value(0)).abs() < f64::EPSILON); - assert!((6.0 - c.value(1)).abs() < f64::EPSILON); - assert!((7.0 - c.value(2)).abs() < f64::EPSILON); - assert!((8.0 - c.value(3)).abs() < f64::EPSILON); - assert!((9.0 - c.value(4)).abs() < f64::EPSILON); -} - -#[test] -fn u16_as_u8_overflow() { - let array = UInt16Array::from_slice([255, 256, 257, 258, 259]); - let b = cast(&array, &DataType::UInt8, CastOptions { - wrapped: true, - ..Default::default() - }) - .unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - let values = c.values().as_slice(); - - assert_eq!(values, &[255, 0, 1, 2, 3]) -} - -#[test] -fn u16_as_u8_no_overflow() { - let array = UInt16Array::from_slice([1, 2, 3, 4, 5]); - let b = cast(&array, &DataType::UInt8, CastOptions { - wrapped: true, - ..Default::default() - }) - .unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - let values = c.values().as_slice(); - assert_eq!(values, &[1, 2, 3, 4, 5]) -} - -#[test] -fn f32_as_u8_overflow() { - let array = Float32Array::from_slice([1.1, 5000.0]); - let b = cast(&array, &DataType::UInt8, CastOptions::default()).unwrap(); - let expected = UInt8Array::from(&[Some(1), None]); - assert_eq!(expected, b.as_ref()); - - let b = cast(&array, &DataType::UInt8, CastOptions { - wrapped: true, - ..Default::default() - }) - .unwrap(); - let expected = UInt8Array::from(&[Some(1), Some(255)]); - assert_eq!(expected, b.as_ref()); -} - -#[test] -fn i32_to_u8() { - let array = Int32Array::from_slice([-5, 6, -7, 8, 100000000]); - let b = cast(&array, &DataType::UInt8, CastOptions::default()).unwrap(); - let expected = UInt8Array::from(&[None, Some(6), None, Some(8), None]); - let c = b.as_any().downcast_ref::().unwrap(); - assert_eq!(c, &expected); -} - -#[test] -fn i32_to_u8_sliced() { - let array = Int32Array::from_slice([-5, 6, -7, 8, 100000000]); - let array = array.sliced(2, 3); - let b = cast(&array, &DataType::UInt8, CastOptions::default()).unwrap(); - let expected = UInt8Array::from(&[None, Some(8), None]); - let c = b.as_any().downcast_ref::().unwrap(); - assert_eq!(c, &expected); -} - -#[test] -fn i32_to_i32() { - let array = Int32Array::from_slice([5, 6, 7, 8, 9]); - let b = cast(&array, &DataType::Int32, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - - let expected = &[5, 6, 7, 8, 9]; - let expected = Int32Array::from_slice(expected); - assert_eq!(c, &expected); -} - -#[test] -fn i32_to_large_list_i32() { - let array = Int32Array::from_slice([5, 6, 7, 8, 9]); - let b = cast( - &array, - &LargeList(Box::new(Field::new("item", DataType::Int32, true))), - CastOptions::default(), - ) - .unwrap(); - - let arr = b.as_any().downcast_ref::>().unwrap(); - assert_eq!(&[0, 1, 2, 3, 4, 5], arr.offsets().as_slice()); - let values = arr.values(); - let c = values - .as_any() - .downcast_ref::>() - .unwrap(); - - let expected = Int32Array::from_slice([5, 6, 7, 8, 9]); - assert_eq!(c, &expected); -} - -#[test] -fn i32_to_list_i32() { - let array = Int32Array::from_slice([5, 6, 7, 8, 9]); - let b = cast( - &array, - &DataType::List(Box::new(Field::new("item", DataType::Int32, true))), - CastOptions::default(), - ) - .unwrap(); - - let arr = b.as_any().downcast_ref::>().unwrap(); - assert_eq!(&[0, 1, 2, 3, 4, 5], arr.offsets().as_slice()); - let values = arr.values(); - let c = values - .as_any() - .downcast_ref::>() - .unwrap(); - - let expected = Int32Array::from_slice([5, 6, 7, 8, 9]); - assert_eq!(c, &expected); -} - -#[test] -fn i32_to_list_i32_nullable() { - let input = [Some(5), None, Some(7), Some(8), Some(9)]; - - let array = Int32Array::from(input); - let b = cast( - &array, - &DataType::List(Box::new(Field::new("item", DataType::Int32, true))), - CastOptions::default(), - ) - .unwrap(); - - let arr = b.as_any().downcast_ref::>().unwrap(); - assert_eq!(&[0, 1, 2, 3, 4, 5], arr.offsets().as_slice()); - let values = arr.values(); - let c = values.as_any().downcast_ref::().unwrap(); - - let expected = &[Some(5), None, Some(7), Some(8), Some(9)]; - let expected = Int32Array::from(expected); - assert_eq!(c, &expected); -} - -#[test] -fn i32_to_list_f64_nullable_sliced() { - let input = [Some(5), None, Some(7), Some(8), None, Some(10)]; - - let array = Int32Array::from(input); - - let array = array.sliced(2, 4); - let b = cast( - &array, - &DataType::List(Box::new(Field::new("item", DataType::Float64, true))), - CastOptions::default(), - ) - .unwrap(); - - let arr = b.as_any().downcast_ref::>().unwrap(); - assert_eq!(&[0, 1, 2, 3, 4], arr.offsets().as_slice()); - let values = arr.values(); - let c = values.as_any().downcast_ref::().unwrap(); - - let expected = &[Some(7.0), Some(8.0), None, Some(10.0)]; - let expected = Float64Array::from(expected); - assert_eq!(c, &expected); -} - -#[test] -fn i32_to_binary() { - let array = Int32Array::from_slice([5, 6, 7]); - let b = cast(&array, &DataType::Binary, CastOptions::default()).unwrap(); - let expected = BinaryArray::::from([Some(b"5"), Some(b"6"), Some(b"7")]); - let c = b.as_any().downcast_ref::>().unwrap(); - assert_eq!(c, &expected); -} - -#[test] -fn binary_to_i32() { - let array = BinaryArray::::from_slice(["5", "6", "seven", "8", "9.1"]); - let b = cast(&array, &DataType::Int32, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = &[Some(5), Some(6), None, Some(8), None]; - let expected = Int32Array::from(expected); - assert_eq!(c, &expected); -} - -#[test] -fn binary_to_i32_partial() { - let array = BinaryArray::::from_slice(["5", "6", "123 abseven", "aaa", "9.1"]); - let b = cast(&array, &DataType::Int32, CastOptions { - partial: true, - ..Default::default() - }) - .unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = &[Some(5), Some(6), Some(123), Some(0), Some(9)]; - let expected = Int32Array::from(expected); - assert_eq!(c, &expected); -} - -#[test] -fn fixed_size_binary_to_binary() { - let slice = [[0, 1], [2, 3]]; - let array = FixedSizeBinaryArray::from_slice(slice); - - // large-binary - let b = cast(&array, &DataType::LargeBinary, CastOptions { - ..Default::default() - }) - .unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - let expected = BinaryArray::::from_slice(slice); - assert_eq!(c, &expected); - - // binary - let b = cast(&array, &DataType::Binary, CastOptions { - ..Default::default() - }) - .unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - let expected = BinaryArray::::from_slice(slice); - assert_eq!(c, &expected); -} - -#[test] -fn utf8_to_i32() { - let array = Utf8Array::::from_slice(["5", "6", "seven", "8", "9.1"]); - let b = cast(&array, &DataType::Int32, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = &[Some(5), Some(6), None, Some(8), None]; - let expected = Int32Array::from(expected); - assert_eq!(c, &expected); -} - -#[test] -fn int32_to_decimal() { - // 10 and -10 can be represented with precision 1 and scale 0 - let array = Int32Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]); - - let b = cast(&array, &DataType::Decimal(1, 0), CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(1, 0)); - assert_eq!(c, &expected) -} - -#[test] -fn float32_to_decimal() { - let array = Float32Array::from(&[ - Some(2.4), - Some(10.0), - Some(1.123_456_8), - Some(-2.0), - Some(-10.0), - Some(-100.01), // can't be represented in (1,0) - None, - ]); - - let b = cast(&array, &DataType::Decimal(10, 2), CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = Int128Array::from(&[ - Some(240), - Some(1000), - Some(112), - Some(-200), - Some(-1000), - Some(-10001), - None, - ]) - .to(DataType::Decimal(10, 2)); - assert_eq!(c, &expected) -} - -#[test] -fn int32_to_decimal_scaled() { - // 10 and -10 can't be represented with precision 1 and scale 1 - let array = Int32Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]); - - let b = cast(&array, &DataType::Decimal(1, 1), CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = - Int128Array::from(&[Some(20), None, Some(-20), None, None]).to(DataType::Decimal(1, 1)); - assert_eq!(c, &expected) -} - -#[test] -fn decimal_to_decimal() { - // increase scale and precision - let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(1, 0)); - - let b = cast(&array, &DataType::Decimal(2, 1), CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = Int128Array::from(&[Some(20), Some(100), Some(-20), Some(-100), None]) - .to(DataType::Decimal(2, 1)); - assert_eq!(c, &expected) -} - -#[test] -fn decimal_to_decimal_scaled() { - // decrease precision - // 10 and -10 can't be represented with precision 1 and scale 1 - let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(1, 0)); - - let b = cast(&array, &DataType::Decimal(1, 1), CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = - Int128Array::from(&[Some(20), None, Some(-20), None, None]).to(DataType::Decimal(1, 1)); - assert_eq!(c, &expected) -} - -#[test] -fn decimal_to_decimal_fast() { - // increase precision - // 10 and -10 can't be represented with precision 1 and scale 1 - let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(1, 1)); - - let b = cast(&array, &DataType::Decimal(2, 1), CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(2, 1)); - assert_eq!(c, &expected) -} - -#[test] -fn decimal_to_float() { - let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(2, 1)); - - let b = cast(&array, &DataType::Float32, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = Float32Array::from(&[Some(0.2), Some(1.0), Some(-0.2), Some(-1.0), None]); - assert_eq!(c, &expected) -} - -#[test] -fn decimal_to_integer() { - let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None, Some(2560)]) - .to(DataType::Decimal(2, 1)); - - let b = cast(&array, &DataType::Int8, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = Int8Array::from(&[Some(0), Some(1), Some(0), Some(-1), None, None]); - assert_eq!(c, &expected) -} - -#[test] -fn utf8_to_i32_partial() { - let array = Utf8Array::::from_slice(["5", "6", "seven", "8aa", "9.1aa"]); - let b = cast(&array, &DataType::Int32, CastOptions { - partial: true, - ..Default::default() - }) - .unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = &[Some(5), Some(6), Some(0), Some(8), Some(9)]; - let expected = Int32Array::from(expected); - assert_eq!(c, &expected); -} - -#[test] -fn bool_to_i32() { - let array = BooleanArray::from(vec![Some(true), Some(false), None]); - let b = cast(&array, &DataType::Int32, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - - let expected = &[Some(1), Some(0), None]; - let expected = Int32Array::from(expected); - assert_eq!(c, &expected); -} - -#[test] -fn bool_to_f64() { - let array = BooleanArray::from(vec![Some(true), Some(false), None]); - let b = cast(&array, &DataType::Float64, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - - let expected = &[Some(1.0), Some(0.0), None]; - let expected = Float64Array::from(expected); - assert_eq!(c, &expected); -} - -#[test] -fn bool_to_utf8() { - let array = BooleanArray::from(vec![Some(true), Some(false), None]); - let b = cast(&array, &DataType::Utf8, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = Utf8Array::::from([Some("1"), Some("0"), Some("0")]); - assert_eq!(c, &expected); -} - -#[test] -fn bool_to_binary() { - let array = BooleanArray::from(vec![Some(true), Some(false), None]); - let b = cast(&array, &DataType::Binary, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::>().unwrap(); - - let expected = BinaryArray::::from([Some("1"), Some("0"), Some("0")]); - assert_eq!(c, &expected); -} - -#[test] -fn int32_to_timestamp() { - let array = Int32Array::from(&[Some(2), Some(10), None]); - assert!( - cast( - &array, - &DataType::Timestamp(TimeUnit::Microsecond, None), - CastOptions::default() - ) - .is_err() - ); -} - -#[test] -fn consistency() { - use DataType::*; - let datatypes = vec![ - Null, - Boolean, - UInt8, - UInt16, - UInt32, - UInt64, - Int8, - Int16, - Int32, - Int64, - Float16, - Float32, - Float64, - Timestamp(TimeUnit::Second, None), - Timestamp(TimeUnit::Millisecond, None), - Timestamp(TimeUnit::Millisecond, Some("+01:00".to_string())), - Timestamp(TimeUnit::Microsecond, None), - Timestamp(TimeUnit::Nanosecond, None), - Time64(TimeUnit::Microsecond), - Time64(TimeUnit::Nanosecond), - Date32, - Time32(TimeUnit::Second), - Time32(TimeUnit::Millisecond), - Decimal(1, 2), - Decimal(2, 2), - Date64, - Utf8, - LargeUtf8, - Binary, - LargeBinary, - Duration(TimeUnit::Second), - Duration(TimeUnit::Millisecond), - Duration(TimeUnit::Microsecond), - Duration(TimeUnit::Nanosecond), - List(Box::new(Field::new("a", Utf8, true))), - LargeList(Box::new(Field::new("a", Utf8, true))), - ]; - for d1 in &datatypes { - for d2 in &datatypes { - let array = new_null_array(d1.clone(), 10); - if can_cast_types(d1, d2) { - let result = cast(array.as_ref(), d2, CastOptions::default()); - if let Ok(result) = result { - assert_eq!(result.data_type(), d2, "type not equal: {d1:?} {d2:?}"); - } else { - panic!("Cast should have not have failed {d1:?} {d2:?}: {result:?}"); - } - } else if cast(array.as_ref(), d2, CastOptions::default()).is_ok() { - panic!("Cast should have failed {d1:?} {d2:?}"); - } - } - } -} - -fn test_primitive_to_primitive( - lhs: &[I], - lhs_type: DataType, - expected: &[O], - expected_type: DataType, -) { - let a = PrimitiveArray::::from_slice(lhs).to(lhs_type); - let b = cast(&a, &expected_type, CastOptions::default()).unwrap(); - let b = b.as_any().downcast_ref::>().unwrap(); - let expected = PrimitiveArray::::from_slice(expected).to(expected_type); - assert_eq!(b, &expected); -} - -#[test] -fn date32_to_date64() { - test_primitive_to_primitive( - &[10000i32, 17890], - DataType::Date32, - &[864000000000i64, 1545696000000], - DataType::Date64, - ); -} - -#[test] -fn days_ms_to_months_days_ns() { - test_primitive_to_primitive( - &[days_ms::new(1, 1), days_ms::new(1, 2)], - DataType::Interval(IntervalUnit::DayTime), - &[ - months_days_ns::new(0, 1, 1000), - months_days_ns::new(0, 1, 2000), - ], - DataType::Interval(IntervalUnit::MonthDayNano), - ); -} - -#[test] -fn months_to_months_days_ns() { - test_primitive_to_primitive( - &[1, 2], - DataType::Interval(IntervalUnit::YearMonth), - &[months_days_ns::new(1, 0, 0), months_days_ns::new(2, 0, 0)], - DataType::Interval(IntervalUnit::MonthDayNano), - ); -} - -#[test] -fn date64_to_date32() { - test_primitive_to_primitive( - &[864000000005i64, 1545696000001], - DataType::Date64, - &[10000i32, 17890], - DataType::Date32, - ); -} - -#[test] -fn date32_to_int32() { - test_primitive_to_primitive( - &[10000i32, 17890], - DataType::Date32, - &[10000i32, 17890], - DataType::Int32, - ); -} - -#[test] -fn date64_to_int32() { - test_primitive_to_primitive( - &[10000i64, 17890], - DataType::Date64, - &[10000i32, 17890], - DataType::Int32, - ); -} - -#[test] -fn date32_to_int64() { - test_primitive_to_primitive( - &[10000i32, 17890], - DataType::Date32, - &[10000i64, 17890], - DataType::Int64, - ); -} - -#[test] -fn int32_to_date32() { - test_primitive_to_primitive( - &[10000i32, 17890], - DataType::Int32, - &[10000i32, 17890], - DataType::Date32, - ); -} - -#[test] -fn timestamp_to_date32() { - test_primitive_to_primitive( - &[864000000005i64, 1545696000001], - DataType::Timestamp(TimeUnit::Millisecond, Some(String::from("UTC"))), - &[10000i32, 17890], - DataType::Date32, - ); -} - -#[test] -fn timestamp_to_date64() { - test_primitive_to_primitive( - &[864000000005i64, 1545696000001], - DataType::Timestamp(TimeUnit::Millisecond, Some(String::from("UTC"))), - &[864000000005i64, 1545696000001i64], - DataType::Date64, - ); -} - -#[test] -fn timestamp_to_i64() { - test_primitive_to_primitive( - &[864000000005i64, 1545696000001], - DataType::Timestamp(TimeUnit::Millisecond, Some(String::from("UTC"))), - &[864000000005i64, 1545696000001i64], - DataType::Int64, - ); -} - -#[test] -fn timestamp_to_timestamp() { - test_primitive_to_primitive( - &[864000003005i64, 1545696002001], - DataType::Timestamp(TimeUnit::Millisecond, None), - &[864000003i64, 1545696002], - DataType::Timestamp(TimeUnit::Second, None), - ); -} - -#[test] -fn utf8_to_dict() { - let array = Utf8Array::::from([Some("one"), None, Some("three"), Some("one")]); - - // Cast to a dictionary (same value type, Utf8) - let cast_type = DataType::Dictionary(u8::KEY_TYPE, Box::new(DataType::Utf8), false); - let result = cast(&array, &cast_type, CastOptions::default()).expect("cast failed"); - - let mut expected = MutableDictionaryArray::>::new(); - expected - .try_extend([Some("one"), None, Some("three"), Some("one")]) - .unwrap(); - let expected: DictionaryArray = expected.into(); - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn dict_to_utf8() { - let mut array = MutableDictionaryArray::>::new(); - array - .try_extend([Some("one"), None, Some("three"), Some("one")]) - .unwrap(); - let array: DictionaryArray = array.into(); - - let result = cast(&array, &DataType::Utf8, CastOptions::default()).expect("cast failed"); - - let expected = Utf8Array::::from([Some("one"), None, Some("three"), Some("one")]); - - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn i32_to_dict() { - let array = Int32Array::from(&[Some(1), None, Some(3), Some(1)]); - - // Cast to a dictionary (same value type, Utf8) - let cast_type = DataType::Dictionary(u8::KEY_TYPE, Box::new(DataType::Int32), false); - let result = cast(&array, &cast_type, CastOptions::default()).expect("cast failed"); - - let mut expected = MutableDictionaryArray::>::new(); - expected - .try_extend([Some(1), None, Some(3), Some(1)]) - .unwrap(); - let expected: DictionaryArray = expected.into(); - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn list_to_list() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(4), None, Some(6)]), - ]; - - let expected_data = data - .iter() - .map(|x| x.as_ref().map(|x| x.iter().map(|x| x.map(|x| x as u16)))); - - let mut array = MutableListArray::>::new(); - array.try_extend(data.clone()).unwrap(); - let array: ListArray = array.into(); - - let mut expected = MutableListArray::>::new(); - expected.try_extend(expected_data).unwrap(); - let expected: ListArray = expected.into(); - - let result = cast(&array, expected.data_type(), CastOptions::default()).unwrap(); - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn list_to_from_fixed_size_list() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - Some(vec![Some(4), Some(5), None]), - Some(vec![Some(6), None, Some(7)]), - ]; - - let fixed_data = data - .iter() - .map(|x| x.as_ref().map(|x| x.iter().map(|x| x.map(|x| x as u16)))); - - let mut list = MutableListArray::>::new(); - list.try_extend(data.clone()).unwrap(); - let list: ListArray = list.into(); - - let inner = MutablePrimitiveArray::::new(); - let mut fixed = MutableFixedSizeListArray::>::new(inner, 3); - fixed.try_extend(fixed_data).unwrap(); - let fixed: FixedSizeListArray = fixed.into(); - - let result = cast(&list, fixed.data_type(), CastOptions::default()).unwrap(); - assert_eq!(fixed, result.as_ref()); - - let result = cast(&fixed, list.data_type(), CastOptions::default()).unwrap(); - assert_eq!(list, result.as_ref()); -} - -#[test] -fn timestamp_with_tz_to_utf8() { - let tz = "-02:00".to_string(); - let expected = - Utf8Array::::from_slice(["1996-12-19T16:39:57-02:00", "1996-12-19T17:39:57-02:00"]); - let array = Int64Array::from_slice([851020797000000000, 851024397000000000]) - .to(DataType::Timestamp(TimeUnit::Nanosecond, Some(tz))); - - let result = cast(&array, expected.data_type(), CastOptions::default()).expect("cast failed"); - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn utf8_to_timestamp_with_tz() { - let tz = "-02:00".to_string(); - let array = - Utf8Array::::from_slice(["1996-12-19T16:39:57-02:00", "1996-12-19T17:39:57-02:00"]); - // the timezone is used to map the time to UTC. - let expected = Int64Array::from_slice([851020797000000000, 851024397000000000]) - .to(DataType::Timestamp(TimeUnit::Nanosecond, Some(tz))); - - let result = cast(&array, expected.data_type(), CastOptions::default()).expect("cast failed"); - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn utf8_to_naive_timestamp() { - let array = - Utf8Array::::from_slice(["1996-12-19T16:39:57-02:00", "1996-12-19T17:39:57-02:00"]); - // the timezone is disregarded from the string and we assume UTC - let expected = Int64Array::from_slice([851013597000000000, 851017197000000000]) - .to(DataType::Timestamp(TimeUnit::Nanosecond, None)); - - let result = cast(&array, expected.data_type(), CastOptions::default()).expect("cast failed"); - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn naive_timestamp_to_utf8() { - let array = Int64Array::from_slice([851013597000000000, 851017197000000000]) - .to(DataType::Timestamp(TimeUnit::Nanosecond, None)); - - let expected = Utf8Array::::from_slice(["1996-12-19 16:39:57", "1996-12-19 17:39:57"]); - - let result = cast(&array, expected.data_type(), CastOptions::default()).expect("cast failed"); - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn null_array_from_and_to_others() { - macro_rules! typed_test { - ($ARR_TYPE:ident, $DATATYPE:ident) => {{ - { - let array = new_null_array(DataType::Null, 6); - let expected = $ARR_TYPE::from(vec![None; 6]); - let cast_type = DataType::$DATATYPE; - let result = - cast(array.as_ref(), &cast_type, CastOptions::default()).expect("cast failed"); - let result = result.as_any().downcast_ref::<$ARR_TYPE>().unwrap(); - assert_eq!(result.data_type(), &cast_type); - assert_eq!(result, &expected); - } - { - let array = $ARR_TYPE::from(vec![None; 4]); - let expected = NullArray::new_null(DataType::Null, 4); - let result = - cast(&array, &DataType::Null, CastOptions::default()).expect("cast failed"); - let result = result.as_any().downcast_ref::().unwrap(); - assert_eq!(result.data_type(), &DataType::Null); - assert_eq!(result, &expected); - } - }}; - } - - typed_test!(Int16Array, Int16); - typed_test!(Int32Array, Int32); - typed_test!(Int64Array, Int64); - - typed_test!(UInt16Array, UInt16); - typed_test!(UInt32Array, UInt32); - typed_test!(UInt64Array, UInt64); - - typed_test!(Float16Array, Float16); - typed_test!(Float32Array, Float32); - typed_test!(Float64Array, Float64); -} - -#[test] -fn utf8_to_date32() { - let array = Utf8Array::::from_slice(["1970-01-01", "1970-01-02"]); - let b = cast(&array, &DataType::Date32, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - - let expected = Int32Array::from_slice([0, 1]).to(DataType::Date32); - - assert_eq!(&expected, c); -} - -#[test] -fn utf8_to_date64() { - let array = Utf8Array::::from_slice(["1970-01-01", "1970-01-02"]); - let b = cast(&array, &DataType::Date64, CastOptions::default()).unwrap(); - let c = b.as_any().downcast_ref::().unwrap(); - - let expected = Int64Array::from_slice([0, 86400000]).to(DataType::Date64); - - assert_eq!(&expected, c); -} - -#[test] -fn dict_keys() { - let mut array = MutableDictionaryArray::>::new(); - array - .try_extend([Some("one"), None, Some("three"), Some("one")]) - .unwrap(); - let array: DictionaryArray = array.into(); - - let result = cast( - &array, - &DataType::Dictionary(IntegerType::Int64, Box::new(DataType::Utf8), false), - CastOptions::default(), - ) - .expect("cast failed"); - - let mut expected = MutableDictionaryArray::>::new(); - expected - .try_extend([Some("one"), None, Some("three"), Some("one")]) - .unwrap(); - let expected: DictionaryArray = expected.into(); - - assert_eq!(expected, result.as_ref()); -} diff --git a/src/common/arrow/tests/it/arrow/compute/merge_sort.rs b/src/common/arrow/tests/it/arrow/compute/merge_sort.rs deleted file mode 100644 index e5df1e605431..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/merge_sort.rs +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::once; - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::compute::merge_sort::*; -use databend_common_arrow::arrow::compute::sort::sort; -use databend_common_arrow::arrow::error::Result; - -#[test] -fn merge_u32() -> Result<()> { - let a0: &dyn Array = &Int32Array::from_slice([0, 1, 2, 3]); - let a1: &dyn Array = &Int32Array::from_slice([2, 3, 4, 5]); - - let options = SortOptions::default(); - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - - // (0, 1, 2) corresponds to slice [1, 2] of a0 - // (1, 2, 2) corresponds to slice [4, 5] of a1 - // slices are already sorted => identity - let result = - merge_sort_slices(once(&(0, 1, 2)), once(&(1, 2, 2)), &comparator).collect::>(); - - assert_eq!(result, vec![(0, 1, 2), (1, 2, 2)]); - - // (0, 2, 2) corresponds to slice [2, 3] of a0 - // (1, 0, 3) corresponds to slice [2, 3, 4] of a1 - let result = - merge_sort_slices(once(&(0, 2, 2)), once(&(1, 0, 3)), &comparator).collect::>(); - - // 2 (a0) , [2, 3] (a1) , 3 (a0) , 4 (a1) - // (0, 2, 1), (1, 0, 2) , (0, 3, 1), (1, 2, 1) - assert_eq!(result, vec![(0, 2, 1), (1, 0, 2), (0, 3, 1), (1, 2, 1)]); - Ok(()) -} - -#[test] -fn merge_null_first() -> Result<()> { - let a0: &dyn Array = &Int32Array::from(&[None, Some(0)]); - let a1: &dyn Array = &Int32Array::from(&[Some(2), Some(3)]); - let options = SortOptions { - descending: false, - nulls_first: true, - }; - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - let result = - merge_sort_slices(once(&(0, 0, 2)), once(&(1, 0, 2)), &comparator).collect::>(); - assert_eq!(result, vec![(0, 0, 2), (1, 0, 2)]); - - let a0: &dyn Array = &Int32Array::from(&[Some(0), None]); - let a1: &dyn Array = &Int32Array::from(&[Some(2), Some(3)]); - let options = SortOptions { - descending: false, - nulls_first: false, - }; - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - let result = - merge_sort_slices(once(&(0, 0, 2)), once(&(1, 0, 2)), &comparator).collect::>(); - assert_eq!(result, vec![(0, 0, 1), (1, 0, 2), (0, 1, 1)]); - - let a0: &dyn Array = &Int32Array::from(&[Some(0), None]); - let a1: &dyn Array = &Int32Array::from(&[Some(3), Some(2)]); - let options = SortOptions { - descending: true, - nulls_first: false, - }; - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - let result = - merge_sort_slices(once(&(0, 0, 2)), once(&(1, 0, 2)), &comparator).collect::>(); - assert_eq!(result, vec![(1, 0, 2), (0, 0, 2)]); - - let a0: &dyn Array = &Int32Array::from(&[None, Some(0)]); - let a1: &dyn Array = &Int32Array::from(&[Some(3), Some(2)]); - let options = SortOptions { - descending: true, - nulls_first: true, - }; - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - let result = - merge_sort_slices(once(&(0, 0, 2)), once(&(1, 0, 2)), &comparator).collect::>(); - assert_eq!(result, vec![(0, 0, 1), (1, 0, 2), (0, 1, 1)]); - - Ok(()) -} - -#[test] -fn merge_with_limit() -> Result<()> { - let a0: &dyn Array = &Int32Array::from_slice([0, 2, 4, 6, 8]); - let a1: &dyn Array = &Int32Array::from_slice([1, 3, 5, 7, 9]); - - let options = SortOptions::default(); - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - - let slices = merge_sort_slices(once(&(0, 0, 5)), once(&(1, 0, 5)), &comparator); - // thus, they can be used to take from the arrays - let array = take_arrays(&arrays, slices, Some(5)); - - let expected = Int32Array::from_slice([0, 1, 2, 3, 4]); - // values are right - assert_eq!(expected, array.as_ref()); - Ok(()) -} - -#[test] -fn merge_slices_to_vec() -> Result<()> { - let a0: &dyn Array = &Int32Array::from_slice([0, 2, 4, 6, 8]); - let a1: &dyn Array = &Int32Array::from_slice([1, 3, 5, 7, 9]); - - let options = SortOptions::default(); - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - - let slices = merge_sort_slices(once(&(0, 0, 5)), once(&(1, 0, 5)), &comparator); - let vec = slices.to_vec(Some(5)); - assert_eq!(vec, [(0, 0, 1), (1, 0, 1), (0, 1, 1), (1, 1, 1), (0, 2, 1)]); - Ok(()) -} - -#[test] -fn merge_4_i32() -> Result<()> { - let a0: &dyn Array = &Int32Array::from_slice([0, 1]); - let a1: &dyn Array = &Int32Array::from_slice([2, 6]); - let a2: &dyn Array = &Int32Array::from_slice([3, 5]); - let a3: &dyn Array = &Int32Array::from_slice([4, 7]); - - let options = SortOptions::default(); - let arrays = vec![a0, a1, a2, a3]; - let pairs = vec![(arrays.as_ref(), &options)]; - let slices = slices(&pairs)?; - - // slices are right. - assert_eq!(slices, vec![ - (0, 0, 2), - (1, 0, 1), - (2, 0, 1), - (3, 0, 1), // 4 - (2, 1, 1), // 5 - (1, 1, 1), // 6 - (3, 1, 1), // 7 - ]); - - // thus, they can be used to take from the arrays - let array = take_arrays(&arrays, slices, None); - - let expected = Int32Array::from_slice([0, 1, 2, 3, 4, 5, 6, 7]); - - // values are right - assert_eq!(expected, array.as_ref()); - Ok(()) -} - -#[test] -fn merge_binary() -> Result<()> { - let a0: &dyn Array = &BinaryArray::::from_slice([b"a", b"c", b"d", b"e"]); - let a1: &dyn Array = &BinaryArray::::from_slice([b"b", b"y", b"z", b"z"]); - - let options = SortOptions::default(); - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - - // (0, 0, 4) corresponds to slice ["a", "c", "d", "e"] of a0 - // (1, 0, 4) corresponds to slice ["b", "y", "z", "z"] of a1 - - let result = - merge_sort_slices(once(&(0, 0, 4)), once(&(1, 0, 4)), &comparator).collect::>(); - - // "a" (a0) , "b" (a1) , ["c", "d", "e"] (a0), ["y", "z", "z"] (a1) - // (0, 0, 1), (1, 0, 1), (0, 1, 3) , (1, 1, 3) - assert_eq!(result, vec![(0, 0, 1), (1, 0, 1), (0, 1, 3), (1, 1, 3)]); - - // (0, 1, 2) corresponds to slice ["c", "d"] of a0 - // (1, 0, 3) corresponds to slice ["b", "y", "z"] of a1 - let result = - merge_sort_slices(once(&(0, 1, 2)), once(&(1, 0, 3)), &comparator).collect::>(); - - // "b" (a1) , ["c", "d"] (a0) , ["y", "z"] - // (1, 0, 1), (0, 1, 2) , (1, 1, 2) - assert_eq!(result, vec![(1, 0, 1), (0, 1, 2), (1, 1, 2)]); - Ok(()) -} - -#[test] -fn merge_string() -> Result<()> { - let a0: &dyn Array = &Utf8Array::::from_slice(["a", "c", "d", "e"]); - let a1: &dyn Array = &Utf8Array::::from_slice(["b", "y", "z", "z"]); - - let options = SortOptions::default(); - let arrays = vec![a0, a1]; - let pairs = vec![(arrays.as_ref(), &options)]; - let comparator = build_comparator(&pairs)?; - - // (0, 0, 4) corresponds to slice ["a", "c", "d", "e"] of a0 - // (1, 0, 4) corresponds to slice ["b", "y", "z", "z"] of a1 - - let result = - merge_sort_slices(once(&(0, 0, 4)), once(&(1, 0, 4)), &comparator).collect::>(); - - // "a" (a0) , "b" (a1) , ["c", "d", "e"] (a0), ["y", "z", "z"] (a1) - // (0, 0, 1), (1, 0, 1), (0, 1, 3) , (1, 1, 3) - assert_eq!(result, vec![(0, 0, 1), (1, 0, 1), (0, 1, 3), (1, 1, 3)]); - - // (0, 1, 2) corresponds to slice ["c", "d"] of a0 - // (1, 0, 3) corresponds to slice ["b", "y", "z"] of a1 - let result = - merge_sort_slices(once(&(0, 1, 2)), once(&(1, 0, 3)), &comparator).collect::>(); - - // "b" (a1) , ["c", "d"] (a0) , ["y", "z"] - // (1, 0, 1), (0, 1, 2) , (1, 1, 2) - assert_eq!(result, vec![(1, 0, 1), (0, 1, 2), (1, 1, 2)]); - Ok(()) -} - -#[test] -fn merge_sort_many() -> Result<()> { - // column 1 - let a00: &dyn Array = &Int32Array::from_slice([0, 1, 2, 3]); - let a01: &dyn Array = &Int32Array::from_slice([2, 3, 4]); - // column 2 - let a10: &dyn Array = &Utf8Array::::from_slice(["a", "c", "d", "e"]); - let a11: &dyn Array = &Utf8Array::::from_slice(["b", "y", "z"]); - // column 3 - // arrays to be sorted via the columns above - let array0: &dyn Array = &Int32Array::from_slice([0, 1, 2, 3]); - let array1: &dyn Array = &Int32Array::from_slice([4, 5, 6]); - - let expected = Int32Array::from_slice([ - 0, // 0 (a00) < 2 (a01) - 1, // 1 (a00) < 2 (a01) - 4, // 2 (a00) == 2 (a01), "d" (a10) > "b" (a11) - 2, // 2 (a00) < 3 (a01) - 3, // 3 (a00) == 3 (a01), "e" (a10) < "y" (a11) - 5, // arrays0 has finished - 6, // arrays0 has finished - ]); - - // merge-sort according to column 1 and then column 2 - let options = SortOptions::default(); - let arrays0 = vec![a00, a01]; - let arrays1 = vec![a10, a11]; - let pairs = vec![(arrays0.as_ref(), &options), (arrays1.as_ref(), &options)]; - let slices = slices(&pairs)?; - - let array = take_arrays(&[array0, array1], slices, None); - - assert_eq!(expected, array.as_ref()); - Ok(()) -} - -#[test] -fn test_sort() -> Result<()> { - let data0 = vec![4, 1, 2, 10, 3, 3]; - let data1 = vec![5, 1, 0, 6, 7]; - - let mut expected_data = [data0.clone(), data1.clone()].concat(); - expected_data.sort_unstable(); - let expected = Int32Array::from_slice(&expected_data); - - let a0: &dyn Array = &Int32Array::from_slice(&data0); - let a1: &dyn Array = &Int32Array::from_slice(&data1); - - let options = SortOptions::default(); - - // sort individually, potentially in parallel. - let a0 = sort(a0, &options, None)?; - let a1 = sort(a1, &options, None)?; - - // merge then. If multiple arrays, this can be applied in parallel. - let result = merge_sort(a0.as_ref(), a1.as_ref(), &options, None)?; - - assert_eq!(expected, result.as_ref()); - Ok(()) -} diff --git a/src/common/arrow/tests/it/arrow/compute/mod.rs b/src/common/arrow/tests/it/arrow/compute/mod.rs index b1683b64ac29..9f3816f2a629 100644 --- a/src/common/arrow/tests/it/arrow/compute/mod.rs +++ b/src/common/arrow/tests/it/arrow/compute/mod.rs @@ -13,15 +13,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "compute_aggregate")] -mod aggregate; -#[cfg(feature = "compute_cast")] -mod cast; #[cfg(feature = "compute_concatenate")] mod concatenate; -#[cfg(feature = "compute_merge_sort")] -mod merge_sort; -#[cfg(feature = "compute_sort")] -mod sort; -#[cfg(feature = "compute_take")] -mod take; diff --git a/src/common/arrow/tests/it/arrow/compute/sort/lex_sort.rs b/src/common/arrow/tests/it/arrow/compute/sort/lex_sort.rs deleted file mode 100644 index 94f9104036b2..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/sort/lex_sort.rs +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::compute::sort::lexsort; -use databend_common_arrow::arrow::compute::sort::SortColumn; -use databend_common_arrow::arrow::compute::sort::SortOptions; - -fn test_lex_sort_arrays(input: Vec, expected: Vec>) { - let sorted = lexsort::(&input, None).unwrap(); - assert_eq!(sorted, expected); - - let sorted = lexsort::(&input, Some(4)).unwrap(); - let expected = expected - .into_iter() - .map(|x| x.sliced(0, 4)) - .collect::>(); - assert_eq!(sorted, expected); - - let sorted = lexsort::(&input, Some(2)).unwrap(); - let expected = expected - .into_iter() - .map(|x| x.sliced(0, 2)) - .collect::>(); - assert_eq!(sorted, expected); -} - -#[test] -fn test_lex_sort_mixed_types() { - let c1 = Int64Array::from(&[Some(0), Some(2), Some(-1), Some(0)]); - let c2 = UInt32Array::from(&[Some(101), Some(8), Some(7), Some(102)]); - let c3 = Int64Array::from(&[Some(-1), Some(-2), Some(-3), Some(-4)]); - - let input = vec![ - SortColumn { - values: &c1, - options: None, - }, - SortColumn { - values: &c2, - options: None, - }, - SortColumn { - values: &c3, - options: None, - }, - ]; - let c1 = Int64Array::from([Some(-1), Some(0), Some(0), Some(2)]); - let c2 = UInt32Array::from([Some(7), Some(101), Some(102), Some(8)]); - let c3 = Int64Array::from([Some(-3), Some(-1), Some(-4), Some(-2)]); - let expected = vec![c1.boxed(), c2.boxed(), c3.boxed()]; - test_lex_sort_arrays(input, expected); -} - -#[test] -fn test_lex_sort_mixed_types2() { - // test mix of string and in64 with option - let c1 = Int64Array::from([Some(0), Some(2), Some(-1), Some(0)]); - let c2 = Utf8Array::::from([Some("foo"), Some("9"), Some("7"), Some("bar")]); - let input = vec![ - SortColumn { - values: &c1, - options: Some(SortOptions { - descending: true, - nulls_first: true, - }), - }, - SortColumn { - values: &c2, - options: Some(SortOptions { - descending: true, - nulls_first: true, - }), - }, - ]; - let expected = vec![ - Int64Array::from([Some(2), Some(0), Some(0), Some(-1)]).boxed(), - Utf8Array::::from([Some("9"), Some("foo"), Some("bar"), Some("7")]).boxed(), - ]; - test_lex_sort_arrays(input, expected); -} - -// test sort with nulls first -// let input = vec![ -// SortColumn { -// values: Arc::new(PrimitiveArray::::from(vec![ -// None, -// Some(-1), -// Some(2), -// None, -// ])) as ArrayRef, -// options: Some(SortOptions { -// descending: true, -// nulls_first: true, -// }), -// }, -// SortColumn { -// values: Arc::new(StringArray::from(vec![ -// Some("foo"), -// Some("world"), -// Some("hello"), -// None, -// ])) as ArrayRef, -// options: Some(SortOptions { -// descending: true, -// nulls_first: true, -// }), -// }, -// ]; -// let expected = vec![ -// Arc::new(PrimitiveArray::::from(vec![ -// None, -// None, -// Some(2), -// Some(-1), -// ])) as ArrayRef, -// Arc::new(StringArray::from(vec![ -// None, -// Some("foo"), -// Some("hello"), -// Some("world"), -// ])) as ArrayRef, -// ]; -// test_lex_sort_arrays(input, expected); -// -// test sort with nulls last -// let input = vec![ -// SortColumn { -// values: Arc::new(PrimitiveArray::::from(vec![ -// None, -// Some(-1), -// Some(2), -// None, -// ])) as ArrayRef, -// options: Some(SortOptions { -// descending: true, -// nulls_first: false, -// }), -// }, -// SortColumn { -// values: Arc::new(StringArray::from(vec![ -// Some("foo"), -// Some("world"), -// Some("hello"), -// None, -// ])) as ArrayRef, -// options: Some(SortOptions { -// descending: true, -// nulls_first: false, -// }), -// }, -// ]; -// let expected = vec![ -// Arc::new(PrimitiveArray::::from(vec![ -// Some(2), -// Some(-1), -// None, -// None, -// ])) as ArrayRef, -// Arc::new(StringArray::from(vec![ -// Some("hello"), -// Some("world"), -// Some("foo"), -// None, -// ])) as ArrayRef, -// ]; -// test_lex_sort_arrays(input, expected); -// -// test sort with opposite options -// let input = vec![ -// SortColumn { -// values: Arc::new(PrimitiveArray::::from(vec![ -// None, -// Some(-1), -// Some(2), -// Some(-1), -// None, -// ])) as ArrayRef, -// options: Some(SortOptions { -// descending: false, -// nulls_first: false, -// }), -// }, -// SortColumn { -// values: Arc::new(StringArray::from(vec![ -// Some("foo"), -// Some("bar"), -// Some("world"), -// Some("hello"), -// None, -// ])) as ArrayRef, -// options: Some(SortOptions { -// descending: true, -// nulls_first: true, -// }), -// }, -// ]; -// let expected = vec![ -// Arc::new(PrimitiveArray::::from(vec![ -// Some(-1), -// Some(-1), -// Some(2), -// None, -// None, -// ])) as ArrayRef, -// Arc::new(StringArray::from(vec![ -// Some("hello"), -// Some("bar"), -// Some("world"), -// None, -// Some("foo"), -// ])) as ArrayRef, -// ]; -// test_lex_sort_arrays(input, expected); -// } diff --git a/src/common/arrow/tests/it/arrow/compute/sort/mod.rs b/src/common/arrow/tests/it/arrow/compute/sort/mod.rs deleted file mode 100644 index 326da7cba549..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/sort/mod.rs +++ /dev/null @@ -1,613 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod lex_sort; -mod row; - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::compute::sort::*; -use databend_common_arrow::arrow::datatypes::*; -use databend_common_arrow::arrow::types::NativeType; - -fn to_indices_boolean_arrays(data: &[Option], options: SortOptions, expected_data: &[i32]) { - let output = BooleanArray::from(data); - let expected = Int32Array::from_slice(expected_data); - let output = sort_to_indices(&output, &options, None).unwrap(); - assert_eq!(output, expected) -} - -fn primitive_arrays( - data: &[Option], - data_type: DataType, - options: SortOptions, - expected_data: &[Option], -) where - T: NativeType, -{ - let input = PrimitiveArray::::from(data).to(data_type.clone()); - let expected = PrimitiveArray::::from(expected_data).to(data_type); - let output = sort(&input, &options, None).unwrap(); - assert_eq!(expected, output.as_ref()) -} - -fn to_indices_string_arrays(data: &[Option<&str>], options: SortOptions, expected_data: &[i32]) { - let input = Utf8Array::::from(data); - let expected = Int32Array::from_slice(expected_data); - let output = sort_to_indices(&input, &options, None).unwrap(); - assert_eq!(output, expected) -} - -fn string_arrays(data: &[Option<&str>], options: SortOptions, expected_data: &[Option<&str>]) { - let input = Utf8Array::::from(data); - let expected = Utf8Array::::from(expected_data); - let output = sort(&input, &options, None).unwrap(); - assert_eq!(expected, output.as_ref()) -} - -fn string_dict_arrays(data: &[Option<&str>], options: SortOptions, expected_data: &[Option<&str>]) { - let mut input = MutableDictionaryArray::>::new(); - input.try_extend(data.iter().copied()).unwrap(); - let input = input.into_arc(); - - let mut expected = MutableDictionaryArray::>::new(); - expected.try_extend(expected_data.iter().copied()).unwrap(); - let expected = expected.into_arc(); - - let output = sort(input.as_ref(), &options, None).unwrap(); - assert_eq!(expected.as_ref(), output.as_ref()) -} - -// fn list_arrays( -// data: Vec>>>, -// options: Option, -// expected_data: Vec>>>, -// fixed_length: Option, -// ) where -// T: ArrowPrimitiveType, -// PrimitiveArray: From>>, -// { -// for FixedSizedList -// if let Some(length) = fixed_length { -// let input = Arc::new(build_fixed_size_list_nullable(data.clone(), length)); -// let sorted = sort(&(input as ArrayRef), options).unwrap(); -// let expected = Arc::new(build_fixed_size_list_nullable( -// expected_data.clone(), -// length, -// )) as ArrayRef; -// -// assert_eq!(&sorted, &expected); -// } -// -// for List -// let input = Arc::new(build_generic_list_nullable::(data.clone())); -// let sorted = sort(&(input as ArrayRef), options).unwrap(); -// let expected = -// Arc::new(build_generic_list_nullable::(expected_data.clone())) -// as ArrayRef; -// -// assert_eq!(&sorted, &expected); -// -// for LargeList -// let input = Arc::new(build_generic_list_nullable::(data)); -// let sorted = sort(&(input as ArrayRef), options).unwrap(); -// let expected = -// Arc::new(build_generic_list_nullable::(expected_data)) as ArrayRef; -// -// assert_eq!(&sorted, &expected); -// } -// -// fn test_lex_sort_arrays(input: Vec, expected_output: Vec) { -// let sorted = lexsort(&input).unwrap(); -// -// for (result, expected) in sorted.iter().zip(expected_output.iter()) { -// assert_eq!(result, expected); -// } -// } - -#[test] -fn boolean() { - // boolean - to_indices_boolean_arrays( - &[None, Some(false), Some(true), Some(true), Some(false), None], - SortOptions { - descending: false, - nulls_first: true, - }, - &[0, 5, 1, 4, 2, 3], - ); - - // boolean, descending - to_indices_boolean_arrays( - &[None, Some(false), Some(true), Some(true), Some(false), None], - SortOptions { - descending: true, - nulls_first: false, - }, - &[2, 3, 1, 4, 5, 0], - ); - - // boolean, descending, nulls first - to_indices_boolean_arrays( - &[None, Some(false), Some(true), Some(true), Some(false), None], - SortOptions { - descending: true, - nulls_first: true, - }, - &[5, 0, 2, 3, 1, 4], - ); -} - -#[test] -#[ignore] // improve equality for NaN values. These are right but the equality fails -fn test_nans() { - primitive_arrays::( - &[None, Some(0.0), Some(2.0), Some(-1.0), Some(f64::NAN), None], - DataType::Float64, - SortOptions { - descending: true, - nulls_first: true, - }, - &[None, None, Some(f64::NAN), Some(2.0), Some(0.0), Some(-1.0)], - ); - primitive_arrays::( - &[Some(f64::NAN), Some(f64::NAN), Some(f64::NAN), Some(1.0)], - DataType::Float64, - SortOptions { - descending: true, - nulls_first: true, - }, - &[Some(f64::NAN), Some(f64::NAN), Some(f64::NAN), Some(1.0)], - ); - - primitive_arrays::( - &[None, Some(0.0), Some(2.0), Some(-1.0), Some(f64::NAN), None], - DataType::Float64, - SortOptions { - descending: false, - nulls_first: true, - }, - &[None, None, Some(-1.0), Some(0.0), Some(2.0), Some(f64::NAN)], - ); - // nans - primitive_arrays::( - &[Some(f64::NAN), Some(f64::NAN), Some(f64::NAN), Some(1.0)], - DataType::Float64, - SortOptions { - descending: false, - nulls_first: true, - }, - &[Some(1.0), Some(f64::NAN), Some(f64::NAN), Some(f64::NAN)], - ); -} - -#[test] -fn to_indices_strings() { - to_indices_string_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: false, - nulls_first: true, - }, - // &[3, 0, 5, 1, 4, 2] is also valid - &[0, 3, 5, 1, 4, 2], - ); - - to_indices_string_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: true, - nulls_first: false, - }, - // &[2, 4, 1, 5, 3, 0] is also valid - &[2, 4, 1, 5, 0, 3], - ); - - to_indices_string_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: false, - nulls_first: true, - }, - // &[3, 0, 5, 1, 4, 2] is also valid - &[0, 3, 5, 1, 4, 2], - ); - - to_indices_string_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: true, - nulls_first: true, - }, - // &[3, 0, 2, 4, 1, 5] is also valid - &[0, 3, 2, 4, 1, 5], - ); -} - -#[test] -fn strings() { - string_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: false, - nulls_first: true, - }, - &[ - None, - None, - Some("-ad"), - Some("bad"), - Some("glad"), - Some("sad"), - ], - ); - - string_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: true, - nulls_first: false, - }, - &[ - Some("sad"), - Some("glad"), - Some("bad"), - Some("-ad"), - None, - None, - ], - ); - - string_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: false, - nulls_first: true, - }, - &[ - None, - None, - Some("-ad"), - Some("bad"), - Some("glad"), - Some("sad"), - ], - ); - - string_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: true, - nulls_first: true, - }, - &[ - None, - None, - Some("sad"), - Some("glad"), - Some("bad"), - Some("-ad"), - ], - ); -} - -#[test] -fn string_dicts() { - string_dict_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: false, - nulls_first: true, - }, - &[ - None, - None, - Some("-ad"), - Some("bad"), - Some("glad"), - Some("sad"), - ], - ); - - string_dict_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: true, - nulls_first: false, - }, - &[ - Some("sad"), - Some("glad"), - Some("bad"), - Some("-ad"), - None, - None, - ], - ); - - string_dict_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: false, - nulls_first: true, - }, - &[ - None, - None, - Some("-ad"), - Some("bad"), - Some("glad"), - Some("sad"), - ], - ); - - string_dict_arrays( - &[ - None, - Some("bad"), - Some("sad"), - None, - Some("glad"), - Some("-ad"), - ], - SortOptions { - descending: true, - nulls_first: true, - }, - &[ - None, - None, - Some("sad"), - Some("glad"), - Some("bad"), - Some("-ad"), - ], - ); -} - -// #[test] -// fn list() { -// list_arrays::( -// vec![ -// Some(vec![Some(1)]), -// Some(vec![Some(4)]), -// Some(vec![Some(2)]), -// Some(vec![Some(3)]), -// ], -// Some(SortOptions { -// descending: false, -// nulls_first: false, -// }), -// vec![ -// Some(vec![Some(1)]), -// Some(vec![Some(2)]), -// Some(vec![Some(3)]), -// Some(vec![Some(4)]), -// ], -// Some(1), -// ); -// -// list_arrays::( -// vec![ -// Some(vec![Some(1), Some(0)]), -// Some(vec![Some(4), Some(3), Some(2), Some(1)]), -// Some(vec![Some(2), Some(3), Some(4)]), -// Some(vec![Some(3), Some(3), Some(3), Some(3)]), -// Some(vec![Some(1), Some(1)]), -// ], -// Some(SortOptions { -// descending: false, -// nulls_first: false, -// }), -// vec![ -// Some(vec![Some(1), Some(0)]), -// Some(vec![Some(1), Some(1)]), -// Some(vec![Some(2), Some(3), Some(4)]), -// Some(vec![Some(3), Some(3), Some(3), Some(3)]), -// Some(vec![Some(4), Some(3), Some(2), Some(1)]), -// ], -// None, -// ); -// -// list_arrays::( -// vec![ -// None, -// Some(vec![Some(4), None, Some(2)]), -// Some(vec![Some(2), Some(3), Some(4)]), -// None, -// Some(vec![Some(3), Some(3), None]), -// ], -// Some(SortOptions { -// descending: false, -// nulls_first: false, -// }), -// vec![ -// Some(vec![Some(2), Some(3), Some(4)]), -// Some(vec![Some(3), Some(3), None]), -// Some(vec![Some(4), None, Some(2)]), -// None, -// None, -// ], -// Some(3), -// ); -// } -// -// #[test] -// fn test_lex_sort_single_column() { -// let input = vec![SortColumn { -// values: Arc::new(PrimitiveArray::::from(vec![ -// Some(17), -// Some(2), -// Some(-1), -// Some(0), -// ])) as ArrayRef, -// options: None, -// }]; -// let expected = vec![Arc::new(PrimitiveArray::::from(vec![ -// Some(-1), -// Some(0), -// Some(2), -// Some(17), -// ])) as ArrayRef]; -// test_lex_sort_arrays(input, expected); -// } -// -// #[test] -// fn test_lex_sort_unaligned_rows() { -// let input = vec![ -// SortColumn { -// values: Arc::new(PrimitiveArray::::from(vec![None, Some(-1)])) -// as ArrayRef, -// options: None, -// }, -// SortColumn { -// values: Arc::new(StringArray::from(vec![Some("foo")])) as ArrayRef, -// options: None, -// }, -// ]; -// assert!( -// lexsort(&input).is_err(), -// "lexsort should reject columns with different row counts" -// ); -// } - -#[test] -fn consistency() { - use databend_common_arrow::arrow::array::new_null_array; - use databend_common_arrow::arrow::datatypes::DataType::*; - use databend_common_arrow::arrow::datatypes::TimeUnit; - - let datatypes = vec![ - Null, - Boolean, - UInt8, - UInt16, - UInt32, - UInt64, - Int8, - Int16, - Int32, - Int64, - Float32, - Float64, - Timestamp(TimeUnit::Second, None), - Timestamp(TimeUnit::Millisecond, None), - Timestamp(TimeUnit::Microsecond, None), - Timestamp(TimeUnit::Nanosecond, None), - Time64(TimeUnit::Microsecond), - Time64(TimeUnit::Nanosecond), - Date32, - Time32(TimeUnit::Second), - Time32(TimeUnit::Millisecond), - Date64, - Utf8, - LargeUtf8, - Binary, - LargeBinary, - Duration(TimeUnit::Second), - Duration(TimeUnit::Millisecond), - Duration(TimeUnit::Microsecond), - Duration(TimeUnit::Nanosecond), - ]; - - datatypes.into_iter().for_each(|d1| { - let array = new_null_array(d1.clone(), 10); - let options = SortOptions { - descending: true, - nulls_first: true, - }; - if can_sort(&d1) { - assert!(sort(array.as_ref(), &options, None).is_ok()); - } else { - assert!(sort(array.as_ref(), &options, None).is_err()); - } - }); -} diff --git a/src/common/arrow/tests/it/arrow/compute/sort/row/mod.rs b/src/common/arrow/tests/it/arrow/compute/sort/row/mod.rs deleted file mode 100644 index b69d9723d3b7..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/sort/row/mod.rs +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::BinaryArray; -use databend_common_arrow::arrow::array::BooleanArray; -use databend_common_arrow::arrow::array::DictionaryArray; -use databend_common_arrow::arrow::array::Float32Array; -use databend_common_arrow::arrow::array::Int128Array; -use databend_common_arrow::arrow::array::Int16Array; -use databend_common_arrow::arrow::array::Int256Array; -use databend_common_arrow::arrow::array::Int32Array; -use databend_common_arrow::arrow::array::MutableDictionaryArray; -use databend_common_arrow::arrow::array::MutablePrimitiveArray; -use databend_common_arrow::arrow::array::MutableUtf8Array; -use databend_common_arrow::arrow::array::NullArray; -use databend_common_arrow::arrow::array::TryExtend; -use databend_common_arrow::arrow::array::TryPush; -use databend_common_arrow::arrow::array::Utf8Array; -use databend_common_arrow::arrow::compute::sort::row::RowConverter; -use databend_common_arrow::arrow::compute::sort::row::SortField; -use databend_common_arrow::arrow::compute::sort::SortOptions; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::IntegerType; -use databend_common_arrow::arrow::types::i256; - -#[test] -fn test_fixed_width() { - let cols = [ - Int16Array::from_iter([Some(1), Some(2), None, Some(-5), Some(2), Some(2), Some(0)]) - .to_boxed(), - Float32Array::from_iter([ - Some(1.3), - Some(2.5), - None, - Some(4.), - Some(0.1), - Some(-4.), - Some(-0.), - ]) - .to_boxed(), - ]; - - let mut converter = RowConverter::new(vec![ - SortField::new(DataType::Int16), - SortField::new(DataType::Float32), - ]); - let rows = converter.convert_columns(&cols).unwrap(); - - assert!(rows.row(3) < rows.row(6)); - assert!(rows.row(0) < rows.row(1)); - assert!(rows.row(3) < rows.row(0)); - assert!(rows.row(4) < rows.row(1)); - assert!(rows.row(5) < rows.row(4)); -} - -#[test] -fn test_decimal128() { - let mut converter = RowConverter::new(vec![SortField::new(DataType::Decimal(38, 7))]); - let col = Int128Array::from_iter([ - None, - Some(i128::MIN), - Some(-13), - Some(46_i128), - Some(5456_i128), - Some(i128::MAX), - ]) - .to(DataType::Decimal(38, 7)) - .to_boxed(); - - let rows = converter.convert_columns(&[col]).unwrap(); - for i in 0..rows.len() - 1 { - assert!(rows.row(i) < rows.row(i + 1)); - } -} - -#[test] -fn test_decimal256() { - let mut converter = RowConverter::new(vec![SortField::new(DataType::Decimal256(76, 7))]); - let col = Int256Array::from_iter([ - None, - Some(i256::from_words(i128::MIN, i128::MIN)), - Some(i256::from_words(0, 46_i128)), - Some(i256::from_words(0, -1)), - Some(i256::from_words(5, 46_i128)), - Some(i256::from_words(i128::MAX, 0)), - Some(i256::from_words(i128::MAX, i128::MAX)), - Some(i256::from_words(i128::MAX, -1)), - ]) - .to(DataType::Decimal256(76, 7)) - .to_boxed(); - - let rows = converter.convert_columns(&[col]).unwrap(); - for i in 0..rows.len() - 1 { - assert!(rows.row(i) < rows.row(i + 1)); - } -} - -#[test] -fn test_bool() { - let mut converter = RowConverter::new(vec![SortField::new(DataType::Boolean)]); - - let col = BooleanArray::from_iter([None, Some(false), Some(true)]).to_boxed(); - - let rows = converter.convert_columns(&[Box::clone(&col)]).unwrap(); - assert!(rows.row(2) > rows.row(1)); - assert!(rows.row(2) > rows.row(0)); - assert!(rows.row(1) > rows.row(0)); - - let mut converter = RowConverter::new(vec![SortField::new_with_options( - DataType::Boolean, - SortOptions { - descending: true, - nulls_first: false, - }, - )]); - - let rows = converter.convert_columns(&[Box::clone(&col)]).unwrap(); - assert!(rows.row(2) < rows.row(1)); - assert!(rows.row(2) < rows.row(0)); - assert!(rows.row(1) < rows.row(0)); -} - -#[test] -fn test_null_encoding() { - let col = NullArray::new(DataType::Null, 10).to_boxed(); - let mut converter = RowConverter::new(vec![SortField::new(DataType::Null)]); - let rows = converter.convert_columns(&[col]).unwrap(); - assert_eq!(rows.len(), 10); -} - -#[test] -fn test_variable_width() { - let col = - Utf8Array::::from([Some("hello"), Some("he"), None, Some("foo"), Some("")]).to_boxed(); - - let mut converter = RowConverter::new(vec![SortField::new(DataType::Utf8)]); - let rows = converter.convert_columns(&[Box::clone(&col)]).unwrap(); - - assert!(rows.row(1) < rows.row(0)); - assert!(rows.row(2) < rows.row(4)); - assert!(rows.row(3) < rows.row(0)); - assert!(rows.row(3) < rows.row(1)); - - let col = BinaryArray::::from_iter([ - None, - Some(vec![0_u8; 0]), - Some(vec![0_u8; 6]), - Some(vec![0_u8; 32]), - Some(vec![0_u8; 33]), - Some(vec![1_u8; 6]), - Some(vec![1_u8; 32]), - Some(vec![1_u8; 33]), - Some(vec![0xFF_u8; 6]), - Some(vec![0xFF_u8; 32]), - Some(vec![0xFF_u8; 33]), - ]) - .to_boxed(); - - let mut converter = RowConverter::new(vec![SortField::new(DataType::Binary)]); - let rows = converter.convert_columns(&[Box::clone(&col)]).unwrap(); - - for i in 0..rows.len() { - for j in i + 1..rows.len() { - assert!( - rows.row(i) < rows.row(j), - "{} < {} - {:?} < {:?}", - i, - j, - rows.row(i), - rows.row(j) - ); - } - } - - let mut converter = RowConverter::new(vec![SortField::new_with_options( - DataType::Binary, - SortOptions { - descending: true, - nulls_first: false, - }, - )]); - let rows = converter.convert_columns(&[Box::clone(&col)]).unwrap(); - - for i in 0..rows.len() { - for j in i + 1..rows.len() { - assert!( - rows.row(i) > rows.row(j), - "{} > {} - {:?} > {:?}", - i, - j, - rows.row(i), - rows.row(j) - ); - } - } -} - -#[test] -fn test_string_dictionary() { - let data = vec![ - Some("foo"), - Some("hello"), - Some("he"), - None, - Some("hello"), - Some(""), - Some("hello"), - Some("hello"), - ]; - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data).unwrap(); - let a: DictionaryArray = array.into(); - let a = a.to_boxed(); - - let mut converter = RowConverter::new(vec![SortField::new(a.data_type().clone())]); - let rows_a = converter.convert_columns(&[Box::clone(&a)]).unwrap(); - - assert!(rows_a.row(3) < rows_a.row(5)); - assert!(rows_a.row(2) < rows_a.row(1)); - assert!(rows_a.row(0) < rows_a.row(1)); - assert!(rows_a.row(3) < rows_a.row(0)); - - assert_eq!(rows_a.row(1), rows_a.row(4)); - assert_eq!(rows_a.row(1), rows_a.row(6)); - assert_eq!(rows_a.row(1), rows_a.row(7)); - - let data = vec![Some("hello"), None, Some("cupcakes")]; - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data).unwrap(); - let b: DictionaryArray = array.into(); - - let rows_b = converter.convert_columns(&[b.to_boxed()]).unwrap(); - assert_eq!(rows_a.row(1), rows_b.row(0)); - assert_eq!(rows_a.row(3), rows_b.row(1)); - assert!(rows_b.row(2) < rows_a.row(0)); - - let mut converter = RowConverter::new(vec![SortField::new_with_options( - a.data_type().clone(), - SortOptions { - descending: true, - nulls_first: false, - }, - )]); - - let rows_c = converter.convert_columns(&[Box::clone(&a)]).unwrap(); - assert!(rows_c.row(3) > rows_c.row(5)); - assert!(rows_c.row(2) > rows_c.row(1)); - assert!(rows_c.row(0) > rows_c.row(1)); - assert!(rows_c.row(3) > rows_c.row(0)); -} - -#[test] -fn test_primitive_dictionary() { - let mut builder = MutableDictionaryArray::>::new(); - builder.try_push(Some(2)).unwrap(); - builder.try_push(Some(3)).unwrap(); - builder.try_push(Some(0)).unwrap(); - builder.push_null(); - builder.try_push(Some(5)).unwrap(); - builder.try_push(Some(3)).unwrap(); - builder.try_push(Some(-1)).unwrap(); - - let a: DictionaryArray = builder.into(); - - let mut converter = RowConverter::new(vec![SortField::new(a.data_type().clone())]); - let rows = converter.convert_columns(&[a.to_boxed()]).unwrap(); - assert!(rows.row(0) < rows.row(1)); - assert!(rows.row(2) < rows.row(0)); - assert!(rows.row(3) < rows.row(2)); - assert!(rows.row(6) < rows.row(2)); - assert!(rows.row(3) < rows.row(6)); -} - -#[test] -fn test_dictionary_nulls() { - let values = Int32Array::from_iter([Some(1), Some(-1), None, Some(4), None]); - let keys = Int32Array::from_iter([Some(0), Some(0), Some(1), Some(2), Some(4), None]); - - let data_type = DataType::Dictionary(IntegerType::Int32, Box::new(DataType::Int32), false); - let data = DictionaryArray::try_from_keys(keys, values.to_boxed()).unwrap(); - - let mut converter = RowConverter::new(vec![SortField::new(data_type)]); - let rows = converter.convert_columns(&[data.to_boxed()]).unwrap(); - - assert_eq!(rows.row(0), rows.row(1)); - assert_eq!(rows.row(3), rows.row(4)); - assert_eq!(rows.row(4), rows.row(5)); - assert!(rows.row(3) < rows.row(0)); -} diff --git a/src/common/arrow/tests/it/arrow/compute/take.rs b/src/common/arrow/tests/it/arrow/compute/take.rs deleted file mode 100644 index 8e9f6049346d..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/take.rs +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::compute::take::can_take; -use databend_common_arrow::arrow::compute::take::take; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::datatypes::IntervalUnit; -use databend_common_arrow::arrow::error::Result; -use databend_common_arrow::arrow::types::NativeType; - -fn test_take_primitive( - data: &[Option], - indices: &Int32Array, - expected_data: &[Option], - data_type: DataType, -) -> Result<()> -where - T: NativeType, -{ - let output = PrimitiveArray::::from(data).to(data_type.clone()); - let expected = PrimitiveArray::::from(expected_data).to(data_type); - let output = take(&output, indices)?; - assert_eq!(expected, output.as_ref()); - Ok(()) -} - -#[test] -fn test_take_primitive_non_null_indices() { - let indices = Int32Array::from_slice([0, 5, 3, 1, 4, 2]); - test_take_primitive::( - &[None, Some(2), Some(4), Some(6), Some(8), None], - &indices, - &[None, None, Some(6), Some(2), Some(8), Some(4)], - DataType::Int8, - ) - .unwrap(); - - test_take_primitive::( - &[Some(0), Some(2), Some(4), Some(6), Some(8), Some(10)], - &indices, - &[Some(0), Some(10), Some(6), Some(2), Some(8), Some(4)], - DataType::Int8, - ) - .unwrap(); -} - -#[test] -fn test_take_primitive_null_values() { - let indices = Int32Array::from(&[Some(0), None, Some(3), Some(1), Some(4), Some(2)]); - test_take_primitive::( - &[Some(0), Some(2), Some(4), Some(6), Some(8), Some(10)], - &indices, - &[Some(0), None, Some(6), Some(2), Some(8), Some(4)], - DataType::Int8, - ) - .unwrap(); - - test_take_primitive::( - &[None, Some(2), Some(4), Some(6), Some(8), Some(10)], - &indices, - &[None, None, Some(6), Some(2), Some(8), Some(4)], - DataType::Int8, - ) - .unwrap(); -} - -fn create_test_struct() -> StructArray { - let boolean = BooleanArray::from_slice([true, false, false, true]); - let int = Int32Array::from_slice([42, 28, 19, 31]); - let validity = vec![true, true, false, true] - .into_iter() - .collect::() - .into(); - let fields = vec![ - Field::new("a", DataType::Boolean, true), - Field::new("b", DataType::Int32, true), - ]; - StructArray::new( - DataType::Struct(fields), - vec![boolean.boxed(), int.boxed()], - validity, - ) -} - -#[test] -fn test_struct_with_nulls() { - let array = create_test_struct(); - - let indices = Int32Array::from(&[None, Some(3), Some(1), None, Some(0)]); - - let output = take(&array, &indices).unwrap(); - - let boolean = BooleanArray::from(&[None, Some(true), Some(false), None, Some(true)]); - let int = Int32Array::from(&[None, Some(31), Some(28), None, Some(42)]); - let validity = vec![false, true, true, false, true] - .into_iter() - .collect::() - .into(); - let expected = StructArray::new( - array.data_type().clone(), - vec![boolean.boxed(), int.boxed()], - validity, - ); - assert_eq!(expected, output.as_ref()); -} - -#[test] -fn consistency() { - use databend_common_arrow::arrow::array::new_null_array; - use databend_common_arrow::arrow::datatypes::DataType::*; - use databend_common_arrow::arrow::datatypes::TimeUnit; - - let datatypes = vec![ - Null, - Boolean, - UInt8, - UInt16, - UInt32, - UInt64, - Int8, - Int16, - Int32, - Int64, - Float32, - Float64, - Timestamp(TimeUnit::Second, None), - Timestamp(TimeUnit::Millisecond, None), - Timestamp(TimeUnit::Microsecond, None), - Timestamp(TimeUnit::Nanosecond, None), - Time64(TimeUnit::Microsecond), - Time64(TimeUnit::Nanosecond), - Interval(IntervalUnit::DayTime), - Interval(IntervalUnit::YearMonth), - Date32, - Time32(TimeUnit::Second), - Time32(TimeUnit::Millisecond), - Date64, - Utf8, - LargeUtf8, - Binary, - LargeBinary, - Duration(TimeUnit::Second), - Duration(TimeUnit::Millisecond), - Duration(TimeUnit::Microsecond), - Duration(TimeUnit::Nanosecond), - ]; - - datatypes.into_iter().for_each(|d1| { - let array = new_null_array(d1.clone(), 10); - let indices = Int32Array::from(&[Some(1), Some(2), None, Some(3)]); - if can_take(&d1) { - assert!(take(array.as_ref(), &indices).is_ok()); - } else { - assert!(take(array.as_ref(), &indices).is_err()); - } - }); -} - -#[test] -fn empty() { - let indices = Int32Array::from_slice([]); - let values = BooleanArray::from(vec![Some(true), Some(false)]); - let a = take(&values, &indices).unwrap(); - assert_eq!(a.len(), 0) -} - -#[test] -fn unsigned_take() { - let indices = UInt32Array::from_slice([]); - let values = BooleanArray::from(vec![Some(true), Some(false)]); - let a = take(&values, &indices).unwrap(); - assert_eq!(a.len(), 0) -} - -#[test] -fn list_with_no_none() { - let values = Buffer::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - let values = PrimitiveArray::::new(DataType::Int32, values, None); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::new( - data_type, - vec![0, 2, 2, 6, 9, 10].try_into().unwrap(), - Box::new(values), - None, - ); - - let indices = PrimitiveArray::from([Some(4i32), Some(1), Some(3)]); - let result = take(&array, &indices).unwrap(); - - let expected_values = Buffer::from(vec![9, 6, 7, 8]); - let expected_values = PrimitiveArray::::new(DataType::Int32, expected_values, None); - let expected_type = ListArray::::default_datatype(DataType::Int32); - let expected = ListArray::::new( - expected_type, - vec![0, 1, 1, 4].try_into().unwrap(), - Box::new(expected_values), - None, - ); - - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn list_with_none() { - let values = Buffer::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - let values = PrimitiveArray::::new(DataType::Int32, values, None); - - let validity_values = vec![true, false, true, true, true]; - let validity = Bitmap::from_trusted_len_iter(validity_values.into_iter()); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::new( - data_type, - vec![0, 2, 2, 6, 9, 10].try_into().unwrap(), - Box::new(values), - Some(validity), - ); - - let indices = PrimitiveArray::from([Some(4i32), None, Some(2), Some(3)]); - let result = take(&array, &indices).unwrap(); - - let data_expected = vec![ - Some(vec![Some(9i32)]), - None, - Some(vec![Some(2i32), Some(3), Some(4), Some(5)]), - Some(vec![Some(6i32), Some(7), Some(8)]), - ]; - - let mut expected = MutableListArray::>::new(); - expected.try_extend(data_expected).unwrap(); - let expected: ListArray = expected.into(); - - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn list_both_validity() { - let values = vec![ - Some(vec![Some(2i32), Some(3), Some(4), Some(5)]), - None, - Some(vec![Some(9i32)]), - Some(vec![Some(6i32), Some(7), Some(8)]), - ]; - - let mut array = MutableListArray::>::new(); - array.try_extend(values).unwrap(); - let array: ListArray = array.into(); - - let indices = PrimitiveArray::from([Some(3i32), None, Some(1), Some(0)]); - let result = take(&array, &indices).unwrap(); - - let data_expected = vec![ - Some(vec![Some(6i32), Some(7), Some(8)]), - None, - None, - Some(vec![Some(2i32), Some(3), Some(4), Some(5)]), - ]; - let mut expected = MutableListArray::>::new(); - expected.try_extend(data_expected).unwrap(); - let expected: ListArray = expected.into(); - - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn fixed_size_list_with_no_none() { - let values = Buffer::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - let values = PrimitiveArray::::new(DataType::Int32, values, None); - - let data_type = FixedSizeListArray::default_datatype(DataType::Int32, 2); - let array = FixedSizeListArray::new(data_type, Box::new(values), None); - - let indices = PrimitiveArray::from([Some(4i32), Some(1), Some(3)]); - let result = take(&array, &indices).unwrap(); - - let expected_values = Buffer::from(vec![8, 9, 2, 3, 6, 7]); - let expected_values = PrimitiveArray::::new(DataType::Int32, expected_values, None); - let expected_type = FixedSizeListArray::default_datatype(DataType::Int32, 2); - let expected = FixedSizeListArray::new(expected_type, Box::new(expected_values), None); - - assert_eq!(expected, result.as_ref()); -} - -#[test] -fn test_nested() { - let values = Buffer::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); - let values = PrimitiveArray::::new(DataType::Int32, values, None); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::new( - data_type, - vec![0, 2, 4, 7, 7, 8, 10].try_into().unwrap(), - Box::new(values), - None, - ); - - let data_type = ListArray::::default_datatype(array.data_type().clone()); - let nested = ListArray::::new( - data_type, - vec![0, 2, 5, 6].try_into().unwrap(), - Box::new(array), - None, - ); - - let indices = PrimitiveArray::from([Some(0i32), Some(1)]); - let result = take(&nested, &indices).unwrap(); - - // expected data - let expected_values = Buffer::from(vec![1, 2, 3, 4, 5, 6, 7, 8]); - let expected_values = PrimitiveArray::::new(DataType::Int32, expected_values, None); - - let expected_data_type = ListArray::::default_datatype(DataType::Int32); - let expected_array = ListArray::::new( - expected_data_type, - vec![0, 2, 4, 7, 7, 8].try_into().unwrap(), - Box::new(expected_values), - None, - ); - - let expected_data_type = ListArray::::default_datatype(expected_array.data_type().clone()); - let expected = ListArray::::new( - expected_data_type, - vec![0, 2, 5].try_into().unwrap(), - Box::new(expected_array), - None, - ); - - assert_eq!(expected, result.as_ref()); -} diff --git a/src/common/arrow/tests/it/arrow/ffi/data.rs b/src/common/arrow/tests/it/arrow/ffi/data.rs deleted file mode 100644 index a2cec83b99b3..000000000000 --- a/src/common/arrow/tests/it/arrow/ffi/data.rs +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::BTreeMap; - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::datatypes::TimeUnit; -use databend_common_arrow::arrow::error::Result; -use databend_common_arrow::arrow::ffi; - -fn _test_round_trip(array: Box, expected: Box) -> Result<()> { - let field = Field::new("a", array.data_type().clone(), true); - - // export array and corresponding data_type - let array_ffi = ffi::export_array_to_c(array); - let schema_ffi = ffi::export_field_to_c(&field); - - // import references - let result_field = unsafe { ffi::import_field_from_c(&schema_ffi)? }; - let result_array = - unsafe { ffi::import_array_from_c(array_ffi, result_field.data_type.clone())? }; - - assert_eq!(&result_array, &expected); - assert_eq!(result_field, field); - Ok(()) -} - -fn test_round_trip(expected: impl Array + Clone + 'static) -> Result<()> { - let array: Box = Box::new(expected.clone()); - let expected = Box::new(expected) as Box; - _test_round_trip(array.clone(), clone(expected.as_ref()))?; - - // sliced - _test_round_trip(array.sliced(1, 2), expected.sliced(1, 2)) -} - -fn test_round_trip_schema(field: Field) -> Result<()> { - let schema_ffi = ffi::export_field_to_c(&field); - - let result = unsafe { ffi::import_field_from_c(&schema_ffi)? }; - - assert_eq!(result, field); - Ok(()) -} - -#[test] -fn bool_nullable() -> Result<()> { - let data = BooleanArray::from(&[Some(true), None, Some(false), None]); - test_round_trip(data) -} - -#[test] -fn bool() -> Result<()> { - let data = BooleanArray::from_slice([true, true, false]); - test_round_trip(data) -} - -#[test] -fn bool_nullable_sliced() -> Result<()> { - let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); - let data = BooleanArray::try_new(DataType::Boolean, [true, true, false].into(), Some(bitmap))?; - test_round_trip(data) -} - -#[test] -fn u32_nullable() -> Result<()> { - let data = Int32Array::from(&[Some(2), None, Some(1), None]); - test_round_trip(data) -} - -#[test] -fn u32() -> Result<()> { - let data = Int32Array::from_slice([2, 0, 1, 0]); - test_round_trip(data) -} - -#[test] -fn u32_sliced() -> Result<()> { - let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); - let data = Int32Array::try_new(DataType::Int32, vec![1, 2, 3].into(), Some(bitmap))?; - test_round_trip(data) -} - -#[test] -fn decimal() -> Result<()> { - let data = Int128Array::from_slice([1, 0, 2, 0]); - test_round_trip(data) -} - -#[test] -fn decimal_nullable() -> Result<()> { - let data = Int128Array::from(&[Some(1), None, Some(2), None]); - test_round_trip(data) -} - -#[test] -fn timestamp_tz() -> Result<()> { - let data = Int64Array::from(&vec![Some(2), None, None]).to(DataType::Timestamp( - TimeUnit::Second, - Some("UTC".to_string()), - )); - test_round_trip(data) -} - -#[test] -fn utf8_nullable() -> Result<()> { - let data = Utf8Array::::from([Some("a"), None, Some("bb"), None]); - test_round_trip(data) -} - -#[test] -fn utf8() -> Result<()> { - let data = Utf8Array::::from_slice(["a", "", "bb", ""]); - test_round_trip(data) -} - -#[test] -fn utf8_sliced() -> Result<()> { - let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); - let data = Utf8Array::::try_new( - DataType::Utf8, - vec![0, 1, 1, 2].try_into().unwrap(), - b"ab".to_vec().into(), - Some(bitmap), - )?; - test_round_trip(data) -} - -#[test] -fn large_utf8() -> Result<()> { - let data = Utf8Array::::from([Some("a"), None, Some("bb"), None]); - test_round_trip(data) -} - -#[test] -fn binary_nullable() -> Result<()> { - let data = BinaryArray::::from([Some(b"a".as_ref()), None, Some(b"bb".as_ref()), None]); - test_round_trip(data) -} - -#[test] -fn binary() -> Result<()> { - let data = BinaryArray::::from_slice([b"a".as_ref(), b"", b"bb", b""]); - test_round_trip(data) -} - -#[test] -fn binary_sliced() -> Result<()> { - let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); - let data = BinaryArray::::try_new( - DataType::Binary, - vec![0, 1, 1, 2].try_into().unwrap(), - b"ab".to_vec().into(), - Some(bitmap), - )?; - test_round_trip(data) -} - -#[test] -fn large_binary() -> Result<()> { - let data = BinaryArray::::from([Some(b"a".as_ref()), None, Some(b"bb".as_ref()), None]); - test_round_trip(data) -} - -#[test] -fn fixed_size_binary() -> Result<()> { - let data = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(2), - vec![1, 2, 3, 4, 5, 6].into(), - None, - ); - test_round_trip(data) -} - -#[test] -fn fixed_size_binary_nullable() -> Result<()> { - let data = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(2), - vec![1, 2, 3, 4, 5, 6].into(), - Some([true, true, false].into()), - ); - test_round_trip(data) -} - -#[test] -fn fixed_size_binary_sliced() -> Result<()> { - let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); - let data = FixedSizeBinaryArray::try_new( - DataType::FixedSizeBinary(2), - b"ababab".to_vec().into(), - Some(bitmap), - )?; - test_round_trip(data) -} - -#[test] -fn list() -> Result<()> { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(4), None, Some(6)]), - ]; - - let mut array = MutableListArray::>::new(); - array.try_extend(data)?; - - let array: ListArray = array.into(); - - test_round_trip(array) -} - -#[test] -fn list_sliced() -> Result<()> { - let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); - - let array = ListArray::::try_new( - DataType::List(Box::new(Field::new("a", DataType::Int32, true))), - vec![0, 1, 1, 2].try_into().unwrap(), - Box::new(PrimitiveArray::::from_slice([1, 2])), - Some(bitmap), - )?; - - test_round_trip(array) -} - -#[test] -fn large_list() -> Result<()> { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(4), None, Some(6)]), - ]; - - let mut array = MutableListArray::>::new(); - array.try_extend(data)?; - - let array: ListArray = array.into(); - - test_round_trip(array) -} - -#[test] -fn fixed_size_list() -> Result<()> { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(4), None, Some(6)]), - ]; - - let mut array = MutableFixedSizeListArray::new(MutablePrimitiveArray::::new(), 3); - array.try_extend(data)?; - - let array: FixedSizeListArray = array.into(); - - test_round_trip(array) -} - -#[test] -fn fixed_size_list_sliced() -> Result<()> { - let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); - - let array = FixedSizeListArray::try_new( - DataType::FixedSizeList(Box::new(Field::new("a", DataType::Int32, true)), 2), - Box::new(PrimitiveArray::::from_vec(vec![1, 2, 3, 4, 5, 6])), - Some(bitmap), - )?; - - test_round_trip(array) -} - -#[test] -fn list_list() -> Result<()> { - let data = vec![ - Some(vec![ - Some(vec![None]), - Some(vec![Some(2)]), - Some(vec![Some(3)]), - ]), - None, - Some(vec![Some(vec![Some(4), None, Some(6)])]), - ]; - - let mut array = - MutableListArray::>>::new(); - array.try_extend(data)?; - - let array: ListArray = array.into(); - - test_round_trip(array) -} - -#[test] -fn struct_() -> Result<()> { - let data_type = DataType::Struct(vec![Field::new("a", DataType::Int32, true)]); - let values = vec![Int32Array::from([Some(1), None, Some(3)]).boxed()]; - let validity = Bitmap::from([true, false, true]); - - let array = StructArray::new(data_type, values, validity.into()); - - test_round_trip(array) -} - -#[test] -fn dict() -> Result<()> { - let data = vec![Some("a"), Some("a"), None, Some("b")]; - - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data)?; - - let array: DictionaryArray = array.into(); - - test_round_trip(array) -} - -#[test] -fn schema() -> Result<()> { - let field = Field::new( - "a", - DataType::List(Box::new(Field::new("a", DataType::UInt32, true))), - true, - ); - test_round_trip_schema(field)?; - - let field = Field::new( - "a", - DataType::Dictionary(u32::KEY_TYPE, Box::new(DataType::Utf8), false), - true, - ); - test_round_trip_schema(field)?; - - let field = Field::new("a", DataType::Int32, true); - let mut metadata = BTreeMap::new(); - metadata.insert("some".to_string(), "stuff".to_string()); - let field = field.with_metadata(metadata); - test_round_trip_schema(field) -} - -#[test] -fn extension() -> Result<()> { - let field = Field::new( - "a", - DataType::Extension( - "a".to_string(), - Box::new(DataType::Int32), - Some("bla".to_string()), - ), - true, - ); - test_round_trip_schema(field) -} - -#[test] -fn extension_children() -> Result<()> { - let field = Field::new( - "a", - DataType::Extension( - "b".to_string(), - Box::new(DataType::Struct(vec![Field::new( - "c", - DataType::Int32, - true, - )])), - None, - ), - true, - ); - test_round_trip_schema(field) -} diff --git a/src/common/arrow/tests/it/arrow/ffi/mod.rs b/src/common/arrow/tests/it/arrow/ffi/mod.rs deleted file mode 100644 index 88cc3aa1eea8..000000000000 --- a/src/common/arrow/tests/it/arrow/ffi/mod.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod data; -mod stream; - -use databend_common_arrow::arrow::ffi::mmap; - -#[test] -fn mmap_slice() { - let slice = &[1, 2, 3]; - let array = unsafe { mmap::slice(slice) }; - assert_eq!(array.values().as_ref(), &[1, 2, 3]); - // note: when `slice` is dropped, array must be dropped as-well since by construction of `slice` they share their lifetimes. -} - -#[test] -fn mmap_bitmap() { - let slice = &[123u8, 255]; - let array = unsafe { mmap::bitmap(slice, 2, 14) }.unwrap(); - assert_eq!(array.values_iter().collect::>(), &[ - false, true, true, true, true, false, true, true, true, true, true, true, true, true - ]); - // note: when `slice` is dropped, array must be dropped as-well since by construction of `slice` they share their lifetimes. -} diff --git a/src/common/arrow/tests/it/arrow/ffi/stream.rs b/src/common/arrow/tests/it/arrow/ffi/stream.rs deleted file mode 100644 index 76c60ffe74f0..000000000000 --- a/src/common/arrow/tests/it/arrow/ffi/stream.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::error::Error; -use databend_common_arrow::arrow::error::Result; -use databend_common_arrow::arrow::ffi; - -fn _test_round_trip(arrays: Vec>) -> Result<()> { - let field = Field::new("a", arrays[0].data_type().clone(), true); - let iter = Box::new(arrays.clone().into_iter().map(Ok)) as _; - - let mut stream = Box::new(ffi::ArrowArrayStream::empty()); - - *stream = ffi::export_iterator(iter, field.clone()); - - // import - let mut stream = unsafe { ffi::ArrowArrayStreamReader::try_new(stream)? }; - - let mut produced_arrays: Vec> = vec![]; - while let Some(array) = unsafe { stream.next() } { - produced_arrays.push(array?); - } - - assert_eq!(produced_arrays, arrays); - assert_eq!(stream.field(), &field); - Ok(()) -} - -#[test] -fn round_trip() -> Result<()> { - let array = Int32Array::from(&[Some(2), None, Some(1), None]); - let array: Box = Box::new(array); - - _test_round_trip(vec![array.clone(), array.clone(), array]) -} - -#[test] -fn stream_reader_try_new_invalid_argument_error_on_released_stream() { - let released_stream = Box::new(ffi::ArrowArrayStream::empty()); - let reader = unsafe { ffi::ArrowArrayStreamReader::try_new(released_stream) }; - // poor man's assert_matches: - match reader { - Err(Error::InvalidArgumentError(_)) => {} - _ => panic!("ArrowArrayStreamReader::try_new did not return an InvalidArgumentError"), - } -} diff --git a/src/common/arrow/tests/it/arrow/mod.rs b/src/common/arrow/tests/it/arrow/mod.rs index 4858c417c9ff..a5d66bee6cbe 100644 --- a/src/common/arrow/tests/it/arrow/mod.rs +++ b/src/common/arrow/tests/it/arrow/mod.rs @@ -23,7 +23,7 @@ mod arrow_data; mod bitmap; mod buffer; mod compute; -mod ffi; + mod scalar; mod temporal_conversions; mod types; diff --git a/src/common/arrow/tests/it/native/io.rs b/src/common/arrow/tests/it/native/io.rs index 2dd0816d73ff..7dbb4f27a2aa 100644 --- a/src/common/arrow/tests/it/native/io.rs +++ b/src/common/arrow/tests/it/native/io.rs @@ -42,13 +42,10 @@ use databend_common_arrow::arrow::compute; use databend_common_arrow::arrow::datatypes::DataType; use databend_common_arrow::arrow::datatypes::Field; use databend_common_arrow::arrow::datatypes::Schema; -use databend_common_arrow::arrow::io::parquet::read::n_columns; -use databend_common_arrow::arrow::io::parquet::read::ColumnDescriptor; -use databend_common_arrow::arrow::io::parquet::write::to_parquet_schema; use databend_common_arrow::arrow::offset::OffsetsBuffer; +use databend_common_arrow::native::n_columns; use databend_common_arrow::native::read::batch_read::batch_read_array; use databend_common_arrow::native::read::deserialize::column_iter_to_arrays; -use databend_common_arrow::native::read::reader::is_primitive; use databend_common_arrow::native::read::reader::NativeReader; use databend_common_arrow::native::write::NativeWriter; use databend_common_arrow::native::write::WriteOptions; @@ -352,14 +349,16 @@ fn create_map(size: usize, null_density: f32) -> MapArray { fn create_struct(size: usize, null_density: f32, uniq: usize) -> StructArray { let dt = DataType::Struct(vec![ - Field::new("name", DataType::LargeBinary, true), Field::new("age", DataType::Int32, true), + Field::new("name", DataType::Utf8View, true), + Field::new("name2", DataType::LargeBinary, true), ]); StructArray::try_new( dt, vec![ - Box::new(create_random_string(size, null_density, uniq)) as _, Box::new(create_random_index(size, null_density, uniq)) as _, + Box::new(create_random_view(size, null_density, uniq)) as _, + Box::new(create_random_string(size, null_density, uniq)) as _, ], None, ) @@ -501,14 +500,11 @@ fn test_write_read_with_options(chunk: Chunk>, options: WriteOpti let mut batch_metas = writer.metas.clone(); let mut metas = writer.metas.clone(); - let schema_descriptor = to_parquet_schema(&schema).unwrap(); - let mut leaves = schema_descriptor.columns().to_vec(); let mut results = Vec::with_capacity(schema.fields.len()); for field in schema.fields.iter() { let n = n_columns(&field.data_type); let curr_metas: Vec = metas.drain(..n).collect(); - let curr_leaves: Vec = leaves.drain(..n).collect(); let mut native_readers = Vec::with_capacity(n); for curr_meta in curr_metas.iter() { @@ -518,10 +514,8 @@ fn test_write_read_with_options(chunk: Chunk>, options: WriteOpti let native_reader = NativeReader::new(range_bytes, curr_meta.pages.clone(), vec![]); native_readers.push(native_reader); } - let is_nested = !is_primitive(field.data_type()); - let mut array_iter = - column_iter_to_arrays(native_readers, curr_leaves, field.clone(), is_nested).unwrap(); + let mut array_iter = column_iter_to_arrays(native_readers, field.clone(), vec![]).unwrap(); let mut arrays = vec![]; for array in array_iter.by_ref() { @@ -536,14 +530,11 @@ fn test_write_read_with_options(chunk: Chunk>, options: WriteOpti assert_eq!(chunk, result_chunk); // test batch read - let schema_descriptor = to_parquet_schema(&schema).unwrap(); - let mut leaves = schema_descriptor.columns().to_vec(); let mut batch_results = Vec::with_capacity(schema.fields.len()); for field in schema.fields.iter() { let n = n_columns(&field.data_type); let curr_metas: Vec = batch_metas.drain(..n).collect(); - let curr_leaves: Vec = leaves.drain(..n).collect(); let mut pages: Vec> = Vec::with_capacity(n); let mut readers = Vec::with_capacity(n); @@ -557,9 +548,7 @@ fn test_write_read_with_options(chunk: Chunk>, options: WriteOpti readers.push(reader); } - let is_nested = !is_primitive(field.data_type()); - let batch_result = - batch_read_array(readers, curr_leaves, field.clone(), is_nested, pages).unwrap(); + let batch_result = batch_read_array(readers, field.clone(), pages).unwrap(); batch_results.push(batch_result); } let batch_result_chunk = Chunk::new(batch_results); diff --git a/src/common/exception/src/exception_into.rs b/src/common/exception/src/exception_into.rs index a8f5a9e686e1..57ec00e482b4 100644 --- a/src/common/exception/src/exception_into.rs +++ b/src/common/exception/src/exception_into.rs @@ -117,12 +117,6 @@ impl From for ErrorCode { } } -impl From for ErrorCode { - fn from(error: databend_common_arrow::parquet::error::Error) -> Self { - ErrorCode::from_std_error(error) - } -} - impl From for ErrorCode { fn from(error: arrow_schema::ArrowError) -> Self { match error { diff --git a/src/common/parquet2/.gitignore b/src/common/parquet2/.gitignore deleted file mode 100644 index fed113e1901f..000000000000 --- a/src/common/parquet2/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -target -Cargo.lock -.idea -venv -fixtures/ diff --git a/src/common/parquet2/Cargo.toml b/src/common/parquet2/Cargo.toml deleted file mode 100644 index df8f39a31e4a..000000000000 --- a/src/common/parquet2/Cargo.toml +++ /dev/null @@ -1,44 +0,0 @@ -[package] -description = "Safe implementation of parquet IO, forked from parquet2." -edition = "2021" -license = "Apache-2.0" -name = "databend-common-parquet2" -version = "0.1.0" - -[lib] -bench = false -name = "parquet2" - -[dependencies] -async-stream = { workspace = true, optional = true } -brotli = { workspace = true, optional = true } -bytes = { workspace = true } -flate2 = { workspace = true, optional = true } -futures = { workspace = true, optional = true } -lz4 = { workspace = true, optional = true } -opendal = { workspace = true } -parquet-format-safe = { workspace = true } -seq-macro = { workspace = true, default-features = false } -serde = { workspace = true, optional = true, features = ["derive"] } -snap = { workspace = true, optional = true } -streaming-decompression = { workspace = true } -xxhash-rust = { workspace = true, optional = true, features = ["xxh64"] } -zstd = { workspace = true, optional = true } - -[dev-dependencies] -criterion = { workspace = true } -rand = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt"] } - -[features] -async = ["async-stream", "futures", "parquet-format-safe/async"] -bloom_filter = ["xxhash-rust"] -default = ["snappy", "gzip", "lz4", "zstd", "brotli", "bloom_filter"] -full = ["snappy", "gzip", "lz4", "zstd", "brotli", "bloom_filter", "async"] -gzip = ["flate2/rust_backend"] -gzip_zlib_ng = ["flate2/zlib-ng"] -serde_types = ["serde"] -snappy = ["snap"] - -[lints] -workspace = true diff --git a/src/common/parquet2/LICENSE b/src/common/parquet2/LICENSE deleted file mode 100644 index d01ab551c8e7..000000000000 --- a/src/common/parquet2/LICENSE +++ /dev/null @@ -1,14 +0,0 @@ -Copyright [2021] [Jorge C Leitao] -Copyright 2021 Datafuse Labs - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/src/common/parquet2/src/bloom_filter/hash.rs b/src/common/parquet2/src/bloom_filter/hash.rs deleted file mode 100644 index 1cb5c24d16bc..000000000000 --- a/src/common/parquet2/src/bloom_filter/hash.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use xxhash_rust::xxh64::xxh64; - -use crate::types::NativeType; - -const SEED: u64 = 0; - -/// (xxh64) hash of a [`NativeType`]. -#[inline] -pub fn hash_native(value: T) -> u64 { - xxh64(value.to_le_bytes().as_ref(), SEED) -} - -/// (xxh64) hash of a sequence of bytes (e.g. ByteArray). -#[inline] -pub fn hash_byte>(value: A) -> u64 { - xxh64(value.as_ref(), SEED) -} diff --git a/src/common/parquet2/src/bloom_filter/mod.rs b/src/common/parquet2/src/bloom_filter/mod.rs deleted file mode 100644 index 05561d908fb2..000000000000 --- a/src/common/parquet2/src/bloom_filter/mod.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! API to read and use bloom filters -mod hash; -mod read; -mod split_block; - -pub use hash::hash_byte; -pub use hash::hash_native; -pub use read::read; -pub use split_block::insert; -pub use split_block::is_in_set; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basics() { - let mut bitset = vec![0; 32]; - - // insert - for a in 0..10i64 { - let hash = hash_native(a); - insert(&mut bitset, hash); - } - - // bloom filter produced by parquet-mr/spark for a column of i64 (0..=10) - // import pyspark.sql // 3.2.1 - // spark = pyspark.sql.SparkSession.builder.getOrCreate() - // spark.conf.set("parquet.bloom.filter.enabled", True) - // spark.conf.set("parquet.bloom.filter.expected.ndv", 10) - // spark.conf.set("parquet.bloom.filter.max.bytes", 32) - // - // data = [(i % 10,) for i in range(100)] - // df = spark.createDataFrame(data, ["id"]).repartition(1) - // - // df.write.parquet("bla.parquet", mode = "overwrite") - let expected: &[u8] = &[ - 24, 130, 24, 8, 134, 8, 68, 6, 2, 101, 128, 10, 64, 2, 38, 78, 114, 1, 64, 38, 1, 192, - 194, 152, 64, 70, 0, 36, 56, 121, 64, 0, - ]; - assert_eq!(bitset, expected); - - // check - for a in 0..11i64 { - let hash = hash_native(a); - - let valid = is_in_set(&bitset, hash); - - assert_eq!(a < 10, valid); - } - } - - #[test] - fn binary() { - let mut bitset = vec![0; 32]; - - // insert - for a in 0..10i64 { - let value = format!("a{}", a); - let hash = hash_byte(value); - insert(&mut bitset, hash); - } - - // bloom filter produced by parquet-mr/spark for a column of i64 f"a{i}" for i in 0..10 - let expected: &[u8] = &[ - 200, 1, 80, 20, 64, 68, 8, 109, 6, 37, 4, 67, 144, 80, 96, 32, 8, 132, 43, 33, 0, 5, - 99, 65, 2, 0, 224, 44, 64, 78, 96, 4, - ]; - assert_eq!(bitset, expected); - } -} diff --git a/src/common/parquet2/src/bloom_filter/read.rs b/src/common/parquet2/src/bloom_filter/read.rs deleted file mode 100644 index 23646642793b..000000000000 --- a/src/common/parquet2/src/bloom_filter/read.rs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; - -use parquet_format_safe::thrift::protocol::TCompactInputProtocol; -use parquet_format_safe::BloomFilterAlgorithm; -use parquet_format_safe::BloomFilterCompression; -use parquet_format_safe::BloomFilterHeader; -use parquet_format_safe::SplitBlockAlgorithm; -use parquet_format_safe::Uncompressed; - -use crate::error::Error; -use crate::metadata::ColumnChunkMetaData; - -/// Reads the bloom filter associated to [`ColumnChunkMetaData`] into `bitset`. -/// Results in an empty `bitset` if there is no associated bloom filter or the algorithm is not supported. -/// # Error -/// Errors if the column contains no metadata or the filter can't be read or deserialized. -pub fn read( - column_metadata: &ColumnChunkMetaData, - mut reader: &mut R, - bitset: &mut Vec, -) -> Result<(), Error> { - let offset = column_metadata.metadata().bloom_filter_offset; - - let offset = if let Some(offset) = offset { - offset as u64 - } else { - bitset.clear(); - return Ok(()); - }; - reader.seek(SeekFrom::Start(offset))?; - - // deserialize header - let mut prot = TCompactInputProtocol::new(&mut reader, usize::MAX); // max is ok since `BloomFilterHeader` never allocates - let header = BloomFilterHeader::read_from_in_protocol(&mut prot)?; - - if header.algorithm != BloomFilterAlgorithm::BLOCK(SplitBlockAlgorithm {}) { - bitset.clear(); - return Ok(()); - } - if header.compression != BloomFilterCompression::UNCOMPRESSED(Uncompressed {}) { - bitset.clear(); - return Ok(()); - } - - let length: usize = header.num_bytes.try_into()?; - - bitset.clear(); - bitset.try_reserve(length)?; - reader.by_ref().take(length as u64).read_to_end(bitset)?; - - Ok(()) -} diff --git a/src/common/parquet2/src/bloom_filter/split_block.rs b/src/common/parquet2/src/bloom_filter/split_block.rs deleted file mode 100644 index 472a299da58a..000000000000 --- a/src/common/parquet2/src/bloom_filter/split_block.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::convert::TryInto; - -/// magic numbers taken from https://github.com/apache/parquet-format/blob/master/BloomFilter.md -const SALT: [u32; 8] = [ - 1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529, -]; - -fn hash_to_block_index(hash: u64, len: usize) -> usize { - let number_of_blocks = len as u64 / 32; - let low_hash = hash >> 32; - let block_index = ((low_hash * number_of_blocks) >> 32) as u32; - block_index as usize -} - -fn new_mask(x: u32) -> [u32; 8] { - let mut a = [0u32; 8]; - for i in 0..8 { - let mask = x.wrapping_mul(SALT[i]); - let mask = mask >> 27; - let mask = 0x1 << mask; - a[i] = mask; - } - a -} - -/// loads a block from the bitset to the stack -#[inline] -fn load_block(bitset: &[u8]) -> [u32; 8] { - let mut a = [0u32; 8]; - let bitset = bitset.chunks_exact(4).take(8); - for (a, chunk) in a.iter_mut().zip(bitset) { - *a = u32::from_le_bytes(chunk.try_into().unwrap()) - } - a -} - -/// assigns a block from the stack to `bitset` -#[inline] -fn unload_block(block: [u32; 8], bitset: &mut [u8]) { - let bitset = bitset.chunks_exact_mut(4).take(8); - for (a, chunk) in block.iter().zip(bitset) { - let a = a.to_le_bytes(); - chunk[0] = a[0]; - chunk[1] = a[1]; - chunk[2] = a[2]; - chunk[3] = a[3]; - } -} - -/// Returns whether the `hash` is in the set -pub fn is_in_set(bitset: &[u8], hash: u64) -> bool { - let block_index = hash_to_block_index(hash, bitset.len()); - let key = hash as u32; - - let mask = new_mask(key); - let slice = &bitset[block_index * 32..(block_index + 1) * 32]; - let block_mask = load_block(slice); - - for i in 0..8 { - if mask[i] & block_mask[i] == 0 { - return false; - } - } - true -} - -/// Inserts a new hash to the set -pub fn insert(bitset: &mut [u8], hash: u64) { - let block_index = hash_to_block_index(hash, bitset.len()); - let key = hash as u32; - - let mask = new_mask(key); - let slice = &bitset[block_index * 32..(block_index + 1) * 32]; - let mut block_mask = load_block(slice); - - for i in 0..8 { - block_mask[i] |= mask[i]; - - let mut_slice = &mut bitset[block_index * 32..(block_index + 1) * 32]; - unload_block(block_mask, mut_slice) - } -} diff --git a/src/common/parquet2/src/compression.rs b/src/common/parquet2/src/compression.rs deleted file mode 100644 index a886cddf0fd9..000000000000 --- a/src/common/parquet2/src/compression.rs +++ /dev/null @@ -1,402 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Functionality to compress and decompress data according to the parquet specification -pub use super::parquet_bridge::BrotliLevel; -pub use super::parquet_bridge::Compression; -pub use super::parquet_bridge::CompressionOptions; -pub use super::parquet_bridge::GzipLevel; -pub use super::parquet_bridge::ZstdLevel; -use crate::error::Error; -use crate::error::Result; - -fn inner_compress Result, F: Fn(&[u8], &mut [u8]) -> Result>( - input: &[u8], - output: &mut Vec, - get_length: G, - compress: F, -) -> Result<()> { - let original_length = output.len(); - let max_required_length = get_length(input.len())?; - - output.resize(original_length + max_required_length, 0); - let compressed_size = compress(input, &mut output[original_length..])?; - - output.truncate(original_length + compressed_size); - Ok(()) -} - -/// Compresses data stored in slice `input_buf` and writes the compressed result -/// to `output_buf`. -/// Note that you'll need to call `clear()` before reusing the same `output_buf` -/// across different `compress` calls. -pub fn compress( - compression: CompressionOptions, - input_buf: &[u8], - output_buf: &mut Vec, -) -> Result<()> { - match compression { - #[cfg(feature = "brotli")] - CompressionOptions::Brotli(level) => { - use std::io::Write; - const BROTLI_DEFAULT_BUFFER_SIZE: usize = 4096; - const BROTLI_DEFAULT_LG_WINDOW_SIZE: u32 = 22; // recommended between 20-22 - - let q = level.unwrap_or_default(); - let mut encoder = brotli::CompressorWriter::new( - output_buf, - BROTLI_DEFAULT_BUFFER_SIZE, - q.compression_level(), - BROTLI_DEFAULT_LG_WINDOW_SIZE, - ); - encoder.write_all(input_buf)?; - encoder.flush().map_err(|e| e.into()) - } - #[cfg(not(feature = "brotli"))] - CompressionOptions::Brotli(_) => Err(Error::FeatureNotActive( - crate::error::Feature::Brotli, - "compress to brotli".to_string(), - )), - #[cfg(feature = "gzip")] - CompressionOptions::Gzip(level) => { - use std::io::Write; - let level = level.unwrap_or_default(); - let mut encoder = flate2::write::GzEncoder::new(output_buf, level.into()); - encoder.write_all(input_buf)?; - encoder.try_finish().map_err(|e| e.into()) - } - #[cfg(not(feature = "gzip"))] - CompressionOptions::Gzip(_) => Err(Error::FeatureNotActive( - crate::error::Feature::Gzip, - "compress to gzip".to_string(), - )), - #[cfg(feature = "snappy")] - CompressionOptions::Snappy => inner_compress( - input_buf, - output_buf, - |len| Ok(snap::raw::max_compress_len(len)), - |input, output| Ok(snap::raw::Encoder::new().compress(input, output)?), - ), - #[cfg(not(feature = "snappy"))] - CompressionOptions::Snappy => Err(Error::FeatureNotActive( - crate::error::Feature::Snappy, - "compress to snappy".to_string(), - )), - #[cfg(feature = "lz4")] - CompressionOptions::Lz4Raw => inner_compress( - input_buf, - output_buf, - |len| Ok(lz4::block::compress_bound(len)?), - |input, output| { - let compressed_size = lz4::block::compress_to_buffer(input, None, false, output)?; - Ok(compressed_size) - }, - ), - #[cfg(not(feature = "lz4"))] - CompressionOptions::Lz4Raw => Err(Error::FeatureNotActive( - crate::error::Feature::Lz4, - "compress to lz4".to_string(), - )), - #[cfg(feature = "zstd")] - CompressionOptions::Zstd(level) => { - use std::io::Write; - let level = level.map(|v| v.compression_level()).unwrap_or_default(); - - let mut encoder = zstd::Encoder::new(output_buf, level)?; - encoder.write_all(input_buf)?; - match encoder.finish() { - Ok(_) => Ok(()), - Err(e) => Err(e.into()), - } - } - #[cfg(not(feature = "zstd"))] - CompressionOptions::Zstd(_) => Err(Error::FeatureNotActive( - crate::error::Feature::Zstd, - "compress to zstd".to_string(), - )), - CompressionOptions::Uncompressed => Err(Error::InvalidParameter( - "Compressing uncompressed".to_string(), - )), - _ => Err(Error::FeatureNotSupported(format!( - "Compression {:?} is not supported", - compression, - ))), - } -} - -/// Decompresses data stored in slice `input_buf` and writes output to `output_buf`. -/// Returns the total number of bytes written. -pub fn decompress(compression: Compression, input_buf: &[u8], output_buf: &mut [u8]) -> Result<()> { - match compression { - #[cfg(feature = "brotli")] - Compression::Brotli => { - use std::io::Read; - const BROTLI_DEFAULT_BUFFER_SIZE: usize = 4096; - brotli::Decompressor::new(input_buf, BROTLI_DEFAULT_BUFFER_SIZE) - .read_exact(output_buf) - .map_err(|e| e.into()) - } - #[cfg(not(feature = "brotli"))] - Compression::Brotli => Err(Error::FeatureNotActive( - crate::error::Feature::Brotli, - "decompress with brotli".to_string(), - )), - #[cfg(feature = "gzip")] - Compression::Gzip => { - use std::io::Read; - let mut decoder = flate2::read::GzDecoder::new(input_buf); - decoder.read_exact(output_buf).map_err(|e| e.into()) - } - #[cfg(not(feature = "gzip"))] - Compression::Gzip => Err(Error::FeatureNotActive( - crate::error::Feature::Gzip, - "decompress with gzip".to_string(), - )), - #[cfg(feature = "snappy")] - Compression::Snappy => { - use snap::raw::decompress_len; - use snap::raw::Decoder; - - let len = decompress_len(input_buf)?; - if len > output_buf.len() { - return Err(Error::OutOfSpec(String::from("snappy header out of spec"))); - } - Decoder::new() - .decompress(input_buf, output_buf) - .map_err(|e| e.into()) - .map(|_| ()) - } - #[cfg(not(feature = "snappy"))] - Compression::Snappy => Err(Error::FeatureNotActive( - crate::error::Feature::Snappy, - "decompress with snappy".to_string(), - )), - #[cfg(feature = "lz4")] - Compression::Lz4Raw => { - lz4::block::decompress_to_buffer(input_buf, Some(output_buf.len() as i32), output_buf) - .map(|_| {}) - .map_err(|e| e.into()) - } - #[cfg(not(feature = "lz4"))] - Compression::Lz4Raw => Err(Error::FeatureNotActive( - crate::error::Feature::Lz4, - "decompress with lz4".to_string(), - )), - - #[cfg(feature = "lz4")] - Compression::Lz4 => try_decompress_hadoop(input_buf, output_buf).or_else(|_| { - lz4_decompress_to_buffer(input_buf, Some(output_buf.len() as i32), output_buf) - .map(|_| {}) - }), - - #[cfg(not(feature = "lz4"))] - Compression::Lz4 => Err(Error::FeatureNotActive( - crate::error::Feature::Lz4, - "decompress with legacy lz4".to_string(), - )), - - #[cfg(feature = "zstd")] - Compression::Zstd => { - use std::io::Read; - let mut decoder = zstd::Decoder::new(input_buf)?; - decoder.read_exact(output_buf).map_err(|e| e.into()) - } - #[cfg(not(feature = "zstd"))] - Compression::Zstd => Err(Error::FeatureNotActive( - crate::error::Feature::Zstd, - "decompress with zstd".to_string(), - )), - Compression::Uncompressed => Err(Error::InvalidParameter( - "Compressing uncompressed".to_string(), - )), - _ => Err(Error::FeatureNotSupported(format!( - "Compression {:?} is not supported", - compression, - ))), - } -} - -/// Try to decompress the buffer as if it was compressed with the Hadoop Lz4Codec. -/// Translated from the apache arrow c++ function [TryDecompressHadoop](https://github.com/apache/arrow/blob/bf18e6e4b5bb6180706b1ba0d597a65a4ce5ca48/cpp/src/arrow/util/compression_lz4.cc#L474). -/// Returns error if decompression failed. -#[cfg(feature = "lz4")] -fn try_decompress_hadoop(input_buf: &[u8], output_buf: &mut [u8]) -> Result<()> { - // Parquet files written with the Hadoop Lz4Codec use their own framing. - // The input buffer can contain an arbitrary number of "frames", each - // with the following structure: - // - bytes 0..3: big-endian uint32_t representing the frame decompressed size - // - bytes 4..7: big-endian uint32_t representing the frame compressed size - // - bytes 8...: frame compressed data - // - // The Hadoop Lz4Codec source code can be found here: - // https://github.com/apache/hadoop/blob/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc - - const SIZE_U32: usize = std::mem::size_of::(); - const PREFIX_LEN: usize = SIZE_U32 * 2; - let mut input_len = input_buf.len(); - let mut input = input_buf; - let mut output_len = output_buf.len(); - let mut output: &mut [u8] = output_buf; - while input_len >= PREFIX_LEN { - let mut bytes = [0; SIZE_U32]; - bytes.copy_from_slice(&input[0..4]); - let expected_decompressed_size = u32::from_be_bytes(bytes); - let mut bytes = [0; SIZE_U32]; - bytes.copy_from_slice(&input[4..8]); - let expected_compressed_size = u32::from_be_bytes(bytes); - input = &input[PREFIX_LEN..]; - input_len -= PREFIX_LEN; - - if input_len < expected_compressed_size as usize { - return Err(Error::oos("Not enough bytes for Hadoop frame")); - } - - if output_len < expected_decompressed_size as usize { - return Err(Error::oos("Not enough bytes to hold advertised output")); - } - let decompressed_size = lz4_decompress_to_buffer( - &input[..expected_compressed_size as usize], - Some(output_len as i32), - output, - )?; - if decompressed_size != expected_decompressed_size as usize { - return Err(Error::oos("unexpected decompressed size")); - } - input_len -= expected_compressed_size as usize; - output_len -= expected_decompressed_size as usize; - if input_len > expected_compressed_size as usize { - input = &input[expected_compressed_size as usize..]; - output = &mut output[expected_decompressed_size as usize..]; - } else { - break; - } - } - if input_len == 0 { - Ok(()) - } else { - Err(Error::oos("Not all input are consumed")) - } -} - -#[cfg(feature = "lz4")] -#[inline] -fn lz4_decompress_to_buffer( - src: &[u8], - uncompressed_size: Option, - buffer: &mut [u8], -) -> Result { - let size = lz4::block::decompress_to_buffer(src, uncompressed_size, buffer)?; - Ok(size) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn test_roundtrip(c: CompressionOptions, data: &[u8]) { - let offset = 2048; - - // Compress to a buffer that already has data is possible - let mut compressed = vec![2; offset]; - compress(c, data, &mut compressed).expect("Error when compressing"); - - // data is compressed... - assert!(compressed.len() - offset < data.len()); - - let mut decompressed = vec![0; data.len()]; - decompress(c.into(), &compressed[offset..], &mut decompressed) - .expect("Error when decompressing"); - assert_eq!(data, decompressed.as_slice()); - } - - fn test_codec(c: CompressionOptions) { - let sizes = vec![1000, 10000, 100000]; - for size in sizes { - let data = (0..size).map(|x| (x % 255) as u8).collect::>(); - test_roundtrip(c, &data); - } - } - - #[test] - fn test_codec_snappy() { - test_codec(CompressionOptions::Snappy); - } - - #[test] - fn test_codec_gzip_default() { - test_codec(CompressionOptions::Gzip(None)); - } - - #[test] - fn test_codec_gzip_low_compression() { - test_codec(CompressionOptions::Gzip(Some( - GzipLevel::try_new(1).unwrap(), - ))); - } - - /// FIXME: this test could fail under all-features enabled. - #[test] - #[ignore] - fn test_codec_gzip_high_compression() { - test_codec(CompressionOptions::Gzip(Some( - GzipLevel::try_new(10).unwrap(), - ))); - } - - #[test] - fn test_codec_brotli_default() { - test_codec(CompressionOptions::Brotli(None)); - } - - #[test] - fn test_codec_brotli_low_compression() { - test_codec(CompressionOptions::Brotli(Some( - BrotliLevel::try_new(1).unwrap(), - ))); - } - - #[test] - fn test_codec_brotli_high_compression() { - test_codec(CompressionOptions::Brotli(Some( - BrotliLevel::try_new(11).unwrap(), - ))); - } - - #[test] - fn test_codec_lz4_raw() { - test_codec(CompressionOptions::Lz4Raw); - } - - #[test] - fn test_codec_zstd_default() { - test_codec(CompressionOptions::Zstd(None)); - } - - #[cfg(feature = "zstd")] - #[test] - fn test_codec_zstd_low_compression() { - test_codec(CompressionOptions::Zstd(Some( - ZstdLevel::try_new(1).unwrap(), - ))); - } - - #[cfg(feature = "zstd")] - #[test] - fn test_codec_zstd_high_compression() { - test_codec(CompressionOptions::Zstd(Some( - ZstdLevel::try_new(21).unwrap(), - ))); - } -} diff --git a/src/common/parquet2/src/deserialize/binary.rs b/src/common/parquet2/src/deserialize/binary.rs deleted file mode 100644 index ee49a3186852..000000000000 --- a/src/common/parquet2/src/deserialize/binary.rs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::utils; -use crate::encoding::hybrid_rle; -use crate::encoding::plain_byte_array::BinaryIter; -use crate::error::Error; -use crate::page::split_buffer; -use crate::page::DataPage; -use crate::parquet_bridge::Encoding; -use crate::parquet_bridge::Repetition; - -#[derive(Debug)] -pub struct Dictionary<'a, P> { - pub indexes: hybrid_rle::HybridRleDecoder<'a>, - pub dict: P, -} - -impl<'a, P> Dictionary<'a, P> { - pub fn try_new(page: &'a DataPage, dict: P) -> Result { - let indexes = utils::dict_indices_decoder(page)?; - - Ok(Self { indexes, dict }) - } - - #[inline] - pub fn len(&self) -> usize { - self.indexes.size_hint().0 - } -} - -#[allow(clippy::large_enum_variant)] -pub enum BinaryPageState<'a, P> { - Optional(utils::DefLevelsDecoder<'a>, BinaryIter<'a>), - Required(BinaryIter<'a>), - RequiredDictionary(Dictionary<'a, P>), - OptionalDictionary(utils::DefLevelsDecoder<'a>, Dictionary<'a, P>), -} - -impl<'a, P> BinaryPageState<'a, P> { - pub fn try_new(page: &'a DataPage, dict: Option

) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - - match (page.encoding(), dict, is_optional) { - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false) => { - Dictionary::try_new(page, dict).map(Self::RequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true) => { - Ok(Self::OptionalDictionary( - utils::DefLevelsDecoder::try_new(page)?, - Dictionary::try_new(page, dict)?, - )) - } - (Encoding::Plain, _, true) => { - let (_, _, values) = split_buffer(page)?; - - let validity = utils::DefLevelsDecoder::try_new(page)?; - let values = BinaryIter::new(values, None); - - Ok(Self::Optional(validity, values)) - } - (Encoding::Plain, _, false) => { - let (_, _, values) = split_buffer(page)?; - let values = BinaryIter::new(values, Some(page.num_values())); - - Ok(Self::Required(values)) - } - _ => Err(Error::FeatureNotSupported(format!( - "Viewing page for encoding {:?} for binary type", - page.encoding(), - ))), - } - } -} diff --git a/src/common/parquet2/src/deserialize/boolean.rs b/src/common/parquet2/src/deserialize/boolean.rs deleted file mode 100644 index 017f1654ef25..000000000000 --- a/src/common/parquet2/src/deserialize/boolean.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::utils; -use crate::encoding::hybrid_rle::BitmapIter; -use crate::error::Error; -use crate::page::split_buffer; -use crate::page::DataPage; -use crate::parquet_bridge::Encoding; -use crate::parquet_bridge::Repetition; - -// The state of a `DataPage` of `Boolean` parquet boolean type -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum BooleanPageState<'a> { - Optional(utils::DefLevelsDecoder<'a>, BitmapIter<'a>), - Required(&'a [u8], usize), -} - -impl<'a> BooleanPageState<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - - match (page.encoding(), is_optional) { - (Encoding::Plain, true) => { - let validity = utils::DefLevelsDecoder::try_new(page)?; - - let (_, _, values) = split_buffer(page)?; - let values = BitmapIter::new(values, 0, values.len() * 8); - - Ok(Self::Optional(validity, values)) - } - (Encoding::Plain, false) => { - let (_, _, values) = split_buffer(page)?; - Ok(Self::Required(values, page.num_values())) - } - _ => Err(Error::InvalidParameter(format!( - "Viewing page for encoding {:?} for boolean type not supported", - page.encoding(), - ))), - } - } -} diff --git a/src/common/parquet2/src/deserialize/filtered_rle.rs b/src/common/parquet2/src/deserialize/filtered_rle.rs deleted file mode 100644 index 4f29f15f32f2..000000000000 --- a/src/common/parquet2/src/deserialize/filtered_rle.rs +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use super::HybridDecoderBitmapIter; -use super::HybridEncoded; -use crate::encoding::hybrid_rle::BitmapIter; -use crate::error::Error; -use crate::indexes::Interval; - -/// Type definition of a [`FilteredHybridBitmapIter`] of [`HybridDecoderBitmapIter`]. -pub type FilteredHybridRleDecoderIter<'a> = - FilteredHybridBitmapIter<'a, HybridDecoderBitmapIter<'a>>; - -/// The decoding state of the hybrid-RLE decoder with a maximum definition level of 1 -/// that can supports skipped runs -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FilteredHybridEncoded<'a> { - /// a bitmap (values, offset, length, skipped_set) - Bitmap { - values: &'a [u8], - offset: usize, - length: usize, - }, - Repeated { - is_set: bool, - length: usize, - }, - /// When the run was skipped - contains the number of set values on the skipped run - Skipped(usize), -} - -fn is_set_count(values: &[u8], offset: usize, length: usize) -> usize { - BitmapIter::new(values, offset, length) - .filter(|x| *x) - .count() -} - -impl<'a> FilteredHybridEncoded<'a> { - /// Returns the length of the run in number of items - #[inline] - pub fn len(&self) -> usize { - match self { - FilteredHybridEncoded::Bitmap { length, .. } => *length, - FilteredHybridEncoded::Repeated { length, .. } => *length, - FilteredHybridEncoded::Skipped(_) => 0, - } - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -/// An [`Iterator`] adapter over [`HybridEncoded`] that yields [`FilteredHybridEncoded`]. -/// -/// This iterator adapter is used in combination with -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FilteredHybridBitmapIter<'a, I: Iterator, Error>>> { - iter: I, - current: Option<(HybridEncoded<'a>, usize)>, - // a run may end in the middle of an interval, in which case we must - // split the interval in parts. This tracks the current interval being computed - current_interval: Option, - selected_rows: VecDeque, - current_items_in_runs: usize, - - total_items: usize, -} - -impl<'a, I: Iterator, Error>>> FilteredHybridBitmapIter<'a, I> { - pub fn new(iter: I, selected_rows: VecDeque) -> Self { - let total_items = selected_rows.iter().map(|x| x.length).sum(); - Self { - iter, - current: None, - current_interval: None, - selected_rows, - current_items_in_runs: 0, - total_items, - } - } - - fn advance_current_interval(&mut self, length: usize) { - if let Some(interval) = &mut self.current_interval { - interval.start += length; - interval.length -= length; - self.total_items -= length; - } - } - - /// Returns the number of elements remaining. Note that each run - /// of the iterator contains more than one element - this is is _not_ equivalent to size_hint. - pub fn len(&self) -> usize { - self.total_items - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -impl<'a, I: Iterator, Error>>> Iterator - for FilteredHybridBitmapIter<'a, I> -{ - type Item = Result, Error>; - - fn next(&mut self) -> Option { - let interval = if let Some(interval) = self.current_interval { - interval - } else { - self.current_interval = self.selected_rows.pop_front(); - self.current_interval?; // case where iteration finishes - return self.next(); - }; - - let (run, offset) = if let Some((run, offset)) = self.current { - (run, offset) - } else { - // a new run - let run = self.iter.next()?; // no run => something wrong since intervals should only slice items up all runs' length - match run { - Ok(run) => { - self.current = Some((run, 0)); - } - Err(e) => return Some(Err(e)), - } - return self.next(); - }; - - // one of three things can happen: - // * the start of the interval is not aligned wirh the start of the run => issue a `Skipped` and advance the run / next run - // * the run contains this interval => consume the interval and keep the run - // * the run contains part of this interval => consume the run and keep the interval - - match run { - HybridEncoded::Repeated(is_set, full_run_length) => { - let run_length = full_run_length - offset; - // interval.start is from the start of the first run; discount `current_items_in_runs` - // to get the start from the current run's offset - let interval_start = interval.start - self.current_items_in_runs; - - if interval_start > 0 { - // we need to skip values from the run - let to_skip = interval_start; - - // we only skip up to a run (yield a single skip per multiple runs) - let max_skip = full_run_length - offset; - let to_skip = to_skip.min(max_skip); - - let set = if is_set { to_skip } else { 0 }; - - self.current_items_in_runs += to_skip; - - self.current = if to_skip == max_skip { - None - } else { - Some((run, offset + to_skip)) - }; - - return Some(Ok(FilteredHybridEncoded::Skipped(set))); - }; - - // slice the bitmap according to current interval - // note that interval start is from the start of the first run. - let new_offset = offset + interval_start; - - if interval_start > run_length { - let set = if is_set { run_length } else { 0 }; - - self.advance_current_interval(run_length); - self.current_items_in_runs += run_length; - self.current = None; - Some(Ok(FilteredHybridEncoded::Skipped(set))) - } else { - let length = if run_length > interval.length { - // interval is fully consumed - self.current_items_in_runs += interval.length; - - // fetch next interval - self.total_items -= interval.length; - self.current_interval = self.selected_rows.pop_front(); - - self.current = Some((run, offset + interval.length)); - - interval.length - } else { - // the run is consumed and the interval is shortened accordingly - self.current_items_in_runs += run_length; - - // the interval may cover two runs; shorten the length - // to its maximum allowed for this run - let length = run_length.min(full_run_length - new_offset); - - self.advance_current_interval(length); - - self.current = None; - length - }; - Some(Ok(FilteredHybridEncoded::Repeated { is_set, length })) - } - } - HybridEncoded::Bitmap(values, full_run_length) => { - let run_length = full_run_length - offset; - // interval.start is from the start of the first run; discount `current_items_in_runs` - // to get the start from the current run's offset - let interval_start = interval.start - self.current_items_in_runs; - - if interval_start > 0 { - // we need to skip values from the run - let to_skip = interval_start; - - // we only skip up to a run (yield a single skip per multiple runs) - let max_skip = full_run_length - offset; - let to_skip = to_skip.min(max_skip); - - let set = is_set_count(values, offset, to_skip); - - self.current_items_in_runs += to_skip; - - self.current = if to_skip == max_skip { - None - } else { - Some((run, offset + to_skip)) - }; - - return Some(Ok(FilteredHybridEncoded::Skipped(set))); - }; - - // slice the bitmap according to current interval - // note that interval start is from the start of the first run. - let new_offset = offset + interval_start; - - if interval_start > run_length { - let set = is_set_count(values, offset, full_run_length); - - self.advance_current_interval(run_length); - self.current_items_in_runs += run_length; - self.current = None; - Some(Ok(FilteredHybridEncoded::Skipped(set))) - } else { - let length = if run_length > interval.length { - // interval is fully consumed - self.current_items_in_runs += interval.length; - - // fetch next interval - self.total_items -= interval.length; - self.current_interval = self.selected_rows.pop_front(); - - self.current = Some((run, offset + interval.length)); - - interval.length - } else { - // the run is consumed and the interval is shortened accordingly - self.current_items_in_runs += run_length; - - // the interval may cover two runs; shorten the length - // to its maximum allowed for this run - let length = run_length.min(full_run_length - new_offset); - - self.advance_current_interval(length); - - self.current = None; - length - }; - Some(Ok(FilteredHybridEncoded::Bitmap { - values, - offset: new_offset, - length, - })) - } - } - } - } -} diff --git a/src/common/parquet2/src/deserialize/fixed_len.rs b/src/common/parquet2/src/deserialize/fixed_len.rs deleted file mode 100644 index 436ff132a1a3..000000000000 --- a/src/common/parquet2/src/deserialize/fixed_len.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::utils; -use crate::encoding::hybrid_rle; -use crate::error::Error; -use crate::page::split_buffer; -use crate::page::DataPage; -use crate::parquet_bridge::Encoding; -use crate::parquet_bridge::Repetition; -use crate::schema::types::PhysicalType; - -#[derive(Debug)] -pub struct FixexBinaryIter<'a> { - values: std::slice::ChunksExact<'a, u8>, -} - -impl<'a> FixexBinaryIter<'a> { - pub fn new(values: &'a [u8], size: usize) -> Self { - let values = values.chunks_exact(size); - Self { values } - } -} - -impl<'a> Iterator for FixexBinaryIter<'a> { - type Item = &'a [u8]; - - #[inline] - fn next(&mut self) -> Option { - self.values.next() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.values.size_hint() - } -} - -#[derive(Debug)] -pub struct Dictionary<'a, P> { - pub indexes: hybrid_rle::HybridRleDecoder<'a>, - pub dict: P, -} - -impl<'a, P> Dictionary<'a, P> { - pub fn try_new(page: &'a DataPage, dict: P) -> Result { - let indexes = utils::dict_indices_decoder(page)?; - - Ok(Self { indexes, dict }) - } - - #[inline] - pub fn len(&self) -> usize { - self.indexes.size_hint().0 - } -} - -#[allow(clippy::large_enum_variant)] -pub enum FixedLenBinaryPageState<'a, P> { - Optional(utils::DefLevelsDecoder<'a>, FixexBinaryIter<'a>), - Required(FixexBinaryIter<'a>), - RequiredDictionary(Dictionary<'a, P>), - OptionalDictionary(utils::DefLevelsDecoder<'a>, Dictionary<'a, P>), -} - -impl<'a, P> FixedLenBinaryPageState<'a, P> { - pub fn try_new(page: &'a DataPage, dict: Option

) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - - let size: usize = if let PhysicalType::FixedLenByteArray(size) = - page.descriptor.primitive_type.physical_type - { - size - } else { - return Err(Error::InvalidParameter( - "FixedLenBinaryPageState must be initialized by pages of FixedLenByteArray" - .to_string(), - )); - }; - - match (page.encoding(), dict, is_optional) { - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false) => { - Dictionary::try_new(page, dict).map(Self::RequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true) => { - Ok(Self::OptionalDictionary( - utils::DefLevelsDecoder::try_new(page)?, - Dictionary::try_new(page, dict)?, - )) - } - (Encoding::Plain, _, true) => { - let (_, _, values) = split_buffer(page)?; - - let validity = utils::DefLevelsDecoder::try_new(page)?; - let values = FixexBinaryIter::new(values, size); - - Ok(Self::Optional(validity, values)) - } - (Encoding::Plain, _, false) => { - let (_, _, values) = split_buffer(page)?; - let values = FixexBinaryIter::new(values, size); - - Ok(Self::Required(values)) - } - _ => Err(Error::FeatureNotSupported(format!( - "Viewing page for encoding {:?} for binary type", - page.encoding(), - ))), - } - } -} diff --git a/src/common/parquet2/src/deserialize/hybrid_rle.rs b/src/common/parquet2/src/deserialize/hybrid_rle.rs deleted file mode 100644 index c1ac803dd045..000000000000 --- a/src/common/parquet2/src/deserialize/hybrid_rle.rs +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::encoding::hybrid_rle::BitmapIter; -use crate::encoding::hybrid_rle::{self}; -use crate::error::Error; - -/// The decoding state of the hybrid-RLE decoder with a maximum definition level of 1 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HybridEncoded<'a> { - /// a bitmap - Bitmap(&'a [u8], usize), - /// A repeated item. The first attribute corresponds to whether the value is set - /// the second attribute corresponds to the number of repetitions. - Repeated(bool, usize), -} - -impl<'a> HybridEncoded<'a> { - /// Returns the length of the run in number of items - #[inline] - pub fn len(&self) -> usize { - match self { - HybridEncoded::Bitmap(_, length) => *length, - HybridEncoded::Repeated(_, length) => *length, - } - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -pub trait HybridRleRunsIterator<'a>: Iterator, Error>> { - /// Number of elements remaining. This may not be the items of the iterator - an item - /// of the iterator may contain more than one element. - fn number_of_elements(&self) -> usize; -} - -/// An iterator of [`HybridEncoded`], adapter over [`hybrid_rle::HybridEncoded`]. -#[derive(Debug, Clone)] -pub struct HybridRleIter<'a, I> -where I: Iterator, Error>> -{ - iter: I, - length: usize, - consumed: usize, -} - -impl<'a, I> HybridRleIter<'a, I> -where I: Iterator, Error>> -{ - /// Returns a new [`HybridRleIter`] - #[inline] - pub fn new(iter: I, length: usize) -> Self { - Self { - iter, - length, - consumed: 0, - } - } - - /// the number of elements in the iterator. Note that this _is not_ the number of runs. - #[inline] - pub fn len(&self) -> usize { - self.length - self.consumed - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -impl<'a, I> HybridRleRunsIterator<'a> for HybridRleIter<'a, I> -where I: Iterator, Error>> -{ - fn number_of_elements(&self) -> usize { - self.len() - } -} - -impl<'a, I> Iterator for HybridRleIter<'a, I> -where I: Iterator, Error>> -{ - type Item = Result, Error>; - - #[inline] - fn next(&mut self) -> Option { - if self.consumed == self.length { - return None; - }; - let run = self.iter.next()?; - - Some(run.map(|run| match run { - hybrid_rle::HybridEncoded::Bitpacked(pack) => { - // a pack has at most `pack.len() * 8` bits - let pack_size = pack.len() * 8; - - let additional = pack_size.min(self.len()); - - self.consumed += additional; - HybridEncoded::Bitmap(pack, additional) - } - hybrid_rle::HybridEncoded::Rle(value, length) => { - let is_set = value[0] == 1; - - let additional = length.min(self.len()); - - self.consumed += additional; - HybridEncoded::Repeated(is_set, additional) - } - })) - } - - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -/// Type definition for a [`HybridRleIter`] using [`hybrid_rle::Decoder`]. -pub type HybridDecoderBitmapIter<'a> = HybridRleIter<'a, hybrid_rle::Decoder<'a>>; - -#[derive(Debug)] -enum HybridBooleanState<'a> { - /// a bitmap - Bitmap(BitmapIter<'a>), - /// A repeated item. The first attribute corresponds to whether the value is set - /// the second attribute corresponds to the number of repetitions. - Repeated(bool, usize), -} - -/// An iterator adapter that maps an iterator of [`HybridEncoded`] into an iterator -/// over [`bool`]. -#[derive(Debug)] -pub struct HybridRleBooleanIter<'a, I> -where I: Iterator, Error>> -{ - iter: I, - current_run: Option>, -} - -impl<'a, I> HybridRleBooleanIter<'a, I> -where I: Iterator, Error>> -{ - pub fn new(iter: I) -> Self { - Self { - iter, - current_run: None, - } - } -} - -impl<'a, I> Iterator for HybridRleBooleanIter<'a, I> -where I: HybridRleRunsIterator<'a> -{ - type Item = Result; - - #[inline] - fn next(&mut self) -> Option { - if let Some(run) = &mut self.current_run { - match run { - HybridBooleanState::Bitmap(bitmap) => bitmap.next().map(Ok), - HybridBooleanState::Repeated(value, remaining) => if *remaining == 0 { - None - } else { - *remaining -= 1; - Some(*value) - } - .map(Ok), - } - } else if let Some(run) = self.iter.next() { - let run = run.map(|run| match run { - HybridEncoded::Bitmap(bitmap, length) => { - HybridBooleanState::Bitmap(BitmapIter::new(bitmap, 0, length)) - } - HybridEncoded::Repeated(value, length) => { - HybridBooleanState::Repeated(value, length) - } - }); - match run { - Ok(run) => { - self.current_run = Some(run); - self.next() - } - Err(e) => Some(Err(e)), - } - } else { - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let exact = self.iter.number_of_elements(); - (exact, Some(exact)) - } -} - -/// Type definition for a [`HybridRleBooleanIter`] using [`hybrid_rle::Decoder`]. -pub type HybridRleDecoderIter<'a> = HybridRleBooleanIter<'a, HybridDecoderBitmapIter<'a>>; diff --git a/src/common/parquet2/src/deserialize/mod.rs b/src/common/parquet2/src/deserialize/mod.rs deleted file mode 100644 index 4b1664fdf052..000000000000 --- a/src/common/parquet2/src/deserialize/mod.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod binary; -mod boolean; -mod filtered_rle; -mod fixed_len; -mod hybrid_rle; -mod native; -mod utils; - -#[allow(ambiguous_glob_reexports)] -pub use binary::*; -pub use boolean::*; -pub use filtered_rle::*; -pub use fixed_len::*; -pub use hybrid_rle::*; -pub use native::*; -pub use utils::DefLevelsDecoder; -pub use utils::OptionalValues; -pub use utils::SliceFilteredIter; diff --git a/src/common/parquet2/src/deserialize/native.rs b/src/common/parquet2/src/deserialize/native.rs deleted file mode 100644 index 3392dcddb7a9..000000000000 --- a/src/common/parquet2/src/deserialize/native.rs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::utils; -use crate::encoding::hybrid_rle; -use crate::error::Error; -use crate::page::split_buffer; -use crate::page::DataPage; -use crate::parquet_bridge::Encoding; -use crate::parquet_bridge::Repetition; -use crate::types::decode; -use crate::types::NativeType; - -/// Typedef of an iterator over PLAIN page values -pub type Casted<'a, T> = std::iter::Map, fn(&'a [u8]) -> T>; - -/// Views the values of the data page as [`Casted`] to [`NativeType`]. -pub fn native_cast(page: &DataPage) -> Result, Error> { - let (_, _, values) = split_buffer(page)?; - if values.len() % std::mem::size_of::() != 0 { - return Err(Error::oos( - "A primitive page data's len must be a multiple of the type", - )); - } - - Ok(values - .chunks_exact(std::mem::size_of::()) - .map(decode::)) -} - -#[derive(Debug)] -pub struct Dictionary<'a, P> { - pub indexes: hybrid_rle::HybridRleDecoder<'a>, - pub dict: P, -} - -impl<'a, P> Dictionary<'a, P> { - pub fn try_new(page: &'a DataPage, dict: P) -> Result { - let indexes = utils::dict_indices_decoder(page)?; - - Ok(Self { dict, indexes }) - } - - pub fn len(&self) -> usize { - self.indexes.size_hint().0 - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -/// The deserialization state of a `DataPage` of `Primitive` parquet primitive type -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum NativePageState<'a, T, P> -where T: NativeType -{ - /// A page of optional values - Optional(utils::DefLevelsDecoder<'a>, Casted<'a, T>), - /// A page of required values - Required(Casted<'a, T>), - /// A page of required, dictionary-encoded values - RequiredDictionary(Dictionary<'a, P>), - /// A page of optional, dictionary-encoded values - OptionalDictionary(utils::DefLevelsDecoder<'a>, Dictionary<'a, P>), -} - -impl<'a, T: NativeType, P> NativePageState<'a, T, P> { - /// Tries to create [`NativePageState`] - /// # Error - /// Errors iff the page is not a `NativePageState` - pub fn try_new(page: &'a DataPage, dict: Option

) -> Result { - let is_optional = - page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - - match (page.encoding(), dict, is_optional) { - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), false) => { - Dictionary::try_new(page, dict).map(Self::RequiredDictionary) - } - (Encoding::PlainDictionary | Encoding::RleDictionary, Some(dict), true) => { - Ok(Self::OptionalDictionary( - utils::DefLevelsDecoder::try_new(page)?, - Dictionary::try_new(page, dict)?, - )) - } - (Encoding::Plain, _, true) => { - let validity = utils::DefLevelsDecoder::try_new(page)?; - let values = native_cast(page)?; - - Ok(Self::Optional(validity, values)) - } - (Encoding::Plain, _, false) => native_cast(page).map(Self::Required), - _ => Err(Error::FeatureNotSupported(format!( - "Viewing page for encoding {:?} for native type {}", - page.encoding(), - std::any::type_name::() - ))), - } - } -} diff --git a/src/common/parquet2/src/deserialize/utils.rs b/src/common/parquet2/src/deserialize/utils.rs deleted file mode 100644 index 5ad8ead9d29f..000000000000 --- a/src/common/parquet2/src/deserialize/utils.rs +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use super::hybrid_rle::HybridDecoderBitmapIter; -use super::hybrid_rle::HybridRleIter; -use crate::encoding::hybrid_rle::HybridRleDecoder; -use crate::encoding::hybrid_rle::{self}; -use crate::error::Error; -use crate::indexes::Interval; -use crate::page::split_buffer; -use crate::page::DataPage; -use crate::read::levels::get_bit_width; - -pub(super) fn dict_indices_decoder(page: &DataPage) -> Result { - let (_, _, indices_buffer) = split_buffer(page)?; - - // SPEC: Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32), - // SPEC: followed by the values encoded using RLE/Bit packed described above (with the given bit width). - let bit_width = indices_buffer[0]; - if bit_width > 32 { - return Err(Error::oos( - "Bit width of dictionary pages cannot be larger than 32", - )); - } - let indices_buffer = &indices_buffer[1..]; - - hybrid_rle::HybridRleDecoder::try_new(indices_buffer, bit_width as u32, page.num_values()) -} - -/// Decoder of definition levels. -#[derive(Debug)] -pub enum DefLevelsDecoder<'a> { - /// When the maximum definition level is 1, the definition levels are RLE-encoded and - /// the bitpacked runs are bitmaps. This variant contains [`HybridDecoderBitmapIter`] - /// that decodes the runs, but not the individual values - Bitmap(HybridDecoderBitmapIter<'a>), - /// When the maximum definition level is larger than 1 - Levels(HybridRleDecoder<'a>, u32), -} - -impl<'a> DefLevelsDecoder<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, def_levels, _) = split_buffer(page)?; - - let max_def_level = page.descriptor.max_def_level; - Ok(if max_def_level == 1 { - let iter = hybrid_rle::Decoder::new(def_levels, 1); - let iter = HybridRleIter::new(iter, page.num_values()); - Self::Bitmap(iter) - } else { - let iter = HybridRleDecoder::try_new( - def_levels, - get_bit_width(max_def_level), - page.num_values(), - )?; - Self::Levels(iter, max_def_level as u32) - }) - } -} - -/// Iterator adapter to convert an iterator of non-null values and an iterator over validity -/// into an iterator of optional values. -#[derive(Debug, Clone)] -pub struct OptionalValues>, I: Iterator> { - validity: V, - values: I, -} - -impl>, I: Iterator> OptionalValues { - pub fn new(validity: V, values: I) -> Self { - Self { validity, values } - } -} - -impl>, I: Iterator> Iterator - for OptionalValues -{ - type Item = Result, Error>; - - #[inline] - fn next(&mut self) -> Option { - self.validity - .next() - .map(|x| x.map(|x| if x { self.values.next() } else { None })) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.validity.size_hint() - } -} - -/// An iterator adapter that converts an iterator over items into an iterator over slices of -/// those N items. -/// -/// This iterator is best used with iterators that implement `nth` since skipping items -/// allows this iterator to skip sequences of items without having to call each of them. -#[derive(Debug, Clone)] -pub struct SliceFilteredIter { - iter: I, - selected_rows: VecDeque, - current_remaining: usize, - current: usize, // position in the slice - total_length: usize, -} - -impl SliceFilteredIter { - /// Return a new [`SliceFilteredIter`] - pub fn new(iter: I, selected_rows: VecDeque) -> Self { - let total_length = selected_rows.iter().map(|i| i.length).sum(); - Self { - iter, - selected_rows, - current_remaining: 0, - current: 0, - total_length, - } - } -} - -impl> Iterator for SliceFilteredIter { - type Item = T; - - #[inline] - fn next(&mut self) -> Option { - if self.current_remaining == 0 { - if let Some(interval) = self.selected_rows.pop_front() { - // skip the hole between the previous start and this start - // (start + length) - start - let item = self.iter.nth(interval.start - self.current); - self.current = interval.start + interval.length; - self.current_remaining = interval.length - 1; - self.total_length -= 1; - item - } else { - None - } - } else { - self.current_remaining -= 1; - self.total_length -= 1; - self.iter.next() - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.total_length, Some(self.total_length)) - } -} - -#[cfg(test)] -mod test { - use std::collections::VecDeque; - - use super::*; - - #[test] - fn basic() { - let iter = 0..=100; - - let intervals = vec![ - Interval::new(0, 2), - Interval::new(20, 11), - Interval::new(31, 1), - ]; - - let a: VecDeque = intervals.clone().into_iter().collect(); - let mut a = SliceFilteredIter::new(iter, a); - - let expected: Vec = intervals - .into_iter() - .flat_map(|interval| interval.start..(interval.start + interval.length)) - .collect(); - - assert_eq!(expected, a.by_ref().collect::>()); - assert_eq!((0, Some(0)), a.size_hint()); - } -} diff --git a/src/common/parquet2/src/encoding/bitpacked/decode.rs b/src/common/parquet2/src/encoding/bitpacked/decode.rs deleted file mode 100644 index db3db039d427..000000000000 --- a/src/common/parquet2/src/encoding/bitpacked/decode.rs +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Packed; -use super::Unpackable; -use super::Unpacked; -use crate::error::Error; - -/// An [`Iterator`] of [`Unpackable`] unpacked from a bitpacked slice of bytes. -/// # Implementation -/// This iterator unpacks bytes in chunks and does not allocate. -#[derive(Debug, Clone)] -pub struct Decoder<'a, T: Unpackable> { - packed: std::slice::Chunks<'a, u8>, - num_bits: usize, - remaining: usize, // in number of items - current_pack_index: usize, // invariant: < T::PACK_LENGTH - unpacked: T::Unpacked, // has the current unpacked values. -} - -#[inline] -fn decode_pack(packed: &[u8], num_bits: usize, unpacked: &mut T::Unpacked) { - if packed.len() < T::Unpacked::LENGTH * num_bits / 8 { - let mut buf = T::Packed::zero(); - buf.as_mut()[..packed.len()].copy_from_slice(packed); - T::unpack(buf.as_ref(), num_bits, unpacked) - } else { - T::unpack(packed, num_bits, unpacked) - } -} - -impl<'a, T: Unpackable> Decoder<'a, T> { - /// Returns a [`Decoder`] with `T` encoded in `packed` with `num_bits`. - pub fn try_new(packed: &'a [u8], num_bits: usize, mut length: usize) -> Result { - let block_size = std::mem::size_of::() * num_bits; - - if num_bits == 0 { - return Err(Error::oos("Bitpacking requires num_bits > 0")); - } - - if packed.len() * 8 < length * num_bits { - return Err(Error::oos(format!( - "Unpacking {length} items with a number of bits {num_bits} requires at least {} bytes.", - length * num_bits / 8 - ))); - } - - let mut packed = packed.chunks(block_size); - let mut unpacked = T::Unpacked::zero(); - if let Some(chunk) = packed.next() { - decode_pack::(chunk, num_bits, &mut unpacked); - } else { - length = 0 - }; - - Ok(Self { - remaining: length, - packed, - num_bits, - unpacked, - current_pack_index: 0, - }) - } -} - -impl<'a, T: Unpackable> Iterator for Decoder<'a, T> { - type Item = T; - - #[inline] // -71% improvement in bench - fn next(&mut self) -> Option { - if self.remaining == 0 { - return None; - } - let result = self.unpacked[self.current_pack_index]; - self.current_pack_index += 1; - self.remaining -= 1; - if self.current_pack_index == T::Unpacked::LENGTH { - if let Some(packed) = self.packed.next() { - decode_pack::(packed, self.num_bits, &mut self.unpacked); - self.current_pack_index = 0; - } - } - Some(result) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.remaining, Some(self.remaining)) - } -} - -#[cfg(test)] -mod tests { - use super::super::tests::case1; - use super::*; - - #[test] - fn test_decode_rle() { - // Test data: 0-7 with bit width 3 - // 0: 000 - // 1: 001 - // 2: 010 - // 3: 011 - // 4: 100 - // 5: 101 - // 6: 110 - // 7: 111 - let num_bits = 3; - let length = 8; - // encoded: 0b10001000u8, 0b11000110, 0b11111010 - let data = vec![0b10001000u8, 0b11000110, 0b11111010]; - - let decoded = Decoder::::try_new(&data, num_bits, length) - .unwrap() - .collect::>(); - assert_eq!(decoded, vec![0, 1, 2, 3, 4, 5, 6, 7]); - } - - #[test] - fn decode_large() { - let (num_bits, expected, data) = case1(); - - let decoded = Decoder::::try_new(&data, num_bits, expected.len()) - .unwrap() - .collect::>(); - assert_eq!(decoded, expected); - } - - #[test] - fn test_decode_bool() { - let num_bits = 1; - let length = 8; - let data = vec![0b10101010]; - - let decoded = Decoder::::try_new(&data, num_bits, length) - .unwrap() - .collect::>(); - assert_eq!(decoded, vec![0, 1, 0, 1, 0, 1, 0, 1]); - } - - #[test] - fn test_decode_u64() { - let num_bits = 1; - let length = 8; - let data = vec![0b10101010]; - - let decoded = Decoder::::try_new(&data, num_bits, length) - .unwrap() - .collect::>(); - assert_eq!(decoded, vec![0, 1, 0, 1, 0, 1, 0, 1]); - } - - #[test] - fn even_case() { - // [0, 1, 2, 3, 4, 5, 6, 0]x99 - let data = &[0b10001000u8, 0b11000110, 0b00011010]; - let num_bits = 3; - let copies = 99; // 8 * 99 % 32 != 0 - let expected = std::iter::repeat(&[0u32, 1, 2, 3, 4, 5, 6, 0]) - .take(copies) - .flatten() - .copied() - .collect::>(); - let data = std::iter::repeat(data) - .take(copies) - .flatten() - .copied() - .collect::>(); - let length = expected.len(); - - let decoded = Decoder::::try_new(&data, num_bits, length) - .unwrap() - .collect::>(); - assert_eq!(decoded, expected); - } - - #[test] - fn odd_case() { - // [0, 1, 2, 3, 4, 5, 6, 0]x4 + [2] - let data = &[0b10001000u8, 0b11000110, 0b00011010]; - let num_bits = 3; - let copies = 4; - let expected = std::iter::repeat(&[0u32, 1, 2, 3, 4, 5, 6, 0]) - .take(copies) - .flatten() - .copied() - .chain(std::iter::once(2)) - .collect::>(); - let data = std::iter::repeat(data) - .take(copies) - .flatten() - .copied() - .chain(std::iter::once(0b00000010u8)) - .collect::>(); - let length = expected.len(); - - let decoded = Decoder::::try_new(&data, num_bits, length) - .unwrap() - .collect::>(); - assert_eq!(decoded, expected); - } - - #[test] - fn test_errors() { - // zero length - assert!(Decoder::::try_new(&[], 1, 0).is_ok()); - // no bytes - assert!(Decoder::::try_new(&[], 1, 1).is_err()); - // too few bytes - assert!(Decoder::::try_new(&[1], 1, 8).is_ok()); - assert!(Decoder::::try_new(&[1, 1], 2, 8).is_ok()); - assert!(Decoder::::try_new(&[1], 1, 9).is_err()); - // zero num_bits - assert!(Decoder::::try_new(&[1], 0, 1).is_err()); - } -} diff --git a/src/common/parquet2/src/encoding/bitpacked/encode.rs b/src/common/parquet2/src/encoding/bitpacked/encode.rs deleted file mode 100644 index 7714f0eab5cd..000000000000 --- a/src/common/parquet2/src/encoding/bitpacked/encode.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::convert::TryInto; - -use super::Packed; -use super::Unpackable; -use super::Unpacked; - -/// Encodes (packs) a slice of [`Unpackable`] into bitpacked bytes `packed`, using `num_bits` per value. -/// -/// This function assumes that the maximum value in `unpacked` fits in `num_bits` bits -/// and saturates higher values. -/// -/// Only the first `ceil8(unpacked.len() * num_bits)` of `packed` are populated. -pub fn encode(unpacked: &[T], num_bits: usize, packed: &mut [u8]) { - let chunks = unpacked.chunks_exact(T::Unpacked::LENGTH); - - let remainder = chunks.remainder(); - - let packed_size = (T::Unpacked::LENGTH * num_bits + 7) / 8; - if !remainder.is_empty() { - let packed_chunks = packed.chunks_mut(packed_size); - let mut last_chunk = T::Unpacked::zero(); - for i in 0..remainder.len() { - last_chunk[i] = remainder[i] - } - - chunks - .chain(std::iter::once(last_chunk.as_ref())) - .zip(packed_chunks) - .for_each(|(unpacked, packed)| { - T::pack(&unpacked.try_into().unwrap(), num_bits, packed); - }); - } else { - let packed_chunks = packed.chunks_exact_mut(packed_size); - chunks.zip(packed_chunks).for_each(|(unpacked, packed)| { - T::pack(&unpacked.try_into().unwrap(), num_bits, packed); - }); - } -} - -/// Encodes (packs) a potentially incomplete pack of [`Unpackable`] into bitpacked -/// bytes `packed`, using `num_bits` per value. -/// -/// This function assumes that the maximum value in `unpacked` fits in `num_bits` bits -/// and saturates higher values. -/// -/// Only the first `ceil8(unpacked.len() * num_bits)` of `packed` are populated. -#[inline] -pub fn encode_pack(unpacked: &[T], num_bits: usize, packed: &mut [u8]) { - if unpacked.len() < T::Packed::LENGTH { - let mut complete_unpacked = T::Unpacked::zero(); - complete_unpacked.as_mut()[..unpacked.len()].copy_from_slice(unpacked); - T::pack(&complete_unpacked, num_bits, packed) - } else { - T::pack(&unpacked.try_into().unwrap(), num_bits, packed) - } -} diff --git a/src/common/parquet2/src/encoding/bitpacked/mod.rs b/src/common/parquet2/src/encoding/bitpacked/mod.rs deleted file mode 100644 index 78d001ed4a17..000000000000 --- a/src/common/parquet2/src/encoding/bitpacked/mod.rs +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod decode; -mod encode; -mod pack; -mod unpack; - -pub use decode::Decoder; -pub use encode::encode; -pub use encode::encode_pack; - -/// A byte slice (e.g. `[u8; 8]`) denoting types that represent complete packs. -pub trait Packed: - Copy - + Sized - + AsRef<[u8]> - + AsMut<[u8]> - + std::ops::IndexMut - + for<'a> TryFrom<&'a [u8]> -{ - const LENGTH: usize; - fn zero() -> Self; -} - -impl Packed for [u8; 8] { - const LENGTH: usize = 8; - #[inline] - fn zero() -> Self { - [0; 8] - } -} - -impl Packed for [u8; 16 * 2] { - const LENGTH: usize = 16 * 2; - #[inline] - fn zero() -> Self { - [0; 16 * 2] - } -} - -impl Packed for [u8; 32 * 4] { - const LENGTH: usize = 32 * 4; - #[inline] - fn zero() -> Self { - [0; 32 * 4] - } -} - -impl Packed for [u8; 64 * 64] { - const LENGTH: usize = 64 * 64; - #[inline] - fn zero() -> Self { - [0; 64 * 64] - } -} - -/// A byte slice of [`Unpackable`] denoting complete unpacked arrays. -pub trait Unpacked: - Copy - + Sized - + AsRef<[T]> - + AsMut<[T]> - + std::ops::Index - + std::ops::IndexMut - + for<'a> TryFrom<&'a [T], Error = std::array::TryFromSliceError> -{ - const LENGTH: usize; - fn zero() -> Self; -} - -impl Unpacked for [u8; 8] { - const LENGTH: usize = 8; - #[inline] - fn zero() -> Self { - [0; 8] - } -} - -impl Unpacked for [u16; 16] { - const LENGTH: usize = 16; - #[inline] - fn zero() -> Self { - [0; 16] - } -} - -impl Unpacked for [u32; 32] { - const LENGTH: usize = 32; - #[inline] - fn zero() -> Self { - [0; 32] - } -} - -impl Unpacked for [u64; 64] { - const LENGTH: usize = 64; - #[inline] - fn zero() -> Self { - [0; 64] - } -} - -/// A type representing a type that can be bitpacked and unpacked by this crate. -pub trait Unpackable: Copy + Sized + Default { - type Packed: Packed; - type Unpacked: Unpacked; - fn unpack(packed: &[u8], num_bits: usize, unpacked: &mut Self::Unpacked); - fn pack(unpacked: &Self::Unpacked, num_bits: usize, packed: &mut [u8]); -} - -impl Unpackable for u8 { - type Packed = [u8; 8]; - type Unpacked = [u8; 8]; - - #[inline] - fn unpack(packed: &[u8], num_bits: usize, unpacked: &mut Self::Unpacked) { - unpack::unpack8(packed, unpacked, num_bits) - } - - #[inline] - fn pack(packed: &Self::Unpacked, num_bits: usize, unpacked: &mut [u8]) { - pack::pack8(packed, unpacked, num_bits) - } -} - -impl Unpackable for u16 { - type Packed = [u8; 16 * 2]; - type Unpacked = [u16; 16]; - - #[inline] - fn unpack(packed: &[u8], num_bits: usize, unpacked: &mut Self::Unpacked) { - unpack::unpack16(packed, unpacked, num_bits) - } - - #[inline] - fn pack(packed: &Self::Unpacked, num_bits: usize, unpacked: &mut [u8]) { - pack::pack16(packed, unpacked, num_bits) - } -} - -impl Unpackable for u32 { - type Packed = [u8; 32 * 4]; - type Unpacked = [u32; 32]; - - #[inline] - fn unpack(packed: &[u8], num_bits: usize, unpacked: &mut Self::Unpacked) { - unpack::unpack32(packed, unpacked, num_bits) - } - - #[inline] - fn pack(packed: &Self::Unpacked, num_bits: usize, unpacked: &mut [u8]) { - pack::pack32(packed, unpacked, num_bits) - } -} - -impl Unpackable for u64 { - type Packed = [u8; 64 * 64]; - type Unpacked = [u64; 64]; - - #[inline] - fn unpack(packed: &[u8], num_bits: usize, unpacked: &mut Self::Unpacked) { - unpack::unpack64(packed, unpacked, num_bits) - } - - #[inline] - fn pack(packed: &Self::Unpacked, num_bits: usize, unpacked: &mut [u8]) { - pack::pack64(packed, unpacked, num_bits) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - pub fn case1() -> (usize, Vec, Vec) { - let num_bits = 3; - let compressed = vec![ - 0b10001000u8, - 0b11000110, - 0b11111010, - 0b10001000u8, - 0b11000110, - 0b11111010, - 0b10001000u8, - 0b11000110, - 0b11111010, - 0b10001000u8, - 0b11000110, - 0b11111010, - 0b10001000u8, - 0b11000110, - 0b11111010, - ]; - let decompressed = vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, - 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, - ]; - (num_bits, decompressed, compressed) - } - - #[test] - fn encode_large() { - let (num_bits, unpacked, expected) = case1(); - let mut packed = vec![0u8; 4 * 32]; - - encode(&unpacked, num_bits, &mut packed); - assert_eq!(&packed[..15], expected); - } - - #[test] - fn test_encode() { - let num_bits = 3; - let unpacked = vec![0, 1, 2, 3, 4, 5, 6, 7]; - - let mut packed = vec![0u8; 4 * 32]; - - encode::(&unpacked, num_bits, &mut packed); - - let expected = vec![0b10001000u8, 0b11000110, 0b11111010]; - - assert_eq!(&packed[..3], expected); - } -} diff --git a/src/common/parquet2/src/encoding/bitpacked/pack.rs b/src/common/parquet2/src/encoding/bitpacked/pack.rs deleted file mode 100644 index 06a6eb80dfd8..000000000000 --- a/src/common/parquet2/src/encoding/bitpacked/pack.rs +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/// Macro that generates a packing function taking the number of bits as a const generic -macro_rules! pack_impl { - ($t:ty, $bytes:literal, $bits:tt) => { - pub fn pack(input: &[$t; $bits], output: &mut [u8]) { - if NUM_BITS == 0 { - for out in output { - *out = 0; - } - return; - } - assert!(NUM_BITS <= $bytes * 8); - assert!(output.len() >= NUM_BITS * $bytes); - - let mask = match NUM_BITS { - $bits => <$t>::MAX, - _ => ((1 << NUM_BITS) - 1), - }; - - for i in 0..$bits { - let start_bit = i * NUM_BITS; - let end_bit = start_bit + NUM_BITS; - - let start_bit_offset = start_bit % $bits; - let end_bit_offset = end_bit % $bits; - let start_byte = start_bit / $bits; - let end_byte = end_bit / $bits; - if start_byte != end_byte && end_bit_offset != 0 { - let a = input[i] << start_bit_offset; - let val_a = <$t>::to_le_bytes(a); - for i in 0..$bytes { - output[start_byte * $bytes + i] |= val_a[i] - } - - let b = (input[i] >> (NUM_BITS - end_bit_offset)) & mask; - let val_b = <$t>::to_le_bytes(b); - for i in 0..$bytes { - output[end_byte * $bytes + i] |= val_b[i] - } - } else { - let val = (input[i] & mask) << start_bit_offset; - let val = <$t>::to_le_bytes(val); - - for i in 0..$bytes { - output[start_byte * $bytes + i] |= val[i] - } - } - } - } - }; -} - -/// Macro that generates pack functions that accept num_bits as a parameter -macro_rules! pack { - ($name:ident, $t:ty, $bytes:literal, $bits:tt) => { - mod $name { - pack_impl!($t, $bytes, $bits); - } - - /// Pack unpacked `input` into `output` with a bit width of `num_bits` - pub fn $name(input: &[$t; $bits], output: &mut [u8], num_bits: usize) { - // This will get optimised into a jump table - seq_macro::seq!(i in 0..=$bits { - if i == num_bits { - return $name::pack::(input, output); - } - }); - unreachable!("invalid num_bits {}", num_bits); - } - }; -} - -pack!(pack8, u8, 1, 8); -pack!(pack16, u16, 2, 16); -pack!(pack32, u32, 4, 32); -pack!(pack64, u64, 8, 64); - -#[cfg(test)] -mod tests { - use super::super::unpack::*; - use super::*; - - #[test] - fn test_basic() { - let input = [0u16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - for num_bits in 4..16 { - let mut output = [0u8; 16 * 2]; - pack16(&input, &mut output, num_bits); - let mut other = [0u16; 16]; - unpack16(&output, &mut other, num_bits); - assert_eq!(other, input); - } - } - - #[test] - fn test_u32() { - let input = [ - 0u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0u32, 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, - ]; - for num_bits in 4..32 { - let mut output = [0u8; 32 * 4]; - pack32(&input, &mut output, num_bits); - let mut other = [0u32; 32]; - unpack32(&output, &mut other, num_bits); - assert_eq!(other, input); - } - } -} diff --git a/src/common/parquet2/src/encoding/bitpacked/unpack.rs b/src/common/parquet2/src/encoding/bitpacked/unpack.rs deleted file mode 100644 index c56466323af5..000000000000 --- a/src/common/parquet2/src/encoding/bitpacked/unpack.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Copied from https://github.com/apache/arrow-rs/blob/6859efa690d4c9530cf8a24053bc6ed81025a164/parquet/src/util/bit_pack.rs - -/// Macro that generates an unpack function taking the number of bits as a const generic -macro_rules! unpack_impl { - ($t:ty, $bytes:literal, $bits:tt) => { - pub fn unpack(input: &[u8], output: &mut [$t; $bits]) { - if NUM_BITS == 0 { - for out in output { - *out = 0; - } - return; - } - - assert!(NUM_BITS <= $bytes * 8); - - let mask = match NUM_BITS { - $bits => <$t>::MAX, - _ => ((1 << NUM_BITS) - 1), - }; - - assert!(input.len() >= NUM_BITS * $bytes); - - let r = |output_idx: usize| { - <$t>::from_le_bytes( - input[output_idx * $bytes..output_idx * $bytes + $bytes] - .try_into() - .unwrap(), - ) - }; - - seq_macro::seq!(i in 0..$bits { - let start_bit = i * NUM_BITS; - let end_bit = start_bit + NUM_BITS; - - let start_bit_offset = start_bit % $bits; - let end_bit_offset = end_bit % $bits; - let start_byte = start_bit / $bits; - let end_byte = end_bit / $bits; - if start_byte != end_byte && end_bit_offset != 0 { - let val = r(start_byte); - let a = val >> start_bit_offset; - let val = r(end_byte); - let b = val << (NUM_BITS - end_bit_offset); - - output[i] = a | (b & mask); - } else { - let val = r(start_byte); - output[i] = (val >> start_bit_offset) & mask; - } - }); - } - }; -} - -/// Macro that generates unpack functions that accept num_bits as a parameter -macro_rules! unpack { - ($name:ident, $t:ty, $bytes:literal, $bits:tt) => { - mod $name { - unpack_impl!($t, $bytes, $bits); - } - - /// Unpack packed `input` into `output` with a bit width of `num_bits` - pub fn $name(input: &[u8], output: &mut [$t; $bits], num_bits: usize) { - // This will get optimised into a jump table - seq_macro::seq!(i in 0..=$bits { - if i == num_bits { - return $name::unpack::(input, output); - } - }); - unreachable!("invalid num_bits {}", num_bits); - } - }; -} - -unpack!(unpack8, u8, 1, 8); -unpack!(unpack16, u16, 2, 16); -unpack!(unpack32, u32, 4, 32); -unpack!(unpack64, u64, 8, 64); - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_basic() { - let input = [0xFF; 4096]; - - for i in 0..=8 { - let mut output = [0; 8]; - unpack8(&input, &mut output, i); - for (idx, out) in output.iter().enumerate() { - assert_eq!(out.trailing_ones() as usize, i, "out[{}] = {}", idx, out); - } - } - - for i in 0..=16 { - let mut output = [0; 16]; - unpack16(&input, &mut output, i); - for (idx, out) in output.iter().enumerate() { - assert_eq!(out.trailing_ones() as usize, i, "out[{}] = {}", idx, out); - } - } - - for i in 0..=32 { - let mut output = [0; 32]; - unpack32(&input, &mut output, i); - for (idx, out) in output.iter().enumerate() { - assert_eq!(out.trailing_ones() as usize, i, "out[{}] = {}", idx, out); - } - } - - for i in 0..=64 { - let mut output = [0; 64]; - unpack64(&input, &mut output, i); - for (idx, out) in output.iter().enumerate() { - assert_eq!(out.trailing_ones() as usize, i, "out[{}] = {}", idx, out); - } - } - } -} diff --git a/src/common/parquet2/src/encoding/delta_bitpacked/decoder.rs b/src/common/parquet2/src/encoding/delta_bitpacked/decoder.rs deleted file mode 100644 index 0d9e0a4fb69e..000000000000 --- a/src/common/parquet2/src/encoding/delta_bitpacked/decoder.rs +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::bitpacked; -use super::super::uleb128; -use super::super::zigzag_leb128; -use crate::encoding::ceil8; -use crate::error::Error; - -/// An [`Iterator`] of [`i64`] -#[derive(Debug)] -struct Block<'a> { - // this is the minimum delta that must be added to every value. - min_delta: i64, - _num_mini_blocks: usize, - /// Number of values that each mini block has. - values_per_mini_block: usize, - bitwidths: std::slice::Iter<'a, u8>, - values: &'a [u8], - remaining: usize, // number of elements - current_index: usize, // invariant: < values_per_mini_block - // None represents a relative delta of zero, in which case there is no miniblock. - current_miniblock: Option>, - // number of bytes consumed. - consumed_bytes: usize, -} - -impl<'a> Block<'a> { - pub fn try_new( - mut values: &'a [u8], - num_mini_blocks: usize, - values_per_mini_block: usize, - length: usize, - ) -> Result { - let length = std::cmp::min(length, num_mini_blocks * values_per_mini_block); - - let mut consumed_bytes = 0; - let (min_delta, consumed) = zigzag_leb128::decode(values)?; - consumed_bytes += consumed; - values = &values[consumed..]; - - if num_mini_blocks > values.len() { - return Err(Error::oos( - "Block must contain at least num_mini_blocks bytes (the bitwidths)", - )); - } - let (bitwidths, remaining) = values.split_at(num_mini_blocks); - consumed_bytes += num_mini_blocks; - values = remaining; - - let mut block = Block { - min_delta, - _num_mini_blocks: num_mini_blocks, - values_per_mini_block, - bitwidths: bitwidths.iter(), - remaining: length, - values, - current_index: 0, - current_miniblock: None, - consumed_bytes, - }; - - // Set up first mini-block - block.advance_miniblock()?; - - Ok(block) - } - - fn advance_miniblock(&mut self) -> Result<(), Error> { - // unwrap is ok: we sliced it by num_mini_blocks in try_new - let num_bits = self.bitwidths.next().copied().unwrap() as usize; - - self.current_miniblock = if num_bits > 0 { - let length = std::cmp::min(self.remaining, self.values_per_mini_block); - - let miniblock_length = ceil8(self.values_per_mini_block * num_bits); - if miniblock_length > self.values.len() { - return Err(Error::oos( - "block must contain at least miniblock_length bytes (the mini block)", - )); - } - let (miniblock, remainder) = self.values.split_at(miniblock_length); - - self.values = remainder; - self.consumed_bytes += miniblock_length; - - Some(bitpacked::Decoder::try_new(miniblock, num_bits, length).unwrap()) - } else { - None - }; - self.current_index = 0; - - Ok(()) - } -} - -impl<'a> Iterator for Block<'a> { - type Item = Result; - - fn next(&mut self) -> Option { - if self.remaining == 0 { - return None; - } - let result = self.min_delta - + self - .current_miniblock - .as_mut() - .map(|x| x.next().unwrap_or_default()) - .unwrap_or(0) as i64; - self.current_index += 1; - self.remaining -= 1; - - if self.remaining > 0 && self.current_index == self.values_per_mini_block { - if let Err(e) = self.advance_miniblock() { - return Some(Err(e)); - } - } - - Some(Ok(result)) - } -} - -/// Decoder of parquets' `DELTA_BINARY_PACKED`. Implements `Iterator`. -/// # Implementation -/// This struct does not allocate on the heap. -#[derive(Debug)] -pub struct Decoder<'a> { - num_mini_blocks: usize, - values_per_mini_block: usize, - values_remaining: usize, - next_value: i64, - values: &'a [u8], - current_block: Option>, - // the total number of bytes consumed up to a given point, excluding the bytes on the current_block - consumed_bytes: usize, -} - -impl<'a> Decoder<'a> { - pub fn try_new(mut values: &'a [u8]) -> Result { - let mut consumed_bytes = 0; - let (block_size, consumed) = uleb128::decode(values)?; - consumed_bytes += consumed; - assert_eq!(block_size % 128, 0); - values = &values[consumed..]; - let (num_mini_blocks, consumed) = uleb128::decode(values)?; - let num_mini_blocks = num_mini_blocks as usize; - consumed_bytes += consumed; - values = &values[consumed..]; - let (total_count, consumed) = uleb128::decode(values)?; - let total_count = total_count as usize; - consumed_bytes += consumed; - values = &values[consumed..]; - let (first_value, consumed) = zigzag_leb128::decode(values)?; - consumed_bytes += consumed; - values = &values[consumed..]; - - let values_per_mini_block = block_size as usize / num_mini_blocks; - assert_eq!(values_per_mini_block % 8, 0); - - // If we only have one value (first_value), there are no blocks. - let current_block = if total_count > 1 { - Some(Block::try_new( - values, - num_mini_blocks, - values_per_mini_block, - total_count - 1, - )?) - } else { - None - }; - - Ok(Self { - num_mini_blocks, - values_per_mini_block, - values_remaining: total_count, - next_value: first_value, - values, - current_block, - consumed_bytes, - }) - } - - /// Returns the total number of bytes consumed up to this point by [`Decoder`]. - pub fn consumed_bytes(&self) -> usize { - self.consumed_bytes + self.current_block.as_ref().map_or(0, |b| b.consumed_bytes) - } - - fn load_delta(&mut self) -> Result { - // At this point we must have at least one block and value available - let current_block = self.current_block.as_mut().unwrap(); - if let Some(x) = current_block.next() { - x - } else { - // load next block - self.values = &self.values[current_block.consumed_bytes..]; - self.consumed_bytes += current_block.consumed_bytes; - - let next_block = Block::try_new( - self.values, - self.num_mini_blocks, - self.values_per_mini_block, - self.values_remaining, - ); - match next_block { - Ok(mut next_block) => { - let delta = next_block - .next() - .ok_or_else(|| Error::oos("Missing block"))?; - self.current_block = Some(next_block); - delta - } - Err(e) => Err(e), - } - } - } -} - -impl<'a> Iterator for Decoder<'a> { - type Item = Result; - - fn next(&mut self) -> Option { - if self.values_remaining == 0 { - return None; - } - - let result = Some(Ok(self.next_value)); - - self.values_remaining -= 1; - if self.values_remaining == 0 { - // do not try to load another block - return result; - } - - let delta = match self.load_delta() { - Ok(delta) => delta, - Err(e) => return Some(Err(e)), - }; - - self.next_value += delta; - result - } - - fn size_hint(&self) -> (usize, Option) { - (self.values_remaining, Some(self.values_remaining)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn single_value() { - // Generated by parquet-rs - // - // header: [128, 1, 4, 1, 2] - // block size: 128, 1 - // mini-blocks: 4 - // elements: 1 - // first_value: 2 <=z> 1 - let data = &[128, 1, 4, 1, 2]; - - let mut decoder = Decoder::try_new(data).unwrap(); - let r = decoder.by_ref().collect::, _>>().unwrap(); - - assert_eq!(&r[..], &[1]); - assert_eq!(decoder.consumed_bytes(), 5); - } - - #[test] - fn test_from_spec() { - let expected = (1..=5).collect::>(); - // VALIDATED FROM SPARK==3.1.1 - // header: [128, 1, 4, 5, 2] - // block size: 128, 1 - // mini-blocks: 4 - // elements: 5 - // first_value: 2 <=z> 1 - // block1: [2, 0, 0, 0, 0] - // min_delta: 2 <=z> 1 - // bit_width: 0 - let data = &[128, 1, 4, 5, 2, 2, 0, 0, 0, 0]; - - let mut decoder = Decoder::try_new(data).unwrap(); - let r = decoder.by_ref().collect::, _>>().unwrap(); - - assert_eq!(expected, r); - - assert_eq!(decoder.consumed_bytes(), 10); - } - - #[test] - fn case2() { - let expected = vec![1, 2, 3, 4, 5, 1]; - // VALIDATED FROM SPARK==3.1.1 - // header: [128, 1, 4, 6, 2] - // block size: 128, 1 <=u> 128 - // mini-blocks: 4 <=u> 4 - // elements: 6 <=u> 6 - // first_value: 2 <=z> 1 - // block1: [7, 3, 0, 0, 0] - // min_delta: 7 <=z> -4 - // bit_widths: [3, 0, 0, 0] - // values: [ - // 0b01101101 - // 0b00001011 - // ... - // ] <=b> [3, 3, 3, 3, 0] - let data = &[ - 128, 1, 4, 6, 2, 7, 3, 0, 0, 0, 0b01101101, 0b00001011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - // these should not be consumed - 1, 2, 3, - ]; - - let mut decoder = Decoder::try_new(data).unwrap(); - let r = decoder.by_ref().collect::, _>>().unwrap(); - - assert_eq!(expected, r); - assert_eq!(decoder.consumed_bytes(), data.len() - 3); - } - - #[test] - fn multiple_miniblocks() { - #[rustfmt::skip] - let data = &[ - // Header: [128, 1, 4, 65, 100] - 128, 1, // block size <=u> 128 - 4, // number of mini-blocks <=u> 4 - 65, // number of elements <=u> 65 - 100, // first_value <=z> 50 - - // Block 1 header: [7, 3, 4, 0, 0] - 7, // min_delta <=z> -4 - 3, 4, 255, 0, // bit_widths (255 should not be used as only two miniblocks are needed) - - // 32 3-bit values of 0 for mini-block 1 (12 bytes) - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - // 32 4-bit values of 8 for mini-block 2 (16 bytes) - 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, - 0x88, 0x88, - - // these should not be consumed - 1, 2, 3, - ]; - - #[rustfmt::skip] - let expected = [ - // First value - 50, - - // Mini-block 1: 32 deltas of -4 - 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, -2, -6, -10, -14, -18, -22, -26, -30, -34, - -38, -42, -46, -50, -54, -58, -62, -66, -70, -74, -78, - - // Mini-block 2: 32 deltas of 4 - -74, -70, -66, -62, -58, -54, -50, -46, -42, -38, -34, -30, -26, -22, -18, -14, -10, -6, - -2, 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, - ]; - - let mut decoder = Decoder::try_new(data).unwrap(); - let r = decoder.by_ref().collect::, _>>().unwrap(); - - assert_eq!(&expected[..], &r[..]); - assert_eq!(decoder.consumed_bytes(), data.len() - 3); - } -} diff --git a/src/common/parquet2/src/encoding/delta_bitpacked/encoder.rs b/src/common/parquet2/src/encoding/delta_bitpacked/encoder.rs deleted file mode 100644 index 19d2280b65b8..000000000000 --- a/src/common/parquet2/src/encoding/delta_bitpacked/encoder.rs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::bitpacked; -use super::super::uleb128; -use super::super::zigzag_leb128; -use crate::encoding::ceil8; - -/// Encodes an iterator of `i64` according to parquet's `DELTA_BINARY_PACKED`. -/// # Implementation -/// * This function does not allocate on the heap. -/// * The number of mini-blocks is always 1. This may change in the future. -pub fn encode>(mut iterator: I, buffer: &mut Vec) { - let block_size = 128; - let mini_blocks = 1; - - let mut container = [0u8; 10]; - let encoded_len = uleb128::encode(block_size, &mut container); - buffer.extend_from_slice(&container[..encoded_len]); - - let encoded_len = uleb128::encode(mini_blocks, &mut container); - buffer.extend_from_slice(&container[..encoded_len]); - - let length = iterator.size_hint().1.unwrap(); - let encoded_len = uleb128::encode(length as u64, &mut container); - buffer.extend_from_slice(&container[..encoded_len]); - - let mut values = [0i64; 128]; - let mut deltas = [0u64; 128]; - - let first_value = iterator.next().unwrap_or_default(); - let (container, encoded_len) = zigzag_leb128::encode(first_value); - buffer.extend_from_slice(&container[..encoded_len]); - - let mut prev = first_value; - let mut length = iterator.size_hint().1.unwrap(); - while length != 0 { - let mut min_delta = i64::MAX; - let mut max_delta = i64::MIN; - let mut num_bits = 0; - for (i, integer) in (0..128).zip(&mut iterator) { - let delta = integer - prev; - min_delta = min_delta.min(delta); - max_delta = max_delta.max(delta); - - num_bits = 64 - (max_delta - min_delta).leading_zeros(); - values[i] = delta; - prev = integer; - } - let consumed = std::cmp::min(length - iterator.size_hint().1.unwrap(), 128); - length = iterator.size_hint().1.unwrap(); - let values = &values[..consumed]; - - values.iter().zip(deltas.iter_mut()).for_each(|(v, delta)| { - *delta = (v - min_delta) as u64; - }); - - // - let (container, encoded_len) = zigzag_leb128::encode(min_delta); - buffer.extend_from_slice(&container[..encoded_len]); - - // one miniblock => 1 byte - buffer.push(num_bits as u8); - write_miniblock(buffer, num_bits as usize, deltas); - } -} - -fn write_miniblock(buffer: &mut Vec, num_bits: usize, deltas: [u64; 128]) { - if num_bits > 0 { - let start = buffer.len(); - - // bitpack encode all (deltas.len = 128 which is a multiple of 32) - let bytes_needed = start + ceil8(deltas.len() * num_bits); - buffer.resize(bytes_needed, 0); - bitpacked::encode(deltas.as_ref(), num_bits, &mut buffer[start..]); - - let bytes_needed = start + ceil8(deltas.len() * num_bits); - buffer.truncate(bytes_needed); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn constant_delta() { - // header: [128, 1, 1, 5, 2]: - // block size: 128 <=u> 128, 1 - // mini-blocks: 1 <=u> 1 - // elements: 5 <=u> 5 - // first_value: 2 <=z> 1 - // block1: [2, 0, 0, 0, 0] - // min_delta: 1 <=z> 2 - // bitwidth: 0 - let data = 1..=5; - let expected = vec![128u8, 1, 1, 5, 2, 2, 0]; - - let mut buffer = vec![]; - encode(data, &mut buffer); - assert_eq!(expected, buffer); - } - - #[test] - fn negative_min_delta() { - // max - min = 1 - -4 = 5 - let data = vec![1, 2, 3, 4, 5, 1]; - // header: [128, 1, 4, 6, 2] - // block size: 128 <=u> 128, 1 - // mini-blocks: 1 <=u> 1 - // elements: 6 <=u> 5 - // first_value: 2 <=z> 1 - // block1: [7, 3, 253, 255] - // min_delta: -4 <=z> 7 - // bitwidth: 3 - // values: [5, 5, 5, 5, 0] <=b> [ - // 0b01101101 - // 0b00001011 - // ] - let mut expected = vec![128u8, 1, 1, 6, 2, 7, 3, 0b01101101, 0b00001011]; - expected.extend(std::iter::repeat(0).take(128 * 3 / 8 - 2)); // 128 values, 3 bits, 2 already used - - let mut buffer = vec![]; - encode(data.into_iter(), &mut buffer); - assert_eq!(expected, buffer); - } -} diff --git a/src/common/parquet2/src/encoding/delta_bitpacked/mod.rs b/src/common/parquet2/src/encoding/delta_bitpacked/mod.rs deleted file mode 100644 index 05de8d058d3a..000000000000 --- a/src/common/parquet2/src/encoding/delta_bitpacked/mod.rs +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod decoder; -mod encoder; - -pub use decoder::Decoder; -pub use encoder::encode; - -#[cfg(test)] -mod tests { - use super::*; - use crate::error::Error; - - #[test] - fn basic() -> Result<(), Error> { - let data = vec![1, 3, 1, 2, 3]; - - let mut buffer = vec![]; - encode(data.clone().into_iter(), &mut buffer); - let iter = Decoder::try_new(&buffer)?; - - let result = iter.collect::, _>>()?; - assert_eq!(result, data); - Ok(()) - } - - #[test] - fn negative_value() -> Result<(), Error> { - let data = vec![1, 3, -1, 2, 3]; - - let mut buffer = vec![]; - encode(data.clone().into_iter(), &mut buffer); - let iter = Decoder::try_new(&buffer)?; - - let result = iter.collect::, _>>()?; - assert_eq!(result, data); - Ok(()) - } - - #[test] - fn some() -> Result<(), Error> { - let data = vec![ - -2147483648, - -1777158217, - -984917788, - -1533539476, - -731221386, - -1322398478, - 906736096, - ]; - - let mut buffer = vec![]; - encode(data.clone().into_iter(), &mut buffer); - let iter = Decoder::try_new(&buffer)?; - - let result = iter.collect::, Error>>()?; - assert_eq!(result, data); - Ok(()) - } - - #[test] - fn more_than_one_block() -> Result<(), Error> { - let mut data = vec![1, 3, -1, 2, 3, 10, 1]; - for x in 0..128 { - data.push(x - 10) - } - - let mut buffer = vec![]; - encode(data.clone().into_iter(), &mut buffer); - let iter = Decoder::try_new(&buffer)?; - - let result = iter.collect::, _>>()?; - assert_eq!(result, data); - Ok(()) - } - - #[test] - fn test_another() -> Result<(), Error> { - let data = vec![2, 3, 1, 2, 1]; - - let mut buffer = vec![]; - encode(data.clone().into_iter(), &mut buffer); - let len = buffer.len(); - let mut iter = Decoder::try_new(&buffer)?; - - let result = iter.by_ref().collect::, _>>()?; - assert_eq!(result, data); - - assert_eq!(iter.consumed_bytes(), len); - Ok(()) - } -} diff --git a/src/common/parquet2/src/encoding/delta_byte_array/decoder.rs b/src/common/parquet2/src/encoding/delta_byte_array/decoder.rs deleted file mode 100644 index 226f0ee039cf..000000000000 --- a/src/common/parquet2/src/encoding/delta_byte_array/decoder.rs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::delta_bitpacked; -use super::super::delta_length_byte_array; -use crate::error::Error; - -/// Decodes according to [Delta strings](https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7), -/// prefixes, lengths and values -/// # Implementation -/// This struct does not allocate on the heap. -#[derive(Debug)] -pub struct Decoder<'a> { - values: &'a [u8], - prefix_lengths: delta_bitpacked::Decoder<'a>, -} - -impl<'a> Decoder<'a> { - pub fn try_new(values: &'a [u8]) -> Result { - let prefix_lengths = delta_bitpacked::Decoder::try_new(values)?; - Ok(Self { - values, - prefix_lengths, - }) - } - - pub fn into_lengths(self) -> Result, Error> { - assert_eq!(self.prefix_lengths.size_hint().0, 0); - delta_length_byte_array::Decoder::try_new( - &self.values[self.prefix_lengths.consumed_bytes()..], - ) - } -} - -impl<'a> Iterator for Decoder<'a> { - type Item = Result; - - fn next(&mut self) -> Option { - self.prefix_lengths.next().map(|x| x.map(|x| x as u32)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_bla() -> Result<(), Error> { - // VALIDATED from spark==3.1.1 - let data = &[ - 128, 1, 4, 2, 0, 0, 0, 0, 0, 0, 128, 1, 4, 2, 10, 0, 0, 0, 0, 0, 72, 101, 108, 108, - 111, 87, 111, 114, 108, 100, - // extra bytes are not from spark, but they should be ignored by the decoder - // because they are beyond the sum of all lengths. - 1, 2, 3, - ]; - // result of encoding - let expected = &["Hello", "World"]; - let expected_lengths = expected.iter().map(|x| x.len() as i32).collect::>(); - let expected_prefixes = vec![0, 0]; - let expected_values = expected.join(""); - let expected_values = expected_values.as_bytes(); - - let mut decoder = Decoder::try_new(data)?; - let prefixes = decoder.by_ref().collect::, _>>()?; - assert_eq!(prefixes, expected_prefixes); - - // move to the lengths - let mut decoder = decoder.into_lengths()?; - - let lengths = decoder.by_ref().collect::, _>>()?; - assert_eq!(lengths, expected_lengths); - - // move to the values - let values = decoder.values(); - assert_eq!(values, expected_values); - Ok(()) - } - - #[test] - fn test_with_prefix() -> Result<(), Error> { - // VALIDATED from spark==3.1.1 - let data = &[ - 128, 1, 4, 2, 0, 6, 0, 0, 0, 0, 128, 1, 4, 2, 10, 4, 0, 0, 0, 0, 72, 101, 108, 108, - 111, 105, 99, 111, 112, 116, 101, 114, - // extra bytes are not from spark, but they should be ignored by the decoder - // because they are beyond the sum of all lengths. - 1, 2, 3, - ]; - // result of encoding - let expected_lengths = vec![5, 7]; - let expected_prefixes = vec![0, 3]; - let expected_values = b"Helloicopter"; - - let mut decoder = Decoder::try_new(data)?; - let prefixes = decoder.by_ref().collect::, _>>()?; - assert_eq!(prefixes, expected_prefixes); - - // move to the lengths - let mut decoder = decoder.into_lengths()?; - - let lengths = decoder.by_ref().collect::, _>>()?; - assert_eq!(lengths, expected_lengths); - - // move to the values - let values = decoder.values(); - assert_eq!(values, expected_values); - Ok(()) - } -} diff --git a/src/common/parquet2/src/encoding/delta_byte_array/encoder.rs b/src/common/parquet2/src/encoding/delta_byte_array/encoder.rs deleted file mode 100644 index e32f8e9e608a..000000000000 --- a/src/common/parquet2/src/encoding/delta_byte_array/encoder.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::delta_bitpacked; -use crate::encoding::delta_length_byte_array; - -/// Encodes an iterator of according to DELTA_BYTE_ARRAY -pub fn encode<'a, I: Iterator + Clone>(iterator: I, buffer: &mut Vec) { - let mut previous = b"".as_ref(); - - let mut sum_lengths = 0; - let prefixes = iterator - .clone() - .map(|item| { - let prefix_length = item - .iter() - .zip(previous.iter()) - .enumerate() - // find first difference - .find_map(|(length, (lhs, rhs))| (lhs != rhs).then_some(length)) - .unwrap_or(previous.len()); - previous = item; - - sum_lengths += item.len() - prefix_length; - prefix_length as i64 - }) - .collect::>(); - delta_bitpacked::encode(prefixes.iter().copied(), buffer); - - let remaining = iterator - .zip(prefixes) - .map(|(item, prefix)| &item[prefix as usize..]); - - delta_length_byte_array::encode(remaining, buffer); -} diff --git a/src/common/parquet2/src/encoding/delta_byte_array/mod.rs b/src/common/parquet2/src/encoding/delta_byte_array/mod.rs deleted file mode 100644 index 17109085e76e..000000000000 --- a/src/common/parquet2/src/encoding/delta_byte_array/mod.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod decoder; -mod encoder; - -pub use decoder::Decoder; -pub use encoder::encode; - -#[cfg(test)] -mod tests { - use super::*; - use crate::error::Error; - - #[test] - fn basic() -> Result<(), Error> { - let data = vec![b"Hello".as_ref(), b"Helicopter"]; - let mut buffer = vec![]; - encode(data.clone().into_iter(), &mut buffer); - - let mut decoder = Decoder::try_new(&buffer)?; - let prefixes = decoder.by_ref().collect::, _>>()?; - assert_eq!(prefixes, vec![0, 3]); - - // move to the lengths - let mut decoder = decoder.into_lengths()?; - - let lengths = decoder.by_ref().collect::, _>>()?; - assert_eq!(lengths, vec![5, 7]); - - // move to the values - let values = decoder.values(); - assert_eq!(values, b"Helloicopter"); - Ok(()) - } -} diff --git a/src/common/parquet2/src/encoding/delta_length_byte_array/decoder.rs b/src/common/parquet2/src/encoding/delta_length_byte_array/decoder.rs deleted file mode 100644 index f1bf7f02d8f7..000000000000 --- a/src/common/parquet2/src/encoding/delta_length_byte_array/decoder.rs +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::delta_bitpacked; -use crate::error::Error; - -/// Decodes [Delta-length byte array](https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6) -/// lengths and values. -/// # Implementation -/// This struct does not allocate on the heap. -/// # Example -/// ``` -/// use parquet2::encoding::delta_length_byte_array::Decoder; -/// -/// let expected = &["Hello", "World"]; -/// let expected_lengths = expected.iter().map(|x| x.len() as i32).collect::>(); -/// let expected_values = expected.join(""); -/// let expected_values = expected_values.as_bytes(); -/// let data = &[ -/// 128, 1, 4, 2, 10, 0, 0, 0, 0, 0, 72, 101, 108, 108, 111, 87, 111, 114, 108, 100, -/// ]; -/// -/// let mut decoder = Decoder::try_new(data).unwrap(); -/// -/// // Extract the lengths -/// let lengths = decoder.by_ref().collect::, _>>().unwrap(); -/// assert_eq!(lengths, expected_lengths); -/// -/// // Extract the values. This _must_ be called after consuming all lengths by reference (see above). -/// let values = decoder.into_values(); -/// -/// assert_eq!(values, expected_values); -#[derive(Debug)] -pub struct Decoder<'a> { - values: &'a [u8], - lengths: delta_bitpacked::Decoder<'a>, - total_length: u32, -} - -impl<'a> Decoder<'a> { - pub fn try_new(values: &'a [u8]) -> Result { - let lengths = delta_bitpacked::Decoder::try_new(values)?; - Ok(Self { - values, - lengths, - total_length: 0, - }) - } - - /// Consumes this decoder and returns the slice of concatenated values. - /// # Panics - /// This function panics if this iterator has not been fully consumed. - pub fn into_values(self) -> &'a [u8] { - assert_eq!(self.lengths.size_hint().0, 0); - let start = self.lengths.consumed_bytes(); - &self.values[start..start + self.total_length as usize] - } - - /// Returns the slice of concatenated values. - /// # Panics - /// This function panics if this iterator has not yet been fully consumed. - pub fn values(&self) -> &'a [u8] { - assert_eq!(self.lengths.size_hint().0, 0); - let start = self.lengths.consumed_bytes(); - &self.values[start..start + self.total_length as usize] - } -} - -impl<'a> Iterator for Decoder<'a> { - type Item = Result; - - fn next(&mut self) -> Option { - let result = self.lengths.next(); - match result { - Some(Ok(v)) => { - self.total_length += v as u32; - Some(Ok(v as i32)) - } - Some(Err(error)) => Some(Err(error)), - None => None, - } - } -} diff --git a/src/common/parquet2/src/encoding/delta_length_byte_array/encoder.rs b/src/common/parquet2/src/encoding/delta_length_byte_array/encoder.rs deleted file mode 100644 index c5fa57dc5911..000000000000 --- a/src/common/parquet2/src/encoding/delta_length_byte_array/encoder.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::encoding::delta_bitpacked; - -/// Encodes a cloneable iterator of `&[u8]` into `buffer`. This does not allocated on the heap. -/// # Implementation -/// This encoding is equivalent to call [`delta_bitpacked::encode`] on the lengths of the items -/// of the iterator followed by extending the buffer from each item of the iterator. -pub fn encode, I: Iterator + Clone>(iterator: I, buffer: &mut Vec) { - let mut total_length = 0; - delta_bitpacked::encode( - iterator.clone().map(|x| { - let len = x.as_ref().len(); - total_length += len; - len as i64 - }), - buffer, - ); - buffer.reserve(total_length); - iterator.for_each(|x| buffer.extend(x.as_ref())) -} diff --git a/src/common/parquet2/src/encoding/delta_length_byte_array/mod.rs b/src/common/parquet2/src/encoding/delta_length_byte_array/mod.rs deleted file mode 100644 index 646903a827ae..000000000000 --- a/src/common/parquet2/src/encoding/delta_length_byte_array/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod decoder; -mod encoder; - -pub use decoder::Decoder; -pub use encoder::encode; - -#[cfg(test)] -mod tests { - use super::*; - use crate::error::Error; - - #[test] - fn basic() -> Result<(), Error> { - let data = vec!["aa", "bbb", "a", "aa", "b"]; - - let mut buffer = vec![]; - encode(data.into_iter().map(|x| x.as_bytes()), &mut buffer); - - let mut iter = Decoder::try_new(&buffer)?; - - let result = iter.by_ref().collect::, _>>()?; - assert_eq!(result, vec![2, 3, 1, 2, 1]); - - let result = iter.values(); - assert_eq!(result, b"aabbbaaab".as_ref()); - Ok(()) - } - - #[test] - fn many_numbers() -> Result<(), Error> { - let mut data = vec![]; - for i in 0..136 { - data.push(format!("a{}", i)) - } - let expected_values = data.join(""); - let expected_lengths = data.iter().map(|x| x.len() as i32).collect::>(); - - let mut buffer = vec![]; - encode(data.into_iter(), &mut buffer); - - let mut iter = Decoder::try_new(&buffer)?; - - let result = iter.by_ref().collect::, _>>()?; - assert_eq!(result, expected_lengths); - - let result = iter.into_values(); - assert_eq!(result, expected_values.as_str().as_bytes()); - Ok(()) - } -} diff --git a/src/common/parquet2/src/encoding/hybrid_rle/bitmap.rs b/src/common/parquet2/src/encoding/hybrid_rle/bitmap.rs deleted file mode 100644 index 20738591a6d6..000000000000 --- a/src/common/parquet2/src/encoding/hybrid_rle/bitmap.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -const BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; - -/// Sets bit at position `i` in `byte` -#[inline] -pub fn set(byte: u8, i: usize) -> u8 { - byte | BIT_MASK[i] -} - -/// An [`Iterator`] of bool that decodes a bitmap. -/// This is a specialization of [`super::super::bitpacked::Decoder`] for `num_bits == 1`. -#[derive(Debug)] -pub struct BitmapIter<'a> { - iter: std::slice::Iter<'a, u8>, - current_byte: &'a u8, - remaining: usize, - mask: u8, -} - -impl<'a> BitmapIter<'a> { - /// Returns a new [`BitmapIter`]. - /// # Panics - /// This function panics iff `offset / 8 > slice.len()` - #[inline] - pub fn new(slice: &'a [u8], offset: usize, len: usize) -> Self { - let bytes = &slice[offset / 8..]; - - let mut iter = bytes.iter(); - - let current_byte = iter.next().unwrap_or(&0); - - Self { - iter, - mask: 1u8.rotate_left(offset as u32), - remaining: len, - current_byte, - } - } -} - -impl<'a> Iterator for BitmapIter<'a> { - type Item = bool; - - #[inline] - fn next(&mut self) -> Option { - // easily predictable in branching - if self.remaining == 0 { - return None; - } else { - self.remaining -= 1; - } - let value = self.current_byte & self.mask != 0; - self.mask = self.mask.rotate_left(1); - if self.mask == 1 { - // reached a new byte => try to fetch it from the iterator - if let Some(v) = self.iter.next() { - self.current_byte = v - } - } - Some(value) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.remaining, Some(self.remaining)) - } -} - -/// Writes an iterator of bools into writer, with LSB first. -pub fn encode_bool>( - writer: &mut W, - mut iterator: I, -) -> std::io::Result<()> { - // the length of the iterator. - let length = iterator.size_hint().1.unwrap(); - - let chunks = length / 8; - let reminder = length % 8; - - (0..chunks).try_for_each(|_| { - let mut byte = 0u8; - (0..8).for_each(|i| { - if iterator.next().unwrap() { - byte = set(byte, i) - } - }); - writer.write_all(&[byte]) - })?; - - if reminder != 0 { - let mut last = 0u8; - iterator.enumerate().for_each(|(i, value)| { - if value { - last = set(last, i) - } - }); - writer.write_all(&[last]) - } else { - Ok(()) - } -} diff --git a/src/common/parquet2/src/encoding/hybrid_rle/decoder.rs b/src/common/parquet2/src/encoding/hybrid_rle/decoder.rs deleted file mode 100644 index 452d813e5209..000000000000 --- a/src/common/parquet2/src/encoding/hybrid_rle/decoder.rs +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::ceil8; -use super::super::uleb128; -use super::HybridEncoded; -use crate::error::Error; - -/// An [`Iterator`] of [`HybridEncoded`]. -#[derive(Debug, Clone)] -pub struct Decoder<'a> { - values: &'a [u8], - num_bits: usize, -} - -impl<'a> Decoder<'a> { - /// Returns a new [`Decoder`] - pub fn new(values: &'a [u8], num_bits: usize) -> Self { - Self { values, num_bits } - } - - /// Returns the number of bits being used by this decoder. - #[inline] - pub fn num_bits(&self) -> usize { - self.num_bits - } -} - -impl<'a> Iterator for Decoder<'a> { - type Item = Result, Error>; - - #[inline] // -18% improvement in bench - fn next(&mut self) -> Option { - if self.num_bits == 0 { - return None; - } - - if self.values.is_empty() { - return None; - } - - let (indicator, consumed) = match uleb128::decode(self.values) { - Ok((indicator, consumed)) => (indicator, consumed), - Err(e) => return Some(Err(e)), - }; - self.values = &self.values[consumed..]; - if self.values.is_empty() { - return None; - }; - - if indicator & 1 == 1 { - // is bitpacking - let bytes = (indicator as usize >> 1) * self.num_bits; - let bytes = std::cmp::min(bytes, self.values.len()); - let (result, remaining) = self.values.split_at(bytes); - self.values = remaining; - Some(Ok(HybridEncoded::Bitpacked(result))) - } else { - // is rle - let run_length = indicator as usize >> 1; - // repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width) - let rle_bytes = ceil8(self.num_bits); - let (result, remaining) = self.values.split_at(rle_bytes); - self.values = remaining; - Some(Ok(HybridEncoded::Rle(result, run_length))) - } - } -} - -#[cfg(test)] -mod tests { - use super::super::super::bitpacked; - use super::*; - - #[test] - fn basics_1() { - let bit_width = 1usize; - let length = 5; - let values = vec![ - 2, 0, 0, 0, // length - 0b00000011, 0b00001011, // data - ]; - - let mut decoder = Decoder::new(&values[4..6], bit_width); - - let run = decoder.next().unwrap(); - - if let HybridEncoded::Bitpacked(values) = run.unwrap() { - assert_eq!(values, &[0b00001011]); - let result = bitpacked::Decoder::::try_new(values, bit_width, length) - .unwrap() - .collect::>(); - assert_eq!(result, &[1, 1, 0, 1, 0]); - } else { - panic!() - }; - } - - #[test] - fn basics_2() { - // This test was validated by the result of what pyarrow3 outputs when - // the bitmap is used. - let bit_width = 1; - let values = vec![ - 3, 0, 0, 0, // length - 0b00000101, 0b11101011, 0b00000010, // data - ]; - let expected = &[1, 1, 0, 1, 0, 1, 1, 1, 0, 1]; - - let mut decoder = Decoder::new(&values[4..4 + 3], bit_width); - - let run = decoder.next().unwrap(); - - if let HybridEncoded::Bitpacked(values) = run.unwrap() { - assert_eq!(values, &[0b11101011, 0b00000010]); - let result = bitpacked::Decoder::::try_new(values, bit_width, 10) - .unwrap() - .collect::>(); - assert_eq!(result, expected); - } else { - panic!() - }; - } - - #[test] - fn basics_3() { - let bit_width = 1; - let length = 8; - let values = vec![ - 2, 0, 0, 0, // length - 0b00010000, // data - 0b00000001, - ]; - - let mut decoder = Decoder::new(&values[4..4 + 2], bit_width); - - let run = decoder.next().unwrap(); - - if let HybridEncoded::Rle(values, items) = run.unwrap() { - assert_eq!(values, &[0b00000001]); - assert_eq!(items, length); - } else { - panic!() - }; - } -} diff --git a/src/common/parquet2/src/encoding/hybrid_rle/encoder.rs b/src/common/parquet2/src/encoding/hybrid_rle/encoder.rs deleted file mode 100644 index fac942e79406..000000000000 --- a/src/common/parquet2/src/encoding/hybrid_rle/encoder.rs +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -use super::bitpacked_encode; -use crate::encoding::bitpacked; -use crate::encoding::ceil8; -use crate::encoding::uleb128; - -/// RLE-hybrid encoding of `u32`. This currently only yields bitpacked values. -pub fn encode_u32>( - writer: &mut W, - iterator: I, - num_bits: u32, -) -> std::io::Result<()> { - let num_bits = num_bits as u8; - // the length of the iterator. - let length = iterator.size_hint().1.unwrap(); - - // write the length + indicator - let mut header = ceil8(length) as u64; - header <<= 1; - header |= 1; // it is bitpacked => first bit is set - let mut container = [0; 10]; - let used = uleb128::encode(header, &mut container); - writer.write_all(&container[..used])?; - - bitpacked_encode_u32(writer, iterator, num_bits as usize)?; - - Ok(()) -} - -const U32_BLOCK_LEN: usize = 32; - -fn bitpacked_encode_u32>( - writer: &mut W, - mut iterator: I, - num_bits: usize, -) -> std::io::Result<()> { - // the length of the iterator. - let length = iterator.size_hint().1.unwrap(); - - let chunks = length / U32_BLOCK_LEN; - let remainder = length - chunks * U32_BLOCK_LEN; - let mut buffer = [0u32; U32_BLOCK_LEN]; - - let compressed_chunk_size = ceil8(U32_BLOCK_LEN * num_bits); - - for _ in 0..chunks { - iterator - .by_ref() - .take(U32_BLOCK_LEN) - .zip(buffer.iter_mut()) - .for_each(|(item, buf)| *buf = item); - - let mut packed = [0u8; 4 * U32_BLOCK_LEN]; - bitpacked::encode_pack::(&buffer, num_bits, packed.as_mut()); - writer.write_all(&packed[..compressed_chunk_size])?; - } - - if remainder != 0 { - let compressed_remainder_size = ceil8(remainder * num_bits); - iterator - .by_ref() - .take(remainder) - .zip(buffer.iter_mut()) - .for_each(|(item, buf)| *buf = item); - - let mut packed = [0u8; 4 * U32_BLOCK_LEN]; - bitpacked::encode_pack(&buffer, num_bits, packed.as_mut()); - writer.write_all(&packed[..compressed_remainder_size])?; - }; - Ok(()) -} - -/// the bitpacked part of the encoder. -pub fn encode_bool>( - writer: &mut W, - iterator: I, -) -> std::io::Result<()> { - // the length of the iterator. - let length = iterator.size_hint().1.unwrap(); - - // write the length + indicator - let mut header = ceil8(length) as u64; - header <<= 1; - header |= 1; // it is bitpacked => first bit is set - let mut container = [0; 10]; - let used = uleb128::encode(header, &mut container); - - writer.write_all(&container[..used])?; - - // encode the iterator - bitpacked_encode(writer, iterator) -} - -#[cfg(test)] -mod tests { - use super::super::bitmap::BitmapIter; - use super::*; - - #[test] - fn bool_basics_1() -> std::io::Result<()> { - let iter = BitmapIter::new(&[0b10011101u8, 0b10011101], 0, 14); - - let mut vec = vec![]; - - encode_bool(&mut vec, iter)?; - - assert_eq!(vec, vec![(2 << 1 | 1), 0b10011101u8, 0b00011101]); - - Ok(()) - } - - #[test] - fn bool_from_iter() -> std::io::Result<()> { - let mut vec = vec![]; - - encode_bool( - &mut vec, - vec![true, true, true, true, true, true, true, true].into_iter(), - )?; - - assert_eq!(vec, vec![(1 << 1 | 1), 0b11111111]); - Ok(()) - } - - #[test] - fn test_encode_u32() -> std::io::Result<()> { - let mut vec = vec![]; - - encode_u32(&mut vec, vec![0, 1, 2, 1, 2, 1, 1, 0, 3].into_iter(), 2)?; - - assert_eq!(vec, vec![ - (2 << 1 | 1), - 0b01_10_01_00, - 0b00_01_01_10, - 0b_00_00_00_11 - ]); - Ok(()) - } - - #[test] - fn test_encode_u32_large() -> std::io::Result<()> { - let mut vec = vec![]; - - let values = (0..128).map(|x| x % 4); - - encode_u32(&mut vec, values, 2)?; - - let length = 128; - let expected = 0b11_10_01_00u8; - - let mut expected = vec![expected; length / 4]; - expected.insert(0, ((length / 8) as u8) << 1 | 1); - - assert_eq!(vec, expected); - Ok(()) - } - - #[test] - fn test_u32_other() -> std::io::Result<()> { - let values = vec![3, 3, 0, 3, 2, 3, 3, 3, 3, 1, 3, 3, 3, 0, 3].into_iter(); - - let mut vec = vec![]; - encode_u32(&mut vec, values, 2)?; - - let expected = vec![5, 207, 254, 247, 51]; - assert_eq!(expected, vec); - Ok(()) - } -} diff --git a/src/common/parquet2/src/encoding/hybrid_rle/mod.rs b/src/common/parquet2/src/encoding/hybrid_rle/mod.rs deleted file mode 100644 index 7c763963612e..000000000000 --- a/src/common/parquet2/src/encoding/hybrid_rle/mod.rs +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// See https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 -mod bitmap; -mod decoder; -mod encoder; -pub use bitmap::encode_bool as bitpacked_encode; -pub use bitmap::BitmapIter; -pub use decoder::Decoder; -pub use encoder::encode_bool; -pub use encoder::encode_u32; - -use super::bitpacked; -use crate::error::Error; - -/// The two possible states of an RLE-encoded run. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HybridEncoded<'a> { - /// A bitpacked slice. The consumer must know its bit-width to unpack it. - Bitpacked(&'a [u8]), - /// A RLE-encoded slice. The first attribute corresponds to the slice (that can be interpreted) - /// the second attribute corresponds to the number of repetitions. - Rle(&'a [u8], usize), -} - -#[derive(Debug, Clone)] -enum State<'a> { - None, - Bitpacked(bitpacked::Decoder<'a, u32>), - Rle(std::iter::Take>), - // Add a special branch for a single value to - // adhere to the strong law of small numbers. - Single(Option), -} - -/// [`Iterator`] of [`u32`] from a byte slice of Hybrid-RLE encoded values -#[derive(Debug, Clone)] -pub struct HybridRleDecoder<'a> { - decoder: Decoder<'a>, - state: State<'a>, - remaining: usize, -} - -#[inline] -fn read_next<'a, 'b>(decoder: &'b mut Decoder<'a>, remaining: usize) -> Result, Error> { - Ok(match decoder.next().transpose()? { - Some(HybridEncoded::Bitpacked(packed)) => { - let num_bits = decoder.num_bits(); - let length = std::cmp::min(packed.len() * 8 / num_bits, remaining); - let decoder = bitpacked::Decoder::::try_new(packed, num_bits, length)?; - State::Bitpacked(decoder) - } - Some(HybridEncoded::Rle(pack, additional)) => { - let mut bytes = [0u8; std::mem::size_of::()]; - pack.iter().zip(bytes.iter_mut()).for_each(|(src, dst)| { - *dst = *src; - }); - let value = u32::from_le_bytes(bytes); - if additional == 1 { - State::Single(Some(value)) - } else { - State::Rle(std::iter::repeat(value).take(additional)) - } - } - None => State::None, - }) -} - -impl<'a> HybridRleDecoder<'a> { - /// Returns a new [`HybridRleDecoder`] - pub fn try_new(data: &'a [u8], num_bits: u32, num_values: usize) -> Result { - let num_bits = num_bits as usize; - let mut decoder = Decoder::new(data, num_bits); - let state = read_next(&mut decoder, num_values)?; - Ok(Self { - decoder, - state, - remaining: num_values, - }) - } -} - -impl<'a> Iterator for HybridRleDecoder<'a> { - type Item = Result; - - fn next(&mut self) -> Option { - if self.remaining == 0 { - return None; - }; - let result = match &mut self.state { - State::Single(opt_val) => { - // make sure to take so that next calls will return 'None' - // indicating that the iterator is finished. - opt_val.take() - } - State::Bitpacked(decoder) => decoder.next(), - State::Rle(iter) => iter.next(), - State::None => Some(0), - }; - if let Some(result) = result { - self.remaining -= 1; - Some(Ok(result)) - } else { - match read_next(&mut self.decoder, self.remaining) { - Ok(state) => { - self.state = state; - self.next() - } - Err(e) => Some(Err(e)), - } - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.remaining, Some(self.remaining)) - } -} - -impl<'a> ExactSizeIterator for HybridRleDecoder<'a> {} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn roundtrip() -> Result<(), Error> { - let mut buffer = vec![]; - let num_bits = 10u32; - - let data = (0..1000).collect::>(); - - encode_u32(&mut buffer, data.iter().cloned(), num_bits).unwrap(); - - let decoder = HybridRleDecoder::try_new(&buffer, num_bits, data.len())?; - - let result = decoder.collect::, _>>()?; - - assert_eq!(result, data); - Ok(()) - } - - #[test] - fn pyarrow_integration() -> Result<(), Error> { - // data encoded from pyarrow representing (0..1000) - let data = vec![ - 127, 0, 4, 32, 192, 0, 4, 20, 96, 192, 1, 8, 36, 160, 192, 2, 12, 52, 224, 192, 3, 16, - 68, 32, 193, 4, 20, 84, 96, 193, 5, 24, 100, 160, 193, 6, 28, 116, 224, 193, 7, 32, - 132, 32, 194, 8, 36, 148, 96, 194, 9, 40, 164, 160, 194, 10, 44, 180, 224, 194, 11, 48, - 196, 32, 195, 12, 52, 212, 96, 195, 13, 56, 228, 160, 195, 14, 60, 244, 224, 195, 15, - 64, 4, 33, 196, 16, 68, 20, 97, 196, 17, 72, 36, 161, 196, 18, 76, 52, 225, 196, 19, - 80, 68, 33, 197, 20, 84, 84, 97, 197, 21, 88, 100, 161, 197, 22, 92, 116, 225, 197, 23, - 96, 132, 33, 198, 24, 100, 148, 97, 198, 25, 104, 164, 161, 198, 26, 108, 180, 225, - 198, 27, 112, 196, 33, 199, 28, 116, 212, 97, 199, 29, 120, 228, 161, 199, 30, 124, - 244, 225, 199, 31, 128, 4, 34, 200, 32, 132, 20, 98, 200, 33, 136, 36, 162, 200, 34, - 140, 52, 226, 200, 35, 144, 68, 34, 201, 36, 148, 84, 98, 201, 37, 152, 100, 162, 201, - 38, 156, 116, 226, 201, 39, 160, 132, 34, 202, 40, 164, 148, 98, 202, 41, 168, 164, - 162, 202, 42, 172, 180, 226, 202, 43, 176, 196, 34, 203, 44, 180, 212, 98, 203, 45, - 184, 228, 162, 203, 46, 188, 244, 226, 203, 47, 192, 4, 35, 204, 48, 196, 20, 99, 204, - 49, 200, 36, 163, 204, 50, 204, 52, 227, 204, 51, 208, 68, 35, 205, 52, 212, 84, 99, - 205, 53, 216, 100, 163, 205, 54, 220, 116, 227, 205, 55, 224, 132, 35, 206, 56, 228, - 148, 99, 206, 57, 232, 164, 163, 206, 58, 236, 180, 227, 206, 59, 240, 196, 35, 207, - 60, 244, 212, 99, 207, 61, 248, 228, 163, 207, 62, 252, 244, 227, 207, 63, 0, 5, 36, - 208, 64, 4, 21, 100, 208, 65, 8, 37, 164, 208, 66, 12, 53, 228, 208, 67, 16, 69, 36, - 209, 68, 20, 85, 100, 209, 69, 24, 101, 164, 209, 70, 28, 117, 228, 209, 71, 32, 133, - 36, 210, 72, 36, 149, 100, 210, 73, 40, 165, 164, 210, 74, 44, 181, 228, 210, 75, 48, - 197, 36, 211, 76, 52, 213, 100, 211, 77, 56, 229, 164, 211, 78, 60, 245, 228, 211, 79, - 64, 5, 37, 212, 80, 68, 21, 101, 212, 81, 72, 37, 165, 212, 82, 76, 53, 229, 212, 83, - 80, 69, 37, 213, 84, 84, 85, 101, 213, 85, 88, 101, 165, 213, 86, 92, 117, 229, 213, - 87, 96, 133, 37, 214, 88, 100, 149, 101, 214, 89, 104, 165, 165, 214, 90, 108, 181, - 229, 214, 91, 112, 197, 37, 215, 92, 116, 213, 101, 215, 93, 120, 229, 165, 215, 94, - 124, 245, 229, 215, 95, 128, 5, 38, 216, 96, 132, 21, 102, 216, 97, 136, 37, 166, 216, - 98, 140, 53, 230, 216, 99, 144, 69, 38, 217, 100, 148, 85, 102, 217, 101, 152, 101, - 166, 217, 102, 156, 117, 230, 217, 103, 160, 133, 38, 218, 104, 164, 149, 102, 218, - 105, 168, 165, 166, 218, 106, 172, 181, 230, 218, 107, 176, 197, 38, 219, 108, 180, - 213, 102, 219, 109, 184, 229, 166, 219, 110, 188, 245, 230, 219, 111, 192, 5, 39, 220, - 112, 196, 21, 103, 220, 113, 200, 37, 167, 220, 114, 204, 53, 231, 220, 115, 208, 69, - 39, 221, 116, 212, 85, 103, 221, 117, 216, 101, 167, 221, 118, 220, 117, 231, 221, 119, - 224, 133, 39, 222, 120, 228, 149, 103, 222, 121, 232, 165, 167, 222, 122, 236, 181, - 231, 222, 123, 240, 197, 39, 223, 124, 244, 213, 103, 223, 125, 125, 248, 229, 167, - 223, 126, 252, 245, 231, 223, 127, 0, 6, 40, 224, 128, 4, 22, 104, 224, 129, 8, 38, - 168, 224, 130, 12, 54, 232, 224, 131, 16, 70, 40, 225, 132, 20, 86, 104, 225, 133, 24, - 102, 168, 225, 134, 28, 118, 232, 225, 135, 32, 134, 40, 226, 136, 36, 150, 104, 226, - 137, 40, 166, 168, 226, 138, 44, 182, 232, 226, 139, 48, 198, 40, 227, 140, 52, 214, - 104, 227, 141, 56, 230, 168, 227, 142, 60, 246, 232, 227, 143, 64, 6, 41, 228, 144, 68, - 22, 105, 228, 145, 72, 38, 169, 228, 146, 76, 54, 233, 228, 147, 80, 70, 41, 229, 148, - 84, 86, 105, 229, 149, 88, 102, 169, 229, 150, 92, 118, 233, 229, 151, 96, 134, 41, - 230, 152, 100, 150, 105, 230, 153, 104, 166, 169, 230, 154, 108, 182, 233, 230, 155, - 112, 198, 41, 231, 156, 116, 214, 105, 231, 157, 120, 230, 169, 231, 158, 124, 246, - 233, 231, 159, 128, 6, 42, 232, 160, 132, 22, 106, 232, 161, 136, 38, 170, 232, 162, - 140, 54, 234, 232, 163, 144, 70, 42, 233, 164, 148, 86, 106, 233, 165, 152, 102, 170, - 233, 166, 156, 118, 234, 233, 167, 160, 134, 42, 234, 168, 164, 150, 106, 234, 169, - 168, 166, 170, 234, 170, 172, 182, 234, 234, 171, 176, 198, 42, 235, 172, 180, 214, - 106, 235, 173, 184, 230, 170, 235, 174, 188, 246, 234, 235, 175, 192, 6, 43, 236, 176, - 196, 22, 107, 236, 177, 200, 38, 171, 236, 178, 204, 54, 235, 236, 179, 208, 70, 43, - 237, 180, 212, 86, 107, 237, 181, 216, 102, 171, 237, 182, 220, 118, 235, 237, 183, - 224, 134, 43, 238, 184, 228, 150, 107, 238, 185, 232, 166, 171, 238, 186, 236, 182, - 235, 238, 187, 240, 198, 43, 239, 188, 244, 214, 107, 239, 189, 248, 230, 171, 239, - 190, 252, 246, 235, 239, 191, 0, 7, 44, 240, 192, 4, 23, 108, 240, 193, 8, 39, 172, - 240, 194, 12, 55, 236, 240, 195, 16, 71, 44, 241, 196, 20, 87, 108, 241, 197, 24, 103, - 172, 241, 198, 28, 119, 236, 241, 199, 32, 135, 44, 242, 200, 36, 151, 108, 242, 201, - 40, 167, 172, 242, 202, 44, 183, 236, 242, 203, 48, 199, 44, 243, 204, 52, 215, 108, - 243, 205, 56, 231, 172, 243, 206, 60, 247, 236, 243, 207, 64, 7, 45, 244, 208, 68, 23, - 109, 244, 209, 72, 39, 173, 244, 210, 76, 55, 237, 244, 211, 80, 71, 45, 245, 212, 84, - 87, 109, 245, 213, 88, 103, 173, 245, 214, 92, 119, 237, 245, 215, 96, 135, 45, 246, - 216, 100, 151, 109, 246, 217, 104, 167, 173, 246, 218, 108, 183, 237, 246, 219, 112, - 199, 45, 247, 220, 116, 215, 109, 247, 221, 120, 231, 173, 247, 222, 124, 247, 237, - 247, 223, 128, 7, 46, 248, 224, 132, 23, 110, 248, 225, 136, 39, 174, 248, 226, 140, - 55, 238, 248, 227, 144, 71, 46, 249, 228, 148, 87, 110, 249, 229, 152, 103, 174, 249, - 230, 156, 119, 238, 249, 231, 160, 135, 46, 250, 232, 164, 151, 110, 250, 233, 168, - 167, 174, 250, 234, 172, 183, 238, 250, 235, 176, 199, 46, 251, 236, 180, 215, 110, - 251, 237, 184, 231, 174, 251, 238, 188, 247, 238, 251, 239, 192, 7, 47, 252, 240, 196, - 23, 111, 252, 241, 200, 39, 175, 252, 242, 204, 55, 239, 252, 243, 208, 71, 47, 253, - 244, 212, 87, 111, 253, 245, 216, 103, 175, 253, 246, 220, 119, 239, 253, 247, 224, - 135, 47, 254, 248, 228, 151, 111, 254, 249, - ]; - let num_bits = 10; - - let decoder = HybridRleDecoder::try_new(&data, num_bits, 1000)?; - - let result = decoder.collect::, _>>()?; - - assert_eq!(result, (0..1000).collect::>()); - Ok(()) - } - - #[test] - fn small() -> Result<(), Error> { - let data = vec![3, 2]; - - let num_bits = 3; - - let decoder = HybridRleDecoder::try_new(&data, num_bits, 1)?; - - let result = decoder.collect::, _>>()?; - - assert_eq!(result, &[2]); - Ok(()) - } - - #[test] - fn zero_bit_width() -> Result<(), Error> { - let data = vec![3]; - - let num_bits = 0; - - let decoder = HybridRleDecoder::try_new(&data, num_bits, 2)?; - - let result = decoder.collect::, _>>()?; - - assert_eq!(result, &[0, 0]); - Ok(()) - } - - #[test] - fn empty_values() -> Result<(), Error> { - let data = []; - - let num_bits = 1; - - let decoder = HybridRleDecoder::try_new(&data, num_bits, 100)?; - - let result = decoder.collect::, _>>()?; - - assert_eq!(result, vec![0; 100]); - Ok(()) - } -} diff --git a/src/common/parquet2/src/encoding/mod.rs b/src/common/parquet2/src/encoding/mod.rs deleted file mode 100644 index 8db844dc98c8..000000000000 --- a/src/common/parquet2/src/encoding/mod.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::convert::TryInto; - -pub mod bitpacked; -pub mod delta_bitpacked; -pub mod delta_byte_array; -pub mod delta_length_byte_array; -pub mod hybrid_rle; -pub mod plain_byte_array; -pub mod uleb128; -pub mod zigzag_leb128; - -pub use crate::parquet_bridge::Encoding; - -/// # Panics -/// This function panics iff `values.len() < 4`. -#[inline] -pub fn get_length(values: &[u8]) -> Option { - values - .get(0..4) - .map(|x| u32::from_le_bytes(x.try_into().unwrap()) as usize) -} - -/// Returns the ceil of value / 8 -#[inline] -pub fn ceil8(value: usize) -> usize { - value / 8 + ((value % 8 != 0) as usize) -} diff --git a/src/common/parquet2/src/encoding/plain_byte_array.rs b/src/common/parquet2/src/encoding/plain_byte_array.rs deleted file mode 100644 index 5b803e74b52f..000000000000 --- a/src/common/parquet2/src/encoding/plain_byte_array.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/// Decodes according to [Plain strings](https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0), -/// prefixes, lengths and values -/// # Implementation -/// This struct does not allocate on the heap. -use crate::error::Error; - -#[derive(Debug)] -pub struct BinaryIter<'a> { - values: &'a [u8], - length: Option, -} - -impl<'a> BinaryIter<'a> { - pub fn new(values: &'a [u8], length: Option) -> Self { - Self { values, length } - } -} - -impl<'a> Iterator for BinaryIter<'a> { - type Item = Result<&'a [u8], Error>; - - #[inline] - fn next(&mut self) -> Option { - if self.values.len() < 4 { - return None; - } - if let Some(x) = self.length.as_mut() { - *x = x.saturating_sub(1) - } - let length = u32::from_le_bytes(self.values[0..4].try_into().unwrap()) as usize; - self.values = &self.values[4..]; - if length > self.values.len() { - return Some(Err(Error::oos( - "A string in plain encoding declares a length that is out of range", - ))); - } - let (result, remaining) = self.values.split_at(length); - self.values = remaining; - Some(Ok(result)) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.length.unwrap_or_default(), self.length) - } -} diff --git a/src/common/parquet2/src/encoding/uleb128.rs b/src/common/parquet2/src/encoding/uleb128.rs deleted file mode 100644 index 575e8acdd490..000000000000 --- a/src/common/parquet2/src/encoding/uleb128.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::error::Error; - -pub fn decode(values: &[u8]) -> Result<(u64, usize), Error> { - let mut result = 0; - let mut shift = 0; - - let mut consumed = 0; - for byte in values { - consumed += 1; - if shift == 63 && *byte > 1 { - panic!() - }; - - result |= u64::from(byte & 0b01111111) << shift; - - if byte & 0b10000000 == 0 { - break; - } - - shift += 7; - } - Ok((result, consumed)) -} - -/// Encodes `value` in ULEB128 into `container`. The exact number of bytes written -/// depends on `value`, and cannot be determined upfront. The maximum number of bytes -/// required are 10. -/// # Panic -/// This function may panic if `container.len() < 10` and `value` requires more bytes. -pub fn encode(mut value: u64, container: &mut [u8]) -> usize { - let mut consumed = 0; - let mut iter = container.iter_mut(); - loop { - let mut byte = (value as u8) & !128; - value >>= 7; - if value != 0 { - byte |= 128; - } - *iter.next().unwrap() = byte; - consumed += 1; - if value == 0 { - break; - } - } - consumed -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn decode_1() { - let data = vec![0xe5, 0x8e, 0x26, 0xDE, 0xAD, 0xBE, 0xEF]; - let (value, len) = decode(&data).unwrap(); - assert_eq!(value, 624_485); - assert_eq!(len, 3); - } - - #[test] - fn decode_2() { - let data = vec![0b00010000, 0b00000001, 0b00000011, 0b00000011]; - let (value, len) = decode(&data).unwrap(); - assert_eq!(value, 16); - assert_eq!(len, 1); - } - - #[test] - fn round_trip() { - let original = 123124234u64; - let mut container = [0u8; 10]; - let encoded_len = encode(original, &mut container); - let (value, len) = decode(&container).unwrap(); - assert_eq!(value, original); - assert_eq!(len, encoded_len); - } - - #[test] - fn min_value() { - let original = u64::MIN; - let mut container = [0u8; 10]; - let encoded_len = encode(original, &mut container); - let (value, len) = decode(&container).unwrap(); - assert_eq!(value, original); - assert_eq!(len, encoded_len); - } - - #[test] - fn max_value() { - let original = u64::MAX; - let mut container = [0u8; 10]; - let encoded_len = encode(original, &mut container); - let (value, len) = decode(&container).unwrap(); - assert_eq!(value, original); - assert_eq!(len, encoded_len); - } -} diff --git a/src/common/parquet2/src/encoding/zigzag_leb128.rs b/src/common/parquet2/src/encoding/zigzag_leb128.rs deleted file mode 100644 index 09847dd71940..000000000000 --- a/src/common/parquet2/src/encoding/zigzag_leb128.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::uleb128; -use crate::error::Error; - -pub fn decode(values: &[u8]) -> Result<(i64, usize), Error> { - let (u, consumed) = uleb128::decode(values)?; - Ok(((u >> 1) as i64 ^ -((u & 1) as i64), consumed)) -} - -pub fn encode(value: i64) -> ([u8; 10], usize) { - let value = ((value << 1) ^ (value >> (64 - 1))) as u64; - let mut a = [0u8; 10]; - let produced = uleb128::encode(value, &mut a); - (a, produced) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_decode() { - // see e.g. https://stackoverflow.com/a/2211086/931303 - let cases = vec![ - (0u8, 0i64), - (1, -1), - (2, 1), - (3, -2), - (4, 2), - (5, -3), - (6, 3), - (7, -4), - (8, 4), - (9, -5), - ]; - for (data, expected) in cases { - let (result, _) = decode(&[data]).unwrap(); - assert_eq!(result, expected) - } - } - - #[test] - fn test_encode() { - let cases = vec![ - (0u8, 0i64), - (1, -1), - (2, 1), - (3, -2), - (4, 2), - (5, -3), - (6, 3), - (7, -4), - (8, 4), - (9, -5), - ]; - for (expected, data) in cases { - let (result, size) = encode(data); - assert_eq!(size, 1); - assert_eq!(result[0], expected) - } - } - - #[test] - fn test_roundtrip() { - let value = -1001212312; - let (data, size) = encode(value); - let (result, _) = decode(&data[..size]).unwrap(); - assert_eq!(value, result); - } -} diff --git a/src/common/parquet2/src/error.rs b/src/common/parquet2/src/error.rs deleted file mode 100644 index b021c6115bd9..000000000000 --- a/src/common/parquet2/src/error.rs +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains [`Error`] - -/// List of features whose non-activation may cause a runtime error. -/// Used to indicate which lack of feature caused [`Error::FeatureNotActive`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[non_exhaustive] -pub enum Feature { - /// Snappy compression and decompression - Snappy, - /// Brotli compression and decompression - Brotli, - /// Gzip compression and decompression - Gzip, - /// Lz4 raw compression and decompression - Lz4, - /// Zstd compression and decompression - Zstd, -} - -/// Errors generated by this crate -#[derive(Debug, Clone)] -#[non_exhaustive] -pub enum Error { - /// When the parquet file is known to be out of spec. - OutOfSpec(String), - /// Error presented when trying to use a code branch that requires activating a feature. - FeatureNotActive(Feature, String), - /// Error presented when trying to use a feature from parquet that is not yet supported - FeatureNotSupported(String), - /// When encoding, the user passed an invalid parameter - InvalidParameter(String), - /// When decoding or decompressing, the page would allocate more memory than allowed - WouldOverAllocate, -} - -impl Error { - pub(crate) fn oos>(message: I) -> Self { - Self::OutOfSpec(message.into()) - } -} - -impl std::error::Error for Error {} - -impl std::fmt::Display for Error { - fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Error::OutOfSpec(message) => { - write!(fmt, "File out of specification: {}", message) - } - Error::FeatureNotActive(feature, reason) => { - write!( - fmt, - "The feature \"{:?}\" needs to be active to {}", - feature, reason - ) - } - Error::FeatureNotSupported(reason) => { - write!(fmt, "Not yet supported: {}", reason) - } - Error::InvalidParameter(message) => { - write!(fmt, "Invalid parameter: {}", message) - } - Error::WouldOverAllocate => { - write!(fmt, "Operation would exceed memory use threshold") - } - } - } -} - -#[cfg(feature = "snappy")] -impl From for Error { - fn from(e: snap::Error) -> Error { - Error::OutOfSpec(format!("underlying snap error: {}", e)) - } -} - -impl From for Error { - fn from(e: parquet_format_safe::thrift::Error) -> Error { - Error::OutOfSpec(format!("Invalid thrift: {}", e)) - } -} - -impl From for Error { - fn from(e: std::io::Error) -> Error { - Error::OutOfSpec(format!("underlying IO error: {}", e)) - } -} - -impl From for Error { - fn from(e: std::collections::TryReserveError) -> Error { - Error::OutOfSpec(format!("OOM: {}", e)) - } -} - -impl From for Error { - fn from(e: std::num::TryFromIntError) -> Error { - Error::OutOfSpec(format!("Number must be zero or positive: {}", e)) - } -} - -impl From for Error { - fn from(e: std::array::TryFromSliceError) -> Error { - Error::OutOfSpec(format!("Can't deserialize to parquet native type: {}", e)) - } -} - -/// A specialized `Result` for Parquet errors. -pub type Result = std::result::Result; diff --git a/src/common/parquet2/src/indexes/index.rs b/src/common/parquet2/src/indexes/index.rs deleted file mode 100644 index 3199847d7e7a..000000000000 --- a/src/common/parquet2/src/indexes/index.rs +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::any::Any; - -use parquet_format_safe::ColumnIndex; - -use crate::error::Error; -use crate::parquet_bridge::BoundaryOrder; -use crate::schema::types::PhysicalType; -use crate::schema::types::PrimitiveType; -use crate::types::NativeType; - -/// Trait object representing a [`ColumnIndex`] in Rust's native format. -/// -/// See [`NativeIndex`], [`ByteIndex`] and [`FixedLenByteIndex`] for concrete implementations. -pub trait Index: Send + Sync + std::fmt::Debug { - fn as_any(&self) -> &dyn Any; - - fn physical_type(&self) -> &PhysicalType; -} - -impl PartialEq for dyn Index + '_ { - fn eq(&self, that: &dyn Index) -> bool { - equal(self, that) - } -} - -impl Eq for dyn Index + '_ {} - -fn equal(lhs: &dyn Index, rhs: &dyn Index) -> bool { - if lhs.physical_type() != rhs.physical_type() { - return false; - } - - match lhs.physical_type() { - PhysicalType::Boolean => { - lhs.as_any().downcast_ref::().unwrap() - == rhs.as_any().downcast_ref::().unwrap() - } - PhysicalType::Int32 => { - lhs.as_any().downcast_ref::>().unwrap() - == rhs.as_any().downcast_ref::>().unwrap() - } - PhysicalType::Int64 => { - lhs.as_any().downcast_ref::>().unwrap() - == rhs.as_any().downcast_ref::>().unwrap() - } - PhysicalType::Int96 => { - lhs.as_any() - .downcast_ref::>() - .unwrap() - == rhs - .as_any() - .downcast_ref::>() - .unwrap() - } - PhysicalType::Float => { - lhs.as_any().downcast_ref::>().unwrap() - == rhs.as_any().downcast_ref::>().unwrap() - } - PhysicalType::Double => { - lhs.as_any().downcast_ref::>().unwrap() - == rhs.as_any().downcast_ref::>().unwrap() - } - PhysicalType::ByteArray => { - lhs.as_any().downcast_ref::().unwrap() - == rhs.as_any().downcast_ref::().unwrap() - } - PhysicalType::FixedLenByteArray(_) => { - lhs.as_any().downcast_ref::().unwrap() - == rhs.as_any().downcast_ref::().unwrap() - } - } -} - -/// An index of a column of [`NativeType`] physical representation -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct NativeIndex { - /// The primitive type - pub primitive_type: PrimitiveType, - /// The indexes, one item per page - pub indexes: Vec>, - /// the order - pub boundary_order: BoundaryOrder, -} - -impl NativeIndex { - /// Creates a new [`NativeIndex`] - pub(crate) fn try_new( - index: ColumnIndex, - primitive_type: PrimitiveType, - ) -> Result { - let len = index.min_values.len(); - - let null_counts = index - .null_counts - .map(|x| x.into_iter().map(Some).collect::>()) - .unwrap_or_else(|| vec![None; len]); - - let indexes = index - .min_values - .iter() - .zip(index.max_values.into_iter()) - .zip(index.null_pages.into_iter()) - .zip(null_counts.into_iter()) - .map(|(((min, max), is_null), null_count)| { - let (min, max) = if is_null { - (None, None) - } else { - let min = min.as_slice().try_into()?; - let max = max.as_slice().try_into()?; - (Some(T::from_le_bytes(min)), Some(T::from_le_bytes(max))) - }; - Ok(PageIndex { - min, - max, - null_count, - }) - }) - .collect::, Error>>()?; - - Ok(Self { - primitive_type, - indexes, - boundary_order: index.boundary_order.try_into()?, - }) - } -} - -/// The index of a page, containing the min and max values of the page. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct PageIndex { - /// The minimum value in the page. It is None when all values are null - pub min: Option, - /// The maximum value in the page. It is None when all values are null - pub max: Option, - /// The number of null values in the page - pub null_count: Option, -} - -impl Index for NativeIndex { - fn as_any(&self) -> &dyn Any { - self - } - - fn physical_type(&self) -> &PhysicalType { - &T::TYPE - } -} - -/// An index of a column of bytes physical type -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ByteIndex { - /// The [`PrimitiveType`]. - pub primitive_type: PrimitiveType, - /// The indexes, one item per page - pub indexes: Vec>>, - pub boundary_order: BoundaryOrder, -} - -impl ByteIndex { - pub(crate) fn try_new( - index: ColumnIndex, - primitive_type: PrimitiveType, - ) -> Result { - let len = index.min_values.len(); - - let null_counts = index - .null_counts - .map(|x| x.into_iter().map(Some).collect::>()) - .unwrap_or_else(|| vec![None; len]); - - let indexes = index - .min_values - .into_iter() - .zip(index.max_values.into_iter()) - .zip(index.null_pages.into_iter()) - .zip(null_counts.into_iter()) - .map(|(((min, max), is_null), null_count)| { - let (min, max) = if is_null { - (None, None) - } else { - (Some(min), Some(max)) - }; - Ok(PageIndex { - min, - max, - null_count, - }) - }) - .collect::, Error>>()?; - - Ok(Self { - primitive_type, - indexes, - boundary_order: index.boundary_order.try_into()?, - }) - } -} - -impl Index for ByteIndex { - fn as_any(&self) -> &dyn Any { - self - } - - fn physical_type(&self) -> &PhysicalType { - &PhysicalType::ByteArray - } -} - -/// An index of a column of fixed len byte physical type -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FixedLenByteIndex { - /// The [`PrimitiveType`]. - pub primitive_type: PrimitiveType, - /// The indexes, one item per page - pub indexes: Vec>>, - pub boundary_order: BoundaryOrder, -} - -impl FixedLenByteIndex { - pub(crate) fn try_new( - index: ColumnIndex, - primitive_type: PrimitiveType, - ) -> Result { - let len = index.min_values.len(); - - let null_counts = index - .null_counts - .map(|x| x.into_iter().map(Some).collect::>()) - .unwrap_or_else(|| vec![None; len]); - - let indexes = index - .min_values - .into_iter() - .zip(index.max_values.into_iter()) - .zip(index.null_pages.into_iter()) - .zip(null_counts.into_iter()) - .map(|(((min, max), is_null), null_count)| { - let (min, max) = if is_null { - (None, None) - } else { - (Some(min), Some(max)) - }; - Ok(PageIndex { - min, - max, - null_count, - }) - }) - .collect::, Error>>()?; - - Ok(Self { - primitive_type, - indexes, - boundary_order: index.boundary_order.try_into()?, - }) - } -} - -impl Index for FixedLenByteIndex { - fn as_any(&self) -> &dyn Any { - self - } - - fn physical_type(&self) -> &PhysicalType { - &self.primitive_type.physical_type - } -} - -/// An index of a column of boolean physical type -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct BooleanIndex { - /// The indexes, one item per page - pub indexes: Vec>, - pub boundary_order: BoundaryOrder, -} - -impl BooleanIndex { - pub(crate) fn try_new(index: ColumnIndex) -> Result { - let len = index.min_values.len(); - - let null_counts = index - .null_counts - .map(|x| x.into_iter().map(Some).collect::>()) - .unwrap_or_else(|| vec![None; len]); - - let indexes = index - .min_values - .into_iter() - .zip(index.max_values.into_iter()) - .zip(index.null_pages.into_iter()) - .zip(null_counts.into_iter()) - .map(|(((min, max), is_null), null_count)| { - let (min, max) = if is_null { - (None, None) - } else { - let min = min[0] == 1; - let max = max[0] == 1; - (Some(min), Some(max)) - }; - Ok(PageIndex { - min, - max, - null_count, - }) - }) - .collect::, Error>>()?; - - Ok(Self { - indexes, - boundary_order: index.boundary_order.try_into()?, - }) - } -} - -impl Index for BooleanIndex { - fn as_any(&self) -> &dyn Any { - self - } - - fn physical_type(&self) -> &PhysicalType { - &PhysicalType::Boolean - } -} diff --git a/src/common/parquet2/src/indexes/intervals.rs b/src/common/parquet2/src/indexes/intervals.rs deleted file mode 100644 index 71d1deed1944..000000000000 --- a/src/common/parquet2/src/indexes/intervals.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::PageLocation; -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use crate::error::Error; - -/// An interval -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct Interval { - /// Its start - pub start: usize, - /// Its length - pub length: usize, -} - -impl Interval { - /// Create a new interal - pub fn new(start: usize, length: usize) -> Self { - Self { start, length } - } -} - -/// Returns the set of (row) intervals of the pages. -/// # Errors -/// This function errors if the locations are not castable to `usize` or such that -/// their ranges of row are larger than `num_rows`. -pub fn compute_page_row_intervals( - locations: &[PageLocation], - num_rows: usize, -) -> Result, Error> { - if locations.is_empty() { - return Ok(vec![]); - }; - - let last = (|| { - let start: usize = locations.last().unwrap().first_row_index.try_into()?; - let length = num_rows - .checked_sub(start) - .ok_or_else(|| Error::oos("Page start cannot be smaller than the number of rows"))?; - Result::<_, Error>::Ok(Interval::new(start, length)) - })(); - - let pages_lengths = locations - .windows(2) - .map(|x| { - let start = x[0].first_row_index.try_into()?; - - let length = x[1] - .first_row_index - .checked_sub(x[0].first_row_index) - .ok_or_else(|| Error::oos("Page start cannot be smaller than the number of rows"))? - .try_into()?; - - Ok(Interval::new(start, length)) - }) - .chain(std::iter::once(last)); - pages_lengths.collect() -} - -/// Returns the set of intervals `(start, len)` containing all the -/// selected rows (for a given column) -pub fn compute_rows( - selected: &[bool], - locations: &[PageLocation], - num_rows: usize, -) -> Result, Error> { - let page_intervals = compute_page_row_intervals(locations, num_rows)?; - - Ok(selected - .iter() - .zip(page_intervals.iter().copied()) - .filter_map( - |(&is_selected, page)| { - if is_selected { Some(page) } else { None } - }, - ) - .collect()) -} - -/// An enum describing a page that was either selected in a filter pushdown or skipped -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct FilteredPage { - /// Location of the page in the file - pub start: u64, - pub length: usize, - /// rows to select from the page - pub selected_rows: Vec, - pub num_rows: usize, -} - -fn is_in(probe: Interval, intervals: &[Interval]) -> Vec { - intervals - .iter() - .filter_map(|interval| { - let interval_end = interval.start + interval.length; - let probe_end = probe.start + probe.length; - let overlaps = (probe.start < interval_end) && (probe_end > interval.start); - if overlaps { - let start = interval.start.max(probe.start); - let end = interval_end.min(probe_end); - Some(Interval::new(start - probe.start, end - start)) - } else { - None - } - }) - .collect() -} - -/// Given a set of selected [Interval]s of rows and the set of [`PageLocation`], returns the -/// a set of [`FilteredPage`] with the same number of items as `locations`. -pub fn select_pages( - intervals: &[Interval], - locations: &[PageLocation], - num_rows: usize, -) -> Result, Error> { - let page_intervals = compute_page_row_intervals(locations, num_rows)?; - - page_intervals - .into_iter() - .zip(locations.iter()) - .map(|(interval, location)| { - let selected_rows = is_in(interval, intervals); - Ok(FilteredPage { - start: location.offset.try_into()?, - length: location.compressed_page_size.try_into()?, - selected_rows, - num_rows: interval.length, - }) - }) - .collect() -} diff --git a/src/common/parquet2/src/indexes/mod.rs b/src/common/parquet2/src/indexes/mod.rs deleted file mode 100644 index 6f462ac609d7..000000000000 --- a/src/common/parquet2/src/indexes/mod.rs +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod index; -mod intervals; - -pub use intervals::compute_rows; -pub use intervals::select_pages; -pub use intervals::FilteredPage; -pub use intervals::Interval; - -pub use self::index::BooleanIndex; -pub use self::index::ByteIndex; -pub use self::index::FixedLenByteIndex; -pub use self::index::Index; -pub use self::index::NativeIndex; -pub use self::index::PageIndex; -pub use crate::parquet_bridge::BoundaryOrder; -pub use crate::thrift_format::PageLocation; - -#[cfg(test)] -mod tests { - use super::*; - use crate::schema::types::PhysicalType; - use crate::schema::types::PrimitiveType; - - #[test] - fn test_basic() { - let locations = &[PageLocation { - offset: 100, - compressed_page_size: 10, - first_row_index: 0, - }]; - let num_rows = 10; - - let row_intervals = compute_rows(&[true; 1], locations, num_rows).unwrap(); - assert_eq!(row_intervals, vec![Interval::new(0, 10)]) - } - - #[test] - fn test_multiple() { - // two pages - let index = ByteIndex { - primitive_type: PrimitiveType::from_physical("c1".to_string(), PhysicalType::ByteArray), - indexes: vec![ - PageIndex { - min: Some(vec![0]), - max: Some(vec![8, 9]), - null_count: Some(0), - }, - PageIndex { - min: Some(vec![20]), - max: Some(vec![98, 99]), - null_count: Some(0), - }, - ], - boundary_order: Default::default(), - }; - let locations = &[ - PageLocation { - offset: 100, - compressed_page_size: 10, - first_row_index: 0, - }, - PageLocation { - offset: 110, - compressed_page_size: 20, - first_row_index: 5, - }, - ]; - let num_rows = 10; - - // filter of the form `x > "a"` - let selector = |page: &PageIndex>| { - page.max - .as_ref() - .map(|x| x.as_slice()[0] > 97) - .unwrap_or(false) // no max is present => all nulls => not selected - }; - let selected = index.indexes.iter().map(selector).collect::>(); - - let rows = compute_rows(&selected, locations, num_rows).unwrap(); - assert_eq!(rows, vec![Interval::new(5, 5)]); - - let pages = select_pages(&rows, locations, num_rows).unwrap(); - - assert_eq!(pages, vec![ - FilteredPage { - start: 100, - length: 10, - selected_rows: vec![], - num_rows: 5 - }, - FilteredPage { - start: 110, - length: 20, - selected_rows: vec![Interval::new(0, 5)], - num_rows: 5 - } - ]); - } - - #[test] - fn test_other_column() { - let locations = &[ - PageLocation { - offset: 100, - compressed_page_size: 20, - first_row_index: 0, - }, - PageLocation { - offset: 120, - compressed_page_size: 20, - first_row_index: 10, - }, - ]; - let num_rows = 100; - - let intervals = &[Interval::new(5, 5)]; - - let pages = select_pages(intervals, locations, num_rows).unwrap(); - - assert_eq!(pages, vec![ - FilteredPage { - start: 100, - length: 20, - selected_rows: vec![Interval::new(5, 5)], - num_rows: 10, - }, - FilteredPage { - start: 120, - length: 20, - selected_rows: vec![], - num_rows: 90 - }, - ]); - } - - #[test] - fn test_other_interval_in_middle() { - let locations = &[ - PageLocation { - offset: 100, - compressed_page_size: 20, - first_row_index: 0, - }, - PageLocation { - offset: 120, - compressed_page_size: 20, - first_row_index: 10, - }, - PageLocation { - offset: 140, - compressed_page_size: 20, - first_row_index: 100, - }, - ]; - let num_rows = 200; - - // interval partially intersects 2 pages (0 and 1) - let intervals = &[Interval::new(5, 6)]; - - let pages = select_pages(intervals, locations, num_rows).unwrap(); - - assert_eq!(pages, vec![ - FilteredPage { - start: 100, - length: 20, - selected_rows: vec![Interval::new(5, 5)], - num_rows: 10, - }, - FilteredPage { - start: 120, - length: 20, - selected_rows: vec![Interval::new(0, 1)], - num_rows: 90, - }, - FilteredPage { - start: 140, - length: 20, - selected_rows: vec![], - num_rows: 100 - }, - ]); - } - - #[test] - fn test_other_column2() { - let locations = &[ - PageLocation { - offset: 100, - compressed_page_size: 20, - first_row_index: 0, - }, - PageLocation { - offset: 120, - compressed_page_size: 20, - first_row_index: 10, - }, - PageLocation { - offset: 140, - compressed_page_size: 20, - first_row_index: 100, - }, - ]; - let num_rows = 200; - - // interval partially intersects 1 page (0) - let intervals = &[Interval::new(0, 1)]; - - let pages = select_pages(intervals, locations, num_rows).unwrap(); - - assert_eq!(pages, vec![ - FilteredPage { - start: 100, - length: 20, - selected_rows: vec![Interval::new(0, 1)], - num_rows: 10, - }, - FilteredPage { - start: 120, - length: 20, - selected_rows: vec![], - num_rows: 90 - }, - FilteredPage { - start: 140, - length: 20, - selected_rows: vec![], - num_rows: 100 - }, - ]); - } -} diff --git a/src/common/parquet2/src/lib.rs b/src/common/parquet2/src/lib.rs deleted file mode 100644 index f19c160ab172..000000000000 --- a/src/common/parquet2/src/lib.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This code is a fork from upstream and will be removed soon, so we allow all clippy lints. -#![allow(clippy::all)] -#![forbid(unsafe_code)] -#![cfg_attr(docsrs, feature(doc_cfg))] -/// Unofficial implementation of parquet IO in Rust. - -#[macro_use] -pub mod error; -#[cfg(feature = "bloom_filter")] -pub mod bloom_filter; -pub mod compression; -pub mod deserialize; -pub mod encoding; -pub mod indexes; -pub mod metadata; -pub mod page; -mod parquet_bridge; -pub mod read; -pub mod schema; -pub mod statistics; -pub mod types; -pub mod write; - -use parquet_format_safe as thrift_format; -pub use streaming_decompression::fallible_streaming_iterator; -pub use streaming_decompression::FallibleStreamingIterator; - -const HEADER_SIZE: u64 = PARQUET_MAGIC.len() as u64; -const FOOTER_SIZE: u64 = 8; -const PARQUET_MAGIC: [u8; 4] = [b'P', b'A', b'R', b'1']; - -/// The number of bytes read at the end of the parquet file on first read -const DEFAULT_FOOTER_READ_SIZE: u64 = 64 * 1024; diff --git a/src/common/parquet2/src/metadata/column_chunk_metadata.rs b/src/common/parquet2/src/metadata/column_chunk_metadata.rs deleted file mode 100644 index 25ecbe9f13b1..000000000000 --- a/src/common/parquet2/src/metadata/column_chunk_metadata.rs +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use parquet_format_safe::ColumnChunk; -use parquet_format_safe::ColumnMetaData; -use parquet_format_safe::Encoding; - -use super::column_descriptor::ColumnDescriptor; -use crate::compression::Compression; -use crate::error::Error; -use crate::error::Result; -use crate::schema::types::PhysicalType; -use crate::statistics::deserialize_statistics; -use crate::statistics::Statistics; - -#[cfg(feature = "serde_types")] -mod serde_types { - pub use std::io::Cursor; - - pub use parquet_format_safe::thrift::protocol::TCompactInputProtocol; - pub use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; - pub use serde::de::Error as DeserializeError; - pub use serde::ser::Error as SerializeError; - pub use serde::Deserialize; - pub use serde::Deserializer; - pub use serde::Serialize; - pub use serde::Serializer; -} -#[cfg(feature = "serde_types")] -use serde_types::*; - -/// Metadata for a column chunk. -// This contains the `ColumnDescriptor` associated with the chunk so that deserializers have -// access to the descriptor (e.g. physical, converted, logical). -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct ColumnChunkMetaData { - #[cfg_attr( - feature = "serde_types", - serde(serialize_with = "serialize_column_chunk") - )] - #[cfg_attr( - feature = "serde_types", - serde(deserialize_with = "deserialize_column_chunk") - )] - column_chunk: ColumnChunk, - column_descr: ColumnDescriptor, -} - -#[cfg(feature = "serde_types")] -fn serialize_column_chunk( - column_chunk: &ColumnChunk, - serializer: S, -) -> std::result::Result -where - S: Serializer, -{ - let mut buf = vec![]; - let cursor = Cursor::new(&mut buf[..]); - let mut protocol = TCompactOutputProtocol::new(cursor); - column_chunk - .write_to_out_protocol(&mut protocol) - .map_err(S::Error::custom)?; - serializer.serialize_bytes(&buf) -} - -#[cfg(feature = "serde_types")] -fn deserialize_column_chunk<'de, D>(deserializer: D) -> std::result::Result -where D: Deserializer<'de> { - let buf = Vec::::deserialize(deserializer)?; - let mut cursor = Cursor::new(&buf[..]); - let mut protocol = TCompactInputProtocol::new(&mut cursor, usize::MAX); - ColumnChunk::read_from_in_protocol(&mut protocol).map_err(D::Error::custom) -} - -// Represents common operations for a column chunk. -impl ColumnChunkMetaData { - /// Returns a new [`ColumnChunkMetaData`] - pub fn new(column_chunk: ColumnChunk, column_descr: ColumnDescriptor) -> Self { - Self { - column_chunk, - column_descr, - } - } - - /// File where the column chunk is stored. - /// - /// If not set, assumed to belong to the same file as the metadata. - /// This path is relative to the current file. - pub fn file_path(&self) -> &Option { - &self.column_chunk.file_path - } - - /// Byte offset in `file_path()`. - pub fn file_offset(&self) -> i64 { - self.column_chunk.file_offset - } - - /// Returns this column's [`ColumnChunk`] - pub fn column_chunk(&self) -> &ColumnChunk { - &self.column_chunk - } - - /// The column's [`ColumnMetaData`] - pub fn metadata(&self) -> &ColumnMetaData { - self.column_chunk.meta_data.as_ref().unwrap() - } - - /// The [`ColumnDescriptor`] for this column. This descriptor contains the physical and logical type - /// of the pages. - pub fn descriptor(&self) -> &ColumnDescriptor { - &self.column_descr - } - - /// The [`PhysicalType`] of this column. - pub fn physical_type(&self) -> PhysicalType { - self.column_descr.descriptor.primitive_type.physical_type - } - - /// Decodes the raw statistics into [`Statistics`]. - pub fn statistics(&self) -> Option>> { - self.metadata() - .statistics - .as_ref() - .map(|x| deserialize_statistics(x, self.column_descr.descriptor.primitive_type.clone())) - } - - /// Total number of values in this column chunk. Note that this is not necessarily the number - /// of rows. E.g. the (nested) array `[[1, 2], [3]]` has 2 rows and 3 values. - pub fn num_values(&self) -> i64 { - self.metadata().num_values - } - - /// [`Compression`] for this column. - pub fn compression(&self) -> Compression { - self.metadata().codec.try_into().unwrap() - } - - /// Returns the total compressed data size of this column chunk. - pub fn compressed_size(&self) -> i64 { - self.metadata().total_compressed_size - } - - /// Returns the total uncompressed data size of this column chunk. - pub fn uncompressed_size(&self) -> i64 { - self.metadata().total_uncompressed_size - } - - /// Returns the offset for the column data. - pub fn data_page_offset(&self) -> i64 { - self.metadata().data_page_offset - } - - /// Returns `true` if this column chunk contains a index page, `false` otherwise. - pub fn has_index_page(&self) -> bool { - self.metadata().index_page_offset.is_some() - } - - /// Returns the offset for the index page. - pub fn index_page_offset(&self) -> Option { - self.metadata().index_page_offset - } - - /// Returns the offset for the dictionary page, if any. - pub fn dictionary_page_offset(&self) -> Option { - self.metadata().dictionary_page_offset - } - - /// Returns the encoding for this column - pub fn column_encoding(&self) -> &Vec { - &self.metadata().encodings - } - - /// Returns the offset and length in bytes of the column chunk within the file - pub fn byte_range(&self) -> (u64, u64) { - let start = if let Some(dict_page_offset) = self.dictionary_page_offset() { - dict_page_offset as u64 - } else { - self.data_page_offset() as u64 - }; - let length = self.compressed_size() as u64; - // this has been validated in [`try_from_thrift`] - (start, length) - } - - /// Method to convert from Thrift. - pub(crate) fn try_from_thrift( - column_descr: ColumnDescriptor, - column_chunk: ColumnChunk, - ) -> Result { - // validate metadata - if let Some(meta) = &column_chunk.meta_data { - let _: u64 = meta.total_compressed_size.try_into()?; - - if let Some(offset) = meta.dictionary_page_offset { - let _: u64 = offset.try_into()?; - } - let _: u64 = meta.data_page_offset.try_into()?; - - let _: Compression = meta.codec.try_into()?; - } else { - return Err(Error::oos("Column chunk requires metdata")); - } - - Ok(Self { - column_chunk, - column_descr, - }) - } - - /// Method to convert to Thrift. - pub fn into_thrift(self) -> ColumnChunk { - self.column_chunk - } -} diff --git a/src/common/parquet2/src/metadata/column_descriptor.rs b/src/common/parquet2/src/metadata/column_descriptor.rs deleted file mode 100644 index d33784d47e40..000000000000 --- a/src/common/parquet2/src/metadata/column_descriptor.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use crate::schema::types::ParquetType; -use crate::schema::types::PrimitiveType; - -/// A descriptor of a parquet column. It contains the necessary information to deserialize -/// a parquet column. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct Descriptor { - /// The [`PrimitiveType`] of this column - pub primitive_type: PrimitiveType, - - /// The maximum definition level - pub max_def_level: i16, - - /// The maximum repetition level - pub max_rep_level: i16, -} - -/// A descriptor for leaf-level primitive columns. -/// This encapsulates information such as definition and repetition levels and is used to -/// re-assemble nested data. -#[derive(Debug, PartialEq, Clone)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct ColumnDescriptor { - /// The descriptor this columns' leaf. - pub descriptor: Descriptor, - - /// The path of this column. For instance, "a.b.c.d". - pub path_in_schema: Vec, - - /// The [`ParquetType`] this descriptor is a leaf of - pub base_type: ParquetType, -} - -impl ColumnDescriptor { - /// Creates new descriptor for leaf-level column. - pub fn new( - descriptor: Descriptor, - path_in_schema: Vec, - base_type: ParquetType, - ) -> Self { - Self { - descriptor, - path_in_schema, - base_type, - } - } -} diff --git a/src/common/parquet2/src/metadata/column_order.rs b/src/common/parquet2/src/metadata/column_order.rs deleted file mode 100644 index f4457d53299b..000000000000 --- a/src/common/parquet2/src/metadata/column_order.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use super::sort::SortOrder; - -/// Column order that specifies what method was used to aggregate min/max values for -/// statistics. -/// -/// If column order is undefined, then it is the legacy behaviour and all values should -/// be compared as signed values/bytes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum ColumnOrder { - /// Column uses the order defined by its logical or physical type - /// (if there is no logical type), parquet-format 2.4.0+. - TypeDefinedOrder(SortOrder), - /// Undefined column order, means legacy behaviour before parquet-format 2.4.0. - /// Sort order is always SIGNED. - Undefined, -} - -impl ColumnOrder { - /// Returns sort order associated with this column order. - pub fn sort_order(&self) -> SortOrder { - match *self { - ColumnOrder::TypeDefinedOrder(order) => order, - ColumnOrder::Undefined => SortOrder::Signed, - } - } -} diff --git a/src/common/parquet2/src/metadata/file_metadata.rs b/src/common/parquet2/src/metadata/file_metadata.rs deleted file mode 100644 index 066dafdc0559..000000000000 --- a/src/common/parquet2/src/metadata/file_metadata.rs +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::ColumnOrder as TColumnOrder; - -use super::column_order::ColumnOrder; -use super::schema_descriptor::SchemaDescriptor; -use super::RowGroupMetaData; -use crate::error::Error; -use crate::metadata::get_sort_order; -pub use crate::thrift_format::KeyValue; - -/// Metadata for a Parquet file. -// This is almost equal to [`parquet_format_safe::FileMetaData`] but contains the descriptors, -// which are crucial to deserialize pages. -#[derive(Debug, Clone)] -pub struct FileMetaData { - /// version of this file. - pub version: i32, - /// number of rows in the file. - pub num_rows: usize, - /// String message for application that wrote this file. - /// - /// This should have the following format: - /// ` version (build )`. - /// - /// ```shell - /// parquet-mr version 1.8.0 (build 0fda28af84b9746396014ad6a415b90592a98b3b) - /// ``` - pub created_by: Option, - /// The row groups of this file - pub row_groups: Vec, - /// key_value_metadata of this file. - pub key_value_metadata: Option>, - /// schema descriptor. - pub schema_descr: SchemaDescriptor, - /// Column (sort) order used for `min` and `max` values of each column in this file. - /// - /// Each column order corresponds to one column, determined by its position in the - /// list, matching the position of the column in the schema. - /// - /// When `None` is returned, there are no column orders available, and each column - /// should be assumed to have undefined (legacy) column order. - pub column_orders: Option>, -} - -impl FileMetaData { - /// Returns the [`SchemaDescriptor`] that describes schema of this file. - pub fn schema(&self) -> &SchemaDescriptor { - &self.schema_descr - } - - /// returns the metadata - pub fn key_value_metadata(&self) -> &Option> { - &self.key_value_metadata - } - - /// Returns column order for `i`th column in this file. - /// If column orders are not available, returns undefined (legacy) column order. - pub fn column_order(&self, i: usize) -> ColumnOrder { - self.column_orders - .as_ref() - .map(|data| data[i]) - .unwrap_or(ColumnOrder::Undefined) - } - - /// Deserializes [`crate::thrift_format::FileMetaData`] into this struct - pub fn try_from_thrift(metadata: parquet_format_safe::FileMetaData) -> Result { - let schema_descr = SchemaDescriptor::try_from_thrift(&metadata.schema)?; - - let row_groups = metadata - .row_groups - .into_iter() - .map(|rg| RowGroupMetaData::try_from_thrift(&schema_descr, rg)) - .collect::>()?; - - let column_orders = metadata - .column_orders - .map(|orders| parse_column_orders(&orders, &schema_descr)); - - Ok(FileMetaData { - version: metadata.version, - num_rows: metadata.num_rows.try_into()?, - created_by: metadata.created_by, - row_groups, - key_value_metadata: metadata.key_value_metadata, - schema_descr, - column_orders, - }) - } - - /// Serializes itself to thrift's [`parquet_format_safe::FileMetaData`]. - pub fn into_thrift(self) -> parquet_format_safe::FileMetaData { - parquet_format_safe::FileMetaData { - version: self.version, - schema: self.schema_descr.into_thrift(), - num_rows: self.num_rows as i64, - row_groups: self - .row_groups - .into_iter() - .map(|v| v.into_thrift()) - .collect(), - key_value_metadata: self.key_value_metadata, - created_by: self.created_by, - column_orders: None, // todo - encryption_algorithm: None, - footer_signing_key_metadata: None, - } - } -} - -/// Parses [`ColumnOrder`] from Thrift definition. -fn parse_column_orders( - orders: &[TColumnOrder], - schema_descr: &SchemaDescriptor, -) -> Vec { - schema_descr - .columns() - .iter() - .zip(orders.iter()) - .map(|(column, order)| match order { - TColumnOrder::TYPEORDER(_) => { - let sort_order = get_sort_order( - &column.descriptor.primitive_type.logical_type, - &column.descriptor.primitive_type.converted_type, - &column.descriptor.primitive_type.physical_type, - ); - ColumnOrder::TypeDefinedOrder(sort_order) - } - }) - .collect() -} diff --git a/src/common/parquet2/src/metadata/mod.rs b/src/common/parquet2/src/metadata/mod.rs deleted file mode 100644 index 5df68c73f658..000000000000 --- a/src/common/parquet2/src/metadata/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod column_chunk_metadata; -mod column_descriptor; -mod column_order; -mod file_metadata; -mod row_metadata; -mod schema_descriptor; -mod sort; - -pub use column_chunk_metadata::ColumnChunkMetaData; -pub use column_descriptor::ColumnDescriptor; -pub use column_descriptor::Descriptor; -pub use column_order::ColumnOrder; -pub use file_metadata::FileMetaData; -pub use file_metadata::KeyValue; -pub use row_metadata::RowGroupMetaData; -pub use schema_descriptor::SchemaDescriptor; -pub use sort::*; - -pub use crate::thrift_format::FileMetaData as ThriftFileMetaData; diff --git a/src/common/parquet2/src/metadata/row_metadata.rs b/src/common/parquet2/src/metadata/row_metadata.rs deleted file mode 100644 index 5de40c817ef0..000000000000 --- a/src/common/parquet2/src/metadata/row_metadata.rs +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::RowGroup; -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use super::column_chunk_metadata::ColumnChunkMetaData; -use super::schema_descriptor::SchemaDescriptor; -use crate::error::Error; -use crate::error::Result; -use crate::write::ColumnOffsetsMetadata; - -/// Metadata for a row group. -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct RowGroupMetaData { - columns: Vec, - num_rows: usize, - total_byte_size: usize, -} - -impl RowGroupMetaData { - /// Create a new [`RowGroupMetaData`] - pub fn new( - columns: Vec, - num_rows: usize, - total_byte_size: usize, - ) -> RowGroupMetaData { - Self { - columns, - num_rows, - total_byte_size, - } - } - - /// Returns slice of column chunk metadata. - pub fn columns(&self) -> &[ColumnChunkMetaData] { - &self.columns - } - - /// Number of rows in this row group. - pub fn num_rows(&self) -> usize { - self.num_rows - } - - /// Total byte size of all uncompressed column data in this row group. - pub fn total_byte_size(&self) -> usize { - self.total_byte_size - } - - /// Total size of all compressed column data in this row group. - pub fn compressed_size(&self) -> usize { - self.columns - .iter() - .map(|c| c.compressed_size() as usize) - .sum::() - } - - /// Method to convert from Thrift. - pub(crate) fn try_from_thrift( - schema_descr: &SchemaDescriptor, - rg: RowGroup, - ) -> Result { - if schema_descr.columns().len() != rg.columns.len() { - return Err(Error::oos(format!( - "The number of columns in the row group ({}) must be equal to the number of columns in the schema ({})", - rg.columns.len(), - schema_descr.columns().len() - ))); - } - let total_byte_size = rg.total_byte_size.try_into()?; - let num_rows = rg.num_rows.try_into()?; - let columns = rg - .columns - .into_iter() - .zip(schema_descr.columns()) - .map(|(column_chunk, descriptor)| { - ColumnChunkMetaData::try_from_thrift(descriptor.clone(), column_chunk) - }) - .collect::>>()?; - - Ok(RowGroupMetaData { - columns, - num_rows, - total_byte_size, - }) - } - - /// Method to convert to Thrift. - pub(crate) fn into_thrift(self) -> RowGroup { - let file_offset = self - .columns - .iter() - .map(|c| { - ColumnOffsetsMetadata::from_column_chunk_metadata(c).calc_row_group_file_offset() - }) - .next() - .unwrap_or(None); - let total_compressed_size = Some(self.compressed_size() as i64); - RowGroup { - columns: self.columns.into_iter().map(|v| v.into_thrift()).collect(), - total_byte_size: self.total_byte_size as i64, - num_rows: self.num_rows as i64, - sorting_columns: None, - file_offset, - total_compressed_size, - ordinal: None, - } - } -} diff --git a/src/common/parquet2/src/metadata/schema_descriptor.rs b/src/common/parquet2/src/metadata/schema_descriptor.rs deleted file mode 100644 index e0ff64e01766..000000000000 --- a/src/common/parquet2/src/metadata/schema_descriptor.rs +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::SchemaElement; -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use super::column_descriptor::ColumnDescriptor; -use super::column_descriptor::Descriptor; -use crate::error::Error; -use crate::error::Result; -use crate::schema::io_message::from_message; -use crate::schema::types::FieldInfo; -use crate::schema::types::ParquetType; -use crate::schema::Repetition; - -/// A schema descriptor. This encapsulates the top-level schemas for all the columns, -/// as well as all descriptors for all the primitive columns. -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct SchemaDescriptor { - name: String, - // The top-level schema (the "message" type). - fields: Vec, - - // All the descriptors for primitive columns in this schema, constructed from - // `schema` in DFS order. - leaves: Vec, -} - -impl SchemaDescriptor { - /// Creates new schema descriptor from Parquet schema. - pub fn new(name: String, fields: Vec) -> Self { - let mut leaves = vec![]; - for f in &fields { - let mut path = vec![]; - build_tree(f, f, 0, 0, &mut leaves, &mut path); - } - - Self { - name, - fields, - leaves, - } - } - - /// The [`ColumnDescriptor`] (leafs) of this schema. - /// - /// Note that, for nested fields, this may contain more entries than the number of fields - /// in the file - e.g. a struct field may have two columns. - pub fn columns(&self) -> &[ColumnDescriptor] { - &self.leaves - } - - /// The schemas' name. - pub fn name(&self) -> &str { - &self.name - } - - /// The schemas' fields. - pub fn fields(&self) -> &[ParquetType] { - &self.fields - } - - pub(crate) fn into_thrift(self) -> Vec { - ParquetType::GroupType { - field_info: FieldInfo { - name: self.name, - repetition: Repetition::Optional, - id: None, - }, - logical_type: None, - converted_type: None, - fields: self.fields, - } - .to_thrift() - } - - fn try_from_type(type_: ParquetType) -> Result { - match type_ { - ParquetType::GroupType { - field_info, fields, .. - } => Ok(Self::new(field_info.name, fields)), - _ => Err(Error::oos("The parquet schema MUST be a group type")), - } - } - - pub(crate) fn try_from_thrift(elements: &[SchemaElement]) -> Result { - let schema = ParquetType::try_from_thrift(elements)?; - Self::try_from_type(schema) - } - - /// Creates a schema from - pub fn try_from_message(message: &str) -> Result { - let schema = from_message(message)?; - Self::try_from_type(schema) - } -} - -fn build_tree<'a>( - tp: &'a ParquetType, - base_tp: &ParquetType, - mut max_rep_level: i16, - mut max_def_level: i16, - leaves: &mut Vec, - path_so_far: &mut Vec<&'a str>, -) { - path_so_far.push(tp.name()); - match tp.get_field_info().repetition { - Repetition::Optional => { - max_def_level += 1; - } - Repetition::Repeated => { - max_def_level += 1; - max_rep_level += 1; - } - _ => {} - } - - match tp { - ParquetType::PrimitiveType(p) => { - let path_in_schema = path_so_far.iter().copied().map(String::from).collect(); - leaves.push(ColumnDescriptor::new( - Descriptor { - primitive_type: p.clone(), - max_def_level, - max_rep_level, - }, - path_in_schema, - base_tp.clone(), - )); - } - ParquetType::GroupType { ref fields, .. } => { - for f in fields { - build_tree( - f, - base_tp, - max_rep_level, - max_def_level, - leaves, - path_so_far, - ); - path_so_far.pop(); - } - } - } -} diff --git a/src/common/parquet2/src/metadata/sort.rs b/src/common/parquet2/src/metadata/sort.rs deleted file mode 100644 index 9b383440a3d0..000000000000 --- a/src/common/parquet2/src/metadata/sort.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use crate::schema::types::IntegerType; -use crate::schema::types::PhysicalType; -use crate::schema::types::PrimitiveConvertedType; -use crate::schema::types::PrimitiveLogicalType; - -/// Sort order for page and column statistics. -/// -/// Types are associated with sort orders and column stats are aggregated using a sort -/// order, and a sort order should be considered when comparing values with statistics -/// min/max. -/// -/// See reference in -/// -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum SortOrder { - /// Signed (either value or legacy byte-wise) comparison. - Signed, - /// Unsigned (depending on physical type either value or byte-wise) comparison. - Unsigned, - /// Comparison is undefined. - Undefined, -} - -/// Returns sort order for a physical/logical type. -pub fn get_sort_order( - logical_type: &Option, - converted_type: &Option, - physical_type: &PhysicalType, -) -> SortOrder { - if let Some(logical_type) = logical_type { - return get_logical_sort_order(logical_type); - }; - if let Some(converted_type) = converted_type { - return get_converted_sort_order(converted_type); - }; - get_physical_sort_order(physical_type) -} - -fn get_logical_sort_order(logical_type: &PrimitiveLogicalType) -> SortOrder { - // TODO: Should this take converted and logical type, for compatibility? - use PrimitiveLogicalType::*; - match logical_type { - String | Enum | Json | Bson => SortOrder::Unsigned, - Integer(t) => match t { - IntegerType::Int8 | IntegerType::Int16 | IntegerType::Int32 | IntegerType::Int64 => { - SortOrder::Signed - } - _ => SortOrder::Unsigned, - }, - Decimal(_, _) => SortOrder::Signed, - Date => SortOrder::Signed, - Time { .. } => SortOrder::Signed, - Timestamp { .. } => SortOrder::Signed, - Unknown => SortOrder::Undefined, - Uuid => SortOrder::Unsigned, - } -} - -fn get_converted_sort_order(converted_type: &PrimitiveConvertedType) -> SortOrder { - use PrimitiveConvertedType::*; - match converted_type { - // Unsigned byte-wise comparison. - Utf8 | Json | Bson | Enum => SortOrder::Unsigned, - Int8 | Int16 | Int32 | Int64 => SortOrder::Signed, - Uint8 | Uint16 | Uint32 | Uint64 => SortOrder::Unsigned, - // Signed comparison of the represented value. - Decimal(_, _) => SortOrder::Signed, - Date => SortOrder::Signed, - TimeMillis | TimeMicros | TimestampMillis | TimestampMicros => SortOrder::Signed, - Interval => SortOrder::Undefined, - } -} - -fn get_physical_sort_order(physical_type: &PhysicalType) -> SortOrder { - use PhysicalType::*; - match physical_type { - // Order: false, true - Boolean => SortOrder::Unsigned, - Int32 | Int64 => SortOrder::Signed, - Int96 => SortOrder::Undefined, - // Notes to remember when comparing float/double values: - // If the min is a NaN, it should be ignored. - // If the max is a NaN, it should be ignored. - // If the min is +0, the row group may contain -0 values as well. - // If the max is -0, the row group may contain +0 values as well. - // When looking for NaN values, min and max should be ignored. - Float | Double => SortOrder::Signed, - // Unsigned byte-wise comparison - ByteArray | FixedLenByteArray(_) => SortOrder::Unsigned, - } -} diff --git a/src/common/parquet2/src/page/mod.rs b/src/common/parquet2/src/page/mod.rs deleted file mode 100644 index bccfeee71e1a..000000000000 --- a/src/common/parquet2/src/page/mod.rs +++ /dev/null @@ -1,440 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use crate::compression::Compression; -use crate::encoding::get_length; -use crate::encoding::Encoding; -use crate::error::Error; -use crate::error::Result; -use crate::indexes::Interval; -use crate::metadata::Descriptor; -pub use crate::parquet_bridge::DataPageHeaderExt; -pub use crate::parquet_bridge::PageType; -use crate::statistics::deserialize_statistics; -use crate::statistics::Statistics; -pub use crate::thrift_format::DataPageHeader as DataPageHeaderV1; -pub use crate::thrift_format::DataPageHeaderV2; -pub use crate::thrift_format::PageHeader as ParquetPageHeader; - -/// A [`CompressedDataPage`] is compressed, encoded representation of a Parquet data page. -/// It holds actual data and thus cloning it is expensive. -#[derive(Debug)] -pub struct CompressedDataPage { - pub(crate) header: DataPageHeader, - pub(crate) buffer: Vec, - pub(crate) compression: Compression, - uncompressed_page_size: usize, - pub(crate) descriptor: Descriptor, - - // The offset and length in rows - pub(crate) selected_rows: Option>, -} - -impl CompressedDataPage { - /// Returns a new [`CompressedDataPage`]. - pub fn new( - header: DataPageHeader, - buffer: Vec, - compression: Compression, - uncompressed_page_size: usize, - descriptor: Descriptor, - rows: Option, - ) -> Self { - Self::new_read( - header, - buffer, - compression, - uncompressed_page_size, - descriptor, - rows.map(|x| vec![Interval::new(0, x)]), - ) - } - - /// Returns a new [`CompressedDataPage`]. - pub(crate) fn new_read( - header: DataPageHeader, - buffer: Vec, - compression: Compression, - uncompressed_page_size: usize, - descriptor: Descriptor, - selected_rows: Option>, - ) -> Self { - Self { - header, - buffer, - compression, - uncompressed_page_size, - descriptor, - selected_rows, - } - } - - pub fn header(&self) -> &DataPageHeader { - &self.header - } - - pub fn uncompressed_size(&self) -> usize { - self.uncompressed_page_size - } - - pub fn compressed_size(&self) -> usize { - self.buffer.len() - } - - /// The compression of the data in this page. - /// Note that what is compressed in a page depends on its version: - /// in V1, the whole data (`[repetition levels][definition levels][values]`) is compressed; in V2 only the values are compressed. - pub fn compression(&self) -> Compression { - self.compression - } - - /// the rows to be selected by this page. - /// When `None`, all rows are to be considered. - pub fn selected_rows(&self) -> Option<&[Interval]> { - self.selected_rows.as_deref() - } - - pub fn num_values(&self) -> usize { - self.header.num_values() - } - - /// Decodes the raw statistics into a statistics - pub fn statistics(&self) -> Option>> { - match &self.header { - DataPageHeader::V1(d) => d - .statistics - .as_ref() - .map(|x| deserialize_statistics(x, self.descriptor.primitive_type.clone())), - DataPageHeader::V2(d) => d - .statistics - .as_ref() - .map(|x| deserialize_statistics(x, self.descriptor.primitive_type.clone())), - } - } - - #[inline] - pub fn select_rows(&mut self, selected_rows: Vec) { - self.selected_rows = Some(selected_rows); - } -} - -#[derive(Debug, Clone)] -pub enum DataPageHeader { - V1(DataPageHeaderV1), - V2(DataPageHeaderV2), -} - -impl DataPageHeader { - pub fn num_values(&self) -> usize { - match &self { - DataPageHeader::V1(d) => d.num_values as usize, - DataPageHeader::V2(d) => d.num_values as usize, - } - } -} - -/// A [`DataPage`] is an uncompressed, encoded representation of a Parquet data page. It holds actual data -/// and thus cloning it is expensive. -#[derive(Debug, Clone)] -pub struct DataPage { - pub(super) header: DataPageHeader, - pub(super) buffer: Vec, - pub descriptor: Descriptor, - pub selected_rows: Option>, -} - -impl DataPage { - pub fn new( - header: DataPageHeader, - buffer: Vec, - descriptor: Descriptor, - rows: Option, - ) -> Self { - Self::new_read( - header, - buffer, - descriptor, - rows.map(|x| vec![Interval::new(0, x)]), - ) - } - - pub(crate) fn new_read( - header: DataPageHeader, - buffer: Vec, - descriptor: Descriptor, - selected_rows: Option>, - ) -> Self { - Self { - header, - buffer, - descriptor, - selected_rows, - } - } - - pub fn header(&self) -> &DataPageHeader { - &self.header - } - - pub fn buffer(&self) -> &[u8] { - &self.buffer - } - - /// the rows to be selected by this page. - /// When `None`, all rows are to be considered. - pub fn selected_rows(&self) -> Option<&[Interval]> { - self.selected_rows.as_deref() - } - - /// Returns a mutable reference to the internal buffer. - /// Useful to recover the buffer after the page has been decoded. - pub fn buffer_mut(&mut self) -> &mut Vec { - &mut self.buffer - } - - pub fn num_values(&self) -> usize { - self.header.num_values() - } - - pub fn encoding(&self) -> Encoding { - match &self.header { - DataPageHeader::V1(d) => d.encoding(), - DataPageHeader::V2(d) => d.encoding(), - } - } - - pub fn definition_level_encoding(&self) -> Encoding { - match &self.header { - DataPageHeader::V1(d) => d.definition_level_encoding(), - DataPageHeader::V2(_) => Encoding::Rle, - } - } - - pub fn repetition_level_encoding(&self) -> Encoding { - match &self.header { - DataPageHeader::V1(d) => d.repetition_level_encoding(), - DataPageHeader::V2(_) => Encoding::Rle, - } - } - - /// Decodes the raw statistics into a statistics - pub fn statistics(&self) -> Option>> { - match &self.header { - DataPageHeader::V1(d) => d - .statistics - .as_ref() - .map(|x| deserialize_statistics(x, self.descriptor.primitive_type.clone())), - DataPageHeader::V2(d) => d - .statistics - .as_ref() - .map(|x| deserialize_statistics(x, self.descriptor.primitive_type.clone())), - } - } -} - -/// A [`Page`] is an uncompressed, encoded representation of a Parquet page. It may hold actual data -/// and thus cloning it may be expensive. -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum Page { - /// A [`DataPage`] - Data(DataPage), - /// A [`DictPage`] - Dict(DictPage), -} - -impl Page { - pub(crate) fn buffer(&mut self) -> &mut Vec { - match self { - Self::Data(page) => &mut page.buffer, - Self::Dict(page) => &mut page.buffer, - } - } -} - -/// A [`CompressedPage`] is a compressed, encoded representation of a Parquet page. It holds actual data -/// and thus cloning it is expensive. -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum CompressedPage { - Data(CompressedDataPage), - Dict(CompressedDictPage), -} - -impl CompressedPage { - pub(crate) fn buffer(&mut self) -> &mut Vec { - match self { - CompressedPage::Data(page) => &mut page.buffer, - CompressedPage::Dict(page) => &mut page.buffer, - } - } - - pub(crate) fn compression(&self) -> Compression { - match self { - CompressedPage::Data(page) => page.compression(), - CompressedPage::Dict(page) => page.compression(), - } - } - - pub(crate) fn selected_rows(&self) -> Option<&[Interval]> { - match self { - CompressedPage::Data(page) => page.selected_rows(), - CompressedPage::Dict(_) => None, - } - } - - pub(crate) fn uncompressed_size(&self) -> usize { - match self { - CompressedPage::Data(page) => page.uncompressed_page_size, - CompressedPage::Dict(page) => page.uncompressed_page_size, - } - } -} - -/// An uncompressed, encoded dictionary page. -#[derive(Debug)] -pub struct DictPage { - pub buffer: Vec, - pub num_values: usize, - pub is_sorted: bool, -} - -impl DictPage { - pub fn new(buffer: Vec, num_values: usize, is_sorted: bool) -> Self { - Self { - buffer, - num_values, - is_sorted, - } - } -} - -/// A compressed, encoded dictionary page. -#[derive(Debug)] -pub struct CompressedDictPage { - pub(crate) buffer: Vec, - compression: Compression, - pub(crate) num_values: usize, - pub(crate) uncompressed_page_size: usize, - pub is_sorted: bool, -} - -impl CompressedDictPage { - pub fn new( - buffer: Vec, - compression: Compression, - uncompressed_page_size: usize, - num_values: usize, - is_sorted: bool, - ) -> Self { - Self { - buffer, - compression, - uncompressed_page_size, - num_values, - is_sorted, - } - } - - /// The compression of the data in this page. - pub fn compression(&self) -> Compression { - self.compression - } -} - -/// Splits the page buffer into 3 slices corresponding to (encoded rep levels, encoded def levels, encoded values) for v1 pages. -#[inline] -pub fn split_buffer_v1( - buffer: &[u8], - has_rep: bool, - has_def: bool, -) -> Result<(&[u8], &[u8], &[u8])> { - let (rep, buffer) = if has_rep { - let level_buffer_length = get_length(buffer).ok_or_else(|| { - Error::oos("The number of bytes declared in v1 rep levels is higher than the page size") - })?; - ( - buffer.get(4..4 + level_buffer_length).ok_or_else(|| { - Error::oos( - "The number of bytes declared in v1 rep levels is higher than the page size", - ) - })?, - buffer.get(4 + level_buffer_length..).ok_or_else(|| { - Error::oos( - "The number of bytes declared in v1 rep levels is higher than the page size", - ) - })?, - ) - } else { - (&[] as &[u8], buffer) - }; - - let (def, buffer) = if has_def { - let level_buffer_length = get_length(buffer).ok_or_else(|| { - Error::oos("The number of bytes declared in v1 rep levels is higher than the page size") - })?; - ( - buffer.get(4..4 + level_buffer_length).ok_or_else(|| { - Error::oos( - "The number of bytes declared in v1 def levels is higher than the page size", - ) - })?, - buffer.get(4 + level_buffer_length..).ok_or_else(|| { - Error::oos( - "The number of bytes declared in v1 def levels is higher than the page size", - ) - })?, - ) - } else { - (&[] as &[u8], buffer) - }; - - Ok((rep, def, buffer)) -} - -/// Splits the page buffer into 3 slices corresponding to (encoded rep levels, encoded def levels, encoded values) for v2 pages. -pub fn split_buffer_v2( - buffer: &[u8], - rep_level_buffer_length: usize, - def_level_buffer_length: usize, -) -> Result<(&[u8], &[u8], &[u8])> { - Ok(( - &buffer[..rep_level_buffer_length], - &buffer[rep_level_buffer_length..rep_level_buffer_length + def_level_buffer_length], - &buffer[rep_level_buffer_length + def_level_buffer_length..], - )) -} - -/// Splits the page buffer into 3 slices corresponding to (encoded rep levels, encoded def levels, encoded values). -pub fn split_buffer(page: &DataPage) -> Result<(&[u8], &[u8], &[u8])> { - match page.header() { - DataPageHeader::V1(_) => split_buffer_v1( - page.buffer(), - page.descriptor.max_rep_level > 0, - page.descriptor.max_def_level > 0, - ), - DataPageHeader::V2(header) => { - let def_level_buffer_length: usize = header.definition_levels_byte_length.try_into()?; - let rep_level_buffer_length: usize = header.repetition_levels_byte_length.try_into()?; - split_buffer_v2( - page.buffer(), - rep_level_buffer_length, - def_level_buffer_length, - ) - } - } -} diff --git a/src/common/parquet2/src/parquet_bridge.rs b/src/common/parquet2/src/parquet_bridge.rs deleted file mode 100644 index adcbf1d1e5be..000000000000 --- a/src/common/parquet2/src/parquet_bridge.rs +++ /dev/null @@ -1,728 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Bridges structs from thrift-generated code to rust enums. -use std::convert::TryFrom; - -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use super::thrift_format::BoundaryOrder as ParquetBoundaryOrder; -use super::thrift_format::CompressionCodec; -use super::thrift_format::DataPageHeader; -use super::thrift_format::DataPageHeaderV2; -use super::thrift_format::DecimalType; -use super::thrift_format::Encoding as ParquetEncoding; -use super::thrift_format::FieldRepetitionType; -use super::thrift_format::IntType; -use super::thrift_format::LogicalType as ParquetLogicalType; -use super::thrift_format::PageType as ParquetPageType; -use super::thrift_format::TimeType; -use super::thrift_format::TimeUnit as ParquetTimeUnit; -use super::thrift_format::TimestampType; -use crate::error::Error; - -/// The repetition of a parquet field -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum Repetition { - /// When the field has no null values - Required, - /// When the field may have null values - Optional, - /// When the field may be repeated (list field) - Repeated, -} - -impl TryFrom for Repetition { - type Error = Error; - - fn try_from(repetition: FieldRepetitionType) -> Result { - Ok(match repetition { - FieldRepetitionType::REQUIRED => Repetition::Required, - FieldRepetitionType::OPTIONAL => Repetition::Optional, - FieldRepetitionType::REPEATED => Repetition::Repeated, - _ => return Err(Error::oos("Thrift out of range")), - }) - } -} - -impl From for FieldRepetitionType { - fn from(repetition: Repetition) -> Self { - match repetition { - Repetition::Required => FieldRepetitionType::REQUIRED, - Repetition::Optional => FieldRepetitionType::OPTIONAL, - Repetition::Repeated => FieldRepetitionType::REPEATED, - } - } -} - -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum Compression { - Uncompressed, - Snappy, - Gzip, - Lzo, - Brotli, - Lz4, - Zstd, - Lz4Raw, -} - -impl TryFrom for Compression { - type Error = Error; - - fn try_from(codec: CompressionCodec) -> Result { - Ok(match codec { - CompressionCodec::UNCOMPRESSED => Compression::Uncompressed, - CompressionCodec::SNAPPY => Compression::Snappy, - CompressionCodec::GZIP => Compression::Gzip, - CompressionCodec::LZO => Compression::Lzo, - CompressionCodec::BROTLI => Compression::Brotli, - CompressionCodec::LZ4 => Compression::Lz4, - CompressionCodec::ZSTD => Compression::Zstd, - CompressionCodec::LZ4_RAW => Compression::Lz4Raw, - _ => return Err(Error::oos("Thrift out of range")), - }) - } -} - -impl From for CompressionCodec { - fn from(codec: Compression) -> Self { - match codec { - Compression::Uncompressed => CompressionCodec::UNCOMPRESSED, - Compression::Snappy => CompressionCodec::SNAPPY, - Compression::Gzip => CompressionCodec::GZIP, - Compression::Lzo => CompressionCodec::LZO, - Compression::Brotli => CompressionCodec::BROTLI, - Compression::Lz4 => CompressionCodec::LZ4, - Compression::Zstd => CompressionCodec::ZSTD, - Compression::Lz4Raw => CompressionCodec::LZ4_RAW, - } - } -} - -/// Defines the compression settings for writing a parquet file. -/// -/// If None is provided as a compression setting, then the default compression level is used. -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -pub enum CompressionOptions { - Uncompressed, - Snappy, - Gzip(Option), - Lzo, - Brotli(Option), - Lz4, - Zstd(Option), - Lz4Raw, -} - -impl From for Compression { - fn from(value: CompressionOptions) -> Self { - match value { - CompressionOptions::Uncompressed => Compression::Uncompressed, - CompressionOptions::Snappy => Compression::Snappy, - CompressionOptions::Gzip(_) => Compression::Gzip, - CompressionOptions::Lzo => Compression::Lzo, - CompressionOptions::Brotli(_) => Compression::Brotli, - CompressionOptions::Lz4 => Compression::Lz4, - CompressionOptions::Zstd(_) => Compression::Zstd, - CompressionOptions::Lz4Raw => Compression::Lz4Raw, - } - } -} - -impl From for CompressionCodec { - fn from(codec: CompressionOptions) -> Self { - match codec { - CompressionOptions::Uncompressed => CompressionCodec::UNCOMPRESSED, - CompressionOptions::Snappy => CompressionCodec::SNAPPY, - CompressionOptions::Gzip(_) => CompressionCodec::GZIP, - CompressionOptions::Lzo => CompressionCodec::LZO, - CompressionOptions::Brotli(_) => CompressionCodec::BROTLI, - CompressionOptions::Lz4 => CompressionCodec::LZ4, - CompressionOptions::Zstd(_) => CompressionCodec::ZSTD, - CompressionOptions::Lz4Raw => CompressionCodec::LZ4_RAW, - } - } -} - -/// Defines valid compression levels. -pub(crate) trait CompressionLevel { - const MINIMUM_LEVEL: T; - const MAXIMUM_LEVEL: T; - - /// Tests if the provided compression level is valid. - fn is_valid_level(level: T) -> Result<(), Error> { - let compression_range = Self::MINIMUM_LEVEL..=Self::MAXIMUM_LEVEL; - if compression_range.contains(&level) { - Ok(()) - } else { - Err(Error::InvalidParameter(format!( - "valid compression range {}..={} exceeded.", - compression_range.start(), - compression_range.end() - ))) - } - } -} - -/// Represents a valid brotli compression level. -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -pub struct BrotliLevel(u32); - -impl Default for BrotliLevel { - fn default() -> Self { - Self(1) - } -} - -impl CompressionLevel for BrotliLevel { - const MINIMUM_LEVEL: u32 = 0; - const MAXIMUM_LEVEL: u32 = 11; -} - -impl BrotliLevel { - /// Attempts to create a brotli compression level. - /// - /// Compression levels must be valid. - pub fn try_new(level: u32) -> Result { - Self::is_valid_level(level).map(|_| Self(level)) - } - - /// Returns the compression level. - pub fn compression_level(&self) -> u32 { - self.0 - } -} - -/// Represents a valid gzip compression level. -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -pub struct GzipLevel(u8); - -impl Default for GzipLevel { - fn default() -> Self { - // The default as of miniz_oxide 0.5.1 is 6 for compression level - // (miniz_oxide::deflate::CompressionLevel::DefaultLevel) - Self(6) - } -} - -impl CompressionLevel for GzipLevel { - const MINIMUM_LEVEL: u8 = 0; - const MAXIMUM_LEVEL: u8 = 10; -} - -impl GzipLevel { - /// Attempts to create a gzip compression level. - /// - /// Compression levels must be valid (i.e. be acceptable for [`flate2::Compression`]). - pub fn try_new(level: u8) -> Result { - Self::is_valid_level(level).map(|_| Self(level)) - } - - /// Returns the compression level. - pub fn compression_level(&self) -> u8 { - self.0 - } -} - -#[cfg(feature = "gzip")] -impl From for flate2::Compression { - fn from(level: GzipLevel) -> Self { - Self::new(level.compression_level() as u32) - } -} - -/// Represents a valid zstd compression level. -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -pub struct ZstdLevel(i32); - -impl CompressionLevel for ZstdLevel { - // zstd binds to C, and hence zstd::compression_level_range() is not const as this calls the - // underlying C library. - const MINIMUM_LEVEL: i32 = 1; - const MAXIMUM_LEVEL: i32 = 22; -} - -impl ZstdLevel { - /// Attempts to create a zstd compression level from a given compression level. - /// - /// Compression levels must be valid (i.e. be acceptable for [`zstd::compression_level_range`]). - pub fn try_new(level: i32) -> Result { - Self::is_valid_level(level).map(|_| Self(level)) - } - - /// Returns the compression level. - pub fn compression_level(&self) -> i32 { - self.0 - } -} - -#[cfg(feature = "zstd")] -impl Default for ZstdLevel { - fn default() -> Self { - Self(zstd::DEFAULT_COMPRESSION_LEVEL) - } -} - -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -pub enum PageType { - DataPage, - DataPageV2, - DictionaryPage, -} - -impl TryFrom for PageType { - type Error = Error; - - fn try_from(type_: ParquetPageType) -> Result { - Ok(match type_ { - ParquetPageType::DATA_PAGE => PageType::DataPage, - ParquetPageType::DATA_PAGE_V2 => PageType::DataPageV2, - ParquetPageType::DICTIONARY_PAGE => PageType::DictionaryPage, - _ => return Err(Error::oos("Thrift out of range")), - }) - } -} - -impl From for ParquetPageType { - fn from(type_: PageType) -> Self { - match type_ { - PageType::DataPage => ParquetPageType::DATA_PAGE, - PageType::DataPageV2 => ParquetPageType::DATA_PAGE_V2, - PageType::DictionaryPage => ParquetPageType::DICTIONARY_PAGE, - } - } -} - -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -pub enum Encoding { - /// Default encoding. - /// BOOLEAN - 1 bit per value. 0 is false; 1 is true. - /// INT32 - 4 bytes per value. Stored as little-endian. - /// INT64 - 8 bytes per value. Stored as little-endian. - /// FLOAT - 4 bytes per value. IEEE. Stored as little-endian. - /// DOUBLE - 8 bytes per value. IEEE. Stored as little-endian. - /// BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes. - /// FIXED_LEN_BYTE_ARRAY - Just the bytes. - Plain, - /// Deprecated: Dictionary encoding. The values in the dictionary are encoded in the - /// plain type. - /// in a data page use RLE_DICTIONARY instead. - /// in a Dictionary page use PLAIN instead - PlainDictionary, - /// Group packed run length encoding. Usable for definition/repetition levels - /// encoding and Booleans (on one bit: 0 is false; 1 is true.) - Rle, - /// Bit packed encoding. This can only be used if the data has a known max - /// width. Usable for definition/repetition levels encoding. - BitPacked, - /// Delta encoding for integers. This can be used for int columns and works best - /// on sorted data - DeltaBinaryPacked, - /// Encoding for byte arrays to separate the length values and the data. The lengths - /// are encoded using DELTA_BINARY_PACKED - DeltaLengthByteArray, - /// Incremental-encoded byte array. Prefix lengths are encoded using DELTA_BINARY_PACKED. - /// Suffixes are stored as delta length byte arrays. - DeltaByteArray, - /// Dictionary encoding: the ids are encoded using the RLE encoding - RleDictionary, - /// Encoding for floating-point data. - /// K byte-streams are created where K is the size in bytes of the data type. - /// The individual bytes of an FP value are scattered to the corresponding stream and - /// the streams are concatenated. - /// This itself does not reduce the size of the data but can lead to better compression - /// afterwards. - ByteStreamSplit, -} - -impl TryFrom for Encoding { - type Error = Error; - - fn try_from(encoding: ParquetEncoding) -> Result { - Ok(match encoding { - ParquetEncoding::PLAIN => Encoding::Plain, - ParquetEncoding::PLAIN_DICTIONARY => Encoding::PlainDictionary, - ParquetEncoding::RLE => Encoding::Rle, - ParquetEncoding::BIT_PACKED => Encoding::BitPacked, - ParquetEncoding::DELTA_BINARY_PACKED => Encoding::DeltaBinaryPacked, - ParquetEncoding::DELTA_LENGTH_BYTE_ARRAY => Encoding::DeltaLengthByteArray, - ParquetEncoding::DELTA_BYTE_ARRAY => Encoding::DeltaByteArray, - ParquetEncoding::RLE_DICTIONARY => Encoding::RleDictionary, - ParquetEncoding::BYTE_STREAM_SPLIT => Encoding::ByteStreamSplit, - _ => return Err(Error::oos("Thrift out of range")), - }) - } -} - -impl From for ParquetEncoding { - fn from(encoding: Encoding) -> Self { - match encoding { - Encoding::Plain => ParquetEncoding::PLAIN, - Encoding::PlainDictionary => ParquetEncoding::PLAIN_DICTIONARY, - Encoding::Rle => ParquetEncoding::RLE, - Encoding::BitPacked => ParquetEncoding::BIT_PACKED, - Encoding::DeltaBinaryPacked => ParquetEncoding::DELTA_BINARY_PACKED, - Encoding::DeltaLengthByteArray => ParquetEncoding::DELTA_LENGTH_BYTE_ARRAY, - Encoding::DeltaByteArray => ParquetEncoding::DELTA_BYTE_ARRAY, - Encoding::RleDictionary => ParquetEncoding::RLE_DICTIONARY, - Encoding::ByteStreamSplit => ParquetEncoding::BYTE_STREAM_SPLIT, - } - } -} - -/// Enum to annotate whether lists of min/max elements inside ColumnIndex -/// are ordered and if so, in which direction. -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] -pub enum BoundaryOrder { - Unordered, - Ascending, - Descending, -} - -impl Default for BoundaryOrder { - fn default() -> Self { - Self::Unordered - } -} - -impl TryFrom for BoundaryOrder { - type Error = Error; - - fn try_from(encoding: ParquetBoundaryOrder) -> Result { - Ok(match encoding { - ParquetBoundaryOrder::UNORDERED => BoundaryOrder::Unordered, - ParquetBoundaryOrder::ASCENDING => BoundaryOrder::Ascending, - ParquetBoundaryOrder::DESCENDING => BoundaryOrder::Descending, - _ => return Err(Error::oos("BoundaryOrder Thrift value out of range")), - }) - } -} - -impl From for ParquetBoundaryOrder { - fn from(encoding: BoundaryOrder) -> Self { - match encoding { - BoundaryOrder::Unordered => ParquetBoundaryOrder::UNORDERED, - BoundaryOrder::Ascending => ParquetBoundaryOrder::ASCENDING, - BoundaryOrder::Descending => ParquetBoundaryOrder::DESCENDING, - } - } -} - -pub trait DataPageHeaderExt { - fn encoding(&self) -> Encoding; - fn repetition_level_encoding(&self) -> Encoding; - fn definition_level_encoding(&self) -> Encoding; -} - -impl DataPageHeaderExt for DataPageHeader { - fn encoding(&self) -> Encoding { - self.encoding.try_into().unwrap() - } - - fn repetition_level_encoding(&self) -> Encoding { - self.repetition_level_encoding.try_into().unwrap() - } - - fn definition_level_encoding(&self) -> Encoding { - self.definition_level_encoding.try_into().unwrap() - } -} - -impl DataPageHeaderExt for DataPageHeaderV2 { - fn encoding(&self) -> Encoding { - self.encoding.try_into().unwrap() - } - - fn repetition_level_encoding(&self) -> Encoding { - Encoding::Rle - } - - fn definition_level_encoding(&self) -> Encoding { - Encoding::Rle - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum TimeUnit { - Milliseconds, - Microseconds, - Nanoseconds, -} - -impl From for TimeUnit { - fn from(encoding: ParquetTimeUnit) -> Self { - match encoding { - ParquetTimeUnit::MILLIS(_) => TimeUnit::Milliseconds, - ParquetTimeUnit::MICROS(_) => TimeUnit::Microseconds, - ParquetTimeUnit::NANOS(_) => TimeUnit::Nanoseconds, - } - } -} - -impl From for ParquetTimeUnit { - fn from(unit: TimeUnit) -> Self { - match unit { - TimeUnit::Milliseconds => ParquetTimeUnit::MILLIS(Default::default()), - TimeUnit::Microseconds => ParquetTimeUnit::MICROS(Default::default()), - TimeUnit::Nanoseconds => ParquetTimeUnit::NANOS(Default::default()), - } - } -} - -/// Enum of all valid logical integer types -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum IntegerType { - Int8, - Int16, - Int32, - Int64, - UInt8, - UInt16, - UInt32, - UInt64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum PrimitiveLogicalType { - String, - Enum, - Decimal(usize, usize), - Date, - Time { - unit: TimeUnit, - is_adjusted_to_utc: bool, - }, - Timestamp { - unit: TimeUnit, - is_adjusted_to_utc: bool, - }, - Integer(IntegerType), - Unknown, - Json, - Bson, - Uuid, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum GroupLogicalType { - Map, - List, -} - -impl From for ParquetLogicalType { - fn from(type_: GroupLogicalType) -> Self { - match type_ { - GroupLogicalType::Map => ParquetLogicalType::MAP(Default::default()), - GroupLogicalType::List => ParquetLogicalType::LIST(Default::default()), - } - } -} - -impl From<(i32, bool)> for IntegerType { - fn from((bit_width, is_signed): (i32, bool)) -> Self { - match (bit_width, is_signed) { - (8, true) => IntegerType::Int8, - (16, true) => IntegerType::Int16, - (32, true) => IntegerType::Int32, - (64, true) => IntegerType::Int64, - (8, false) => IntegerType::UInt8, - (16, false) => IntegerType::UInt16, - (32, false) => IntegerType::UInt32, - (64, false) => IntegerType::UInt64, - // The above are the only possible annotations for parquet's int32. Anything else - // is a deviation to the parquet specification and we ignore - _ => IntegerType::Int32, - } - } -} - -impl From for (usize, bool) { - fn from(type_: IntegerType) -> (usize, bool) { - match type_ { - IntegerType::Int8 => (8, true), - IntegerType::Int16 => (16, true), - IntegerType::Int32 => (32, true), - IntegerType::Int64 => (64, true), - IntegerType::UInt8 => (8, false), - IntegerType::UInt16 => (16, false), - IntegerType::UInt32 => (32, false), - IntegerType::UInt64 => (64, false), - } - } -} - -impl TryFrom for PrimitiveLogicalType { - type Error = Error; - - fn try_from(type_: ParquetLogicalType) -> Result { - Ok(match type_ { - ParquetLogicalType::STRING(_) => PrimitiveLogicalType::String, - ParquetLogicalType::ENUM(_) => PrimitiveLogicalType::Enum, - ParquetLogicalType::DECIMAL(decimal) => PrimitiveLogicalType::Decimal( - decimal.precision.try_into()?, - decimal.scale.try_into()?, - ), - ParquetLogicalType::DATE(_) => PrimitiveLogicalType::Date, - ParquetLogicalType::TIME(time) => PrimitiveLogicalType::Time { - unit: time.unit.into(), - is_adjusted_to_utc: time.is_adjusted_to_u_t_c, - }, - ParquetLogicalType::TIMESTAMP(time) => PrimitiveLogicalType::Timestamp { - unit: time.unit.into(), - is_adjusted_to_utc: time.is_adjusted_to_u_t_c, - }, - ParquetLogicalType::INTEGER(int) => { - PrimitiveLogicalType::Integer((int.bit_width as i32, int.is_signed).into()) - } - ParquetLogicalType::UNKNOWN(_) => PrimitiveLogicalType::Unknown, - ParquetLogicalType::JSON(_) => PrimitiveLogicalType::Json, - ParquetLogicalType::BSON(_) => PrimitiveLogicalType::Bson, - ParquetLogicalType::UUID(_) => PrimitiveLogicalType::Uuid, - _ => return Err(Error::oos("LogicalType value out of range")), - }) - } -} - -impl TryFrom for GroupLogicalType { - type Error = Error; - - fn try_from(type_: ParquetLogicalType) -> Result { - Ok(match type_ { - ParquetLogicalType::LIST(_) => GroupLogicalType::List, - ParquetLogicalType::MAP(_) => GroupLogicalType::Map, - _ => return Err(Error::oos("LogicalType value out of range")), - }) - } -} - -impl From for ParquetLogicalType { - fn from(type_: PrimitiveLogicalType) -> Self { - match type_ { - PrimitiveLogicalType::String => ParquetLogicalType::STRING(Default::default()), - PrimitiveLogicalType::Enum => ParquetLogicalType::ENUM(Default::default()), - PrimitiveLogicalType::Decimal(precision, scale) => { - ParquetLogicalType::DECIMAL(DecimalType { - precision: precision as i32, - scale: scale as i32, - }) - } - PrimitiveLogicalType::Date => ParquetLogicalType::DATE(Default::default()), - PrimitiveLogicalType::Time { - unit, - is_adjusted_to_utc, - } => ParquetLogicalType::TIME(TimeType { - unit: unit.into(), - is_adjusted_to_u_t_c: is_adjusted_to_utc, - }), - PrimitiveLogicalType::Timestamp { - unit, - is_adjusted_to_utc, - } => ParquetLogicalType::TIMESTAMP(TimestampType { - unit: unit.into(), - is_adjusted_to_u_t_c: is_adjusted_to_utc, - }), - PrimitiveLogicalType::Integer(integer) => { - let (bit_width, is_signed) = integer.into(); - ParquetLogicalType::INTEGER(IntType { - bit_width: bit_width as i8, - is_signed, - }) - } - PrimitiveLogicalType::Unknown => ParquetLogicalType::UNKNOWN(Default::default()), - PrimitiveLogicalType::Json => ParquetLogicalType::JSON(Default::default()), - PrimitiveLogicalType::Bson => ParquetLogicalType::BSON(Default::default()), - PrimitiveLogicalType::Uuid => ParquetLogicalType::UUID(Default::default()), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn round_trip_primitive() -> Result<(), Error> { - use PrimitiveLogicalType::*; - let a = vec![ - String, - Enum, - Decimal(3, 1), - Date, - Time { - unit: TimeUnit::Milliseconds, - is_adjusted_to_utc: true, - }, - Timestamp { - unit: TimeUnit::Milliseconds, - is_adjusted_to_utc: true, - }, - Integer(IntegerType::Int16), - Unknown, - Json, - Bson, - Uuid, - ]; - for a in a { - let c: ParquetLogicalType = a.into(); - let e: PrimitiveLogicalType = c.try_into()?; - assert_eq!(e, a); - } - Ok(()) - } - - #[test] - fn round_trip_encoding() -> Result<(), Error> { - use Encoding::*; - let a = vec![ - Plain, - PlainDictionary, - Rle, - BitPacked, - DeltaBinaryPacked, - DeltaLengthByteArray, - DeltaByteArray, - RleDictionary, - ByteStreamSplit, - ]; - for a in a { - let c: ParquetEncoding = a.into(); - let e: Encoding = c.try_into()?; - assert_eq!(e, a); - } - Ok(()) - } - - #[test] - fn round_compression() -> Result<(), Error> { - use Compression::*; - let a = vec![Uncompressed, Snappy, Gzip, Lzo, Brotli, Lz4, Zstd, Lz4Raw]; - for a in a { - let c: CompressionCodec = a.into(); - let e: Compression = c.try_into()?; - assert_eq!(e, a); - } - Ok(()) - } -} diff --git a/src/common/parquet2/src/read/column/mod.rs b/src/common/parquet2/src/read/column/mod.rs deleted file mode 100644 index ebb3d12a641a..000000000000 --- a/src/common/parquet2/src/read/column/mod.rs +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Read; -use std::io::Seek; -use std::vec::IntoIter; - -use super::get_field_columns; -use super::get_page_iterator; -use super::PageFilter; -use super::PageReader; -use crate::error::Error; -use crate::metadata::ColumnChunkMetaData; -use crate::metadata::RowGroupMetaData; -use crate::page::CompressedPage; -use crate::schema::types::ParquetType; - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -mod stream; - -/// Returns a [`ColumnIterator`] of column chunks corresponding to `field`. -/// -/// Contrarily to [`get_page_iterator`] that returns a single iterator of pages, this iterator -/// iterates over columns, one by one, and returns a [`PageReader`] per column. -/// For primitive fields (e.g. `i64`), [`ColumnIterator`] yields exactly one column. -/// For complex fields, it yields multiple columns. -/// `max_page_size` is the maximum number of bytes allowed. -pub fn get_column_iterator( - reader: R, - row_group: &RowGroupMetaData, - field_name: &str, - page_filter: Option, - scratch: Vec, - max_page_size: usize, -) -> ColumnIterator { - let columns = get_field_columns(row_group.columns(), field_name) - .cloned() - .collect::>(); - - ColumnIterator::new(reader, columns, page_filter, scratch, max_page_size) -} - -/// State of [`MutStreamingIterator`]. -#[derive(Debug)] -pub enum State { - /// Iterator still has elements - Some(T), - /// Iterator finished - Finished(Vec), -} - -/// A special kind of fallible streaming iterator where `advance` consumes the iterator. -pub trait MutStreamingIterator: Sized { - type Item; - type Error; - - fn advance(self) -> std::result::Result, Self::Error>; - fn get(&mut self) -> Option<&mut Self::Item>; -} - -/// A [`MutStreamingIterator`] that reads column chunks one by one, -/// returning a [`PageReader`] per column. -pub struct ColumnIterator { - reader: Option, - columns: Vec, - page_filter: Option, - current: Option<(PageReader, ColumnChunkMetaData)>, - scratch: Vec, - max_page_size: usize, -} - -impl ColumnIterator { - /// Returns a new [`ColumnIterator`] - /// `max_page_size` is the maximum allowed page size - pub fn new( - reader: R, - mut columns: Vec, - page_filter: Option, - scratch: Vec, - max_page_size: usize, - ) -> Self { - columns.reverse(); - Self { - reader: Some(reader), - scratch, - columns, - page_filter, - current: None, - max_page_size, - } - } -} - -impl MutStreamingIterator for ColumnIterator { - type Item = (PageReader, ColumnChunkMetaData); - type Error = Error; - - fn advance(mut self) -> Result, Error> { - let (reader, scratch) = if let Some((iter, _)) = self.current { - iter.into_inner() - } else { - (self.reader.unwrap(), self.scratch) - }; - if self.columns.is_empty() { - return Ok(State::Finished(scratch)); - }; - let column = self.columns.pop().unwrap(); - - let iter = get_page_iterator( - &column, - reader, - self.page_filter.clone(), - scratch, - self.max_page_size, - )?; - let current = Some((iter, column)); - Ok(State::Some(Self { - reader: None, - columns: self.columns, - page_filter: self.page_filter, - current, - scratch: vec![], - max_page_size: self.max_page_size, - })) - } - - fn get(&mut self) -> Option<&mut Self::Item> { - self.current.as_mut() - } -} - -/// A [`MutStreamingIterator`] of pre-read column chunks -#[derive(Debug)] -pub struct ReadColumnIterator { - field: ParquetType, - chunks: Vec<(Vec>, ColumnChunkMetaData)>, - current: Option<(IntoIter>, ColumnChunkMetaData)>, -} - -impl ReadColumnIterator { - /// Returns a new [`ReadColumnIterator`] - pub fn new( - field: ParquetType, - chunks: Vec<(Vec>, ColumnChunkMetaData)>, - ) -> Self { - Self { - field, - chunks, - current: None, - } - } -} - -impl MutStreamingIterator for ReadColumnIterator { - type Item = (IntoIter>, ColumnChunkMetaData); - type Error = Error; - - fn advance(mut self) -> Result, Error> { - if self.chunks.is_empty() { - return Ok(State::Finished(vec![])); - } - self.current = self - .chunks - .pop() - .map(|(pages, meta)| (pages.into_iter(), meta)); - Ok(State::Some(Self { - field: self.field, - chunks: self.chunks, - current: self.current, - })) - } - - fn get(&mut self) -> Option<&mut Self::Item> { - self.current.as_mut() - } -} - -/// Reads all columns that are part of the parquet field `field_name` -/// # Implementation -/// This operation is IO-bounded `O(C)` where C is the number of columns associated to -/// the field (one for non-nested types) -/// It reads the columns sequentially. Use [`read_column`] to fork this operation to multiple -/// readers. -pub fn read_columns<'a, R: Read + Seek>( - reader: &mut R, - columns: &'a [ColumnChunkMetaData], - field_name: &'a str, -) -> Result)>, Error> { - get_field_columns(columns, field_name) - .map(|column| read_column(reader, column).map(|c| (column, c))) - .collect() -} - -/// Reads a column chunk into memory -/// This operation is IO-bounded and allocates the column's `compressed_size`. -pub fn read_column(reader: &mut R, column: &ColumnChunkMetaData) -> Result, Error> -where R: Read + Seek { - let (start, length) = column.byte_range(); - reader.seek(std::io::SeekFrom::Start(start))?; - - let mut chunk = vec![]; - chunk.try_reserve(length as usize)?; - reader.by_ref().take(length).read_to_end(&mut chunk)?; - Ok(chunk) -} - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub use stream::read_column_async; -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub use stream::read_columns_async; diff --git a/src/common/parquet2/src/read/column/stream.rs b/src/common/parquet2/src/read/column/stream.rs deleted file mode 100644 index 5b9e6d67cb56..000000000000 --- a/src/common/parquet2/src/read/column/stream.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use futures::future::try_join_all; -use futures::future::BoxFuture; -use futures::AsyncRead; -use futures::AsyncReadExt; -use futures::AsyncSeek; -use futures::AsyncSeekExt; - -use crate::error::Error; -use crate::metadata::ColumnChunkMetaData; -use crate::read::get_field_columns; - -/// Reads a single column chunk into memory asynchronously -pub async fn read_column_async<'b, R, F>( - factory: F, - meta: &ColumnChunkMetaData, -) -> Result, Error> -where - R: AsyncRead + AsyncSeek + Send + Unpin, - F: Fn() -> BoxFuture<'b, std::io::Result>, -{ - let mut reader = factory().await?; - let (start, length) = meta.byte_range(); - reader.seek(std::io::SeekFrom::Start(start)).await?; - - let mut chunk = vec![]; - chunk.try_reserve(length as usize)?; - reader.take(length as u64).read_to_end(&mut chunk).await?; - Result::Ok(chunk) -} - -/// Reads all columns that are part of the parquet field `field_name` -/// # Implementation -/// This operation is IO-bounded `O(C)` where C is the number of columns associated to -/// the field (one for non-nested types) -/// -/// It does so asynchronously via a single `join_all` over all the necessary columns for -/// `field_name`. -pub async fn read_columns_async< - 'a, - 'b, - R: AsyncRead + AsyncSeek + Send + Unpin, - F: Fn() -> BoxFuture<'b, std::io::Result> + Clone, ->( - factory: F, - columns: &'a [ColumnChunkMetaData], - field_name: &'a str, -) -> Result)>, Error> { - let fields = get_field_columns(columns, field_name).collect::>(); - let futures = fields - .iter() - .map(|meta| async { read_column_async(factory.clone(), meta).await }); - - let columns = try_join_all(futures).await?; - Ok(fields.into_iter().zip(columns).collect()) -} diff --git a/src/common/parquet2/src/read/compression.rs b/src/common/parquet2/src/read/compression.rs deleted file mode 100644 index 372df0b0c8c9..000000000000 --- a/src/common/parquet2/src/read/compression.rs +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::DataPageHeaderV2; -use streaming_decompression; - -use super::page::PageIterator; -use crate::compression::Compression; -use crate::compression::{self}; -use crate::error::Error; -use crate::error::Result; -use crate::page::CompressedPage; -use crate::page::DataPage; -use crate::page::DataPageHeader; -use crate::page::DictPage; -use crate::page::Page; -use crate::FallibleStreamingIterator; - -fn decompress_v1(compressed: &[u8], compression: Compression, buffer: &mut [u8]) -> Result<()> { - compression::decompress(compression, compressed, buffer) -} - -fn decompress_v2( - compressed: &[u8], - page_header: &DataPageHeaderV2, - compression: Compression, - buffer: &mut [u8], -) -> Result<()> { - // When processing data page v2, depending on enabled compression for the - // page, we should account for uncompressed data ('offset') of - // repetition and definition levels. - // - // We always use 0 offset for other pages other than v2, `true` flag means - // that compression will be applied if decompressor is defined - let offset = (page_header.definition_levels_byte_length - + page_header.repetition_levels_byte_length) as usize; - // When is_compressed flag is missing the page is considered compressed - let can_decompress = page_header.is_compressed.unwrap_or(true); - - if can_decompress { - if offset > buffer.len() || offset > compressed.len() { - return Err(Error::OutOfSpec( - "V2 Page Header reported incorrect offset to compressed data".to_string(), - )); - } - - (buffer[..offset]).copy_from_slice(&compressed[..offset]); - - compression::decompress(compression, &compressed[offset..], &mut buffer[offset..])?; - } else { - if buffer.len() != compressed.len() { - return Err(Error::OutOfSpec( - "V2 Page Header reported incorrect decompressed size".to_string(), - )); - } - buffer.copy_from_slice(compressed); - } - Ok(()) -} - -/// decompresses a [`CompressedDataPage`] into `buffer`. -/// If the page is un-compressed, `buffer` is swapped instead. -/// Returns whether the page was decompressed. -pub fn decompress_buffer( - compressed_page: &mut CompressedPage, - buffer: &mut Vec, -) -> Result { - if compressed_page.compression() != Compression::Uncompressed { - // prepare the compression buffer - let read_size = compressed_page.uncompressed_size(); - - if read_size > buffer.capacity() { - // dealloc and ignore region, replacing it by a new region. - // This won't reallocate - it frees and calls `alloc_zeroed` - *buffer = vec![0; read_size]; - } else if read_size > buffer.len() { - // fill what we need with zeros so that we can use them in `Read`. - // This won't reallocate - buffer.resize(read_size, 0); - } else { - buffer.truncate(read_size); - } - match compressed_page { - CompressedPage::Data(compressed_page) => match compressed_page.header() { - DataPageHeader::V1(_) => { - decompress_v1(&compressed_page.buffer, compressed_page.compression, buffer)? - } - DataPageHeader::V2(header) => decompress_v2( - &compressed_page.buffer, - header, - compressed_page.compression, - buffer, - )?, - }, - CompressedPage::Dict(page) => decompress_v1(&page.buffer, page.compression(), buffer)?, - } - Ok(true) - } else { - // page.buffer is already decompressed => swap it with `buffer`, making `page.buffer` the - // decompression buffer and `buffer` the decompressed buffer - std::mem::swap(compressed_page.buffer(), buffer); - Ok(false) - } -} - -fn create_page(compressed_page: CompressedPage, buffer: Vec) -> Page { - match compressed_page { - CompressedPage::Data(page) => Page::Data(DataPage::new_read( - page.header, - buffer, - page.descriptor, - page.selected_rows, - )), - CompressedPage::Dict(page) => Page::Dict(DictPage { - buffer, - num_values: page.num_values, - is_sorted: page.is_sorted, - }), - } -} - -/// Decompresses the page, using `buffer` for decompression. -/// If `page.buffer.len() == 0`, there was no decompression and the buffer was moved. -/// Else, decompression took place. -pub fn decompress(mut compressed_page: CompressedPage, buffer: &mut Vec) -> Result { - decompress_buffer(&mut compressed_page, buffer)?; - Ok(create_page(compressed_page, std::mem::take(buffer))) -} - -fn decompress_reuse( - mut compressed_page: CompressedPage, - iterator: &mut P, - buffer: &mut Vec, -) -> Result<(Page, bool)> { - let was_decompressed = decompress_buffer(&mut compressed_page, buffer)?; - - if was_decompressed { - iterator.swap_buffer(compressed_page.buffer()) - }; - - let new_page = create_page(compressed_page, std::mem::take(buffer)); - - Ok((new_page, was_decompressed)) -} - -/// Decompressor that allows re-using the page buffer of [`PageIterator`]. -/// # Implementation -/// The implementation depends on whether a page is compressed or not. -/// > `PageReader(a)`, `CompressedPage(b)`, `Decompressor(c)`, `DecompressedPage(d)` -/// ### un-compressed pages: -/// > page iter: `a` is swapped with `b` -/// > decompress iter: `b` is swapped with `d`, `b` is swapped with `a` -/// therefore: -/// * `PageReader` has its buffer back -/// * `Decompressor`'s buffer is un-used -/// * `DecompressedPage` has the same data as `CompressedPage` had -/// ### compressed pages: -/// > page iter: `a` is swapped with `b` -/// > decompress iter: -/// > * `b` is decompressed into `c` -/// > * `b` is swapped with `a` -/// > * `c` is moved to `d` -/// > * (next iteration): `d` is moved to `c` -/// therefore, while the page is available: -/// * `PageReader` has its buffer back -/// * `Decompressor`'s buffer empty -/// * `DecompressedPage` has the decompressed buffer -/// after the page is used: -/// * `PageReader` has its buffer back -/// * `Decompressor` has its buffer back -/// * `DecompressedPage` has an empty buffer -pub struct Decompressor { - iter: P, - buffer: Vec, - current: Option, - was_decompressed: bool, -} - -impl Decompressor

{ - /// Creates a new [`Decompressor`]. - pub fn new(iter: P, buffer: Vec) -> Self { - Self { - iter, - buffer, - current: None, - was_decompressed: false, - } - } - - /// Returns two buffers: the first buffer corresponds to the page buffer, - /// the second to the decompression buffer. - pub fn into_buffers(mut self) -> (Vec, Vec) { - let mut page_buffer = vec![]; - self.iter.swap_buffer(&mut page_buffer); - (page_buffer, self.buffer) - } -} - -impl FallibleStreamingIterator for Decompressor

{ - type Item = Page; - type Error = Error; - - fn advance(&mut self) -> Result<()> { - if let Some(page) = self.current.as_mut() { - if self.was_decompressed { - self.buffer = std::mem::take(page.buffer()); - } else { - self.iter.swap_buffer(page.buffer()); - } - } - - let next = self - .iter - .next() - .map(|x| { - x.and_then(|x| { - let (page, was_decompressed) = - decompress_reuse(x, &mut self.iter, &mut self.buffer)?; - self.was_decompressed = was_decompressed; - Ok(page) - }) - }) - .transpose()?; - self.current = next; - Ok(()) - } - - fn get(&self) -> Option<&Self::Item> { - self.current.as_ref() - } -} - -type _Decompressor = streaming_decompression::Decompressor< - CompressedPage, - Page, - fn(CompressedPage, &mut Vec) -> Result, - Error, - I, ->; - -impl streaming_decompression::Compressed for CompressedPage { - #[inline] - fn is_compressed(&self) -> bool { - self.compression() != Compression::Uncompressed - } -} - -impl streaming_decompression::Decompressed for Page { - #[inline] - fn buffer_mut(&mut self) -> &mut Vec { - self.buffer() - } -} - -/// A [`FallibleStreamingIterator`] that decompresses [`CompressedPage`] into [`DataPage`]. -/// # Implementation -/// This decompressor uses an internal [`Vec`] to perform decompressions which -/// is re-used across pages, so that a single allocation is required. -/// If the pages are not compressed, the internal buffer is not used. -pub struct BasicDecompressor>> { - iter: _Decompressor, -} - -impl BasicDecompressor -where I: Iterator> -{ - /// Returns a new [`BasicDecompressor`]. - pub fn new(iter: I, buffer: Vec) -> Self { - Self { - iter: _Decompressor::new(iter, buffer, decompress), - } - } - - /// Returns its internal buffer, consuming itself. - pub fn into_inner(self) -> Vec { - self.iter.into_inner() - } -} - -impl FallibleStreamingIterator for BasicDecompressor -where I: Iterator> -{ - type Item = Page; - type Error = Error; - - fn advance(&mut self) -> Result<()> { - self.iter.advance() - } - - fn get(&self) -> Option<&Self::Item> { - self.iter.get() - } -} diff --git a/src/common/parquet2/src/read/indexes/deserialize.rs b/src/common/parquet2/src/read/indexes/deserialize.rs deleted file mode 100644 index 11239bb61fa3..000000000000 --- a/src/common/parquet2/src/read/indexes/deserialize.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::thrift::protocol::TCompactInputProtocol; -use parquet_format_safe::ColumnIndex; - -use crate::error::Error; -use crate::indexes::BooleanIndex; -use crate::indexes::ByteIndex; -use crate::indexes::FixedLenByteIndex; -use crate::indexes::Index; -use crate::indexes::NativeIndex; -use crate::schema::types::PhysicalType; -use crate::schema::types::PrimitiveType; - -pub fn deserialize(data: &[u8], primitive_type: PrimitiveType) -> Result, Error> { - let mut prot = TCompactInputProtocol::new(data, data.len() * 2 + 1024); - - let index = ColumnIndex::read_from_in_protocol(&mut prot)?; - - let index = match primitive_type.physical_type { - PhysicalType::Boolean => Box::new(BooleanIndex::try_new(index)?) as Box, - PhysicalType::Int32 => Box::new(NativeIndex::::try_new(index, primitive_type)?), - PhysicalType::Int64 => Box::new(NativeIndex::::try_new(index, primitive_type)?), - PhysicalType::Int96 => Box::new(NativeIndex::<[u32; 3]>::try_new(index, primitive_type)?), - PhysicalType::Float => Box::new(NativeIndex::::try_new(index, primitive_type)?), - PhysicalType::Double => Box::new(NativeIndex::::try_new(index, primitive_type)?), - PhysicalType::ByteArray => Box::new(ByteIndex::try_new(index, primitive_type)?), - PhysicalType::FixedLenByteArray(_) => { - Box::new(FixedLenByteIndex::try_new(index, primitive_type)?) - } - }; - - Ok(index) -} diff --git a/src/common/parquet2/src/read/indexes/mod.rs b/src/common/parquet2/src/read/indexes/mod.rs deleted file mode 100644 index c8f54b6bb615..000000000000 --- a/src/common/parquet2/src/read/indexes/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod deserialize; -mod read; - -pub use read::*; diff --git a/src/common/parquet2/src/read/indexes/read.rs b/src/common/parquet2/src/read/indexes/read.rs deleted file mode 100644 index a6bcd2fb8bc2..000000000000 --- a/src/common/parquet2/src/read/indexes/read.rs +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::convert::TryInto; -use std::io::Cursor; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; - -use parquet_format_safe::thrift::protocol::TCompactInputProtocol; -use parquet_format_safe::ColumnChunk; -use parquet_format_safe::OffsetIndex; -use parquet_format_safe::PageLocation; - -use super::deserialize::deserialize; -use crate::error::Error; -use crate::indexes::Index; -use crate::metadata::ColumnChunkMetaData; - -fn prepare_read Option, G: Fn(&ColumnChunk) -> Option>( - chunks: &[ColumnChunkMetaData], - get_offset: F, - get_length: G, -) -> Result<(u64, Vec), Error> { - // c1: [start, length] - // ... - // cN: [start, length] - - let first_chunk = if let Some(chunk) = chunks.first() { - chunk - } else { - return Ok((0, vec![])); - }; - let metadata = first_chunk.column_chunk(); - - let offset: u64 = if let Some(offset) = get_offset(metadata) { - offset.try_into()? - } else { - return Ok((0, vec![])); - }; - - let lengths = chunks - .iter() - .map(|x| get_length(x.column_chunk())) - .map(|maybe_length| { - let index_length = maybe_length.ok_or_else(|| { - Error::oos("The column length must exist if column offset exists") - })?; - - Ok(index_length.try_into()?) - }) - .collect::, Error>>()?; - - Ok((offset, lengths)) -} - -fn prepare_column_index_read(chunks: &[ColumnChunkMetaData]) -> Result<(u64, Vec), Error> { - prepare_read(chunks, |x| x.column_index_offset, |x| x.column_index_length) -} - -fn prepare_offset_index_read(chunks: &[ColumnChunkMetaData]) -> Result<(u64, Vec), Error> { - prepare_read(chunks, |x| x.offset_index_offset, |x| x.offset_index_length) -} - -fn deserialize_column_indexes( - chunks: &[ColumnChunkMetaData], - data: &[u8], - lengths: Vec, -) -> Result>, Error> { - let mut start = 0; - let data = lengths.into_iter().map(|length| { - let r = &data[start..start + length]; - start += length; - r - }); - - chunks - .iter() - .zip(data) - .map(|(chunk, data)| { - let primitive_type = chunk.descriptor().descriptor.primitive_type.clone(); - deserialize(data, primitive_type) - }) - .collect() -} - -/// Reads the column indexes of all [`ColumnChunkMetaData`] and deserializes them into [`Index`]. -/// Returns an empty vector if indexes are not available -pub fn read_columns_indexes( - reader: &mut R, - chunks: &[ColumnChunkMetaData], -) -> Result>, Error> { - let (offset, lengths) = prepare_column_index_read(chunks)?; - - let length = lengths.iter().sum::(); - - reader.seek(SeekFrom::Start(offset))?; - - let mut data = vec![]; - data.try_reserve(length)?; - reader.by_ref().take(length as u64).read_to_end(&mut data)?; - - deserialize_column_indexes(chunks, &data, lengths) -} - -fn deserialize_page_locations( - data: &[u8], - column_number: usize, -) -> Result>, Error> { - let len = data.len() * 2 + 1024; - let mut reader = Cursor::new(data); - - (0..column_number) - .map(|_| { - let mut prot = TCompactInputProtocol::new(&mut reader, len); - let offset = OffsetIndex::read_from_in_protocol(&mut prot)?; - Ok(offset.page_locations) - }) - .collect() -} - -/// Read [`PageLocation`]s from the [`ColumnChunkMetaData`]s. -/// Returns an empty vector if indexes are not available -pub fn read_pages_locations( - reader: &mut R, - chunks: &[ColumnChunkMetaData], -) -> Result>, Error> { - let (offset, lengths) = prepare_offset_index_read(chunks)?; - - let length = lengths.iter().sum::(); - - reader.seek(SeekFrom::Start(offset))?; - - let mut data = vec![]; - data.try_reserve(length)?; - reader.by_ref().take(length as u64).read_to_end(&mut data)?; - - deserialize_page_locations(&data, chunks.len()) -} diff --git a/src/common/parquet2/src/read/levels.rs b/src/common/parquet2/src/read/levels.rs deleted file mode 100644 index 1452524cef66..000000000000 --- a/src/common/parquet2/src/read/levels.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/// Returns the number of bits needed to store the given maximum definition or repetition level. -#[inline] -pub fn get_bit_width(max_level: i16) -> u32 { - 16 - max_level.leading_zeros() -} - -#[cfg(test)] -mod tests { - use super::get_bit_width; - - #[test] - fn test_get_bit_width() { - assert_eq!(0, get_bit_width(0)); - assert_eq!(1, get_bit_width(1)); - assert_eq!(2, get_bit_width(2)); - assert_eq!(2, get_bit_width(3)); - assert_eq!(3, get_bit_width(4)); - assert_eq!(3, get_bit_width(5)); - assert_eq!(3, get_bit_width(6)); - assert_eq!(3, get_bit_width(7)); - assert_eq!(4, get_bit_width(8)); - assert_eq!(4, get_bit_width(15)); - - assert_eq!(8, get_bit_width(255)); - assert_eq!(9, get_bit_width(256)); - } -} diff --git a/src/common/parquet2/src/read/metadata.rs b/src/common/parquet2/src/read/metadata.rs deleted file mode 100644 index 40dded834fc7..000000000000 --- a/src/common/parquet2/src/read/metadata.rs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::cmp::min; -use std::convert::TryInto; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; - -use parquet_format_safe::thrift::protocol::TCompactInputProtocol; -use parquet_format_safe::FileMetaData as TFileMetaData; - -use super::super::metadata::FileMetaData; -use super::super::DEFAULT_FOOTER_READ_SIZE; -use super::super::FOOTER_SIZE; -use super::super::HEADER_SIZE; -use super::super::PARQUET_MAGIC; -use crate::error::Error; -use crate::error::Result; - -pub(super) fn metadata_len(buffer: &[u8], len: usize) -> i32 { - i32::from_le_bytes(buffer[len - 8..len - 4].try_into().unwrap()) -} - -// see (unstable) Seek::stream_len -fn stream_len(seek: &mut impl Seek) -> std::result::Result { - let old_pos = seek.seek(SeekFrom::Current(0))?; - let len = seek.seek(SeekFrom::End(0))?; - - // Avoid seeking a third time when we were already at the end of the - // stream. The branch is usually way cheaper than a seek operation. - if old_pos != len { - seek.seek(SeekFrom::Start(old_pos))?; - } - - Ok(len) -} - -/// Reads a [`FileMetaData`] from the reader, located at the end of the file. -pub fn read_metadata(reader: &mut R) -> Result { - // check file is large enough to hold footer - let file_size = stream_len(reader)?; - read_metadata_with_size(reader, file_size) -} - -/// Reads a [`FileMetaData`] from the reader, located at the end of the file, with known file size. -pub fn read_metadata_with_size( - reader: &mut R, - file_size: u64, -) -> Result { - if file_size < HEADER_SIZE + FOOTER_SIZE { - return Err(Error::oos( - "A parquet file must containt a header and footer with at least 12 bytes", - )); - } - - // read and cache up to DEFAULT_FOOTER_READ_SIZE bytes from the end and process the footer - let default_end_len = min(DEFAULT_FOOTER_READ_SIZE, file_size) as usize; - reader.seek(SeekFrom::End(-(default_end_len as i64)))?; - - let mut buffer = Vec::with_capacity(default_end_len); - reader - .by_ref() - .take(default_end_len as u64) - .read_to_end(&mut buffer)?; - - // check this is indeed a parquet file - if buffer[default_end_len - 4..] != PARQUET_MAGIC { - return Err(Error::oos("The file must end with PAR1")); - } - - let metadata_len = metadata_len(&buffer, default_end_len); - - let metadata_len: u64 = metadata_len.try_into()?; - - let footer_len = FOOTER_SIZE + metadata_len; - if footer_len > file_size { - return Err(Error::oos( - "The footer size must be smaller or equal to the file's size", - )); - } - - let reader: &[u8] = if (footer_len as usize) < buffer.len() { - // the whole metadata is in the bytes we already read - let remaining = buffer.len() - footer_len as usize; - &buffer[remaining..] - } else { - // the end of file read by default is not long enough, read again including the metadata. - reader.seek(SeekFrom::End(-(footer_len as i64)))?; - - buffer.clear(); - buffer.try_reserve(footer_len as usize)?; - reader.take(footer_len).read_to_end(&mut buffer)?; - - &buffer - }; - - // a highly nested but sparse struct could result in many allocations - let max_size = reader.len() * 2 + 1024; - - deserialize_metadata(reader, max_size) -} - -/// Parse loaded metadata bytes -pub fn deserialize_metadata(reader: R, max_size: usize) -> Result { - let mut prot = TCompactInputProtocol::new(reader, max_size); - let metadata = TFileMetaData::read_from_in_protocol(&mut prot)?; - - FileMetaData::try_from_thrift(metadata) -} diff --git a/src/common/parquet2/src/read/mod.rs b/src/common/parquet2/src/read/mod.rs deleted file mode 100644 index 6aa2a3d0af30..000000000000 --- a/src/common/parquet2/src/read/mod.rs +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod column; -mod compression; -mod indexes; -pub mod levels; -mod metadata; -mod page; -#[cfg(feature = "async")] -mod stream; - -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; -use std::sync::Arc; - -pub use column::*; -pub use compression::decompress; -pub use compression::BasicDecompressor; -pub use compression::Decompressor; -pub use indexes::read_columns_indexes; -pub use indexes::read_pages_locations; -pub use metadata::deserialize_metadata; -pub use metadata::read_metadata; -pub use metadata::read_metadata_with_size; -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub use page::get_page_stream; -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub use page::get_page_stream_from_column_start; -pub use page::IndexedPageReader; -pub use page::PageFilter; -pub use page::PageIterator; -pub use page::PageMetaData; -pub use page::PageReader; -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub use stream::read_metadata as read_metadata_async; - -use crate::error::Result; -use crate::metadata::ColumnChunkMetaData; -use crate::metadata::FileMetaData; -use crate::metadata::RowGroupMetaData; - -/// Filters row group metadata to only those row groups, -/// for which the predicate function returns true -pub fn filter_row_groups( - metadata: &FileMetaData, - predicate: &dyn Fn(&RowGroupMetaData, usize) -> bool, -) -> FileMetaData { - let mut filtered_row_groups = Vec::::new(); - for (i, row_group_metadata) in metadata.row_groups.iter().enumerate() { - if predicate(row_group_metadata, i) { - filtered_row_groups.push(row_group_metadata.clone()); - } - } - let mut metadata = metadata.clone(); - metadata.row_groups = filtered_row_groups; - metadata -} - -/// Returns a new [`PageReader`] by seeking `reader` to the begining of `column_chunk`. -pub fn get_page_iterator( - column_chunk: &ColumnChunkMetaData, - mut reader: R, - pages_filter: Option, - scratch: Vec, - max_page_size: usize, -) -> Result> { - let pages_filter = pages_filter.unwrap_or_else(|| Arc::new(|_, _| true)); - - let (col_start, _) = column_chunk.byte_range(); - reader.seek(SeekFrom::Start(col_start))?; - Ok(PageReader::new( - reader, - column_chunk, - pages_filter, - scratch, - max_page_size, - )) -} - -/// Returns all [`ColumnChunkMetaData`] associated to `field_name`. -/// For non-nested types, this returns an iterator with a single column -pub fn get_field_columns<'a>( - columns: &'a [ColumnChunkMetaData], - field_name: &'a str, -) -> impl Iterator { - columns - .iter() - .filter(move |x| x.descriptor().path_in_schema[0] == field_name) -} diff --git a/src/common/parquet2/src/read/page/indexed_reader.rs b/src/common/parquet2/src/read/page/indexed_reader.rs deleted file mode 100644 index 0615afb09517..000000000000 --- a/src/common/parquet2/src/read/page/indexed_reader.rs +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; -use std::io::Cursor; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; - -use super::reader::finish_page; -use super::reader::read_page_header; -use super::reader::PageMetaData; -use crate::error::Error; -use crate::indexes::FilteredPage; -use crate::indexes::Interval; -use crate::metadata::ColumnChunkMetaData; -use crate::metadata::Descriptor; -use crate::page::CompressedDictPage; -use crate::page::CompressedPage; -use crate::page::ParquetPageHeader; -use crate::parquet_bridge::Compression; - -#[derive(Debug, Clone, Copy)] -enum State { - MaybeDict, - Data, -} - -/// A fallible [`Iterator`] of [`CompressedPage`]. This iterator leverages page indexes -/// to skip pages that are not needed. Consequently, the pages from this -/// iterator always have [`Some`] [`crate::page::CompressedDataPage::selected_rows()`] -pub struct IndexedPageReader { - // The source - reader: R, - - column_start: u64, - compression: Compression, - - // used to deserialize dictionary pages and attach the descriptor to every read page - descriptor: Descriptor, - - // buffer to read the whole page [header][data] into memory - buffer: Vec, - - // buffer to store the data [data] and re-use across pages - data_buffer: Vec, - - pages: VecDeque, - - state: State, -} - -fn read_page( - reader: &mut R, - start: u64, - length: usize, - buffer: &mut Vec, - data: &mut Vec, -) -> Result { - // seek to the page - reader.seek(SeekFrom::Start(start))?; - - // read [header][data] to buffer - buffer.clear(); - buffer.try_reserve(length)?; - reader.by_ref().take(length as u64).read_to_end(buffer)?; - - // deserialize [header] - let mut reader = Cursor::new(buffer); - let page_header = read_page_header(&mut reader, 1024 * 1024)?; - let header_size = reader.seek(SeekFrom::Current(0)).unwrap() as usize; - let buffer = reader.into_inner(); - - // copy [data] - data.clear(); - data.extend_from_slice(&buffer[header_size..]); - Ok(page_header) -} - -fn read_dict_page( - reader: &mut R, - start: u64, - length: usize, - buffer: &mut Vec, - data: &mut Vec, - compression: Compression, - descriptor: &Descriptor, -) -> Result { - let page_header = read_page(reader, start, length, buffer, data)?; - - let page = finish_page(page_header, data, compression, descriptor, None)?; - if let CompressedPage::Dict(page) = page { - Ok(page) - } else { - Err(Error::oos( - "The first page is not a dictionary page but it should", - )) - } -} - -impl IndexedPageReader { - /// Returns a new [`IndexedPageReader`]. - pub fn new( - reader: R, - column: &ColumnChunkMetaData, - pages: Vec, - buffer: Vec, - data_buffer: Vec, - ) -> Self { - Self::new_with_page_meta(reader, column.into(), pages, buffer, data_buffer) - } - - /// Returns a new [`IndexedPageReader`] with [`PageMetaData`]. - pub fn new_with_page_meta( - reader: R, - column: PageMetaData, - pages: Vec, - buffer: Vec, - data_buffer: Vec, - ) -> Self { - let pages = pages.into_iter().collect(); - Self { - reader, - column_start: column.column_start, - compression: column.compression, - descriptor: column.descriptor, - buffer, - data_buffer, - pages, - state: State::MaybeDict, - } - } - - /// consumes self into the reader and the two internal buffers - pub fn into_inner(self) -> (R, Vec, Vec) { - (self.reader, self.buffer, self.data_buffer) - } - - fn read_page( - &mut self, - start: u64, - length: usize, - selected_rows: Vec, - ) -> Result { - // it will be read - take buffer - let mut data = std::mem::take(&mut self.data_buffer); - - let page_header = read_page(&mut self.reader, start, length, &mut self.buffer, &mut data)?; - - finish_page( - page_header, - &mut data, - self.compression, - &self.descriptor, - Some(selected_rows), - ) - } - - fn read_dict(&mut self) -> Option> { - // a dictionary page exists iff the first data page is not at the start of - // the column - let (start, length) = match self.pages.get(0) { - Some(page) => { - let length = (page.start - self.column_start) as usize; - if length > 0 { - (self.column_start, length) - } else { - return None; - } - } - None => return None, - }; - - // it will be read - take buffer - let mut data = std::mem::take(&mut self.data_buffer); - - let maybe_page = read_dict_page( - &mut self.reader, - start, - length, - &mut self.buffer, - &mut data, - self.compression, - &self.descriptor, - ); - Some(maybe_page.map(CompressedPage::Dict)) - } -} - -impl Iterator for IndexedPageReader { - type Item = Result; - - fn next(&mut self) -> Option { - match self.state { - State::MaybeDict => { - self.state = State::Data; - if let Some(dict) = self.read_dict() { - Some(dict) - } else { - self.next() - } - } - State::Data => { - if let Some(page) = self.pages.pop_front() { - if page.selected_rows.is_empty() { - self.next() - } else { - Some(self.read_page(page.start, page.length, page.selected_rows)) - } - } else { - None - } - } - } - } -} diff --git a/src/common/parquet2/src/read/page/mod.rs b/src/common/parquet2/src/read/page/mod.rs deleted file mode 100644 index 9992450ee855..000000000000 --- a/src/common/parquet2/src/read/page/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod indexed_reader; -mod reader; -#[cfg(feature = "async")] -mod stream; - -pub use indexed_reader::IndexedPageReader; -pub use reader::PageFilter; -pub use reader::PageMetaData; -pub use reader::PageReader; - -use crate::error::Error; -use crate::page::CompressedPage; - -pub trait PageIterator: Iterator> { - fn swap_buffer(&mut self, buffer: &mut Vec); -} - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub use stream::get_page_stream; -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub use stream::get_page_stream_from_column_start; diff --git a/src/common/parquet2/src/read/page/reader.rs b/src/common/parquet2/src/read/page/reader.rs deleted file mode 100644 index 4c7521a9b5ba..000000000000 --- a/src/common/parquet2/src/read/page/reader.rs +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::convert::TryInto; -use std::io::Read; -use std::sync::Arc; - -use parquet_format_safe::thrift::protocol::TCompactInputProtocol; - -use super::PageIterator; -use crate::compression::Compression; -use crate::error::Error; -use crate::error::Result; -use crate::indexes::Interval; -use crate::metadata::ColumnChunkMetaData; -use crate::metadata::Descriptor; -use crate::page::CompressedDataPage; -use crate::page::CompressedDictPage; -use crate::page::CompressedPage; -use crate::page::DataPageHeader; -use crate::page::PageType; -use crate::page::ParquetPageHeader; -use crate::parquet_bridge::Encoding; - -/// This meta is a small part of [`ColumnChunkMetaData`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PageMetaData { - /// The start offset of this column chunk in file. - pub column_start: u64, - /// The number of values in this column chunk. - pub num_values: i64, - /// Compression type - pub compression: Compression, - /// The descriptor of this parquet column - pub descriptor: Descriptor, -} - -impl PageMetaData { - /// Returns a new [`PageMetaData`]. - pub fn new( - column_start: u64, - num_values: i64, - compression: Compression, - descriptor: Descriptor, - ) -> Self { - Self { - column_start, - num_values, - compression, - descriptor, - } - } -} - -impl From<&ColumnChunkMetaData> for PageMetaData { - fn from(column: &ColumnChunkMetaData) -> Self { - Self { - column_start: column.byte_range().0, - num_values: column.num_values(), - compression: column.compression(), - descriptor: column.descriptor().descriptor.clone(), - } - } -} - -/// Type declaration for a page filter -pub type PageFilter = Arc bool + Send + Sync>; - -/// A fallible [`Iterator`] of [`CompressedDataPage`]. This iterator reads pages back -/// to back until all pages have been consumed. -/// The pages from this iterator always have [`None`] [`crate::page::CompressedDataPage::selected_rows()`] since -/// filter pushdown is not supported without a -/// pre-computed [page index](https://github.com/apache/parquet-format/blob/master/PageIndex.md). -pub struct PageReader { - // The source - reader: R, - - compression: Compression, - - // The number of values we have seen so far. - seen_num_values: i64, - - // The number of total values in this column chunk. - total_num_values: i64, - - pages_filter: PageFilter, - - descriptor: Descriptor, - - // The currently allocated buffer. - pub(crate) scratch: Vec, - - // Maximum page size (compressed or uncompressed) to limit allocations - max_page_size: usize, -} - -impl PageReader { - /// Returns a new [`PageReader`]. - /// - /// It assumes that the reader has been `seeked` to the beginning of `column`. - /// The parameter `max_header_size` - pub fn new( - reader: R, - column: &ColumnChunkMetaData, - pages_filter: PageFilter, - scratch: Vec, - max_page_size: usize, - ) -> Self { - Self::new_with_page_meta(reader, column.into(), pages_filter, scratch, max_page_size) - } - - /// Create a a new [`PageReader`] with [`PageMetaData`]. - /// - /// It assumes that the reader has been `seeked` to the beginning of `column`. - pub fn new_with_page_meta( - reader: R, - reader_meta: PageMetaData, - pages_filter: PageFilter, - scratch: Vec, - max_page_size: usize, - ) -> Self { - Self { - reader, - total_num_values: reader_meta.num_values, - compression: reader_meta.compression, - seen_num_values: 0, - descriptor: reader_meta.descriptor, - pages_filter, - scratch, - max_page_size, - } - } - - /// Returns the reader and this Readers' interval buffer - pub fn into_inner(self) -> (R, Vec) { - (self.reader, self.scratch) - } -} - -impl PageIterator for PageReader { - fn swap_buffer(&mut self, scratch: &mut Vec) { - std::mem::swap(&mut self.scratch, scratch) - } -} - -impl Iterator for PageReader { - type Item = Result; - - fn next(&mut self) -> Option { - let mut buffer = std::mem::take(&mut self.scratch); - let maybe_maybe_page = next_page(self, &mut buffer).transpose(); - if let Some(ref maybe_page) = maybe_maybe_page { - if let Ok(CompressedPage::Data(page)) = maybe_page { - // check if we should filter it (only valid for data pages) - let to_consume = (self.pages_filter)(&self.descriptor, page.header()); - if !to_consume { - self.scratch = std::mem::take(&mut buffer); - return self.next(); - } - } - } else { - // no page => we take back the buffer - self.scratch = std::mem::take(&mut buffer); - } - maybe_maybe_page - } -} - -/// Reads Page header from Thrift. -pub(super) fn read_page_header( - reader: &mut R, - max_size: usize, -) -> Result { - let mut prot = TCompactInputProtocol::new(reader, max_size); - let page_header = ParquetPageHeader::read_from_in_protocol(&mut prot)?; - Ok(page_header) -} - -/// This function is lightweight and executes a minimal amount of work so that it is IO bounded. -// Any un-necessary CPU-intensive tasks SHOULD be executed on individual pages. -fn next_page( - reader: &mut PageReader, - buffer: &mut Vec, -) -> Result> { - if reader.seen_num_values >= reader.total_num_values { - return Ok(None); - }; - build_page(reader, buffer) -} - -pub(super) fn build_page( - reader: &mut PageReader, - buffer: &mut Vec, -) -> Result> { - let page_header = read_page_header(&mut reader.reader, reader.max_page_size)?; - - reader.seen_num_values += get_page_header(&page_header)? - .map(|x| x.num_values() as i64) - .unwrap_or_default(); - - let read_size: usize = page_header.compressed_page_size.try_into()?; - - if read_size > reader.max_page_size { - return Err(Error::WouldOverAllocate); - } - - buffer.clear(); - buffer.try_reserve(read_size)?; - let bytes_read = reader - .reader - .by_ref() - .take(read_size as u64) - .read_to_end(buffer)?; - - if bytes_read != read_size { - return Err(Error::oos( - "The page header reported the wrong page size".to_string(), - )); - } - - finish_page( - page_header, - buffer, - reader.compression, - &reader.descriptor, - None, - ) - .map(Some) -} - -pub(super) fn finish_page( - page_header: ParquetPageHeader, - data: &mut Vec, - compression: Compression, - descriptor: &Descriptor, - selected_rows: Option>, -) -> Result { - let type_ = page_header.type_.try_into()?; - let uncompressed_page_size = page_header.uncompressed_page_size.try_into()?; - match type_ { - PageType::DictionaryPage => { - let dict_header = page_header.dictionary_page_header.as_ref().ok_or_else(|| { - Error::oos( - "The page header type is a dictionary page but the dictionary header is empty", - ) - })?; - let is_sorted = dict_header.is_sorted.unwrap_or(false); - - // move the buffer to `dict_page` - let page = CompressedDictPage::new( - std::mem::take(data), - compression, - uncompressed_page_size, - dict_header.num_values.try_into()?, - is_sorted, - ); - - Ok(CompressedPage::Dict(page)) - } - PageType::DataPage => { - let header = page_header.data_page_header.ok_or_else(|| { - Error::oos("The page header type is a v1 data page but the v1 data header is empty") - })?; - - Ok(CompressedPage::Data(CompressedDataPage::new_read( - DataPageHeader::V1(header), - std::mem::take(data), - compression, - uncompressed_page_size, - descriptor.clone(), - selected_rows, - ))) - } - PageType::DataPageV2 => { - let header = page_header.data_page_header_v2.ok_or_else(|| { - Error::oos("The page header type is a v2 data page but the v2 data header is empty") - })?; - - Ok(CompressedPage::Data(CompressedDataPage::new_read( - DataPageHeader::V2(header), - std::mem::take(data), - compression, - uncompressed_page_size, - descriptor.clone(), - selected_rows, - ))) - } - } -} - -pub(super) fn get_page_header(header: &ParquetPageHeader) -> Result> { - let type_ = header.type_.try_into()?; - Ok(match type_ { - PageType::DataPage => { - let header = header.data_page_header.clone().ok_or_else(|| { - Error::oos("The page header type is a v1 data page but the v1 header is empty") - })?; - let _: Encoding = header.encoding.try_into()?; - let _: Encoding = header.repetition_level_encoding.try_into()?; - let _: Encoding = header.definition_level_encoding.try_into()?; - - Some(DataPageHeader::V1(header)) - } - PageType::DataPageV2 => { - let header = header.data_page_header_v2.clone().ok_or_else(|| { - Error::oos("The page header type is a v1 data page but the v1 header is empty") - })?; - let _: Encoding = header.encoding.try_into()?; - Some(DataPageHeader::V2(header)) - } - _ => None, - }) -} diff --git a/src/common/parquet2/src/read/page/stream.rs b/src/common/parquet2/src/read/page/stream.rs deleted file mode 100644 index 7f07b3a804f7..000000000000 --- a/src/common/parquet2/src/read/page/stream.rs +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::SeekFrom; - -use async_stream::try_stream; -use futures::io::copy; -use futures::io::sink; -use futures::AsyncRead; -use futures::AsyncReadExt; -use futures::AsyncSeek; -use futures::AsyncSeekExt; -use futures::Stream; -use parquet_format_safe::thrift::protocol::TCompactInputStreamProtocol; - -use super::reader::finish_page; -use super::reader::get_page_header; -use super::reader::PageMetaData; -use super::PageFilter; -use crate::compression::Compression; -use crate::error::Error; -use crate::error::Result; -use crate::metadata::ColumnChunkMetaData; -use crate::metadata::Descriptor; -use crate::page::CompressedPage; -use crate::page::ParquetPageHeader; - -/// Returns a stream of compressed data pages -pub async fn get_page_stream<'a, RR: AsyncRead + Unpin + Send + AsyncSeek>( - column_metadata: &'a ColumnChunkMetaData, - reader: &'a mut RR, - scratch: Vec, - pages_filter: PageFilter, - max_page_size: usize, -) -> Result> + 'a> { - get_page_stream_with_page_meta( - column_metadata.into(), - reader, - scratch, - pages_filter, - max_page_size, - ) - .await -} - -/// Returns a stream of compressed data pages from a reader that begins at the start of the column -pub async fn get_page_stream_from_column_start<'a, R: AsyncRead + Unpin + Send>( - column_metadata: &'a ColumnChunkMetaData, - reader: &'a mut R, - scratch: Vec, - pages_filter: PageFilter, - max_header_size: usize, -) -> Result> + 'a> { - let page_metadata: PageMetaData = column_metadata.into(); - Ok(_get_page_stream( - reader, - page_metadata.num_values, - page_metadata.compression, - page_metadata.descriptor, - scratch, - pages_filter, - max_header_size, - )) -} - -/// Returns a stream of compressed data pages with [`PageMetaData`] -pub async fn get_page_stream_with_page_meta( - page_metadata: PageMetaData, - reader: &mut RR, - scratch: Vec, - pages_filter: PageFilter, - max_page_size: usize, -) -> Result> + '_> { - let column_start = page_metadata.column_start; - reader.seek(SeekFrom::Start(column_start)).await?; - Ok(_get_page_stream( - reader, - page_metadata.num_values, - page_metadata.compression, - page_metadata.descriptor, - scratch, - pages_filter, - max_page_size, - )) -} - -fn _get_page_stream( - reader: &mut R, - total_num_values: i64, - compression: Compression, - descriptor: Descriptor, - mut scratch: Vec, - pages_filter: PageFilter, - max_page_size: usize, -) -> impl Stream> + '_ { - let mut seen_values = 0i64; - try_stream! { - while seen_values < total_num_values { - // the header - let page_header = read_page_header(reader, max_page_size).await?; - - let data_header = get_page_header(&page_header)?; - seen_values += data_header.as_ref().map(|x| x.num_values() as i64).unwrap_or_default(); - - let read_size: usize = page_header.compressed_page_size.try_into()?; - - if let Some(data_header) = data_header { - if !pages_filter(&descriptor, &data_header) { - // page to be skipped, we sill need to seek - copy(reader.take(read_size as u64), &mut sink()).await?; - continue - } - } - - if read_size > max_page_size { - Err(Error::WouldOverAllocate)? - } - - // followed by the buffer - scratch.clear(); - scratch.try_reserve(read_size)?; - let bytes_read = reader - .take(read_size as u64) - .read_to_end(&mut scratch).await?; - - if bytes_read != read_size { - Err(Error::oos( - "The page header reported the wrong page size".to_string(), - ))? - } - - yield finish_page( - page_header, - &mut scratch, - compression, - &descriptor, - None, - )?; - } - } -} - -/// Reads Page header from Thrift. -async fn read_page_header( - reader: &mut R, - max_page_size: usize, -) -> Result { - let mut prot = TCompactInputStreamProtocol::new(reader, max_page_size); - let page_header = ParquetPageHeader::stream_from_in_protocol(&mut prot).await?; - Ok(page_header) -} diff --git a/src/common/parquet2/src/read/stream.rs b/src/common/parquet2/src/read/stream.rs deleted file mode 100644 index 38a3c3919fd3..000000000000 --- a/src/common/parquet2/src/read/stream.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use bytes::Buf; - -use super::super::metadata::FileMetaData; -use super::super::DEFAULT_FOOTER_READ_SIZE; -use super::super::FOOTER_SIZE; -use super::super::PARQUET_MAGIC; -use super::metadata::deserialize_metadata; -use crate::error::Error; -use crate::error::Result; - -/// Decodes the footer returning the metadata length in bytes -pub fn decode_footer(slice: &[u8]) -> Result { - assert_eq!(slice.len(), FOOTER_SIZE as usize, "Invalid footer size"); - - // check this is indeed a parquet file - if slice[4..] != PARQUET_MAGIC { - return Err(Error::oos("Invalid Parquet file. Corrupt footer")); - } - - // get the metadata length from the footer - let metadata_len = u32::from_le_bytes(slice[..4].try_into().unwrap()); - // u32 won't be larger than usize in most cases - Ok(metadata_len as usize) -} - -/// Asynchronously reads the files' metadata -/// -/// This implementation is based on the implementation in `parquet`: https://docs.rs/parquet/latest/src/parquet/file/footer.rs.html#69 -pub async fn read_metadata(reader: opendal::Reader, file_size: u64) -> Result { - if file_size < 8 { - return Err(Error::oos(format!( - "file size of {file_size} is less than footer" - ))); - } - - // If a size hint is provided, read more than the minimum size - // to try and avoid a second fetch. - let footer_start = file_size.saturating_sub(DEFAULT_FOOTER_READ_SIZE); - - let suffix = reader - .read(footer_start..file_size) - .await - .map_err(|err| Error::oos(err.to_string()))?; - let suffix_len = suffix.len(); - - let footer = suffix.slice(suffix_len - 8..suffix_len).to_vec(); - let length = decode_footer(&footer)?; - - if file_size < length as u64 + 8 { - return Err(Error::oos(format!( - "file size of {} is less than footer + metadata {}", - file_size, - length + 8 - ))); - } - - // Did not fetch the entire file metadata in the initial read, need to make a second request - if length > suffix_len - 8 { - let metadata_start = file_size as usize - length - 8; - let meta = reader - .read(metadata_start as _..file_size - 8) - .await - .map_err(|err| Error::oos(err.to_string()))?; - - // a highly nested but sparse struct could result in many allocations - let max_size = meta.len() * 2 + 1024; - deserialize_metadata(meta.reader(), max_size) - } else { - let metadata_start = file_size as usize - length - 8 - footer_start as usize; - - let slice = suffix.slice(metadata_start as _..suffix_len - 8); - - // a highly nested but sparse struct could result in many allocations - let max_size = slice.len() * 2 + 1024; - deserialize_metadata(slice.reader(), max_size) - } -} diff --git a/src/common/parquet2/src/schema/io_message/from_message.rs b/src/common/parquet2/src/schema/io_message/from_message.rs deleted file mode 100644 index b1ff3588efc3..000000000000 --- a/src/common/parquet2/src/schema/io_message/from_message.rs +++ /dev/null @@ -1,1160 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Parquet schema parser. -//! Provides methods to parse and validate string message type into Parquet -//! [`ParquetType`](crate::schema::types::ParquetType). -//! -//! # Example -//! -//! ```rust -//! use parquet2::schema::io_message::from_message; -//! -//! let message_type = " -//! message spark_schema { -//! OPTIONAL BYTE_ARRAY a (UTF8); -//! REQUIRED INT32 b; -//! REQUIRED DOUBLE c; -//! REQUIRED BOOLEAN d; -//! OPTIONAL group e (LIST) { -//! REPEATED group list { -//! REQUIRED INT32 element; -//! } -//! } -//! } -//! "; -//! -//! let schema = from_message(message_type).expect("Expected valid schema"); -//! println!("{:?}", schema); -//! ``` - -use parquet_format_safe::Type; -use types::PrimitiveLogicalType; - -use super::super::types::ParquetType; -use super::super::types::TimeUnit; -use super::super::*; -use crate::error::Error; -use crate::error::Result; -use crate::schema::types::GroupConvertedType; -use crate::schema::types::PrimitiveConvertedType; - -fn is_logical_type(s: &str) -> bool { - matches!( - s, - "INTEGER" - | "MAP" - | "LIST" - | "ENUM" - | "DECIMAL" - | "DATE" - | "TIME" - | "TIMESTAMP" - | "STRING" - | "JSON" - | "BSON" - | "UUID" - | "UNKNOWN" - | "INTERVAL" - ) -} - -fn is_converted_type(s: &str) -> bool { - matches!( - s, - "UTF8" - | "ENUM" - | "DECIMAL" - | "DATE" - | "TIME_MILLIS" - | "TIME_MICROS" - | "TIMESTAMP_MILLIS" - | "TIMESTAMP_MICROS" - | "UINT_8" - | "UINT_16" - | "UINT_32" - | "UINT_64" - | "INT_8" - | "INT_16" - | "INT_32" - | "INT_64" - | "JSON" - | "BSON" - | "INTERVAL" - ) -} - -fn converted_group_from_str(s: &str) -> Result { - Ok(match s { - "MAP" => GroupConvertedType::Map, - "MAP_KEY_VALUE" => GroupConvertedType::MapKeyValue, - "LIST" => GroupConvertedType::List, - other => return Err(Error::oos(format!("Invalid converted type {}", other))), - }) -} - -fn converted_primitive_from_str(s: &str) -> Option { - use PrimitiveConvertedType::*; - Some(match s { - "UTF8" => Utf8, - "ENUM" => Enum, - "DECIMAL" => Decimal(0, 0), - "DATE" => Date, - "TIME_MILLIS" => TimeMillis, - "TIME_MICROS" => TimeMicros, - "TIMESTAMP_MILLIS" => TimestampMillis, - "TIMESTAMP_MICROS" => TimestampMicros, - "UINT_8" => Uint8, - "UINT_16" => Uint16, - "UINT_32" => Uint32, - "UINT_64" => Uint64, - "INT_8" => Int8, - "INT_16" => Int16, - "INT_32" => Int32, - "INT_64" => Int64, - "JSON" => Json, - "BSON" => Bson, - "INTERVAL" => Interval, - _ => return None, - }) -} - -fn repetition_from_str(s: &str) -> Result { - Ok(match s { - "REQUIRED" => Repetition::Required, - "OPTIONAL" => Repetition::Optional, - "REPEATED" => Repetition::Repeated, - other => return Err(Error::oos(format!("Invalid repetition {}", other))), - }) -} - -fn type_from_str(s: &str) -> Result { - match s { - "BOOLEAN" => Ok(Type::BOOLEAN), - "INT32" => Ok(Type::INT32), - "INT64" => Ok(Type::INT64), - "INT96" => Ok(Type::INT96), - "FLOAT" => Ok(Type::FLOAT), - "DOUBLE" => Ok(Type::DOUBLE), - "BYTE_ARRAY" | "BINARY" => Ok(Type::BYTE_ARRAY), - "FIXED_LEN_BYTE_ARRAY" => Ok(Type::FIXED_LEN_BYTE_ARRAY), - other => Err(Error::oos(format!("Invalid type {}", other))), - } -} - -/// Parses message type as string into a Parquet [`ParquetType`](crate::schema::types::ParquetType) -/// which, for example, could be used to extract individual columns. Returns Parquet -/// general error when parsing or validation fails. -pub fn from_message(message_type: &str) -> Result { - let mut parser = Parser { - tokenizer: &mut Tokenizer::from_str(message_type), - }; - parser.parse_message_type() -} - -/// Tokenizer to split message type string into tokens that are separated using characters -/// defined in `is_schema_delim` method. Tokenizer also preserves delimiters as tokens. -/// Tokenizer provides Iterator interface to process tokens; it also allows to step back -/// to reprocess previous tokens. -struct Tokenizer<'a> { - // List of all tokens for a string - tokens: Vec<&'a str>, - // Current index of vector - index: usize, -} - -impl<'a> Tokenizer<'a> { - // Create tokenizer from message type string - pub fn from_str(string: &'a str) -> Self { - let vec = string - .split_whitespace() - .flat_map(Self::split_token) - .collect(); - Tokenizer { - tokens: vec, - index: 0, - } - } - - // List of all special characters in schema - fn is_schema_delim(c: char) -> bool { - c == ';' || c == '{' || c == '}' || c == '(' || c == ')' || c == '=' || c == ',' - } - - /// Splits string into tokens; input string can already be token or can contain - /// delimiters, e.g. required" -> Vec("required") and - /// "(UTF8);" -> Vec("(", "UTF8", ")", ";") - fn split_token(string: &str) -> Vec<&str> { - let mut buffer: Vec<&str> = Vec::new(); - let mut tail = string; - while let Some(index) = tail.find(Self::is_schema_delim) { - let (h, t) = tail.split_at(index); - if !h.is_empty() { - buffer.push(h); - } - buffer.push(&t[0..1]); - tail = &t[1..]; - } - if !tail.is_empty() { - buffer.push(tail); - } - buffer - } - - // Move pointer to a previous element - fn backtrack(&mut self) { - self.index -= 1; - } -} - -impl<'a> Iterator for Tokenizer<'a> { - type Item = &'a str; - - fn next(&mut self) -> Option<&'a str> { - if self.index < self.tokens.len() { - self.index += 1; - Some(self.tokens[self.index - 1]) - } else { - None - } - } -} - -/// Internal Schema parser. -/// Traverses message type using tokenizer and parses each group/primitive type -/// recursively. -struct Parser<'a> { - tokenizer: &'a mut Tokenizer<'a>, -} - -// Utility function to assert token on validity. -fn assert_token(token: Option<&str>, expected: &str) -> Result<()> { - match token { - Some(value) if value == expected => Ok(()), - Some(other) => Err(Error::oos(format!( - "Expected '{}', found token '{}'", - expected, other - ))), - None => Err(Error::oos(format!( - "Expected '{}', but no token found (None)", - expected - ))), - } -} - -// Utility function to parse i32 or return general error. -fn parse_i32(value: Option<&str>, not_found_msg: &str, parse_fail_msg: &str) -> Result { - value - .ok_or_else(|| Error::oos(not_found_msg)) - .and_then(|v| v.parse::().map_err(|_| Error::oos(parse_fail_msg))) -} - -// Utility function to parse boolean or return general error. -#[inline] -fn parse_bool(value: Option<&str>, not_found_msg: &str, parse_fail_msg: &str) -> Result { - value - .ok_or_else(|| Error::oos(not_found_msg)) - .and_then(|v| { - v.to_lowercase() - .parse::() - .map_err(|_| Error::oos(parse_fail_msg)) - }) -} - -// Utility function to parse TimeUnit or return general error. -fn parse_timeunit( - value: Option<&str>, - not_found_msg: &str, - parse_fail_msg: &str, -) -> Result { - value - .ok_or_else(|| Error::oos(not_found_msg)) - .and_then(|v| match v.to_uppercase().as_str() { - "MILLIS" => Ok(TimeUnit::Milliseconds), - "MICROS" => Ok(TimeUnit::Microseconds), - "NANOS" => Ok(TimeUnit::Nanoseconds), - _ => Err(Error::oos(parse_fail_msg)), - }) -} - -impl<'a> Parser<'a> { - // Entry function to parse message type, uses internal tokenizer. - fn parse_message_type(&mut self) -> Result { - // Check that message type starts with "message". - match self.tokenizer.next() { - Some("message") => { - let name = self - .tokenizer - .next() - .ok_or_else(|| Error::oos("Expected name, found None"))?; - let fields = self.parse_child_types()?; - Ok(ParquetType::new_root(name.to_string(), fields)) - } - _ => Err(Error::oos("Message type does not start with 'message'")), - } - } - - // Parses child types for a current group type. - // This is only invoked on root and group types. - fn parse_child_types(&mut self) -> Result> { - assert_token(self.tokenizer.next(), "{")?; - let mut vec = Vec::new(); - while let Some(value) = self.tokenizer.next() { - if value == "}" { - break; - } else { - self.tokenizer.backtrack(); - vec.push(self.add_type()?); - } - } - Ok(vec) - } - - fn add_type(&mut self) -> Result { - // Parse repetition - let repetition = self - .tokenizer - .next() - .ok_or_else(|| Error::oos("Expected repetition, found None")) - .and_then(|v| repetition_from_str(&v.to_uppercase()))?; - - match self.tokenizer.next() { - Some(group) if group.to_uppercase() == "GROUP" => self.add_group_type(repetition), - Some(type_string) => { - let physical_type = type_from_str(&type_string.to_uppercase())?; - self.add_primitive_type(repetition, physical_type) - } - None => Err(Error::oos("Invalid type, could not extract next token")), - } - } - - fn add_group_type(&mut self, repetition: Repetition) -> Result { - // Parse name of the group type - let name = self - .tokenizer - .next() - .ok_or_else(|| Error::oos("Expected name, found None"))?; - - // Parse converted type if exists - let converted_type = if let Some("(") = self.tokenizer.next() { - let converted_type = self - .tokenizer - .next() - .ok_or_else(|| Error::oos("Expected converted type, found None")) - .and_then(|v| converted_group_from_str(&v.to_uppercase()))?; - assert_token(self.tokenizer.next(), ")")?; - Some(converted_type) - } else { - self.tokenizer.backtrack(); - None - }; - - // Parse optional id - let id = if let Some("=") = self.tokenizer.next() { - self.tokenizer.next().and_then(|v| v.parse::().ok()) - } else { - self.tokenizer.backtrack(); - None - }; - - let fields = self.parse_child_types()?; - - Ok(ParquetType::from_converted( - name.to_string(), - fields, - repetition, - converted_type, - id, - )) - } - - fn add_primitive_type( - &mut self, - repetition: Repetition, - physical_type: Type, - ) -> Result { - // Read type length if the type is FIXED_LEN_BYTE_ARRAY. - let length = if physical_type == Type::FIXED_LEN_BYTE_ARRAY { - assert_token(self.tokenizer.next(), "(")?; - let length = parse_i32( - self.tokenizer.next(), - "Expected length for FIXED_LEN_BYTE_ARRAY, found None", - "Failed to parse length for FIXED_LEN_BYTE_ARRAY", - )?; - assert_token(self.tokenizer.next(), ")")?; - Some(length) - } else { - None - }; - - // Parse name of the primitive type - let name = self - .tokenizer - .next() - .ok_or_else(|| Error::oos("Expected name, found None"))?; - - // Parse logical types - let (converted_type, logical_type) = if let Some("(") = self.tokenizer.next() { - let (is_logical_type, converted_type, token) = self - .tokenizer - .next() - .ok_or_else(|| Error::oos("Expected converted or logical type, found None")) - .and_then(|v| { - let string = v.to_uppercase(); - Ok(if is_logical_type(&string) { - (true, None, string) - } else if is_converted_type(&string) { - (false, converted_primitive_from_str(&string), string) - } else { - return Err(Error::oos(format!( - "Expected converted or logical type, found {}", - string - ))); - }) - })?; - - let logical_type = if is_logical_type { - Some(self.parse_logical_type(&token)?) - } else { - None - }; - - // converted type decimal - let converted_type = match converted_type { - Some(PrimitiveConvertedType::Decimal(_, _)) => { - Some(self.parse_converted_decimal()?) - } - other => other, - }; - - assert_token(self.tokenizer.next(), ")")?; - (converted_type, logical_type) - } else { - self.tokenizer.backtrack(); - (None, None) - }; - - // Parse optional id - let id = if let Some("=") = self.tokenizer.next() { - self.tokenizer.next().and_then(|v| v.parse::().ok()) - } else { - self.tokenizer.backtrack(); - None - }; - assert_token(self.tokenizer.next(), ";")?; - - ParquetType::try_from_primitive( - name.to_string(), - (physical_type, length).try_into()?, - repetition, - converted_type, - logical_type, - id, - ) - } - - fn parse_converted_decimal(&mut self) -> Result { - assert_token(self.tokenizer.next(), "(")?; - // Parse precision - let precision = parse_i32( - self.tokenizer.next(), - "Expected precision, found None", - "Failed to parse precision for DECIMAL type", - )?; - - // Parse scale - let scale = if let Some(",") = self.tokenizer.next() { - parse_i32( - self.tokenizer.next(), - "Expected scale, found None", - "Failed to parse scale for DECIMAL type", - )? - } else { - // Scale is not provided, set it to 0. - self.tokenizer.backtrack(); - 0 - }; - - assert_token(self.tokenizer.next(), ")")?; - Ok(PrimitiveConvertedType::Decimal( - precision.try_into()?, - scale.try_into()?, - )) - } - - fn parse_logical_type(&mut self, tpe: &str) -> Result { - Ok(match tpe { - "ENUM" => PrimitiveLogicalType::Enum, - "DATE" => PrimitiveLogicalType::Date, - "DECIMAL" => { - let (precision, scale) = if let Some("(") = self.tokenizer.next() { - let precision = parse_i32( - self.tokenizer.next(), - "Expected precision, found None", - "Failed to parse precision for DECIMAL type", - )?; - let scale = if let Some(",") = self.tokenizer.next() { - parse_i32( - self.tokenizer.next(), - "Expected scale, found None", - "Failed to parse scale for DECIMAL type", - )? - } else { - self.tokenizer.backtrack(); - 0 - }; - assert_token(self.tokenizer.next(), ")")?; - (precision, scale) - } else { - self.tokenizer.backtrack(); - (0, 0) - }; - PrimitiveLogicalType::Decimal(precision.try_into()?, scale.try_into()?) - } - "TIME" => { - let (unit, is_adjusted_to_utc) = if let Some("(") = self.tokenizer.next() { - let unit = parse_timeunit( - self.tokenizer.next(), - "Invalid timeunit found", - "Failed to parse timeunit for TIME type", - )?; - let is_adjusted_to_utc = if let Some(",") = self.tokenizer.next() { - parse_bool( - self.tokenizer.next(), - "Invalid boolean found", - "Failed to parse timezone info for TIME type", - )? - } else { - self.tokenizer.backtrack(); - false - }; - assert_token(self.tokenizer.next(), ")")?; - (unit, is_adjusted_to_utc) - } else { - self.tokenizer.backtrack(); - (TimeUnit::Milliseconds, false) - }; - PrimitiveLogicalType::Time { - is_adjusted_to_utc, - unit, - } - } - "TIMESTAMP" => { - let (unit, is_adjusted_to_utc) = if let Some("(") = self.tokenizer.next() { - let unit = parse_timeunit( - self.tokenizer.next(), - "Invalid timeunit found", - "Failed to parse timeunit for TIMESTAMP type", - )?; - let is_adjusted_to_utc = if let Some(",") = self.tokenizer.next() { - parse_bool( - self.tokenizer.next(), - "Invalid boolean found", - "Failed to parse timezone info for TIMESTAMP type", - )? - } else { - // Invalid token for unit - self.tokenizer.backtrack(); - false - }; - assert_token(self.tokenizer.next(), ")")?; - (unit, is_adjusted_to_utc) - } else { - self.tokenizer.backtrack(); - (TimeUnit::Milliseconds, false) - }; - PrimitiveLogicalType::Timestamp { - is_adjusted_to_utc, - unit, - } - } - "INTEGER" => { - let (bit_width, is_signed) = if let Some("(") = self.tokenizer.next() { - let bit_width = parse_i32( - self.tokenizer.next(), - "Invalid bit_width found", - "Failed to parse bit_width for INTEGER type", - )?; - let is_signed = if let Some(",") = self.tokenizer.next() { - parse_bool( - self.tokenizer.next(), - "Invalid boolean found", - "Failed to parse is_signed for INTEGER type", - )? - } else { - // Invalid token for unit - self.tokenizer.backtrack(); - return Err(Error::oos("INTEGER requires sign")); - }; - assert_token(self.tokenizer.next(), ")")?; - (bit_width, is_signed) - } else { - // Invalid token for unit - self.tokenizer.backtrack(); - return Err(Error::oos("INTEGER requires width and sign")); - }; - PrimitiveLogicalType::Integer((bit_width, is_signed).into()) - } - "STRING" => PrimitiveLogicalType::String, - "JSON" => PrimitiveLogicalType::Json, - "BSON" => PrimitiveLogicalType::Bson, - "UUID" => PrimitiveLogicalType::Uuid, - "UNKNOWN" => PrimitiveLogicalType::Unknown, - "INTERVAL" => return Err(Error::oos("Interval logical type not yet supported")), - _ => unreachable!(), - }) - } -} - -#[cfg(test)] -mod tests { - use types::IntegerType; - use types::PrimitiveLogicalType; - - use super::*; - use crate::schema::types::GroupConvertedType; - use crate::schema::types::PhysicalType; - use crate::schema::types::PrimitiveConvertedType; - - #[test] - fn test_tokenize_empty_string() { - assert_eq!(Tokenizer::from_str("").next(), None); - } - - #[test] - fn test_tokenize_delimiters() { - let mut iter = Tokenizer::from_str(",;{}()="); - assert_eq!(iter.next(), Some(",")); - assert_eq!(iter.next(), Some(";")); - assert_eq!(iter.next(), Some("{")); - assert_eq!(iter.next(), Some("}")); - assert_eq!(iter.next(), Some("(")); - assert_eq!(iter.next(), Some(")")); - assert_eq!(iter.next(), Some("=")); - assert_eq!(iter.next(), None); - } - - #[test] - fn test_tokenize_delimiters_with_whitespaces() { - let mut iter = Tokenizer::from_str(" , ; { } ( ) = "); - assert_eq!(iter.next(), Some(",")); - assert_eq!(iter.next(), Some(";")); - assert_eq!(iter.next(), Some("{")); - assert_eq!(iter.next(), Some("}")); - assert_eq!(iter.next(), Some("(")); - assert_eq!(iter.next(), Some(")")); - assert_eq!(iter.next(), Some("=")); - assert_eq!(iter.next(), None); - } - - #[test] - fn test_tokenize_words() { - let mut iter = Tokenizer::from_str("abc def ghi jkl mno"); - assert_eq!(iter.next(), Some("abc")); - assert_eq!(iter.next(), Some("def")); - assert_eq!(iter.next(), Some("ghi")); - assert_eq!(iter.next(), Some("jkl")); - assert_eq!(iter.next(), Some("mno")); - assert_eq!(iter.next(), None); - } - - #[test] - fn test_tokenize_backtrack() { - let mut iter = Tokenizer::from_str("abc;"); - assert_eq!(iter.next(), Some("abc")); - assert_eq!(iter.next(), Some(";")); - iter.backtrack(); - assert_eq!(iter.next(), Some(";")); - assert_eq!(iter.next(), None); - } - - #[test] - fn test_tokenize_message_type() { - let schema = " - message schema { - required int32 a; - optional binary c (UTF8); - required group d { - required int32 a; - optional binary c (UTF8); - } - required group e (LIST) { - repeated group list { - required int32 element; - } - } - } - "; - let iter = Tokenizer::from_str(schema); - let mut res = Vec::new(); - for token in iter { - res.push(token); - } - assert_eq!(res, vec![ - "message", "schema", "{", "required", "int32", "a", ";", "optional", "binary", "c", - "(", "UTF8", ")", ";", "required", "group", "d", "{", "required", "int32", "a", ";", - "optional", "binary", "c", "(", "UTF8", ")", ";", "}", "required", "group", "e", "(", - "LIST", ")", "{", "repeated", "group", "list", "{", "required", "int32", "element", - ";", "}", "}", "}" - ]); - } - - #[test] - fn test_assert_token() { - assert!(assert_token(Some("a"), "a").is_ok()); - assert!(assert_token(Some("a"), "b").is_err()); - assert!(assert_token(None, "b").is_err()); - } - - #[test] - fn test_parse_message_type_invalid() { - let mut iter = Tokenizer::from_str("test"); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_err()); - assert_eq!( - result.unwrap_err().to_string(), - "File out of specification: Message type does not start with 'message'" - ); - } - - #[test] - fn test_parse_message_type_no_name() { - let mut iter = Tokenizer::from_str("message"); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_err()); - assert_eq!( - result.unwrap_err().to_string(), - "File out of specification: Expected name, found None" - ); - } - - #[test] - fn test_parse_message_type_fixed_byte_array() { - let schema = " - message schema { - REQUIRED FIXED_LEN_BYTE_ARRAY col; - } - "; - let mut iter = Tokenizer::from_str(schema); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_err()); - - let schema = " - message schema { - REQUIRED FIXED_LEN_BYTE_ARRAY(16) col; - } - "; - let mut iter = Tokenizer::from_str(schema); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_ok()); - } - - #[test] - fn test_parse_message_type_decimal() { - // It is okay for decimal to omit precision and scale with right syntax. - // Here we test wrong syntax of decimal type - - // Invalid decimal syntax - let schema = " - message root { - optional int32 f1 (DECIMAL(); - } - "; - let mut iter = Tokenizer::from_str(schema); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_err()); - - // Invalid decimal, need precision and scale - let schema = " - message root { - optional int32 f1 (DECIMAL()); - } - "; - let mut iter = Tokenizer::from_str(schema); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_err()); - - // Invalid decimal because of `,` - has precision, needs scale - let schema = " - message root { - optional int32 f1 (DECIMAL(8,)); - } - "; - let mut iter = Tokenizer::from_str(schema); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_err()); - } - - #[test] - fn test_parse_decimal_wrong() { - // Invalid decimal because, we always require either precision or scale to be - // specified as part of converted type - let schema = " - message root { - optional int32 f3 (DECIMAL); - } - "; - let mut iter = Tokenizer::from_str(schema); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_err()); - - // Valid decimal (precision, scale) - let schema = " - message root { - optional int32 f1 (DECIMAL(8, 3)); - optional int32 f2 (DECIMAL(8)); - } - "; - let mut iter = Tokenizer::from_str(schema); - let result = Parser { - tokenizer: &mut iter, - } - .parse_message_type(); - assert!(result.is_ok()); - } - - #[test] - fn test_parse_message_type_compare_1() -> Result<()> { - let schema = " - message root { - optional fixed_len_byte_array(5) f1 (DECIMAL(9, 3)); - optional fixed_len_byte_array (16) f2 (DECIMAL (38, 18)); - } - "; - let mut iter = Tokenizer::from_str(schema); - let message = Parser { - tokenizer: &mut iter, - } - .parse_message_type() - .unwrap(); - - let fields = vec![ - ParquetType::try_from_primitive( - "f1".to_string(), - PhysicalType::FixedLenByteArray(5), - Repetition::Optional, - None, - Some(PrimitiveLogicalType::Decimal(9, 3)), - None, - )?, - ParquetType::try_from_primitive( - "f2".to_string(), - PhysicalType::FixedLenByteArray(16), - Repetition::Optional, - None, - Some(PrimitiveLogicalType::Decimal(38, 18)), - None, - )?, - ]; - - let expected = ParquetType::new_root("root".to_string(), fields); - - assert_eq!(message, expected); - Ok(()) - } - - #[test] - fn test_parse_message_type_compare_2() -> Result<()> { - let schema = " - message root { - required group a0 { - optional group a1 (LIST) { - repeated binary a2 (UTF8); - } - - optional group b1 (LIST) { - repeated group b2 { - optional int32 b3; - optional double b4; - } - } - } - } - "; - let mut iter = Tokenizer::from_str(schema); - let message = Parser { - tokenizer: &mut iter, - } - .parse_message_type() - .unwrap(); - - let a2 = ParquetType::try_from_primitive( - "a2".to_string(), - PhysicalType::ByteArray, - Repetition::Repeated, - Some(PrimitiveConvertedType::Utf8), - None, - None, - )?; - let a1 = ParquetType::from_converted( - "a1".to_string(), - vec![a2], - Repetition::Optional, - Some(GroupConvertedType::List), - None, - ); - let b2 = ParquetType::from_converted( - "b2".to_string(), - vec![ - ParquetType::from_physical("b3".to_string(), PhysicalType::Int32), - ParquetType::from_physical("b4".to_string(), PhysicalType::Double), - ], - Repetition::Repeated, - None, - None, - ); - let b1 = ParquetType::from_converted( - "b1".to_string(), - vec![b2], - Repetition::Optional, - Some(GroupConvertedType::List), - None, - ); - let a0 = ParquetType::from_converted( - "a0".to_string(), - vec![a1, b1], - Repetition::Required, - None, - None, - ); - - let expected = ParquetType::new_root("root".to_string(), vec![a0]); - - assert_eq!(message, expected); - Ok(()) - } - - #[test] - fn test_parse_message_type_compare_3() -> Result<()> { - let schema = " - message root { - required int32 _1 (INT_8); - required int32 _2 (INT_16); - required float _3; - required double _4; - optional int32 _5 (DATE); - optional binary _6 (UTF8); - } - "; - let mut iter = Tokenizer::from_str(schema); - let message = Parser { - tokenizer: &mut iter, - } - .parse_message_type() - .unwrap(); - - let f1 = ParquetType::try_from_primitive( - "_1".to_string(), - PhysicalType::Int32, - Repetition::Required, - Some(PrimitiveConvertedType::Int8), - None, - None, - )?; - let f2 = ParquetType::try_from_primitive( - "_2".to_string(), - PhysicalType::Int32, - Repetition::Required, - Some(PrimitiveConvertedType::Int16), - None, - None, - )?; - let f3 = ParquetType::try_from_primitive( - "_3".to_string(), - PhysicalType::Float, - Repetition::Required, - None, - None, - None, - )?; - let f4 = ParquetType::try_from_primitive( - "_4".to_string(), - PhysicalType::Double, - Repetition::Required, - None, - None, - None, - )?; - let f5 = ParquetType::try_from_primitive( - "_5".to_string(), - PhysicalType::Int32, - Repetition::Optional, - None, - Some(PrimitiveLogicalType::Date), - None, - )?; - let f6 = ParquetType::try_from_primitive( - "_6".to_string(), - PhysicalType::ByteArray, - Repetition::Optional, - Some(PrimitiveConvertedType::Utf8), - None, - None, - )?; - - let fields = vec![f1, f2, f3, f4, f5, f6]; - - let expected = ParquetType::new_root("root".to_string(), fields); - assert_eq!(message, expected); - Ok(()) - } - - #[test] - fn test_parse_message_type_compare_4() -> Result<()> { - let schema = " - message root { - required int32 _1 (INTEGER(8,true)); - required int32 _2 (INTEGER(16,false)); - required float _3; - required double _4; - optional int32 _5 (DATE); - optional int32 _6 (TIME(MILLIS,false)); - optional int64 _7 (TIME(MICROS,true)); - optional int64 _8 (TIMESTAMP(MILLIS,true)); - optional int64 _9 (TIMESTAMP(NANOS,false)); - optional binary _10 (STRING); - } - "; - let mut iter = Tokenizer::from_str(schema); - let message = Parser { - tokenizer: &mut iter, - } - .parse_message_type()?; - - let f1 = ParquetType::try_from_primitive( - "_1".to_string(), - PhysicalType::Int32, - Repetition::Required, - None, - Some(PrimitiveLogicalType::Integer(IntegerType::Int8)), - None, - )?; - let f2 = ParquetType::try_from_primitive( - "_2".to_string(), - PhysicalType::Int32, - Repetition::Required, - None, - Some(PrimitiveLogicalType::Integer(IntegerType::UInt16)), - None, - )?; - let f3 = ParquetType::try_from_primitive( - "_3".to_string(), - PhysicalType::Float, - Repetition::Required, - None, - None, - None, - )?; - let f4 = ParquetType::try_from_primitive( - "_4".to_string(), - PhysicalType::Double, - Repetition::Required, - None, - None, - None, - )?; - let f5 = ParquetType::try_from_primitive( - "_5".to_string(), - PhysicalType::Int32, - Repetition::Optional, - None, - Some(PrimitiveLogicalType::Date), - None, - )?; - let f6 = ParquetType::try_from_primitive( - "_6".to_string(), - PhysicalType::Int32, - Repetition::Optional, - None, - Some(PrimitiveLogicalType::Time { - is_adjusted_to_utc: false, - unit: TimeUnit::Milliseconds, - }), - None, - )?; - let f7 = ParquetType::try_from_primitive( - "_7".to_string(), - PhysicalType::Int64, - Repetition::Optional, - None, - Some(PrimitiveLogicalType::Time { - is_adjusted_to_utc: true, - unit: TimeUnit::Microseconds, - }), - None, - )?; - let f8 = ParquetType::try_from_primitive( - "_8".to_string(), - PhysicalType::Int64, - Repetition::Optional, - None, - Some(PrimitiveLogicalType::Timestamp { - is_adjusted_to_utc: true, - unit: TimeUnit::Milliseconds, - }), - None, - )?; - let f9 = ParquetType::try_from_primitive( - "_9".to_string(), - PhysicalType::Int64, - Repetition::Optional, - None, - Some(PrimitiveLogicalType::Timestamp { - is_adjusted_to_utc: false, - unit: TimeUnit::Nanoseconds, - }), - None, - )?; - - let f10 = ParquetType::try_from_primitive( - "_10".to_string(), - PhysicalType::ByteArray, - Repetition::Optional, - None, - Some(PrimitiveLogicalType::String), - None, - )?; - - let fields = vec![f1, f2, f3, f4, f5, f6, f7, f8, f9, f10]; - - let expected = ParquetType::new_root("root".to_string(), fields); - assert_eq!(message, expected); - Ok(()) - } -} diff --git a/src/common/parquet2/src/schema/io_message/mod.rs b/src/common/parquet2/src/schema/io_message/mod.rs deleted file mode 100644 index dcaa8cbc8170..000000000000 --- a/src/common/parquet2/src/schema/io_message/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod from_message; - -pub use from_message::from_message; diff --git a/src/common/parquet2/src/schema/io_thrift/from_thrift.rs b/src/common/parquet2/src/schema/io_thrift/from_thrift.rs deleted file mode 100644 index 3c7f96456159..000000000000 --- a/src/common/parquet2/src/schema/io_thrift/from_thrift.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::SchemaElement; - -use super::super::types::ParquetType; -use crate::error::Error; -use crate::error::Result; -use crate::schema::types::FieldInfo; - -impl ParquetType { - /// Method to convert from Thrift. - pub fn try_from_thrift(elements: &[SchemaElement]) -> Result { - let mut index = 0; - let mut schema_nodes = Vec::new(); - while index < elements.len() { - let t = from_thrift_helper(elements, index)?; - index = t.0; - schema_nodes.push(t.1); - } - if schema_nodes.len() != 1 { - return Err(Error::oos(format!( - "Expected exactly one root node, but found {}", - schema_nodes.len() - ))); - } - - Ok(schema_nodes.remove(0)) - } -} - -/// Constructs a new Type from the `elements`, starting at index `index`. -/// The first result is the starting index for the next Type after this one. If it is -/// equal to `elements.len()`, then this Type is the last one. -/// The second result is the result Type. -fn from_thrift_helper(elements: &[SchemaElement], index: usize) -> Result<(usize, ParquetType)> { - // Whether or not the current node is root (message type). - // There is only one message type node in the schema tree. - let is_root_node = index == 0; - - let element = elements - .get(index) - .ok_or_else(|| Error::oos(format!("index {} on SchemaElement is not valid", index)))?; - let name = element.name.clone(); - let converted_type = element.converted_type; - - let id = element.field_id; - match element.num_children { - // From parquet-format: - // The children count is used to construct the nested relationship. - // This field is not set when the element is a primitive type - // Sometimes parquet-cpp sets num_children field to 0 for primitive types, so we - // have to handle this case too. - None | Some(0) => { - // primitive type - let repetition = element - .repetition_type - .ok_or_else(|| Error::oos("Repetition level must be defined for a primitive type"))? - .try_into()?; - let physical_type = element - .type_ - .ok_or_else(|| Error::oos("Physical type must be defined for a primitive type"))?; - - let converted_type = converted_type - .map(|converted_type| { - let maybe_decimal = match (element.precision, element.scale) { - (Some(precision), Some(scale)) => Some((precision, scale)), - (None, None) => None, - _ => { - return Err(Error::oos( - "When precision or scale are defined, both must be defined", - )); - } - }; - (converted_type, maybe_decimal).try_into() - }) - .transpose()?; - - let logical_type = element - .logical_type - .clone() - .map(|x| x.try_into()) - .transpose()?; - - let tp = ParquetType::try_from_primitive( - name, - (physical_type, element.type_length).try_into()?, - repetition, - converted_type, - logical_type, - id, - )?; - - Ok((index + 1, tp)) - } - Some(n) => { - let mut fields = vec![]; - let mut next_index = index + 1; - for _ in 0..n { - let child_result = from_thrift_helper(elements, next_index)?; - next_index = child_result.0; - fields.push(child_result.1); - } - - let tp = if is_root_node { - ParquetType::new_root(name, fields) - } else { - let repetition = if let Some(repetition) = element.repetition_type { - repetition.try_into()? - } else { - return Err(Error::oos( - "The repetition level of a non-root must be non-null", - )); - }; - - let converted_type = converted_type.map(|x| x.try_into()).transpose()?; - - let logical_type = element - .logical_type - .clone() - .map(|x| x.try_into()) - .transpose()?; - - ParquetType::GroupType { - field_info: FieldInfo { - name, - repetition, - id, - }, - fields, - converted_type, - logical_type, - } - }; - Ok((next_index, tp)) - } - } -} diff --git a/src/common/parquet2/src/schema/io_thrift/mod.rs b/src/common/parquet2/src/schema/io_thrift/mod.rs deleted file mode 100644 index 70bd785c7d1d..000000000000 --- a/src/common/parquet2/src/schema/io_thrift/mod.rs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod from_thrift; -#[allow(unused_imports)] -pub use from_thrift::*; - -mod to_thrift; -#[allow(unused_imports)] -pub use to_thrift::*; - -#[cfg(test)] -mod tests { - use crate::error::Result; - use crate::schema::io_message::from_message; - use crate::schema::types::ParquetType; - - fn test_round_trip(message: &str) -> Result<()> { - let expected_schema = from_message(message)?; - let thrift_schema = expected_schema.to_thrift(); - let thrift_schema = thrift_schema.into_iter().collect::>(); - let result_schema = ParquetType::try_from_thrift(&thrift_schema)?; - assert_eq!(result_schema, expected_schema); - Ok(()) - } - - #[test] - fn test_schema_type_thrift_conversion() { - let message_type = " - message conversions { - REQUIRED INT64 id; - OPTIONAL group int_array_Array (LIST) { - REPEATED group list { - OPTIONAL group element (LIST) { - REPEATED group list { - OPTIONAL INT32 element; - } - } - } - } - OPTIONAL group int_map (MAP) { - REPEATED group map (MAP_KEY_VALUE) { - REQUIRED BYTE_ARRAY key (UTF8); - OPTIONAL INT32 value; - } - } - OPTIONAL group int_Map_Array (LIST) { - REPEATED group list { - OPTIONAL group g (MAP) { - REPEATED group map (MAP_KEY_VALUE) { - REQUIRED BYTE_ARRAY key (UTF8); - OPTIONAL group value { - OPTIONAL group H { - OPTIONAL group i (LIST) { - REPEATED group list { - OPTIONAL DOUBLE element; - } - } - } - } - } - } - } - } - OPTIONAL group nested_struct { - OPTIONAL INT32 A; - OPTIONAL group b (LIST) { - REPEATED group list { - REQUIRED FIXED_LEN_BYTE_ARRAY (16) element; - } - } - } - } - "; - test_round_trip(message_type).unwrap(); - } - - #[test] - fn test_schema_type_thrift_conversion_decimal() { - let message_type = " - message decimals { - OPTIONAL INT32 field0; - OPTIONAL INT64 field1 (DECIMAL (18, 2)); - OPTIONAL FIXED_LEN_BYTE_ARRAY (16) field2 (DECIMAL (38, 18)); - OPTIONAL BYTE_ARRAY field3 (DECIMAL (9)); - } - "; - test_round_trip(message_type).unwrap(); - } -} diff --git a/src/common/parquet2/src/schema/io_thrift/to_thrift.rs b/src/common/parquet2/src/schema/io_thrift/to_thrift.rs deleted file mode 100644 index 78520c71a0ac..000000000000 --- a/src/common/parquet2/src/schema/io_thrift/to_thrift.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::ConvertedType; -use parquet_format_safe::SchemaElement; - -use super::super::types::ParquetType; -use crate::schema::types::PrimitiveType; - -impl ParquetType { - /// Method to convert to Thrift. - pub(crate) fn to_thrift(&self) -> Vec { - let mut elements: Vec = Vec::new(); - to_thrift_helper(self, &mut elements, true); - elements - } -} - -/// Constructs list of `SchemaElement` from the schema using depth-first traversal. -/// Here we assume that schema is always valid and starts with group type. -fn to_thrift_helper(schema: &ParquetType, elements: &mut Vec, is_root: bool) { - match schema { - ParquetType::PrimitiveType(PrimitiveType { - field_info, - logical_type, - converted_type, - physical_type, - }) => { - let (type_, type_length) = (*physical_type).into(); - let (converted_type, maybe_decimal) = converted_type - .map(|x| x.into()) - .map(|x: (ConvertedType, Option<(i32, i32)>)| (Some(x.0), x.1)) - .unwrap_or((None, None)); - - let element = SchemaElement { - type_: Some(type_), - type_length, - repetition_type: Some(field_info.repetition.into()), - name: field_info.name.clone(), - num_children: None, - converted_type, - precision: maybe_decimal.map(|x| x.0), - scale: maybe_decimal.map(|x| x.1), - field_id: field_info.id, - logical_type: logical_type.map(|x| x.into()), - }; - - elements.push(element); - } - ParquetType::GroupType { - field_info, - fields, - logical_type, - converted_type, - } => { - let converted_type = converted_type.map(|x| x.into()); - - let repetition_type = if is_root { - // https://github.com/apache/parquet-format/blob/7f06e838cbd1b7dbd722ff2580b9c2525e37fc46/src/main/thrift/parquet.thrift#L363 - None - } else { - Some(field_info.repetition) - }; - - let element = SchemaElement { - type_: None, - type_length: None, - repetition_type: repetition_type.map(|x| x.into()), - name: field_info.name.clone(), - num_children: Some(fields.len() as i32), - converted_type, - scale: None, - precision: None, - field_id: field_info.id, - logical_type: logical_type.map(|x| x.into()), - }; - - elements.push(element); - - // Add child elements for a group - for field in fields { - to_thrift_helper(field, elements, false); - } - } - } -} diff --git a/src/common/parquet2/src/schema/mod.rs b/src/common/parquet2/src/schema/mod.rs deleted file mode 100644 index c91f94f55dc7..000000000000 --- a/src/common/parquet2/src/schema/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub use super::thrift_format::SchemaElement; -pub use crate::parquet_bridge::Repetition; - -pub mod io_message; -pub mod io_thrift; - -pub mod types; diff --git a/src/common/parquet2/src/schema/types/basic_type.rs b/src/common/parquet2/src/schema/types/basic_type.rs deleted file mode 100644 index 8663a0ab179f..000000000000 --- a/src/common/parquet2/src/schema/types/basic_type.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use super::super::Repetition; - -/// Common type information. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct FieldInfo { - /// The field name - pub name: String, - /// The repetition - pub repetition: Repetition, - /// the optional id, to select fields by id - pub id: Option, -} diff --git a/src/common/parquet2/src/schema/types/converted_type.rs b/src/common/parquet2/src/schema/types/converted_type.rs deleted file mode 100644 index 3c18925e7c2c..000000000000 --- a/src/common/parquet2/src/schema/types/converted_type.rs +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::ConvertedType; -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use crate::error::Error; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum PrimitiveConvertedType { - Utf8, - /// an enum is converted into a binary field - Enum, - /// A decimal value. - /// - /// This may be used to annotate binary or fixed primitive types. The - /// underlying byte array stores the unscaled value encoded as two's - /// complement using big-endian byte order (the most significant byte is the - /// zeroth element). The value of the decimal is the value * 10^{-scale}. - /// - /// This must be accompanied by a (maximum) precision and a scale in the - /// SchemaElement. The precision specifies the number of digits in the decimal - /// and the scale stores the location of the decimal point. For example 1.23 - /// would have precision 3 (3 total digits) and scale 2 (the decimal point is - /// 2 digits over). - // (precision, scale) - Decimal(usize, usize), - /// A Date - /// - /// Stored as days since Unix epoch, encoded as the INT32 physical type. - Date, - /// A time - /// - /// The total number of milliseconds since midnight. The value is stored - /// as an INT32 physical type. - TimeMillis, - /// A time. - /// - /// The total number of microseconds since midnight. The value is stored as - /// an INT64 physical type. - TimeMicros, - /// A date/time combination - /// - /// Date and time recorded as milliseconds since the Unix epoch. Recorded as - /// a physical type of INT64. - TimestampMillis, - /// A date/time combination - /// - /// Date and time recorded as microseconds since the Unix epoch. The value is - /// stored as an INT64 physical type. - TimestampMicros, - /// An unsigned integer value. - /// - /// The number describes the maximum number of meainful data bits in - /// the stored value. 8, 16 and 32 bit values are stored using the - /// INT32 physical type. 64 bit values are stored using the INT64 - /// physical type. - Uint8, - Uint16, - Uint32, - Uint64, - /// A signed integer value. - /// - /// The number describes the maximum number of meainful data bits in - /// the stored value. 8, 16 and 32 bit values are stored using the - /// INT32 physical type. 64 bit values are stored using the INT64 - /// physical type. - Int8, - Int16, - Int32, - Int64, - /// An embedded JSON document - /// - /// A JSON document embedded within a single UTF8 column. - Json, - /// An embedded BSON document - /// - /// A BSON document embedded within a single BINARY column. - Bson, - /// An interval of time - /// - /// This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12 - /// This data is composed of three separate little endian unsigned - /// integers. Each stores a component of a duration of time. The first - /// integer identifies the number of months associated with the duration, - /// the second identifies the number of days associated with the duration - /// and the third identifies the number of milliseconds associated with - /// the provided duration. This duration of time is independent of any - /// particular timezone or date. - Interval, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum GroupConvertedType { - /// a map is converted as an optional field containing a repeated key/value pair - Map, - /// a key/value pair is converted into a group of two fields - MapKeyValue, - /// a list is converted into an optional field containing a repeated field for its - /// values - List, -} - -impl TryFrom<(ConvertedType, Option<(i32, i32)>)> for PrimitiveConvertedType { - type Error = Error; - - fn try_from( - (ty, maybe_decimal): (ConvertedType, Option<(i32, i32)>), - ) -> Result { - use PrimitiveConvertedType::*; - Ok(match ty { - ConvertedType::UTF8 => Utf8, - ConvertedType::ENUM => Enum, - ConvertedType::DECIMAL => { - if let Some((precision, scale)) = maybe_decimal { - Decimal(precision.try_into()?, scale.try_into()?) - } else { - return Err(Error::oos("Decimal requires a precision and scale")); - } - } - ConvertedType::DATE => Date, - ConvertedType::TIME_MILLIS => TimeMillis, - ConvertedType::TIME_MICROS => TimeMicros, - ConvertedType::TIMESTAMP_MILLIS => TimestampMillis, - ConvertedType::TIMESTAMP_MICROS => TimestampMicros, - ConvertedType::UINT_8 => Uint8, - ConvertedType::UINT_16 => Uint16, - ConvertedType::UINT_32 => Uint32, - ConvertedType::UINT_64 => Uint64, - ConvertedType::INT_8 => Int8, - ConvertedType::INT_16 => Int16, - ConvertedType::INT_32 => Int32, - ConvertedType::INT_64 => Int64, - ConvertedType::JSON => Json, - ConvertedType::BSON => Bson, - ConvertedType::INTERVAL => Interval, - _ => { - return Err(Error::oos(format!( - "Converted type \"{:?}\" cannot be applied to a primitive type", - ty - ))); - } - }) - } -} - -impl TryFrom for GroupConvertedType { - type Error = Error; - - fn try_from(type_: ConvertedType) -> Result { - Ok(match type_ { - ConvertedType::LIST => GroupConvertedType::List, - ConvertedType::MAP => GroupConvertedType::Map, - ConvertedType::MAP_KEY_VALUE => GroupConvertedType::MapKeyValue, - _ => return Err(Error::oos("LogicalType value out of range")), - }) - } -} - -impl From for ConvertedType { - fn from(type_: GroupConvertedType) -> Self { - match type_ { - GroupConvertedType::Map => ConvertedType::MAP, - GroupConvertedType::List => ConvertedType::LIST, - GroupConvertedType::MapKeyValue => ConvertedType::MAP_KEY_VALUE, - } - } -} - -impl From for (ConvertedType, Option<(i32, i32)>) { - fn from(ty: PrimitiveConvertedType) -> Self { - use PrimitiveConvertedType::*; - match ty { - Utf8 => (ConvertedType::UTF8, None), - Enum => (ConvertedType::ENUM, None), - Decimal(precision, scale) => ( - ConvertedType::DECIMAL, - Some((precision as i32, scale as i32)), - ), - Date => (ConvertedType::DATE, None), - TimeMillis => (ConvertedType::TIME_MILLIS, None), - TimeMicros => (ConvertedType::TIME_MICROS, None), - TimestampMillis => (ConvertedType::TIMESTAMP_MILLIS, None), - TimestampMicros => (ConvertedType::TIMESTAMP_MICROS, None), - Uint8 => (ConvertedType::UINT_8, None), - Uint16 => (ConvertedType::UINT_16, None), - Uint32 => (ConvertedType::UINT_32, None), - Uint64 => (ConvertedType::UINT_64, None), - Int8 => (ConvertedType::INT_8, None), - Int16 => (ConvertedType::INT_16, None), - Int32 => (ConvertedType::INT_32, None), - Int64 => (ConvertedType::INT_64, None), - Json => (ConvertedType::JSON, None), - Bson => (ConvertedType::BSON, None), - Interval => (ConvertedType::INTERVAL, None), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn round_trip() -> Result<(), Error> { - use PrimitiveConvertedType::*; - let a = vec![ - Utf8, - Enum, - Decimal(3, 1), - Date, - TimeMillis, - TimeMicros, - TimestampMillis, - TimestampMicros, - Uint8, - Uint16, - Uint32, - Uint64, - Int8, - Int16, - Int32, - Int64, - Json, - Bson, - Interval, - ]; - for a in a { - let (c, d): (ConvertedType, Option<(i32, i32)>) = a.into(); - let e: PrimitiveConvertedType = (c, d).try_into()?; - assert_eq!(e, a); - } - Ok(()) - } -} diff --git a/src/common/parquet2/src/schema/types/mod.rs b/src/common/parquet2/src/schema/types/mod.rs deleted file mode 100644 index 61096fad7444..000000000000 --- a/src/common/parquet2/src/schema/types/mod.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod spec; - -mod physical_type; -pub use physical_type::*; - -mod basic_type; -pub use basic_type::*; - -mod converted_type; -pub use converted_type::*; - -mod parquet_type; -pub use parquet_type::*; - -pub use crate::parquet_bridge::GroupLogicalType; -pub use crate::parquet_bridge::IntegerType; -pub use crate::parquet_bridge::PrimitiveLogicalType; -pub use crate::parquet_bridge::TimeUnit; diff --git a/src/common/parquet2/src/schema/types/parquet_type.rs b/src/common/parquet2/src/schema/types/parquet_type.rs deleted file mode 100644 index 9088941e5fc2..000000000000 --- a/src/common/parquet2/src/schema/types/parquet_type.rs +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// see https://github.com/apache/parquet-format/blob/master/LogicalTypes.md -use std::collections::HashMap; - -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use super::super::Repetition; -use super::spec; -use super::FieldInfo; -use super::GroupConvertedType; -use super::GroupLogicalType; -use super::PhysicalType; -use super::PrimitiveConvertedType; -use super::PrimitiveLogicalType; -use crate::error::Result; - -/// The complete description of a parquet column -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub struct PrimitiveType { - /// The fields' generic information - pub field_info: FieldInfo, - /// The optional logical type - pub logical_type: Option, - /// The optional converted type - pub converted_type: Option, - /// The physical type - pub physical_type: PhysicalType, -} - -impl PrimitiveType { - /// Helper method to create an optional field with no logical or converted types. - pub fn from_physical(name: String, physical_type: PhysicalType) -> Self { - let field_info = FieldInfo { - name, - repetition: Repetition::Optional, - id: None, - }; - Self { - field_info, - converted_type: None, - logical_type: None, - physical_type, - } - } -} - -/// Representation of a Parquet type describing primitive and nested fields, -/// including the top-level schema of the parquet file. -#[derive(Clone, Debug, PartialEq)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum ParquetType { - PrimitiveType(PrimitiveType), - GroupType { - field_info: FieldInfo, - logical_type: Option, - converted_type: Option, - fields: Vec, - }, -} - -/// Accessors -impl ParquetType { - /// Returns [`FieldInfo`] information about the type. - pub fn get_field_info(&self) -> &FieldInfo { - match self { - Self::PrimitiveType(primitive) => &primitive.field_info, - Self::GroupType { field_info, .. } => field_info, - } - } - - /// Returns this type's field name. - pub fn name(&self) -> &str { - &self.get_field_info().name - } - - /// Checks if `sub_type` schema is part of current schema. - /// This method can be used to check if projected columns are part of the root schema. - pub fn check_contains(&self, sub_type: &ParquetType) -> bool { - let basic_match = self.get_field_info() == sub_type.get_field_info(); - - match (self, sub_type) { - ( - Self::PrimitiveType(PrimitiveType { physical_type, .. }), - Self::PrimitiveType(PrimitiveType { - physical_type: other_physical_type, - .. - }), - ) => basic_match && physical_type == other_physical_type, - ( - Self::GroupType { fields, .. }, - Self::GroupType { - fields: other_fields, - .. - }, - ) => { - // build hashmap of name -> Type - let mut field_map = HashMap::new(); - for field in fields { - field_map.insert(field.name(), field); - } - - for field in other_fields { - if !field_map - .get(field.name()) - .map(|tpe| tpe.check_contains(field)) - .unwrap_or(false) - { - return false; - } - } - true - } - _ => false, - } - } -} - -/// Constructors -impl ParquetType { - pub(crate) fn new_root(name: String, fields: Vec) -> Self { - let field_info = FieldInfo { - name, - repetition: Repetition::Optional, - id: None, - }; - ParquetType::GroupType { - field_info, - fields, - logical_type: None, - converted_type: None, - } - } - - pub fn from_converted( - name: String, - fields: Vec, - repetition: Repetition, - converted_type: Option, - id: Option, - ) -> Self { - let field_info = FieldInfo { - name, - repetition, - id, - }; - - ParquetType::GroupType { - field_info, - fields, - converted_type, - logical_type: None, - } - } - - /// # Error - /// Errors iff the combination of physical, logical and coverted type is not valid. - pub fn try_from_primitive( - name: String, - physical_type: PhysicalType, - repetition: Repetition, - converted_type: Option, - logical_type: Option, - id: Option, - ) -> Result { - spec::check_converted_invariants(&physical_type, &converted_type)?; - spec::check_logical_invariants(&physical_type, &logical_type)?; - - let field_info = FieldInfo { - name, - repetition, - id, - }; - - Ok(ParquetType::PrimitiveType(PrimitiveType { - field_info, - converted_type, - logical_type, - physical_type, - })) - } - - /// Helper method to create a [`ParquetType::PrimitiveType`] optional field - /// with no logical or converted types. - pub fn from_physical(name: String, physical_type: PhysicalType) -> Self { - ParquetType::PrimitiveType(PrimitiveType::from_physical(name, physical_type)) - } - - pub fn from_group( - name: String, - repetition: Repetition, - converted_type: Option, - logical_type: Option, - fields: Vec, - id: Option, - ) -> Self { - let field_info = FieldInfo { - name, - repetition, - id, - }; - - ParquetType::GroupType { - field_info, - logical_type, - converted_type, - fields, - } - } -} diff --git a/src/common/parquet2/src/schema/types/physical_type.rs b/src/common/parquet2/src/schema/types/physical_type.rs deleted file mode 100644 index 931dd0b9737f..000000000000 --- a/src/common/parquet2/src/schema/types/physical_type.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::Type; -#[cfg(feature = "serde_types")] -use serde::Deserialize; -#[cfg(feature = "serde_types")] -use serde::Serialize; - -use crate::error::Error; - -/// The set of all physical types representable in Parquet -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Deserialize, Serialize))] -pub enum PhysicalType { - Boolean, - Int32, - Int64, - Int96, - Float, - Double, - ByteArray, - FixedLenByteArray(usize), -} - -impl TryFrom<(Type, Option)> for PhysicalType { - type Error = Error; - - fn try_from((type_, length): (Type, Option)) -> Result { - Ok(match type_ { - Type::BOOLEAN => PhysicalType::Boolean, - Type::INT32 => PhysicalType::Int32, - Type::INT64 => PhysicalType::Int64, - Type::INT96 => PhysicalType::Int96, - Type::FLOAT => PhysicalType::Float, - Type::DOUBLE => PhysicalType::Double, - Type::BYTE_ARRAY => PhysicalType::ByteArray, - Type::FIXED_LEN_BYTE_ARRAY => { - let length = length - .ok_or_else(|| Error::oos("Length must be defined for FixedLenByteArray"))?; - PhysicalType::FixedLenByteArray(length.try_into()?) - } - _ => return Err(Error::oos("Unknown type")), - }) - } -} - -impl From for (Type, Option) { - fn from(physical_type: PhysicalType) -> Self { - match physical_type { - PhysicalType::Boolean => (Type::BOOLEAN, None), - PhysicalType::Int32 => (Type::INT32, None), - PhysicalType::Int64 => (Type::INT64, None), - PhysicalType::Int96 => (Type::INT96, None), - PhysicalType::Float => (Type::FLOAT, None), - PhysicalType::Double => (Type::DOUBLE, None), - PhysicalType::ByteArray => (Type::BYTE_ARRAY, None), - PhysicalType::FixedLenByteArray(length) => { - (Type::FIXED_LEN_BYTE_ARRAY, Some(length as i32)) - } - } - } -} diff --git a/src/common/parquet2/src/schema/types/spec.rs b/src/common/parquet2/src/schema/types/spec.rs deleted file mode 100644 index d6bf69331037..000000000000 --- a/src/common/parquet2/src/schema/types/spec.rs +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// see https://github.com/apache/parquet-format/blob/master/LogicalTypes.md -use super::IntegerType; -use super::PhysicalType; -use super::PrimitiveConvertedType; -use super::PrimitiveLogicalType; -use super::TimeUnit; -use crate::error::Error; -use crate::error::Result; - -fn check_decimal_invariants( - physical_type: &PhysicalType, - precision: usize, - scale: usize, -) -> Result<()> { - if precision < 1 { - return Err(Error::oos(format!( - "DECIMAL precision must be larger than 0; It is {}", - precision, - ))); - } - if scale > precision { - return Err(Error::oos(format!( - "Invalid DECIMAL: scale ({}) cannot be greater than precision \ - ({})", - scale, precision - ))); - } - - match physical_type { - PhysicalType::Int32 => { - if !(1..=9).contains(&precision) { - return Err(Error::oos(format!( - "Cannot represent INT32 as DECIMAL with precision {}", - precision - ))); - } - } - PhysicalType::Int64 => { - if !(1..=18).contains(&precision) { - return Err(Error::oos(format!( - "Cannot represent INT64 as DECIMAL with precision {}", - precision - ))); - } - } - PhysicalType::FixedLenByteArray(length) => { - let oos_error = || Error::oos(format!("Byte Array length {} out of spec", length)); - let max_precision = (2f64.powi( - (*length as i32) - .checked_mul(8) - .ok_or_else(oos_error)? - .checked_sub(1) - .ok_or_else(oos_error)?, - ) - 1f64) - .log10() - .floor() as usize; - - if precision > max_precision { - return Err(Error::oos(format!( - "Cannot represent FIXED_LEN_BYTE_ARRAY as DECIMAL with length {} and \ - precision {}. The max precision can only be {}", - length, precision, max_precision - ))); - } - } - PhysicalType::ByteArray => {} - _ => { - return Err(Error::oos( - "DECIMAL can only annotate INT32, INT64, BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY" - .to_string(), - )); - } - }; - Ok(()) -} - -pub fn check_converted_invariants( - physical_type: &PhysicalType, - converted_type: &Option, -) -> Result<()> { - if converted_type.is_none() { - return Ok(()); - }; - let converted_type = converted_type.as_ref().unwrap(); - - use PrimitiveConvertedType::*; - match converted_type { - Utf8 | Bson | Json => { - if physical_type != &PhysicalType::ByteArray { - return Err(Error::oos(format!( - "{:?} can only annotate BYTE_ARRAY fields", - converted_type - ))); - } - } - Decimal(precision, scale) => { - check_decimal_invariants(physical_type, *precision, *scale)?; - } - Date | TimeMillis | Uint8 | Uint16 | Uint32 | Int8 | Int16 | Int32 => { - if physical_type != &PhysicalType::Int32 { - return Err(Error::oos(format!( - "{:?} can only annotate INT32", - converted_type - ))); - } - } - TimeMicros | TimestampMillis | TimestampMicros | Uint64 | Int64 => { - if physical_type != &PhysicalType::Int64 { - return Err(Error::oos(format!( - "{:?} can only annotate INT64", - converted_type - ))); - } - } - Interval => { - if physical_type != &PhysicalType::FixedLenByteArray(12) { - return Err(Error::oos( - "INTERVAL can only annotate FIXED_LEN_BYTE_ARRAY(12)".to_string(), - )); - } - } - Enum => { - if physical_type != &PhysicalType::ByteArray { - return Err(Error::oos( - "ENUM can only annotate BYTE_ARRAY fields".to_string(), - )); - } - } - }; - Ok(()) -} - -pub fn check_logical_invariants( - physical_type: &PhysicalType, - logical_type: &Option, -) -> Result<()> { - if logical_type.is_none() { - return Ok(()); - }; - let logical_type = logical_type.unwrap(); - - // Check that logical type and physical type are compatible - use PrimitiveLogicalType::*; - match (logical_type, physical_type) { - (Enum, PhysicalType::ByteArray) => {} - (Decimal(precision, scale), _) => { - check_decimal_invariants(physical_type, precision, scale)?; - } - (Date, PhysicalType::Int32) => {} - ( - Time { - unit: TimeUnit::Milliseconds, - .. - }, - PhysicalType::Int32, - ) => {} - (Time { unit, .. }, PhysicalType::Int64) => { - if unit == TimeUnit::Milliseconds { - return Err(Error::oos( - "Cannot use millisecond unit on INT64 type".to_string(), - )); - } - } - (Timestamp { .. }, PhysicalType::Int64) => {} - (Integer(IntegerType::Int8), PhysicalType::Int32) => {} - (Integer(IntegerType::Int16), PhysicalType::Int32) => {} - (Integer(IntegerType::Int32), PhysicalType::Int32) => {} - (Integer(IntegerType::UInt8), PhysicalType::Int32) => {} - (Integer(IntegerType::UInt16), PhysicalType::Int32) => {} - (Integer(IntegerType::UInt32), PhysicalType::Int32) => {} - (Integer(IntegerType::UInt64), PhysicalType::Int64) => {} - (Integer(IntegerType::Int64), PhysicalType::Int64) => {} - // Null type - (Unknown, PhysicalType::Int32) => {} - (String | Json | Bson, PhysicalType::ByteArray) => {} - // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid - (Uuid, PhysicalType::FixedLenByteArray(16)) => {} - (a, b) => { - return Err(Error::oos(format!( - "Cannot annotate {:?} from {:?} fields", - a, b - ))); - } - }; - Ok(()) -} diff --git a/src/common/parquet2/src/statistics/binary.rs b/src/common/parquet2/src/statistics/binary.rs deleted file mode 100644 index 8af2369808a6..000000000000 --- a/src/common/parquet2/src/statistics/binary.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use parquet_format_safe::Statistics as ParquetStatistics; - -use super::Statistics; -use crate::error::Result; -use crate::schema::types::PhysicalType; -use crate::schema::types::PrimitiveType; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BinaryStatistics { - pub primitive_type: PrimitiveType, - pub null_count: Option, - pub distinct_count: Option, - pub max_value: Option>, - pub min_value: Option>, -} - -impl Statistics for BinaryStatistics { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn physical_type(&self) -> &PhysicalType { - &PhysicalType::ByteArray - } - - fn null_count(&self) -> Option { - self.null_count - } -} - -pub fn read(v: &ParquetStatistics, primitive_type: PrimitiveType) -> Result> { - Ok(Arc::new(BinaryStatistics { - primitive_type, - null_count: v.null_count, - distinct_count: v.distinct_count, - max_value: v.max_value.clone(), - min_value: v.min_value.clone(), - })) -} - -pub fn write(v: &BinaryStatistics) -> ParquetStatistics { - ParquetStatistics { - null_count: v.null_count, - distinct_count: v.distinct_count, - max_value: v.max_value.clone(), - min_value: v.min_value.clone(), - min: None, - max: None, - } -} diff --git a/src/common/parquet2/src/statistics/boolean.rs b/src/common/parquet2/src/statistics/boolean.rs deleted file mode 100644 index 14dc720bf5ad..000000000000 --- a/src/common/parquet2/src/statistics/boolean.rs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use parquet_format_safe::Statistics as ParquetStatistics; - -use super::Statistics; -use crate::error::Error; -use crate::error::Result; -use crate::schema::types::PhysicalType; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BooleanStatistics { - pub null_count: Option, - pub distinct_count: Option, - pub max_value: Option, - pub min_value: Option, -} - -impl Statistics for BooleanStatistics { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn physical_type(&self) -> &PhysicalType { - &PhysicalType::Boolean - } - - fn null_count(&self) -> Option { - self.null_count - } -} - -pub fn read(v: &ParquetStatistics) -> Result> { - if let Some(ref v) = v.max_value { - if v.len() != std::mem::size_of::() { - return Err(Error::oos( - "The max_value of statistics MUST be plain encoded", - )); - } - }; - if let Some(ref v) = v.min_value { - if v.len() != std::mem::size_of::() { - return Err(Error::oos( - "The min_value of statistics MUST be plain encoded", - )); - } - }; - - Ok(Arc::new(BooleanStatistics { - null_count: v.null_count, - distinct_count: v.distinct_count, - max_value: v - .max_value - .as_ref() - .and_then(|x| x.first()) - .map(|x| *x != 0), - min_value: v - .min_value - .as_ref() - .and_then(|x| x.first()) - .map(|x| *x != 0), - })) -} - -pub fn write(v: &BooleanStatistics) -> ParquetStatistics { - ParquetStatistics { - null_count: v.null_count, - distinct_count: v.distinct_count, - max_value: v.max_value.map(|x| vec![x as u8]), - min_value: v.min_value.map(|x| vec![x as u8]), - min: None, - max: None, - } -} diff --git a/src/common/parquet2/src/statistics/fixed_len_binary.rs b/src/common/parquet2/src/statistics/fixed_len_binary.rs deleted file mode 100644 index 40b676e9d5ad..000000000000 --- a/src/common/parquet2/src/statistics/fixed_len_binary.rs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use parquet_format_safe::Statistics as ParquetStatistics; - -use super::Statistics; -use crate::error::Error; -use crate::error::Result; -use crate::schema::types::PhysicalType; -use crate::schema::types::PrimitiveType; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FixedLenStatistics { - pub primitive_type: PrimitiveType, - pub null_count: Option, - pub distinct_count: Option, - pub max_value: Option>, - pub min_value: Option>, -} - -impl Statistics for FixedLenStatistics { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn physical_type(&self) -> &PhysicalType { - &self.primitive_type.physical_type - } - - fn null_count(&self) -> Option { - self.null_count - } -} - -pub fn read( - v: &ParquetStatistics, - size: usize, - primitive_type: PrimitiveType, -) -> Result> { - if let Some(ref v) = v.max_value { - if v.len() != size { - return Err(Error::oos( - "The max_value of statistics MUST be plain encoded", - )); - } - }; - if let Some(ref v) = v.min_value { - if v.len() != size { - return Err(Error::oos( - "The min_value of statistics MUST be plain encoded", - )); - } - }; - - Ok(Arc::new(FixedLenStatistics { - primitive_type, - null_count: v.null_count, - distinct_count: v.distinct_count, - max_value: v.max_value.clone().map(|mut x| { - x.truncate(size); - x - }), - min_value: v.min_value.clone().map(|mut x| { - x.truncate(size); - x - }), - })) -} - -pub fn write(v: &FixedLenStatistics) -> ParquetStatistics { - ParquetStatistics { - null_count: v.null_count, - distinct_count: v.distinct_count, - max_value: v.max_value.clone(), - min_value: v.min_value.clone(), - min: None, - max: None, - } -} diff --git a/src/common/parquet2/src/statistics/mod.rs b/src/common/parquet2/src/statistics/mod.rs deleted file mode 100644 index 1d61340407eb..000000000000 --- a/src/common/parquet2/src/statistics/mod.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod binary; -mod boolean; -mod fixed_len_binary; -mod primitive; - -use std::any::Any; -use std::sync::Arc; - -pub use binary::BinaryStatistics; -pub use boolean::BooleanStatistics; -pub use fixed_len_binary::FixedLenStatistics; -pub use primitive::PrimitiveStatistics; - -use crate::error::Result; -use crate::schema::types::PhysicalType; -use crate::schema::types::PrimitiveType; -pub use crate::thrift_format::Statistics as ParquetStatistics; - -/// A trait used to describe specific statistics. Each physical type has its own struct. -/// Match the [`Statistics::physical_type`] to each type and downcast accordingly. -pub trait Statistics: Send + Sync + std::fmt::Debug { - fn as_any(&self) -> &dyn Any; - - fn physical_type(&self) -> &PhysicalType; - - fn null_count(&self) -> Option; -} - -impl PartialEq for &dyn Statistics { - fn eq(&self, other: &Self) -> bool { - self.physical_type() == other.physical_type() && { - match self.physical_type() { - PhysicalType::Boolean => { - self.as_any().downcast_ref::().unwrap() - == other.as_any().downcast_ref::().unwrap() - } - PhysicalType::Int32 => { - self.as_any() - .downcast_ref::>() - .unwrap() - == other - .as_any() - .downcast_ref::>() - .unwrap() - } - PhysicalType::Int64 => { - self.as_any() - .downcast_ref::>() - .unwrap() - == other - .as_any() - .downcast_ref::>() - .unwrap() - } - PhysicalType::Int96 => { - self.as_any() - .downcast_ref::>() - .unwrap() - == other - .as_any() - .downcast_ref::>() - .unwrap() - } - PhysicalType::Float => { - self.as_any() - .downcast_ref::>() - .unwrap() - == other - .as_any() - .downcast_ref::>() - .unwrap() - } - PhysicalType::Double => { - self.as_any() - .downcast_ref::>() - .unwrap() - == other - .as_any() - .downcast_ref::>() - .unwrap() - } - PhysicalType::ByteArray => { - self.as_any().downcast_ref::().unwrap() - == other.as_any().downcast_ref::().unwrap() - } - PhysicalType::FixedLenByteArray(_) => { - self.as_any().downcast_ref::().unwrap() - == other.as_any().downcast_ref::().unwrap() - } - } - } - } -} - -/// Deserializes a raw parquet statistics into [`Statistics`]. -/// # Error -/// This function errors if it is not possible to read the statistics to the -/// corresponding `physical_type`. -pub fn deserialize_statistics( - statistics: &ParquetStatistics, - primitive_type: PrimitiveType, -) -> Result> { - match primitive_type.physical_type { - PhysicalType::Boolean => boolean::read(statistics), - PhysicalType::Int32 => primitive::read::(statistics, primitive_type), - PhysicalType::Int64 => primitive::read::(statistics, primitive_type), - PhysicalType::Int96 => primitive::read::<[u32; 3]>(statistics, primitive_type), - PhysicalType::Float => primitive::read::(statistics, primitive_type), - PhysicalType::Double => primitive::read::(statistics, primitive_type), - PhysicalType::ByteArray => binary::read(statistics, primitive_type), - PhysicalType::FixedLenByteArray(size) => { - fixed_len_binary::read(statistics, size, primitive_type) - } - } -} - -/// Serializes [`Statistics`] into a raw parquet statistics. -pub fn serialize_statistics(statistics: &dyn Statistics) -> ParquetStatistics { - match statistics.physical_type() { - PhysicalType::Boolean => boolean::write(statistics.as_any().downcast_ref().unwrap()), - PhysicalType::Int32 => primitive::write::(statistics.as_any().downcast_ref().unwrap()), - PhysicalType::Int64 => primitive::write::(statistics.as_any().downcast_ref().unwrap()), - PhysicalType::Int96 => { - primitive::write::<[u32; 3]>(statistics.as_any().downcast_ref().unwrap()) - } - PhysicalType::Float => primitive::write::(statistics.as_any().downcast_ref().unwrap()), - PhysicalType::Double => { - primitive::write::(statistics.as_any().downcast_ref().unwrap()) - } - PhysicalType::ByteArray => binary::write(statistics.as_any().downcast_ref().unwrap()), - PhysicalType::FixedLenByteArray(_) => { - fixed_len_binary::write(statistics.as_any().downcast_ref().unwrap()) - } - } -} diff --git a/src/common/parquet2/src/statistics/primitive.rs b/src/common/parquet2/src/statistics/primitive.rs deleted file mode 100644 index 88fed7ebfe2f..000000000000 --- a/src/common/parquet2/src/statistics/primitive.rs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use parquet_format_safe::Statistics as ParquetStatistics; - -use super::Statistics; -use crate::error::Error; -use crate::error::Result; -use crate::schema::types::PhysicalType; -use crate::schema::types::PrimitiveType; -use crate::types; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PrimitiveStatistics { - pub primitive_type: PrimitiveType, - pub null_count: Option, - pub distinct_count: Option, - pub min_value: Option, - pub max_value: Option, -} - -impl Statistics for PrimitiveStatistics { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn physical_type(&self) -> &PhysicalType { - &T::TYPE - } - - fn null_count(&self) -> Option { - self.null_count - } -} - -pub fn read( - v: &ParquetStatistics, - primitive_type: PrimitiveType, -) -> Result> { - if let Some(ref v) = v.max_value { - if v.len() != std::mem::size_of::() { - return Err(Error::oos( - "The max_value of statistics MUST be plain encoded", - )); - } - }; - if let Some(ref v) = v.min_value { - if v.len() != std::mem::size_of::() { - return Err(Error::oos( - "The min_value of statistics MUST be plain encoded", - )); - } - }; - - Ok(Arc::new(PrimitiveStatistics:: { - primitive_type, - null_count: v.null_count, - distinct_count: v.distinct_count, - max_value: v.max_value.as_ref().map(|x| types::decode(x)), - min_value: v.min_value.as_ref().map(|x| types::decode(x)), - })) -} - -pub fn write(v: &PrimitiveStatistics) -> ParquetStatistics { - ParquetStatistics { - null_count: v.null_count, - distinct_count: v.distinct_count, - max_value: v.max_value.map(|x| x.to_le_bytes().as_ref().to_vec()), - min_value: v.min_value.map(|x| x.to_le_bytes().as_ref().to_vec()), - min: None, - max: None, - } -} diff --git a/src/common/parquet2/src/types.rs b/src/common/parquet2/src/types.rs deleted file mode 100644 index 50a99cf09eb1..000000000000 --- a/src/common/parquet2/src/types.rs +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::convert::TryFrom; - -use crate::schema::types::PhysicalType; - -/// A physical native representation of a Parquet fixed-sized type. -pub trait NativeType: std::fmt::Debug + Send + Sync + 'static + Copy + Clone { - type Bytes: AsRef<[u8]> + for<'a> TryFrom<&'a [u8], Error = std::array::TryFromSliceError>; - - fn to_le_bytes(&self) -> Self::Bytes; - - fn from_le_bytes(bytes: Self::Bytes) -> Self; - - fn ord(&self, other: &Self) -> std::cmp::Ordering; - - const TYPE: PhysicalType; -} - -macro_rules! native { - ($type:ty, $physical_type:expr) => { - impl NativeType for $type { - type Bytes = [u8; std::mem::size_of::()]; - #[inline] - fn to_le_bytes(&self) -> Self::Bytes { - Self::to_le_bytes(*self) - } - - #[inline] - fn from_le_bytes(bytes: Self::Bytes) -> Self { - Self::from_le_bytes(bytes) - } - - #[inline] - fn ord(&self, other: &Self) -> std::cmp::Ordering { - self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal) - } - - const TYPE: PhysicalType = $physical_type; - } - }; -} - -native!(i32, PhysicalType::Int32); -native!(i64, PhysicalType::Int64); -native!(f32, PhysicalType::Float); -native!(f64, PhysicalType::Double); - -impl NativeType for [u32; 3] { - const TYPE: PhysicalType = PhysicalType::Int96; - - type Bytes = [u8; std::mem::size_of::()]; - #[inline] - fn to_le_bytes(&self) -> Self::Bytes { - let mut bytes = [0; 12]; - let first = self[0].to_le_bytes(); - bytes[0] = first[0]; - bytes[1] = first[1]; - bytes[2] = first[2]; - bytes[3] = first[3]; - let second = self[1].to_le_bytes(); - bytes[4] = second[0]; - bytes[5] = second[1]; - bytes[6] = second[2]; - bytes[7] = second[3]; - let third = self[2].to_le_bytes(); - bytes[8] = third[0]; - bytes[9] = third[1]; - bytes[10] = third[2]; - bytes[11] = third[3]; - bytes - } - - #[inline] - fn from_le_bytes(bytes: Self::Bytes) -> Self { - let mut first = [0; 4]; - first[0] = bytes[0]; - first[1] = bytes[1]; - first[2] = bytes[2]; - first[3] = bytes[3]; - let mut second = [0; 4]; - second[0] = bytes[4]; - second[1] = bytes[5]; - second[2] = bytes[6]; - second[3] = bytes[7]; - let mut third = [0; 4]; - third[0] = bytes[8]; - third[1] = bytes[9]; - third[2] = bytes[10]; - third[3] = bytes[11]; - [ - u32::from_le_bytes(first), - u32::from_le_bytes(second), - u32::from_le_bytes(third), - ] - } - - #[inline] - fn ord(&self, other: &Self) -> std::cmp::Ordering { - int96_to_i64_ns(*self).ord(&int96_to_i64_ns(*other)) - } -} - -#[inline] -pub fn int96_to_i64_ns(value: [u32; 3]) -> i64 { - const JULIAN_DAY_OF_EPOCH: i64 = 2_440_588; - const SECONDS_PER_DAY: i64 = 86_400; - const NANOS_PER_SECOND: i64 = 1_000_000_000; - - let day = value[2] as i64; - let nanoseconds = ((value[1] as i64) << 32) + value[0] as i64; - let seconds = (day - JULIAN_DAY_OF_EPOCH) * SECONDS_PER_DAY; - - seconds * NANOS_PER_SECOND + nanoseconds -} - -/// Returns the ordering of two binary values. -pub fn ord_binary<'a>(a: &'a [u8], b: &'a [u8]) -> std::cmp::Ordering { - use std::cmp::Ordering::*; - match (a.is_empty(), b.is_empty()) { - (true, true) => return Equal, - (true, false) => return Less, - (false, true) => return Greater, - (false, false) => {} - } - - for (v1, v2) in a.iter().zip(b.iter()) { - match v1.cmp(v2) { - Equal => continue, - other => return other, - } - } - Equal -} - -#[inline] -pub fn decode(chunk: &[u8]) -> T { - let chunk: ::Bytes = match chunk.try_into() { - Ok(v) => v, - Err(_) => panic!(), - }; - T::from_le_bytes(chunk) -} diff --git a/src/common/parquet2/src/write/column_chunk.rs b/src/common/parquet2/src/write/column_chunk.rs deleted file mode 100644 index b02afbbe2790..000000000000 --- a/src/common/parquet2/src/write/column_chunk.rs +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashSet; -use std::io::Write; - -#[cfg(feature = "async")] -use futures::AsyncWrite; -use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; -#[cfg(feature = "async")] -use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol; -use parquet_format_safe::ColumnChunk; -use parquet_format_safe::ColumnMetaData; -use parquet_format_safe::Type; - -use super::page::write_page; -#[cfg(feature = "async")] -use super::page::write_page_async; -use super::page::PageWriteSpec; -use super::statistics::reduce; -use super::DynStreamingIterator; -use crate::compression::Compression; -use crate::encoding::Encoding; -use crate::error::Error; -use crate::error::Result; -use crate::metadata::ColumnDescriptor; -use crate::page::CompressedPage; -use crate::page::PageType; -use crate::statistics::serialize_statistics; -use crate::FallibleStreamingIterator; - -pub fn write_column_chunk<'a, W, E>( - writer: &mut W, - mut offset: u64, - descriptor: &ColumnDescriptor, - mut compressed_pages: DynStreamingIterator<'a, CompressedPage, E>, -) -> Result<(ColumnChunk, Vec, u64)> -where - W: Write, - Error: From, - E: std::error::Error, -{ - // write every page - - let initial = offset; - - let mut specs = vec![]; - while let Some(compressed_page) = compressed_pages.next()? { - let spec = write_page(writer, offset, compressed_page)?; - offset += spec.bytes_written; - specs.push(spec); - } - let mut bytes_written = offset - initial; - - let column_chunk = build_column_chunk(&specs, descriptor)?; - - // write metadata - let mut protocol = TCompactOutputProtocol::new(writer); - bytes_written += column_chunk - .meta_data - .as_ref() - .unwrap() - .write_to_out_protocol(&mut protocol)? as u64; - - Ok((column_chunk, specs, bytes_written)) -} - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub async fn write_column_chunk_async( - writer: &mut W, - mut offset: u64, - descriptor: &ColumnDescriptor, - mut compressed_pages: DynStreamingIterator<'_, CompressedPage, E>, -) -> Result<(ColumnChunk, Vec, u64)> -where - W: AsyncWrite + Unpin + Send, - Error: From, - E: std::error::Error, -{ - let initial = offset; - // write every page - let mut specs = vec![]; - while let Some(compressed_page) = compressed_pages.next()? { - let spec = write_page_async(writer, offset, compressed_page).await?; - offset += spec.bytes_written; - specs.push(spec); - } - let mut bytes_written = offset - initial; - - let column_chunk = build_column_chunk(&specs, descriptor)?; - - // write metadata - let mut protocol = TCompactOutputStreamProtocol::new(writer); - bytes_written += column_chunk - .meta_data - .as_ref() - .unwrap() - .write_to_out_stream_protocol(&mut protocol) - .await? as u64; - - Ok((column_chunk, specs, bytes_written)) -} - -fn build_column_chunk( - specs: &[PageWriteSpec], - descriptor: &ColumnDescriptor, -) -> Result { - // compute stats to build header at the end of the chunk - - let compression = specs - .iter() - .map(|spec| spec.compression) - .collect::>(); - if compression.len() > 1 { - return Err(crate::error::Error::oos( - "All pages within a column chunk must be compressed with the same codec", - )); - } - let compression = compression - .into_iter() - .next() - .unwrap_or(Compression::Uncompressed); - - // SPEC: the total compressed size is the total compressed size of each page + the header size - let total_compressed_size = specs - .iter() - .map(|x| x.header_size as i64 + x.header.compressed_page_size as i64) - .sum(); - // SPEC: the total compressed size is the total compressed size of each page + the header size - let total_uncompressed_size = specs - .iter() - .map(|x| x.header_size as i64 + x.header.uncompressed_page_size as i64) - .sum(); - let data_page_offset = specs.first().map(|spec| spec.offset).unwrap_or(0) as i64; - let num_values = specs - .iter() - .map(|spec| { - let type_ = spec.header.type_.try_into().unwrap(); - match type_ { - PageType::DataPage => { - spec.header.data_page_header.as_ref().unwrap().num_values as i64 - } - PageType::DataPageV2 => { - spec.header.data_page_header_v2.as_ref().unwrap().num_values as i64 - } - _ => 0, // only data pages contribute - } - }) - .sum(); - let mut encodings = specs - .iter() - .flat_map(|spec| { - let type_ = spec.header.type_.try_into().unwrap(); - match type_ { - PageType::DataPage => vec![ - spec.header.data_page_header.as_ref().unwrap().encoding, - Encoding::Rle.into(), - ], - PageType::DataPageV2 => { - vec![ - spec.header.data_page_header_v2.as_ref().unwrap().encoding, - Encoding::Rle.into(), - ] - } - PageType::DictionaryPage => vec![ - spec.header - .dictionary_page_header - .as_ref() - .unwrap() - .encoding, - ], - } - }) - .collect::>() // unique - .into_iter() // to vec - .collect::>(); - - // Sort the encodings to have deterministic metadata - encodings.sort(); - - let statistics = specs.iter().map(|x| &x.statistics).collect::>(); - let statistics = reduce(&statistics)?; - let statistics = statistics.map(|x| serialize_statistics(x.as_ref())); - - let (type_, _): (Type, Option) = descriptor.descriptor.primitive_type.physical_type.into(); - - let metadata = ColumnMetaData { - type_, - encodings, - path_in_schema: descriptor.path_in_schema.clone(), - codec: compression.into(), - num_values, - total_uncompressed_size, - total_compressed_size, - key_value_metadata: None, - data_page_offset, - index_page_offset: None, - dictionary_page_offset: None, - statistics, - encoding_stats: None, - bloom_filter_offset: None, - }; - - Ok(ColumnChunk { - file_path: None, // same file for now. - file_offset: data_page_offset + total_compressed_size, - meta_data: Some(metadata), - offset_index_offset: None, - offset_index_length: None, - column_index_offset: None, - column_index_length: None, - crypto_metadata: None, - encrypted_column_metadata: None, - }) -} diff --git a/src/common/parquet2/src/write/compression.rs b/src/common/parquet2/src/write/compression.rs deleted file mode 100644 index c2147ec0f8ca..000000000000 --- a/src/common/parquet2/src/write/compression.rs +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::compression; -use crate::compression::CompressionOptions; -use crate::error::Error; -use crate::error::Result; -use crate::page::CompressedDataPage; -use crate::page::CompressedDictPage; -use crate::page::CompressedPage; -use crate::page::DataPage; -use crate::page::DataPageHeader; -use crate::page::DictPage; -use crate::page::Page; -use crate::FallibleStreamingIterator; - -/// Compresses a [`DataPage`] into a [`CompressedDataPage`]. -fn compress_data( - page: DataPage, - mut compressed_buffer: Vec, - compression: CompressionOptions, -) -> Result { - let DataPage { - mut buffer, - header, - descriptor, - selected_rows, - } = page; - let uncompressed_page_size = buffer.len(); - if compression != CompressionOptions::Uncompressed { - match &header { - DataPageHeader::V1(_) => { - compression::compress(compression, &buffer, &mut compressed_buffer)?; - } - DataPageHeader::V2(header) => { - let levels_byte_length = (header.repetition_levels_byte_length - + header.definition_levels_byte_length) - as usize; - compressed_buffer.extend_from_slice(&buffer[..levels_byte_length]); - compression::compress( - compression, - &buffer[levels_byte_length..], - &mut compressed_buffer, - )?; - } - }; - } else { - std::mem::swap(&mut buffer, &mut compressed_buffer); - }; - Ok(CompressedDataPage::new_read( - header, - compressed_buffer, - compression.into(), - uncompressed_page_size, - descriptor, - selected_rows, - )) -} - -fn compress_dict( - page: DictPage, - mut compressed_buffer: Vec, - compression: CompressionOptions, -) -> Result { - let DictPage { - mut buffer, - num_values, - is_sorted, - } = page; - let uncompressed_page_size = buffer.len(); - if compression != CompressionOptions::Uncompressed { - compression::compress(compression, &buffer, &mut compressed_buffer)?; - } else { - std::mem::swap(&mut buffer, &mut compressed_buffer); - } - Ok(CompressedDictPage::new( - compressed_buffer, - compression.into(), - uncompressed_page_size, - num_values, - is_sorted, - )) -} - -/// Compresses an [`EncodedPage`] into a [`CompressedPage`] using `compressed_buffer` as the -/// intermediary buffer. -/// -/// `compressed_buffer` is taken by value because it becomes owned by [`CompressedPage`] -/// -/// # Errors -/// Errors if the compressor fails -pub fn compress( - page: Page, - compressed_buffer: Vec, - compression: CompressionOptions, -) -> Result { - match page { - Page::Data(page) => { - compress_data(page, compressed_buffer, compression).map(CompressedPage::Data) - } - Page::Dict(page) => { - compress_dict(page, compressed_buffer, compression).map(CompressedPage::Dict) - } - } -} - -/// A [`FallibleStreamingIterator`] that consumes [`Page`] and yields [`CompressedPage`] -/// holding a reusable buffer ([`Vec`]) for compression. -pub struct Compressor>> { - iter: I, - compression: CompressionOptions, - buffer: Vec, - current: Option, -} - -impl>> Compressor { - /// Creates a new [`Compressor`] - pub fn new(iter: I, compression: CompressionOptions, buffer: Vec) -> Self { - Self { - iter, - compression, - buffer, - current: None, - } - } - - /// Creates a new [`Compressor`] (same as `new`) - pub fn new_from_vec(iter: I, compression: CompressionOptions, buffer: Vec) -> Self { - Self::new(iter, compression, buffer) - } - - /// Deconstructs itself into its iterator and scratch buffer. - pub fn into_inner(mut self) -> (I, Vec) { - let mut buffer = if let Some(page) = self.current.as_mut() { - std::mem::take(page.buffer()) - } else { - std::mem::take(&mut self.buffer) - }; - buffer.clear(); - (self.iter, buffer) - } -} - -impl>> FallibleStreamingIterator for Compressor { - type Item = CompressedPage; - type Error = Error; - - fn advance(&mut self) -> std::result::Result<(), Self::Error> { - let mut compressed_buffer = if let Some(page) = self.current.as_mut() { - std::mem::take(page.buffer()) - } else { - std::mem::take(&mut self.buffer) - }; - compressed_buffer.clear(); - - let next = self - .iter - .next() - .map(|x| x.and_then(|page| compress(page, compressed_buffer, self.compression))) - .transpose()?; - self.current = next; - Ok(()) - } - - fn get(&self) -> Option<&Self::Item> { - self.current.as_ref() - } -} diff --git a/src/common/parquet2/src/write/dyn_iter.rs b/src/common/parquet2/src/write/dyn_iter.rs deleted file mode 100644 index 30ea2290077f..000000000000 --- a/src/common/parquet2/src/write/dyn_iter.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::FallibleStreamingIterator; - -/// [`DynIter`] is an implementation of a single-threaded, dynamically-typed iterator. -/// -/// This implementation is object safe. -pub struct DynIter<'a, V> { - iter: Box + 'a + Send + Sync>, -} - -impl<'a, V> Iterator for DynIter<'a, V> { - type Item = V; - fn next(&mut self) -> Option { - self.iter.next() - } - - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -impl<'a, V> DynIter<'a, V> { - /// Returns a new [`DynIter`], boxing the incoming iterator - pub fn new(iter: I) -> Self - where I: Iterator + 'a + Send + Sync { - Self { - iter: Box::new(iter), - } - } -} - -/// Dynamically-typed [`FallibleStreamingIterator`]. -pub struct DynStreamingIterator<'a, V, E> { - iter: Box + 'a + Send + Sync>, -} - -impl<'a, V, E> FallibleStreamingIterator for DynStreamingIterator<'a, V, E> { - type Item = V; - type Error = E; - - fn advance(&mut self) -> Result<(), Self::Error> { - self.iter.advance() - } - - fn get(&self) -> Option<&Self::Item> { - self.iter.get() - } - - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -impl<'a, V, E> DynStreamingIterator<'a, V, E> { - /// Returns a new [`DynStreamingIterator`], boxing the incoming iterator - pub fn new(iter: I) -> Self - where I: FallibleStreamingIterator + 'a + Send + Sync { - Self { - iter: Box::new(iter), - } - } -} diff --git a/src/common/parquet2/src/write/file.rs b/src/common/parquet2/src/write/file.rs deleted file mode 100644 index 4f7296e238ac..000000000000 --- a/src/common/parquet2/src/write/file.rs +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; -use parquet_format_safe::RowGroup; - -use super::indexes::write_column_index; -use super::indexes::write_offset_index; -use super::page::PageWriteSpec; -use super::row_group::write_row_group; -use super::RowGroupIter; -use super::WriteOptions; -use crate::error::Error; -use crate::error::Result; -pub use crate::metadata::KeyValue; -use crate::metadata::SchemaDescriptor; -use crate::metadata::ThriftFileMetaData; -use crate::write::State; -use crate::FOOTER_SIZE; -use crate::PARQUET_MAGIC; - -pub(super) fn start_file(writer: &mut W) -> Result { - writer.write_all(&PARQUET_MAGIC)?; - Ok(PARQUET_MAGIC.len() as u64) -} - -pub(super) fn end_file(mut writer: &mut W, metadata: &ThriftFileMetaData) -> Result { - // Write metadata - let mut protocol = TCompactOutputProtocol::new(&mut writer); - let metadata_len = metadata.write_to_out_protocol(&mut protocol)? as i32; - - // Write footer - let metadata_bytes = metadata_len.to_le_bytes(); - let mut footer_buffer = [0u8; FOOTER_SIZE as usize]; - (0..4).for_each(|i| { - footer_buffer[i] = metadata_bytes[i]; - }); - - (&mut footer_buffer[4..]).write_all(&PARQUET_MAGIC)?; - writer.write_all(&footer_buffer)?; - writer.flush()?; - Ok(metadata_len as u64 + FOOTER_SIZE) -} - -/// An interface to write a parquet file. -/// Use `start` to write the header, `write` to write a row group, -/// and `end` to write the footer. -pub struct FileWriter { - writer: W, - schema: SchemaDescriptor, - options: WriteOptions, - created_by: Option, - - offset: u64, - row_groups: Vec, - page_specs: Vec>>, - /// Used to store the current state for writing the file - state: State, - // when the file is written, metadata becomes available - metadata: Option, -} - -/// Writes a parquet file containing only the header and footer -/// -/// This is used to write the metadata as a separate Parquet file, usually when data -/// is partitioned across multiple files. -/// -/// Note: Recall that when combining row groups from [`ThriftFileMetaData`], the `file_path` on each -/// of their column chunks must be updated with their path relative to where they are written to. -pub fn write_metadata_sidecar( - writer: &mut W, - metadata: &ThriftFileMetaData, -) -> Result { - let mut len = start_file(writer)?; - len += end_file(writer, metadata)?; - Ok(len) -} - -// Accessors -impl FileWriter { - /// The options assigned to the file - pub fn options(&self) -> &WriteOptions { - &self.options - } - - /// The [`SchemaDescriptor`] assigned to this file - pub fn schema(&self) -> &SchemaDescriptor { - &self.schema - } - - /// Returns the [`ThriftFileMetaData`]. This is Some iff the [`Self::end`] has been called. - /// - /// This is used to write the metadata as a separate Parquet file, usually when data - /// is partitioned across multiple files - pub fn metadata(&self) -> Option<&ThriftFileMetaData> { - self.metadata.as_ref() - } -} - -impl FileWriter { - /// Returns a new [`FileWriter`]. - pub fn new( - writer: W, - schema: SchemaDescriptor, - options: WriteOptions, - created_by: Option, - ) -> Self { - Self { - writer, - schema, - options, - created_by, - offset: 0, - row_groups: vec![], - page_specs: vec![], - state: State::Initialised, - metadata: None, - } - } - - /// Writes the header of the file. - /// - /// This is automatically called by [`Self::write`] if not called following [`Self::new`]. - /// - /// # Errors - /// Returns an error if data has been written to the file. - fn start(&mut self) -> Result<()> { - if self.offset == 0 { - self.offset = start_file(&mut self.writer)?; - self.state = State::Started; - Ok(()) - } else { - Err(Error::InvalidParameter( - "Start cannot be called twice".to_string(), - )) - } - } - - /// Writes a row group to the file. - /// - /// This call is IO-bounded - pub fn write(&mut self, row_group: RowGroupIter<'_, E>) -> Result<()> - where - Error: From, - E: std::error::Error, - { - if self.offset == 0 { - self.start()?; - } - let ordinal = self.row_groups.len(); - let (group, specs, size) = write_row_group( - &mut self.writer, - self.offset, - self.schema.columns(), - row_group, - ordinal, - )?; - self.offset += size; - self.row_groups.push(group); - self.page_specs.push(specs); - Ok(()) - } - - /// Writes the footer of the parquet file. Returns the total size of the file and the - /// underlying writer. - pub fn end(&mut self, key_value_metadata: Option>) -> Result { - if self.offset == 0 { - self.start()?; - } - - if self.state != State::Started { - return Err(Error::InvalidParameter( - "End cannot be called twice".to_string(), - )); - } - // compute file stats - let num_rows = self.row_groups.iter().map(|group| group.num_rows).sum(); - - if self.options.write_statistics { - // write column indexes (require page statistics) - self.row_groups - .iter_mut() - .zip(self.page_specs.iter()) - .try_for_each(|(group, pages)| { - group.columns.iter_mut().zip(pages.iter()).try_for_each( - |(column, pages)| { - let offset = self.offset; - column.column_index_offset = Some(offset as i64); - self.offset += write_column_index(&mut self.writer, pages)?; - let length = self.offset - offset; - column.column_index_length = Some(length as i32); - Result::Ok(()) - }, - )?; - Result::Ok(()) - })?; - }; - - // write offset index - self.row_groups - .iter_mut() - .zip(self.page_specs.iter()) - .try_for_each(|(group, pages)| { - group - .columns - .iter_mut() - .zip(pages.iter()) - .try_for_each(|(column, pages)| { - let offset = self.offset; - column.offset_index_offset = Some(offset as i64); - self.offset += write_offset_index(&mut self.writer, pages)?; - column.offset_index_length = Some((self.offset - offset) as i32); - Result::Ok(()) - })?; - Result::Ok(()) - })?; - - let metadata = ThriftFileMetaData::new( - self.options.version.into(), - self.schema.clone().into_thrift(), - num_rows, - self.row_groups.clone(), - key_value_metadata, - self.created_by.clone(), - None, - None, - None, - ); - - let len = end_file(&mut self.writer, &metadata)?; - self.state = State::Finished; - self.metadata = Some(metadata); - Ok(self.offset + len) - } - - /// Returns the underlying writer. - pub fn into_inner(self) -> W { - self.writer - } - - /// Returns the underlying writer and [`ThriftFileMetaData`] - /// # Panics - /// This function panics if [`Self::end`] has not yet been called - pub fn into_inner_and_metadata(self) -> (W, ThriftFileMetaData) { - (self.writer, self.metadata.expect("File to have ended")) - } -} diff --git a/src/common/parquet2/src/write/indexes/mod.rs b/src/common/parquet2/src/write/indexes/mod.rs deleted file mode 100644 index b0ff65b7d7f0..000000000000 --- a/src/common/parquet2/src/write/indexes/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod serialize; -mod write; - -pub use write::*; diff --git a/src/common/parquet2/src/write/indexes/serialize.rs b/src/common/parquet2/src/write/indexes/serialize.rs deleted file mode 100644 index 7300326f6cb1..000000000000 --- a/src/common/parquet2/src/write/indexes/serialize.rs +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use parquet_format_safe::BoundaryOrder; -use parquet_format_safe::ColumnIndex; -use parquet_format_safe::OffsetIndex; -use parquet_format_safe::PageLocation; - -use crate::error::Error; -use crate::error::Result; -#[allow(unused_imports)] -pub use crate::metadata::KeyValue; -use crate::statistics::serialize_statistics; -use crate::write::page::is_data_page; -use crate::write::page::PageWriteSpec; - -pub fn serialize_column_index(pages: &[PageWriteSpec]) -> Result { - let mut null_pages = Vec::with_capacity(pages.len()); - let mut min_values = Vec::with_capacity(pages.len()); - let mut max_values = Vec::with_capacity(pages.len()); - let mut null_counts = Vec::with_capacity(pages.len()); - - pages - .iter() - .filter(|x| is_data_page(x)) - .try_for_each(|spec| { - if let Some(stats) = &spec.statistics { - let stats = serialize_statistics(stats.as_ref()); - - let null_count = stats - .null_count - .ok_or_else(|| Error::oos("null count of a page is required"))?; - null_counts.push(null_count); - - if let Some(min_value) = stats.min_value { - min_values.push(min_value); - max_values.push( - stats - .max_value - .ok_or_else(|| Error::oos("max value of a page is required"))?, - ); - null_pages.push(false) - } else { - min_values.push(vec![0]); - max_values.push(vec![0]); - null_pages.push(true) - } - - Result::Ok(()) - } else { - Err(Error::oos( - "options were set to write statistics but some pages miss them", - )) - } - })?; - Ok(ColumnIndex { - null_pages, - min_values, - max_values, - boundary_order: BoundaryOrder::UNORDERED, - null_counts: Some(null_counts), - }) -} - -pub fn serialize_offset_index(pages: &[PageWriteSpec]) -> Result { - let mut first_row_index = 0; - let page_locations = pages - .iter() - .filter(|x| is_data_page(x)) - .map(|spec| { - let location = PageLocation { - offset: spec.offset.try_into()?, - compressed_page_size: spec.bytes_written.try_into()?, - first_row_index, - }; - let num_rows = spec.num_rows.ok_or_else(|| { - Error::oos( - "options were set to write statistics but some data pages miss number of rows", - ) - })?; - first_row_index += num_rows as i64; - Ok(location) - }) - .collect::>>()?; - - Ok(OffsetIndex { page_locations }) -} diff --git a/src/common/parquet2/src/write/indexes/write.rs b/src/common/parquet2/src/write/indexes/write.rs deleted file mode 100644 index 829974d554a5..000000000000 --- a/src/common/parquet2/src/write/indexes/write.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -#[cfg(feature = "async")] -use futures::AsyncWrite; -use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; -#[cfg(feature = "async")] -use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol; - -use super::serialize::serialize_column_index; -use super::serialize::serialize_offset_index; -use crate::error::Result; -#[allow(unused_imports)] -pub use crate::metadata::KeyValue; -use crate::write::page::PageWriteSpec; - -pub fn write_column_index(writer: &mut W, pages: &[PageWriteSpec]) -> Result { - let index = serialize_column_index(pages)?; - let mut protocol = TCompactOutputProtocol::new(writer); - Ok(index.write_to_out_protocol(&mut protocol)? as u64) -} - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub async fn write_column_index_async( - writer: &mut W, - pages: &[PageWriteSpec], -) -> Result { - let index = serialize_column_index(pages)?; - let mut protocol = TCompactOutputStreamProtocol::new(writer); - Ok(index.write_to_out_stream_protocol(&mut protocol).await? as u64) -} - -pub fn write_offset_index(writer: &mut W, pages: &[PageWriteSpec]) -> Result { - let index = serialize_offset_index(pages)?; - let mut protocol = TCompactOutputProtocol::new(&mut *writer); - Ok(index.write_to_out_protocol(&mut protocol)? as u64) -} - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub async fn write_offset_index_async( - writer: &mut W, - pages: &[PageWriteSpec], -) -> Result { - let index = serialize_offset_index(pages)?; - let mut protocol = TCompactOutputStreamProtocol::new(&mut *writer); - Ok(index.write_to_out_stream_protocol(&mut protocol).await? as u64) -} diff --git a/src/common/parquet2/src/write/mod.rs b/src/common/parquet2/src/write/mod.rs deleted file mode 100644 index f60880d3cd16..000000000000 --- a/src/common/parquet2/src/write/mod.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod column_chunk; -mod compression; -mod file; -mod indexes; -pub(crate) mod page; -mod row_group; -pub(self) mod statistics; - -#[cfg(feature = "async")] -mod stream; -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub use stream::FileStreamer; - -mod dyn_iter; -pub use compression::compress; -pub use compression::Compressor; -pub use dyn_iter::DynIter; -pub use dyn_iter::DynStreamingIterator; -pub use file::write_metadata_sidecar; -pub use file::FileWriter; -pub use row_group::ColumnOffsetsMetadata; - -use crate::page::CompressedPage; - -pub type RowGroupIter<'a, E> = - DynIter<'a, std::result::Result, E>>; - -/// Write options of different interfaces on this crate -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct WriteOptions { - /// Whether to write statistics, including indexes - pub write_statistics: bool, - /// Which Parquet version to use - pub version: Version, -} - -/// The parquet version to use -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum Version { - V1, - V2, -} - -/// Used to recall the state of the parquet writer - whether sync or async. -#[derive(PartialEq)] -enum State { - Initialised, - Started, - Finished, -} - -impl From for i32 { - fn from(version: Version) -> Self { - match version { - Version::V1 => 1, - Version::V2 => 2, - } - } -} diff --git a/src/common/parquet2/src/write/page.rs b/src/common/parquet2/src/write/page.rs deleted file mode 100644 index 5f20ab960189..000000000000 --- a/src/common/parquet2/src/write/page.rs +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::convert::TryInto; -use std::io::Write; -use std::sync::Arc; - -#[cfg(feature = "async")] -use futures::AsyncWrite; -#[cfg(feature = "async")] -use futures::AsyncWriteExt; -use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; -#[cfg(feature = "async")] -use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol; -use parquet_format_safe::DictionaryPageHeader; -use parquet_format_safe::Encoding; -use parquet_format_safe::PageType; - -use crate::compression::Compression; -use crate::error::Error; -use crate::error::Result; -use crate::page::CompressedDataPage; -use crate::page::CompressedDictPage; -use crate::page::CompressedPage; -use crate::page::DataPageHeader; -use crate::page::ParquetPageHeader; -use crate::statistics::Statistics; - -pub(crate) fn is_data_page(page: &PageWriteSpec) -> bool { - page.header.type_ == PageType::DATA_PAGE || page.header.type_ == PageType::DATA_PAGE_V2 -} - -fn maybe_bytes(uncompressed: usize, compressed: usize) -> Result<(i32, i32)> { - let uncompressed_page_size: i32 = uncompressed.try_into().map_err(|_| { - Error::oos(format!( - "A page can only contain i32::MAX uncompressed bytes. This one contains {}", - uncompressed - )) - })?; - - let compressed_page_size: i32 = compressed.try_into().map_err(|_| { - Error::oos(format!( - "A page can only contain i32::MAX compressed bytes. This one contains {}", - compressed - )) - })?; - - Ok((uncompressed_page_size, compressed_page_size)) -} - -/// Contains page write metrics. -pub struct PageWriteSpec { - pub header: ParquetPageHeader, - pub num_rows: Option, - pub header_size: u64, - pub offset: u64, - pub bytes_written: u64, - pub compression: Compression, - pub statistics: Option>, -} - -pub fn write_page( - writer: &mut W, - offset: u64, - compressed_page: &CompressedPage, -) -> Result { - let selected_rows = compressed_page.selected_rows(); - - let header = match &compressed_page { - CompressedPage::Data(compressed_page) => assemble_data_page_header(compressed_page), - CompressedPage::Dict(compressed_page) => assemble_dict_page_header(compressed_page), - }?; - - let header_size = write_page_header(writer, &header)?; - let mut bytes_written = header_size; - - bytes_written += match &compressed_page { - CompressedPage::Data(compressed_page) => { - writer.write_all(&compressed_page.buffer)?; - compressed_page.buffer.len() as u64 - } - CompressedPage::Dict(compressed_page) => { - writer.write_all(&compressed_page.buffer)?; - compressed_page.buffer.len() as u64 - } - }; - - let statistics = match &compressed_page { - CompressedPage::Data(compressed_page) => compressed_page.statistics().transpose()?, - CompressedPage::Dict(_) => None, - }; - - Ok(PageWriteSpec { - header, - header_size, - offset, - bytes_written, - compression: compressed_page.compression(), - statistics, - num_rows: selected_rows.map(|x| x.last().unwrap().length), - }) -} - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub async fn write_page_async( - writer: &mut W, - offset: u64, - compressed_page: &CompressedPage, -) -> Result { - let selected_rows = compressed_page.selected_rows(); - - let header = match &compressed_page { - CompressedPage::Data(compressed_page) => assemble_data_page_header(compressed_page), - CompressedPage::Dict(compressed_page) => assemble_dict_page_header(compressed_page), - }?; - - let header_size = write_page_header_async(writer, &header).await?; - let mut bytes_written = header_size as u64; - - bytes_written += match &compressed_page { - CompressedPage::Data(compressed_page) => { - writer.write_all(&compressed_page.buffer).await?; - compressed_page.buffer.len() as u64 - } - CompressedPage::Dict(compressed_page) => { - writer.write_all(&compressed_page.buffer).await?; - compressed_page.buffer.len() as u64 - } - }; - - let statistics = match &compressed_page { - CompressedPage::Data(compressed_page) => compressed_page.statistics().transpose()?, - CompressedPage::Dict(_) => None, - }; - - Ok(PageWriteSpec { - header, - header_size, - offset, - bytes_written, - compression: compressed_page.compression(), - statistics, - num_rows: selected_rows.map(|x| x.last().unwrap().length), - }) -} - -fn assemble_data_page_header(page: &CompressedDataPage) -> Result { - let (uncompressed_page_size, compressed_page_size) = - maybe_bytes(page.uncompressed_size(), page.compressed_size())?; - - let mut page_header = ParquetPageHeader { - type_: match page.header() { - DataPageHeader::V1(_) => PageType::DATA_PAGE, - DataPageHeader::V2(_) => PageType::DATA_PAGE_V2, - }, - uncompressed_page_size, - compressed_page_size, - crc: None, - data_page_header: None, - index_page_header: None, - dictionary_page_header: None, - data_page_header_v2: None, - }; - - match page.header() { - DataPageHeader::V1(header) => { - page_header.data_page_header = Some(header.clone()); - } - DataPageHeader::V2(header) => { - page_header.data_page_header_v2 = Some(header.clone()); - } - } - Ok(page_header) -} - -fn assemble_dict_page_header(page: &CompressedDictPage) -> Result { - let (uncompressed_page_size, compressed_page_size) = - maybe_bytes(page.uncompressed_page_size, page.buffer.len())?; - - let num_values: i32 = page.num_values.try_into().map_err(|_| { - Error::oos(format!( - "A dictionary page can only contain i32::MAX items. This one contains {}", - page.num_values - )) - })?; - - Ok(ParquetPageHeader { - type_: PageType::DICTIONARY_PAGE, - uncompressed_page_size, - compressed_page_size, - crc: None, - data_page_header: None, - index_page_header: None, - dictionary_page_header: Some(DictionaryPageHeader { - num_values, - encoding: Encoding::PLAIN, - is_sorted: None, - }), - data_page_header_v2: None, - }) -} - -/// writes the page header into `writer`, returning the number of bytes used in the process. -fn write_page_header(mut writer: &mut W, header: &ParquetPageHeader) -> Result { - let mut protocol = TCompactOutputProtocol::new(&mut writer); - Ok(header.write_to_out_protocol(&mut protocol)? as u64) -} - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -/// writes the page header into `writer`, returning the number of bytes used in the process. -async fn write_page_header_async( - mut writer: &mut W, - header: &ParquetPageHeader, -) -> Result { - let mut protocol = TCompactOutputStreamProtocol::new(&mut writer); - Ok(header.write_to_out_stream_protocol(&mut protocol).await? as u64) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dict_too_large() { - let page = CompressedDictPage::new( - vec![], - Compression::Uncompressed, - i32::MAX as usize + 1, - 100, - false, - ); - assert!(assemble_dict_page_header(&page).is_err()); - } - - #[test] - fn dict_too_many_values() { - let page = CompressedDictPage::new( - vec![], - Compression::Uncompressed, - 0, - i32::MAX as usize + 1, - false, - ); - assert!(assemble_dict_page_header(&page).is_err()); - } -} diff --git a/src/common/parquet2/src/write/row_group.rs b/src/common/parquet2/src/write/row_group.rs deleted file mode 100644 index 626a71b8edfc..000000000000 --- a/src/common/parquet2/src/write/row_group.rs +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -#[cfg(feature = "async")] -use futures::AsyncWrite; -use parquet_format_safe::ColumnChunk; -use parquet_format_safe::RowGroup; - -use super::column_chunk::write_column_chunk; -#[cfg(feature = "async")] -use super::column_chunk::write_column_chunk_async; -use super::page::is_data_page; -use super::page::PageWriteSpec; -use super::DynIter; -use super::DynStreamingIterator; -use crate::error::Error; -use crate::error::Result; -use crate::metadata::ColumnChunkMetaData; -use crate::metadata::ColumnDescriptor; -use crate::page::CompressedPage; - -pub struct ColumnOffsetsMetadata { - pub dictionary_page_offset: Option, - pub data_page_offset: Option, -} - -impl ColumnOffsetsMetadata { - pub fn from_column_chunk(column_chunk: &ColumnChunk) -> ColumnOffsetsMetadata { - ColumnOffsetsMetadata { - dictionary_page_offset: column_chunk - .meta_data - .as_ref() - .map(|meta| meta.dictionary_page_offset) - .unwrap_or(None), - data_page_offset: column_chunk - .meta_data - .as_ref() - .map(|meta| meta.data_page_offset), - } - } - - pub fn from_column_chunk_metadata( - column_chunk_metadata: &ColumnChunkMetaData, - ) -> ColumnOffsetsMetadata { - ColumnOffsetsMetadata { - dictionary_page_offset: column_chunk_metadata.dictionary_page_offset(), - data_page_offset: Some(column_chunk_metadata.data_page_offset()), - } - } - - pub fn calc_row_group_file_offset(&self) -> Option { - self.dictionary_page_offset - .filter(|x| *x > 0_i64) - .or(self.data_page_offset) - } -} - -fn compute_num_rows(columns: &[(ColumnChunk, Vec)]) -> Result { - columns - .get(0) - .map(|(_, specs)| { - let mut num_rows = 0; - specs - .iter() - .filter(|x| is_data_page(x)) - .try_for_each(|spec| { - num_rows += spec.num_rows.ok_or_else(|| { - Error::oos("All data pages must declare the number of rows on it") - })? as i64; - Result::Ok(()) - })?; - Result::Ok(num_rows) - }) - .unwrap_or(Ok(0)) -} - -pub fn write_row_group< - 'a, - W, - E, // external error any of the iterators may emit ->( - writer: &mut W, - mut offset: u64, - descriptors: &[ColumnDescriptor], - columns: DynIter<'a, std::result::Result, E>>, - ordinal: usize, -) -> Result<(RowGroup, Vec>, u64)> -where - W: Write, - Error: From, - E: std::error::Error, -{ - let column_iter = descriptors.iter().zip(columns); - - let initial = offset; - let columns = column_iter - .map(|(descriptor, page_iter)| { - let (column, page_specs, size) = - write_column_chunk(writer, offset, descriptor, page_iter?)?; - offset += size; - Ok((column, page_specs)) - }) - .collect::>>()?; - let bytes_written = offset - initial; - - let num_rows = compute_num_rows(&columns)?; - - // compute row group stats - let file_offset = columns - .get(0) - .map(|(column_chunk, _)| { - ColumnOffsetsMetadata::from_column_chunk(column_chunk).calc_row_group_file_offset() - }) - .unwrap_or(None); - - let total_byte_size = columns - .iter() - .map(|(c, _)| c.meta_data.as_ref().unwrap().total_uncompressed_size) - .sum(); - let total_compressed_size = columns - .iter() - .map(|(c, _)| c.meta_data.as_ref().unwrap().total_compressed_size) - .sum(); - - let (columns, specs) = columns.into_iter().unzip(); - - Ok(( - RowGroup { - columns, - total_byte_size, - num_rows, - sorting_columns: None, - file_offset, - total_compressed_size: Some(total_compressed_size), - ordinal: ordinal.try_into().ok(), - }, - specs, - bytes_written, - )) -} - -#[cfg(feature = "async")] -#[cfg_attr(docsrs, doc(cfg(feature = "async")))] -pub async fn write_row_group_async< - 'a, - W, - E, // external error any of the iterators may emit ->( - writer: &mut W, - mut offset: u64, - descriptors: &[ColumnDescriptor], - columns: DynIter<'a, std::result::Result, E>>, - ordinal: usize, -) -> Result<(RowGroup, Vec>, u64)> -where - W: AsyncWrite + Unpin + Send, - Error: From, - E: std::error::Error, -{ - let column_iter = descriptors.iter().zip(columns); - - let initial = offset; - let mut columns = vec![]; - for (descriptor, page_iter) in column_iter { - let (column, page_specs, size) = - write_column_chunk_async(writer, offset, descriptor, page_iter?).await?; - offset += size; - columns.push((column, page_specs)); - } - let bytes_written = offset - initial; - - let num_rows = compute_num_rows(&columns)?; - - // compute row group stats - let file_offset = columns - .get(0) - .map(|(column_chunk, _)| { - ColumnOffsetsMetadata::from_column_chunk(column_chunk).calc_row_group_file_offset() - }) - .unwrap_or(None); - - let total_byte_size = columns - .iter() - .map(|(c, _)| c.meta_data.as_ref().unwrap().total_uncompressed_size) - .sum(); - let total_compressed_size = columns - .iter() - .map(|(c, _)| c.meta_data.as_ref().unwrap().total_compressed_size) - .sum(); - - let (columns, specs) = columns.into_iter().unzip(); - - Ok(( - RowGroup { - columns, - total_byte_size, - num_rows: num_rows as i64, - sorting_columns: None, - file_offset, - total_compressed_size: Some(total_compressed_size), - ordinal: ordinal.try_into().ok(), - }, - specs, - bytes_written, - )) -} diff --git a/src/common/parquet2/src/write/statistics.rs b/src/common/parquet2/src/write/statistics.rs deleted file mode 100644 index 8359dcf592bf..000000000000 --- a/src/common/parquet2/src/write/statistics.rs +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use crate::error::Error; -use crate::error::Result; -use crate::schema::types::PhysicalType; -use crate::statistics::*; -use crate::types::NativeType; - -#[inline] -fn reduce_single T>(lhs: Option, rhs: Option, op: F) -> Option { - match (lhs, rhs) { - (None, None) => None, - (Some(x), None) => Some(x), - (None, Some(x)) => Some(x), - (Some(x), Some(y)) => Some(op(x, y)), - } -} - -#[inline] -fn reduce_vec8(lhs: Option>, rhs: &Option>, max: bool) -> Option> { - match (lhs, rhs) { - (None, None) => None, - (Some(x), None) => Some(x), - (None, Some(x)) => Some(x.clone()), - (Some(x), Some(y)) => Some(ord_binary(x, y.clone(), max)), - } -} - -pub fn reduce(stats: &[&Option>]) -> Result>> { - if stats.is_empty() { - return Ok(None); - } - let stats = stats - .iter() - .filter_map(|x| x.as_ref()) - .map(|x| x.as_ref()) - .collect::>(); - if stats.is_empty() { - return Ok(None); - }; - - let same_type = stats - .iter() - .skip(1) - .all(|x| x.physical_type() == stats[0].physical_type()); - if !same_type { - return Err(Error::oos("The statistics do not have the same data_type")); - }; - Ok(match stats[0].physical_type() { - PhysicalType::Boolean => { - let stats = stats.iter().map(|x| x.as_any().downcast_ref().unwrap()); - Some(Arc::new(reduce_boolean(stats))) - } - PhysicalType::Int32 => { - let stats = stats.iter().map(|x| x.as_any().downcast_ref().unwrap()); - Some(Arc::new(reduce_primitive::(stats))) - } - PhysicalType::Int64 => { - let stats = stats.iter().map(|x| x.as_any().downcast_ref().unwrap()); - Some(Arc::new(reduce_primitive::(stats))) - } - PhysicalType::Float => { - let stats = stats.iter().map(|x| x.as_any().downcast_ref().unwrap()); - Some(Arc::new(reduce_primitive::(stats))) - } - PhysicalType::Double => { - let stats = stats.iter().map(|x| x.as_any().downcast_ref().unwrap()); - Some(Arc::new(reduce_primitive::(stats))) - } - PhysicalType::ByteArray => { - let stats = stats.iter().map(|x| x.as_any().downcast_ref().unwrap()); - Some(Arc::new(reduce_binary(stats))) - } - PhysicalType::FixedLenByteArray(_) => { - let stats = stats.iter().map(|x| x.as_any().downcast_ref().unwrap()); - Some(Arc::new(reduce_fix_len_binary(stats))) - } - _ => todo!(), - }) -} - -fn reduce_binary<'a, I: Iterator>(mut stats: I) -> BinaryStatistics { - let initial = stats.next().unwrap().clone(); - stats.fold(initial, |mut acc, new| { - acc.min_value = reduce_vec8(acc.min_value, &new.min_value, false); - acc.max_value = reduce_vec8(acc.max_value, &new.max_value, true); - acc.null_count = reduce_single(acc.null_count, new.null_count, |x, y| x + y); - acc.distinct_count = None; - acc - }) -} - -fn reduce_fix_len_binary<'a, I: Iterator>( - mut stats: I, -) -> FixedLenStatistics { - let initial = stats.next().unwrap().clone(); - stats.fold(initial, |mut acc, new| { - acc.min_value = reduce_vec8(acc.min_value, &new.min_value, false); - acc.max_value = reduce_vec8(acc.max_value, &new.max_value, true); - acc.null_count = reduce_single(acc.null_count, new.null_count, |x, y| x + y); - acc.distinct_count = None; - acc - }) -} - -fn ord_binary(a: Vec, b: Vec, max: bool) -> Vec { - for (v1, v2) in a.iter().zip(b.iter()) { - match v1.cmp(v2) { - std::cmp::Ordering::Greater => { - if max { - return a; - } else { - return b; - } - } - std::cmp::Ordering::Less => { - if max { - return b; - } else { - return a; - } - } - _ => {} - } - } - a -} - -fn reduce_boolean<'a, I: Iterator>( - mut stats: I, -) -> BooleanStatistics { - let initial = stats.next().unwrap().clone(); - stats.fold(initial, |mut acc, new| { - acc.min_value = reduce_single( - acc.min_value, - new.min_value, - |x, y| if x & !(y) { y } else { x }, - ); - acc.max_value = reduce_single( - acc.max_value, - new.max_value, - |x, y| if x & !(y) { x } else { y }, - ); - acc.null_count = reduce_single(acc.null_count, new.null_count, |x, y| x + y); - acc.distinct_count = None; - acc - }) -} - -fn reduce_primitive< - 'a, - T: NativeType + std::cmp::PartialOrd, - I: Iterator>, ->( - mut stats: I, -) -> PrimitiveStatistics { - let initial = stats.next().unwrap().clone(); - stats.fold(initial, |mut acc, new| { - acc.min_value = reduce_single( - acc.min_value, - new.min_value, - |x, y| if x > y { y } else { x }, - ); - acc.max_value = reduce_single( - acc.max_value, - new.max_value, - |x, y| if x > y { x } else { y }, - ); - acc.null_count = reduce_single(acc.null_count, new.null_count, |x, y| x + y); - acc.distinct_count = None; - acc - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::schema::types::PrimitiveType; - - #[test] - fn binary() -> Result<()> { - let iter = vec![ - BinaryStatistics { - primitive_type: PrimitiveType::from_physical( - "bla".to_string(), - PhysicalType::ByteArray, - ), - null_count: Some(0), - distinct_count: None, - min_value: Some(vec![1, 2]), - max_value: Some(vec![3, 4]), - }, - BinaryStatistics { - primitive_type: PrimitiveType::from_physical( - "bla".to_string(), - PhysicalType::ByteArray, - ), - null_count: Some(0), - distinct_count: None, - min_value: Some(vec![4, 5]), - max_value: None, - }, - ]; - let a = reduce_binary(iter.iter()); - - assert_eq!(a, BinaryStatistics { - primitive_type: PrimitiveType::from_physical( - "bla".to_string(), - PhysicalType::ByteArray, - ), - null_count: Some(0), - distinct_count: None, - min_value: Some(vec![1, 2]), - max_value: Some(vec![3, 4]), - },); - - Ok(()) - } - - #[test] - fn fixed_len_binary() -> Result<()> { - let iter = vec![ - FixedLenStatistics { - primitive_type: PrimitiveType::from_physical( - "bla".to_string(), - PhysicalType::FixedLenByteArray(2), - ), - null_count: Some(0), - distinct_count: None, - min_value: Some(vec![1, 2]), - max_value: Some(vec![3, 4]), - }, - FixedLenStatistics { - primitive_type: PrimitiveType::from_physical( - "bla".to_string(), - PhysicalType::FixedLenByteArray(2), - ), - null_count: Some(0), - distinct_count: None, - min_value: Some(vec![4, 5]), - max_value: None, - }, - ]; - let a = reduce_fix_len_binary(iter.iter()); - - assert_eq!(a, FixedLenStatistics { - primitive_type: PrimitiveType::from_physical( - "bla".to_string(), - PhysicalType::FixedLenByteArray(2), - ), - null_count: Some(0), - distinct_count: None, - min_value: Some(vec![1, 2]), - max_value: Some(vec![3, 4]), - },); - - Ok(()) - } - - #[test] - fn boolean() -> Result<()> { - let iter = vec![ - BooleanStatistics { - null_count: Some(0), - distinct_count: None, - min_value: Some(false), - max_value: Some(false), - }, - BooleanStatistics { - null_count: Some(0), - distinct_count: None, - min_value: Some(true), - max_value: Some(true), - }, - ]; - let a = reduce_boolean(iter.iter()); - - assert_eq!(a, BooleanStatistics { - null_count: Some(0), - distinct_count: None, - min_value: Some(false), - max_value: Some(true), - },); - - Ok(()) - } - - #[test] - fn primitive() -> Result<()> { - let iter = vec![PrimitiveStatistics { - null_count: Some(2), - distinct_count: None, - min_value: Some(30), - max_value: Some(70), - primitive_type: PrimitiveType::from_physical("bla".to_string(), PhysicalType::Int32), - }]; - let a = reduce_primitive(iter.iter()); - - assert_eq!(a, PrimitiveStatistics { - null_count: Some(2), - distinct_count: None, - min_value: Some(30), - max_value: Some(70), - primitive_type: PrimitiveType::from_physical("bla".to_string(), PhysicalType::Int32,), - },); - - Ok(()) - } -} diff --git a/src/common/parquet2/src/write/stream.rs b/src/common/parquet2/src/write/stream.rs deleted file mode 100644 index 8c9efccbd521..000000000000 --- a/src/common/parquet2/src/write/stream.rs +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright [2021] [Jorge C Leitao] -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -use futures::AsyncWrite; -use futures::AsyncWriteExt; -use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol; -use parquet_format_safe::FileMetaData; -use parquet_format_safe::RowGroup; - -use super::row_group::write_row_group_async; -use super::RowGroupIter; -use super::WriteOptions; -use crate::error::Error; -use crate::error::Result; -use crate::metadata::KeyValue; -use crate::metadata::SchemaDescriptor; -use crate::write::indexes::write_column_index_async; -use crate::write::indexes::write_offset_index_async; -use crate::write::page::PageWriteSpec; -use crate::write::State; -use crate::FOOTER_SIZE; -use crate::PARQUET_MAGIC; - -async fn start_file(writer: &mut W) -> Result { - writer.write_all(&PARQUET_MAGIC).await?; - Ok(PARQUET_MAGIC.len() as u64) -} - -async fn end_file( - mut writer: &mut W, - metadata: FileMetaData, -) -> Result { - // Write file metadata - let mut protocol = TCompactOutputStreamProtocol::new(&mut writer); - let metadata_len = metadata.write_to_out_stream_protocol(&mut protocol).await? as i32; - - // Write footer - let metadata_bytes = metadata_len.to_le_bytes(); - let mut footer_buffer = [0u8; FOOTER_SIZE as usize]; - (0..4).for_each(|i| { - footer_buffer[i] = metadata_bytes[i]; - }); - - (&mut footer_buffer[4..]).write_all(&PARQUET_MAGIC)?; - writer.write_all(&footer_buffer).await?; - writer.flush().await?; - Ok(metadata_len as u64 + FOOTER_SIZE) -} - -/// An interface to write a parquet file asynchronously. -/// Use `start` to write the header, `write` to write a row group, -/// and `end` to write the footer. -pub struct FileStreamer { - writer: W, - schema: SchemaDescriptor, - options: WriteOptions, - created_by: Option, - - offset: u64, - row_groups: Vec, - page_specs: Vec>>, - /// Used to store the current state for writing the file - state: State, -} - -// Accessors -impl FileStreamer { - /// The options assigned to the file - pub fn options(&self) -> &WriteOptions { - &self.options - } - - /// The [`SchemaDescriptor`] assigned to this file - pub fn schema(&self) -> &SchemaDescriptor { - &self.schema - } -} - -impl FileStreamer { - /// Returns a new [`FileStreamer`]. - pub fn new( - writer: W, - schema: SchemaDescriptor, - options: WriteOptions, - created_by: Option, - ) -> Self { - Self { - writer, - schema, - options, - created_by, - offset: 0, - row_groups: vec![], - page_specs: vec![], - state: State::Initialised, - } - } - - /// Writes the header of the file. - /// - /// This is automatically called by [`Self::write`] if not called following [`Self::new`]. - /// - /// # Errors - /// Returns an error if data has been written to the file. - async fn start(&mut self) -> Result<()> { - if self.offset == 0 { - self.offset = start_file(&mut self.writer).await? as u64; - self.state = State::Started; - Ok(()) - } else { - Err(Error::InvalidParameter( - "Start cannot be called twice".to_string(), - )) - } - } - - /// Writes a row group to the file. - pub async fn write(&mut self, row_group: RowGroupIter<'_, E>) -> Result<()> - where - Error: From, - E: std::error::Error, - { - if self.offset == 0 { - self.start().await?; - } - - let ordinal = self.row_groups.len(); - let (group, specs, size) = write_row_group_async( - &mut self.writer, - self.offset, - self.schema.columns(), - row_group, - ordinal, - ) - .await?; - self.offset += size; - self.row_groups.push(group); - self.page_specs.push(specs); - Ok(()) - } - - /// Writes the footer of the parquet file. Returns the total size of the file and the - /// underlying writer. - pub async fn end(&mut self, key_value_metadata: Option>) -> Result { - if self.offset == 0 { - self.start().await?; - } - - if self.state != State::Started { - return Err(Error::InvalidParameter( - "End cannot be called twice".to_string(), - )); - } - // compute file stats - let num_rows = self.row_groups.iter().map(|group| group.num_rows).sum(); - - if self.options.write_statistics { - // write column indexes (require page statistics) - for (group, pages) in self.row_groups.iter_mut().zip(self.page_specs.iter()) { - for (column, pages) in group.columns.iter_mut().zip(pages.iter()) { - let offset = self.offset; - column.column_index_offset = Some(offset as i64); - self.offset += write_column_index_async(&mut self.writer, pages).await?; - let length = self.offset - offset; - column.column_index_length = Some(length as i32); - } - } - }; - - // write offset index - for (group, pages) in self.row_groups.iter_mut().zip(self.page_specs.iter()) { - for (column, pages) in group.columns.iter_mut().zip(pages.iter()) { - let offset = self.offset; - column.offset_index_offset = Some(offset as i64); - self.offset += write_offset_index_async(&mut self.writer, pages).await?; - column.offset_index_length = Some((self.offset - offset) as i32); - } - } - - let metadata = FileMetaData::new( - self.options.version.into(), - self.schema.clone().into_thrift(), - num_rows, - self.row_groups.clone(), - key_value_metadata, - self.created_by.clone(), - None, - None, - None, - ); - - let len = end_file(&mut self.writer, metadata).await?; - Ok(self.offset + len) - } - - /// Returns the underlying writer. - pub fn into_inner(self) -> W { - self.writer - } -} diff --git a/src/common/storage/src/column_node.rs b/src/common/storage/src/column_node.rs index ca1e9c7643b0..93860057b9d6 100644 --- a/src/common/storage/src/column_node.rs +++ b/src/common/storage/src/column_node.rs @@ -18,7 +18,7 @@ use databend_common_arrow::arrow::datatypes::DataType as ArrowType; use databend_common_arrow::arrow::datatypes::Field as ArrowField; use databend_common_arrow::arrow::datatypes::Schema as ArrowSchema; -use databend_common_arrow::arrow::io::parquet::read::InitNested; +use databend_common_arrow::native::nested::InitNested; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::ColumnId; @@ -68,6 +68,7 @@ impl ColumnNodes { for inner_field in inner_fields { let mut inner_init = init.clone(); inner_init.push(InitNested::Struct(field.is_nullable)); + let child_column_node = Self::traverse_fields_dfs(inner_field, true, inner_init, leaf_id); child_leaf_ids.extend(child_column_node.leaf_indices.clone()); @@ -86,6 +87,7 @@ impl ColumnNodes { | ArrowType::FixedSizeList(inner_field, _) => { let mut inner_init = init.clone(); inner_init.push(InitNested::List(field.is_nullable)); + let mut child_column_nodes = Vec::with_capacity(1); let mut child_leaf_ids = Vec::with_capacity(1); let child_column_node = @@ -103,6 +105,7 @@ impl ColumnNodes { ArrowType::Map(inner_field, _) => { let mut inner_init = init.clone(); inner_init.push(InitNested::List(field.is_nullable)); + let mut child_column_nodes = Vec::with_capacity(1); let mut child_leaf_ids = Vec::with_capacity(1); let child_column_node = diff --git a/src/meta/client/Cargo.toml b/src/meta/client/Cargo.toml index 0a9f431bc7ab..6b75fed5bd69 100644 --- a/src/meta/client/Cargo.toml +++ b/src/meta/client/Cargo.toml @@ -15,8 +15,8 @@ test = true [dependencies] anyerror = { workspace = true } +arrow-flight = { workspace = true } async-backtrace = { workspace = true } -databend-common-arrow = { workspace = true } databend-common-base = { workspace = true } databend-common-grpc = { workspace = true } databend-common-meta-api = { workspace = true } diff --git a/src/meta/client/src/grpc_client.rs b/src/meta/client/src/grpc_client.rs index 6d9bbfe011d2..7c019f070a7e 100644 --- a/src/meta/client/src/grpc_client.rs +++ b/src/meta/client/src/grpc_client.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use std::time::Duration; use std::time::Instant; -use databend_common_arrow::arrow_format::flight::data::BasicAuth; +use arrow_flight::BasicAuth; use databend_common_base::base::tokio::select; use databend_common_base::base::tokio::sync::mpsc; use databend_common_base::base::tokio::sync::mpsc::UnboundedReceiver; diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index 6b5b2b8fcf91..2ea2d38a0f46 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -23,6 +23,7 @@ io-uring = [ [dependencies] anyerror = { workspace = true } anyhow = { workspace = true } +arrow-flight = { workspace = true } async-trait = { workspace = true } backon = { workspace = true } clap = { workspace = true } diff --git a/src/meta/service/src/api/grpc/grpc_service.rs b/src/meta/service/src/api/grpc/grpc_service.rs index 817b15ae6aeb..a10e3171032c 100644 --- a/src/meta/service/src/api/grpc/grpc_service.rs +++ b/src/meta/service/src/api/grpc/grpc_service.rs @@ -16,7 +16,7 @@ use std::io; use std::pin::Pin; use std::sync::Arc; -use databend_common_arrow::arrow_format::flight::data::BasicAuth; +use arrow_flight::BasicAuth; use databend_common_base::base::tokio::sync::mpsc; use databend_common_base::future::TimedFutureExt; use databend_common_base::runtime::ThreadTracker; diff --git a/src/query/expression/src/kernels/take_chunks.rs b/src/query/expression/src/kernels/take_chunks.rs index d532a67fabb4..90bafc210d2f 100644 --- a/src/query/expression/src/kernels/take_chunks.rs +++ b/src/query/expression/src/kernels/take_chunks.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use binary::BinaryColumnBuilder; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::compute::merge_sort::MergeSlice; use databend_common_hashtable::RowPtr; use itertools::Itertools; use string::StringColumnBuilder; @@ -49,6 +48,8 @@ use crate::Value; // Block idx, row idx in the block, repeat times pub type BlockRowIndex = (u32, u32, usize); +pub type MergeSlice = (usize, usize, usize); + impl DataBlock { pub fn take_blocks( blocks: &[DataBlock], diff --git a/src/query/expression/tests/it/common.rs b/src/query/expression/tests/it/common.rs index 38a3bdaf16ad..f7ffc5344e8f 100644 --- a/src/query/expression/tests/it/common.rs +++ b/src/query/expression/tests/it/common.rs @@ -14,13 +14,14 @@ use std::io::Write; -use databend_common_arrow::arrow::compute::merge_sort::MergeSlice; use databend_common_expression::BlockEntry; use databend_common_expression::BlockRowIndex; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::Value; +type MergeSlice = (usize, usize, usize); + pub fn new_block(columns: &[Column]) -> DataBlock { let len = columns.first().map_or(1, |c| c.len()); let columns = columns diff --git a/src/query/expression/tests/it/serde.rs b/src/query/expression/tests/it/serde.rs index f07bae3157da..59934888268d 100644 --- a/src/query/expression/tests/it/serde.rs +++ b/src/query/expression/tests/it/serde.rs @@ -84,34 +84,9 @@ fn test_serde_bin_column() -> Result<()> { let t = deserialize_column(&data).unwrap(); assert_eq!(col, &t); } - - for col in &columns { - let data = serialize_column_old(col); - let t = deserialize_column(&data).unwrap(); - assert_eq!(col, &t); - } Ok(()) } -fn serialize_column_old(col: &Column) -> Vec { - use databend_common_arrow::arrow::chunk::Chunk; - use databend_common_arrow::arrow::datatypes::Schema; - use databend_common_arrow::arrow::io::ipc::write::FileWriter; - use databend_common_arrow::arrow::io::ipc::write::WriteOptions; - - let mut buffer = Vec::new(); - - let schema = Schema::from(vec![col.arrow_field()]); - let mut writer = FileWriter::new(&mut buffer, schema, None, WriteOptions::default()); - writer.start().unwrap(); - writer - .write(&Chunk::new(vec![col.as_arrow()]), None) - .unwrap(); - writer.finish().unwrap(); - - buffer -} - #[test] fn test_borsh_serde_column() -> Result<()> { #[derive(BorshSerialize, BorshDeserialize, Eq, PartialEq, Debug)] diff --git a/src/query/service/tests/it/servers/flight/flight_service.rs b/src/query/service/tests/it/servers/flight/flight_service.rs index 015efc53c7de..ca84366f35b8 100644 --- a/src/query/service/tests/it/servers/flight/flight_service.rs +++ b/src/query/service/tests/it/servers/flight/flight_service.rs @@ -17,8 +17,8 @@ use std::net::TcpListener; use std::str::FromStr; use std::sync::Arc; -use databend_common_arrow::arrow_format::flight::data::Empty; -use databend_common_arrow::arrow_format::flight::service::flight_service_client::FlightServiceClient; +use arrow_flight::flight_service_client::FlightServiceClient; +use arrow_flight::Empty; use databend_common_base::base::tokio; use databend_common_exception::ErrorCode; use databend_common_exception::Result; diff --git a/src/query/storages/common/table_meta/src/table/table_compression.rs b/src/query/storages/common/table_meta/src/table/table_compression.rs index 71cab1e43516..b542555fc8d8 100644 --- a/src/query/storages/common/table_meta/src/table/table_compression.rs +++ b/src/query/storages/common/table_meta/src/table/table_compression.rs @@ -13,7 +13,6 @@ // limitations under the License. use databend_common_arrow::native; -use databend_common_arrow::parquet as databend_parquet; use databend_common_exception::ErrorCode; use parquet::basic::Compression as ParquetCompression; use parquet::basic::GzipLevel; @@ -49,20 +48,6 @@ impl TryFrom<&str> for TableCompression { } } -/// Convert to parquet CompressionOptions. -impl From for databend_parquet::compression::CompressionOptions { - fn from(value: TableCompression) -> Self { - match value { - TableCompression::None => { - databend_parquet::compression::CompressionOptions::Uncompressed - } - TableCompression::LZ4 => databend_parquet::compression::CompressionOptions::Lz4Raw, - TableCompression::Snappy => databend_parquet::compression::CompressionOptions::Snappy, - TableCompression::Zstd => databend_parquet::compression::CompressionOptions::Zstd(None), - } - } -} - /// Convert to native Compression. impl From for native::CommonCompression { fn from(value: TableCompression) -> Self { diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs index d8aada0bdd7b..d656758174bb 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs @@ -169,13 +169,12 @@ impl BlockReader { fn chunks_to_native_array( &self, - column_node: &ColumnNode, + _column_node: &ColumnNode, metas: Vec<&ColumnMeta>, chunks: Vec<&[u8]>, - leaf_ids: Vec, + _leaf_ids: Vec, field: ArrowField, ) -> Result> { - let is_nested = column_node.is_nested; let mut page_metas = Vec::with_capacity(chunks.len()); let mut readers = Vec::with_capacity(chunks.len()); for (chunk, meta) in chunks.into_iter().zip(metas.into_iter()) { @@ -187,7 +186,7 @@ impl BlockReader { match self .native_columns_reader - .batch_read_array(readers, &leaf_ids, field, is_nested, page_metas) + .batch_read_array(readers, field, page_metas) { Ok(array) => Ok(array), Err(err) => Err(err.into()), @@ -276,13 +275,11 @@ impl BlockReader { readers: Vec>>, ) -> Result> { let field = column_node.field.clone(); - let is_nested = column_node.is_nested; match self.native_columns_reader.column_iter_to_arrays( readers, - &column_node.leaf_indices, field, - is_nested, + column_node.init.clone(), ) { Ok(array_iter) => Ok(array_iter), Err(err) => Err(err.into()), @@ -304,7 +301,7 @@ impl BlockReader { ); let schema = ArrowSchema::from(vec![field.clone()]); let native_column_reader = NativeColumnsReader::new(schema)?; - match native_column_reader.column_iter_to_arrays(readers, &[0], field, false) { + match native_column_reader.column_iter_to_arrays(readers, field, vec![]) { Ok(array_iter) => Ok(array_iter), Err(err) => Err(err.into()), } diff --git a/src/query/storages/fuse/src/io/read/meta/meta_readers.rs b/src/query/storages/fuse/src/io/read/meta/meta_readers.rs index ddb2dabbcf56..3e0a65ded9e2 100644 --- a/src/query/storages/fuse/src/io/read/meta/meta_readers.rs +++ b/src/query/storages/fuse/src/io/read/meta/meta_readers.rs @@ -264,7 +264,7 @@ async fn bytes_reader(op: &Operator, path: &str, len_hint: Option) -> Resul } mod thrift_file_meta_read { - use databend_common_arrow::parquet::error::Error; + use parquet::errors::ParquetError; use thrift::protocol::TCompactInputProtocol; use super::*; @@ -308,7 +308,7 @@ mod thrift_file_meta_read { let meta = op .stat(path) .await - .map_err(|err| Error::OutOfSpec(err.to_string()))?; + .map_err(|err| ParquetError::General(err.to_string()))?; meta.content_length() }; @@ -326,7 +326,7 @@ mod thrift_file_meta_read { .read_with(path) .range(file_size - default_end_len as u64..file_size) .await - .map_err(|err| Error::OutOfSpec(err.to_string()))? + .map_err(|err| ParquetError::General(err.to_string()))? .to_vec(); // check this is indeed a parquet file diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0027_func_fuse_encoding.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0027_func_fuse_encoding.test index af4c570443ac..7798250dc0f1 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0027_func_fuse_encoding.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0027_func_fuse_encoding.test @@ -16,7 +16,7 @@ insert into t select number from numbers(2048); query III select * from fuse_encoding('db_09_0027'); ---- -t c INT NULL 663567 2592 8192 DeltaBitpack NULL +t c INT NULL 2048 2592 8192 DeltaBitpack NULL query III select level_one,level_two,count(*) from fuse_encoding('db_09_0027') group by level_one,level_two; diff --git a/tests/suites/1_stateful/09_http_handler/09_0007_token.py b/tests/suites/1_stateful/09_http_handler/09_0007_token.py index 8a8bcfb4a8e4..ee404695af13 100755 --- a/tests/suites/1_stateful/09_http_handler/09_0007_token.py +++ b/tests/suites/1_stateful/09_http_handler/09_0007_token.py @@ -19,15 +19,17 @@ verify_url = "http://localhost:8000/v1/verify" auth = ("root", "") + class GlobalCookieJar(RequestsCookieJar): def __init__(self): super().__init__() def set_cookie(self, cookie: Cookie, *args, **kwargs): - cookie.domain = '' - cookie.path = '/' + cookie.domain = "" + cookie.path = "/" super().set_cookie(cookie, *args, **kwargs) + client = requests.session() client.cookies = GlobalCookieJar() client.cookies.set("cookie_enabled", "true") From e35bacc136965ffda58c3e8287820ad33b9b52b7 Mon Sep 17 00:00:00 2001 From: Sky Fan <3374614481@qq.com> Date: Tue, 12 Nov 2024 17:28:43 +0800 Subject: [PATCH 23/92] fix: purge may not work on tables after a flash back operation (#16812) * fix: purge may not work on tables after a flash back operation * fix * fix test * fix test --- .../it/storages/fuse/operations/navigate.rs | 4 +-- .../storages/fuse/src/operations/navigate.rs | 27 +++---------------- .../09_0008_fuse_optimize_table.test | 2 -- .../20+_others/20_0011_purge_before.result | 2 +- .../20+_others/20_0011_purge_before.sh | 4 +-- 5 files changed, 9 insertions(+), 30 deletions(-) diff --git a/src/query/service/tests/it/storages/fuse/operations/navigate.rs b/src/query/service/tests/it/storages/fuse/operations/navigate.rs index a2b0b70ebbdb..e1e56a3ef29c 100644 --- a/src/query/service/tests/it/storages/fuse/operations/navigate.rs +++ b/src/query/service/tests/it/storages/fuse/operations/navigate.rs @@ -168,7 +168,7 @@ async fn test_navigate_for_purge() -> Result<()> { // keep the first snapshot of the insertion let table = fixture.latest_default_table().await?; - let first_snapshot = FuseTable::try_from_table(table.as_ref())? + let _first_snapshot = FuseTable::try_from_table(table.as_ref())? .snapshot_loc() .await? .unwrap(); @@ -233,7 +233,7 @@ async fn test_navigate_for_purge() -> Result<()> { // navigate from the instant that is just one ms before the timestamp of the latest snapshot. let (navigate, files) = fuse_table.list_by_time_point(time_point).await?; assert_eq!(2, files.len()); - assert_eq!(navigate, first_snapshot); + assert_eq!(navigate, third_snapshot); // 5. navigate by snapshot id. let snapshot_id = snapshots[1].0.snapshot_id.simple().to_string(); diff --git a/src/query/storages/fuse/src/operations/navigate.rs b/src/query/storages/fuse/src/operations/navigate.rs index f7818def2cf7..376ea880f5f8 100644 --- a/src/query/storages/fuse/src/operations/navigate.rs +++ b/src/query/storages/fuse/src/operations/navigate.rs @@ -24,7 +24,6 @@ use databend_common_exception::Result; use databend_common_exception::ResultExt; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableStatistics; -use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_SOURCE_TABLE_ID; @@ -274,29 +273,11 @@ impl FuseTable { )); } - let location = files[0].clone(); - let reader = MetaReaders::table_snapshot_reader(self.get_operator()); - let ver = TableMetaLocationGenerator::snapshot_version(location.as_str()); - let load_params = LoadParams { - location, - len_hint: None, - ver, - put_cache: false, + let Some(location) = self.snapshot_loc().await? else { + return Err(ErrorCode::TableHistoricalDataNotFound("No historical data")); }; - let snapshot = reader.read(&load_params).await?; - // Take the prev snapshot as base snapshot to avoid get orphan snapshot. - let prev = snapshot.prev_snapshot_id; - match prev { - Some((id, v)) => { - let new_loc = self - .meta_location_generator() - .snapshot_location_from_uuid(&id, v)?; - Ok((new_loc, files)) - } - None => Err(ErrorCode::TableHistoricalDataNotFound( - "No historical data found at given point", - )), - } + + Ok((location, files)) } #[async_backtrace::framed] diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0008_fuse_optimize_table.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0008_fuse_optimize_table.test index 847b0de373de..0d9d0ea33133 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0008_fuse_optimize_table.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0008_fuse_optimize_table.test @@ -102,8 +102,6 @@ query II select segment_count,block_count from fuse_snapshot('db_09_0008', 't') limit 2 ---- 1 1 -4 4 - query I diff --git a/tests/suites/0_stateless/20+_others/20_0011_purge_before.result b/tests/suites/0_stateless/20+_others/20_0011_purge_before.result index 545c465ecfe6..5cb94f5c24ca 100644 --- a/tests/suites/0_stateless/20+_others/20_0011_purge_before.result +++ b/tests/suites/0_stateless/20+_others/20_0011_purge_before.result @@ -6,7 +6,7 @@ checking that after purge (by snapshot id) there should be 4 rows left true checking that there should are 3 snapshots before purge true -checking that after purge (by timestamp) there should be at least 2 snapshots left +checking that after purge (by timestamp) there should be 1 snapshot left true checking that after purge (by timestamp) there should be 4 rows left true diff --git a/tests/suites/0_stateless/20+_others/20_0011_purge_before.sh b/tests/suites/0_stateless/20+_others/20_0011_purge_before.sh index 2ea59f8a797f..20149973f248 100755 --- a/tests/suites/0_stateless/20+_others/20_0011_purge_before.sh +++ b/tests/suites/0_stateless/20+_others/20_0011_purge_before.sh @@ -52,8 +52,8 @@ TIMEPOINT=$(echo "select timestamp from fuse_snapshot('default', 't20_0011') whe ## verify echo "set data_retention_time_in_days=0; optimize table t20_0011 purge before (TIMESTAMP => '$TIMEPOINT'::TIMESTAMP)" | $BENDSQL_CLIENT_CONNECT -echo "checking that after purge (by timestamp) there should be at least 2 snapshots left" -echo "select count(*)>=2 from fuse_snapshot('default', 't20_0011')" | $BENDSQL_CLIENT_CONNECT +echo "checking that after purge (by timestamp) there should be 1 snapshot left" +echo "select count(*)=1 from fuse_snapshot('default', 't20_0011')" | $BENDSQL_CLIENT_CONNECT echo "checking that after purge (by timestamp) there should be 4 rows left" echo "select count(*)=4 from t20_0011" | $BENDSQL_CLIENT_CONNECT From 3023243b1d86a10dd8745c61f14a8fec28cbced9 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 12 Nov 2024 22:24:24 +0800 Subject: [PATCH 24/92] ci: Enable incremental back to see how it works (#16817) build: Enable incremental back to see how it works Signed-off-by: Xuanwo --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 113ad08c77ca..94a00b803105 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -567,7 +567,7 @@ debug = 1 lto = "thin" overflow-checks = false opt-level = "s" ## defaults to be 3 -incremental = false +incremental = true # [profile.release.package] # databend-common-arrow = { codegen-units = 16 } @@ -581,8 +581,8 @@ overflow-checks = false [profile.dev] split-debuginfo = "unpacked" overflow-checks = false -# wait until https://github.com/rust-lang/rust/issues/100142 fixed -incremental = false +# Report to https://github.com/rust-lang/rust/issues/100142 if incremental works well +incremental = true [profile.dev.package] addr2line = { opt-level = 3 } From 5e2095c1217dcb6fe972e504c4f8b2daa06ae797 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Wed, 13 Nov 2024 00:23:30 +0800 Subject: [PATCH 25/92] chore: purge dropped view metadata during vacuum (#16819) * chore: purge view metadata during vacuum * tweak logic test --- .../interpreter_vacuum_drop_tables.rs | 18 +++++-- .../ee/03_ee_vacuum/03_0002_vacuum_views.test | 52 +++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 tests/sqllogictests/suites/ee/03_ee_vacuum/03_0002_vacuum_views.test diff --git a/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs b/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs index 70e2a15c729e..1aad2e680b70 100644 --- a/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs +++ b/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs @@ -31,6 +31,7 @@ use databend_common_meta_app::schema::DroppedId; use databend_common_meta_app::schema::GcDroppedTableReq; use databend_common_meta_app::schema::ListDroppedTableReq; use databend_common_sql::plans::VacuumDropTablePlan; +use databend_common_storages_view::view_table::VIEW_ENGINE; use databend_enterprise_vacuum_handler::get_vacuum_handler; use log::info; @@ -156,13 +157,17 @@ impl Interpreter for VacuumDropTablesInterpreter { drop_ids ); - // TODO buggy, table as catalog obj should be allowed to drop - // also drop ids - // filter out read-only tables - let tables = tables + // Filter out read-only tables and views. + // Note: The drop_ids list still includes view IDs + let (views, tables): (Vec<_>, Vec<_>) = tables .into_iter() .filter(|tbl| !tbl.as_ref().is_read_only()) - .collect::>(); + .partition(|tbl| tbl.get_table_info().meta.engine == VIEW_ENGINE); + + { + let view_ids = views.into_iter().map(|v| v.get_id()).collect::>(); + info!("view ids excluded from purging data: {:?}", view_ids); + } let handler = get_vacuum_handler(); let threads_nums = self.ctx.get_settings().get_max_threads()? as usize; @@ -187,6 +192,8 @@ impl Interpreter for VacuumDropTablesInterpreter { // gc metadata only when not dry run if self.plan.option.dry_run.is_none() { let mut success_dropped_ids = vec![]; + // Since drop_ids contains view IDs, any views (if present) will be added to + // the success_dropped_id list, with removal from the meta-server attempted later. for drop_id in drop_ids { match &drop_id { DroppedId::Db { db_id, db_name: _ } => { @@ -205,6 +212,7 @@ impl Interpreter for VacuumDropTablesInterpreter { "failed dbs:{:?}, failed_tables:{:?}, success_drop_ids:{:?}", failed_db_ids, failed_tables, success_dropped_ids ); + self.gc_drop_tables(catalog, success_dropped_ids).await?; } diff --git a/tests/sqllogictests/suites/ee/03_ee_vacuum/03_0002_vacuum_views.test b/tests/sqllogictests/suites/ee/03_ee_vacuum/03_0002_vacuum_views.test new file mode 100644 index 000000000000..8acb9f69838d --- /dev/null +++ b/tests/sqllogictests/suites/ee/03_ee_vacuum/03_0002_vacuum_views.test @@ -0,0 +1,52 @@ +## Copyright 2023 Databend Cloud +## +## Licensed under the Elastic License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## https://www.elastic.co/licensing/elastic-license +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +statement ok +drop database if exists vacuum_view_test; + +statement ok +create database vacuum_view_test; + +statement ok +use vacuum_view_test; + +statement ok +create table t as select * from numbers(1); + +statement ok +create view v as select * from t; + +statement ok +drop view v; + +# the dropped view vacuum_view_test.v should be there +query I +select count() from system.tables_with_history where database = 'vacuum_view_test' and name = 'v'; +---- +1 + +statement ok +set data_retention_time_in_days = 0; + +statement ok +vacuum drop table from vacuum_view_test; + +# the dropped view vacuum_view_test.v should be vacuumed +query I +select count() from system.tables_with_history where database = 'vacuum_view_test' and name = 'v'; +---- +0 + +statement ok +drop database vacuum_view_test; From aca2fe4f01867d73a9b5715f04ed988056067b34 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Wed, 13 Nov 2024 01:09:08 +0800 Subject: [PATCH 26/92] chore(query): keep queries detail and profile log format (#16822) --- src/common/tracing/src/init.rs | 4 ++-- src/common/tracing/src/loggers.rs | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/common/tracing/src/init.rs b/src/common/tracing/src/init.rs index 734f4b093edb..1f47cca45539 100644 --- a/src/common/tracing/src/init.rs +++ b/src/common/tracing/src/init.rs @@ -270,7 +270,7 @@ pub fn init_logging( EnvFilterBuilder::new() .filter(Some("databend::log::query"), LevelFilter::Trace), )) - .append(query_log_file); + .append(query_log_file.with_layout(get_layout("identical"))); logger = logger.dispatch(dispatch); } if let Some(endpoint) = &cfg.query.otlp { @@ -312,7 +312,7 @@ pub fn init_logging( EnvFilterBuilder::new() .filter(Some("databend::log::profile"), LevelFilter::Trace), )) - .append(profile_log_file); + .append(profile_log_file.with_layout(get_layout("identical"))); logger = logger.dispatch(dispatch); } if let Some(endpoint) = &cfg.profile.otlp { diff --git a/src/common/tracing/src/loggers.rs b/src/common/tracing/src/loggers.rs index b49cf37f7ed2..d5dd662edabe 100644 --- a/src/common/tracing/src/loggers.rs +++ b/src/common/tracing/src/loggers.rs @@ -49,10 +49,15 @@ pub fn get_layout(format: &str) -> Layout { match format { "text" => text_layout(), "json" => json_layout(), + "identical" => identical_layout(), _ => unimplemented!("file logging format {format} is not supported"), } } +fn identical_layout() -> Layout { + CustomLayout::new(|record: &Record| Ok(format!("{}\n", record.args()).into_bytes())).into() +} + fn text_layout() -> Layout { CustomLayout::new(|record: &Record| { let s = match ThreadTracker::query_id() { From aab0f85c1bb04b0682b164ad8580302f916933fb Mon Sep 17 00:00:00 2001 From: coldWater Date: Wed, 13 Nov 2024 08:35:38 +0800 Subject: [PATCH 27/92] feat(query): TopN window operator (#16726) * transform_window_partial_top_n Signed-off-by: coldWater * test Signed-off-by: coldWater * builder Signed-off-by: coldWater * RulePushDownFilterWindowRank Signed-off-by: coldWater * WindowPartitionTopNFunc Signed-off-by: coldWater * SortCompareEquality Signed-off-by: coldWater * name Signed-off-by: coldWater * group_hash_value_spread Signed-off-by: coldWater * group_hash_columns_slice Signed-off-by: coldWater * WindowPartitionTopNExchange Signed-off-by: coldWater * builder Signed-off-by: coldWater * group_hash test Signed-off-by: coldWater * fix Signed-off-by: coldWater * test Signed-off-by: coldWater * refine Signed-off-by: coldWater --------- Signed-off-by: coldWater --- .../expression/src/aggregate/group_hash.rs | 361 +++++++++++++++++- .../expression/src/kernels/sort_compare.rs | 23 +- src/query/expression/src/types.rs | 1 + src/query/expression/src/types/geography.rs | 6 + src/query/expression/src/utils/visitor.rs | 24 +- .../core/src/processors/shuffle_processor.rs | 5 +- .../src/pipelines/builders/builder_window.rs | 22 +- .../transforms/window/partition/mod.rs | 2 + .../partition/window_partition_exchange.rs | 23 +- ...window_partition_partial_top_n_exchange.rs | 336 ++++++++++++++++ .../sql/src/executor/physical_plan_visitor.rs | 1 + .../sql/src/executor/physical_plans/mod.rs | 4 +- .../executor/physical_plans/physical_sort.rs | 28 +- .../physical_window_partition.rs | 14 + .../sql/src/planner/binder/bind_query/bind.rs | 2 +- src/query/sql/src/planner/binder/sort.rs | 2 +- src/query/sql/src/planner/binder/window.rs | 13 +- .../planner/format/display_rel_operator.rs | 21 +- .../decorrelate/subquery_rewriter.rs | 11 +- .../sql/src/planner/optimizer/rule/factory.rs | 48 +-- .../src/planner/optimizer/rule/rewrite/mod.rs | 2 + .../rule/rewrite/rule_eliminate_sort.rs | 4 +- .../rule_push_down_filter_window_top_n.rs | 185 +++++++++ .../rewrite/rule_push_down_limit_aggregate.rs | 2 +- .../sql/src/planner/optimizer/rule/rule.rs | 3 + src/query/sql/src/planner/plans/sort.rs | 52 +-- src/query/sql/src/planner/plans/window.rs | 7 + .../mode/standalone/explain/window.test | 44 +-- tests/sqllogictests/suites/tpcds/spill.test | 16 +- 29 files changed, 1113 insertions(+), 149 deletions(-) create mode 100644 src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_partial_top_n_exchange.rs create mode 100644 src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_window_top_n.rs diff --git a/src/query/expression/src/aggregate/group_hash.rs b/src/query/expression/src/aggregate/group_hash.rs index 660fc11ace3b..15f36900af8f 100644 --- a/src/query/expression/src/aggregate/group_hash.rs +++ b/src/query/expression/src/aggregate/group_hash.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_arrow::arrow::types::Index; use databend_common_base::base::OrderedFloat; +use databend_common_exception::Result; use ethnum::i256; -use crate::types::decimal::DecimalType; -use crate::types::geometry::GeometryType; use crate::types::AnyType; use crate::types::ArgType; use crate::types::BinaryType; @@ -24,17 +25,29 @@ use crate::types::BitmapType; use crate::types::BooleanType; use crate::types::DataType; use crate::types::DateType; +use crate::types::DecimalColumn; use crate::types::DecimalDataType; +use crate::types::DecimalScalar; +use crate::types::DecimalType; +use crate::types::GeographyType; +use crate::types::GeometryType; +use crate::types::NumberColumn; use crate::types::NumberDataType; +use crate::types::NumberScalar; use crate::types::NumberType; use crate::types::StringType; use crate::types::TimestampType; use crate::types::ValueType; use crate::types::VariantType; +use crate::visitor::ValueVisitor; +use crate::with_decimal_type; use crate::with_number_mapped_type; +use crate::with_number_type; use crate::Column; use crate::InputColumns; +use crate::Scalar; use crate::ScalarRef; +use crate::Value; const NULL_HASH_VAL: u64 = 0xd1cefa08eb382d69; @@ -47,15 +60,6 @@ pub fn group_hash_columns(cols: InputColumns, values: &mut [u64]) { } } -pub fn group_hash_columns_slice(cols: &[Column], values: &mut [u64]) { - debug_assert!(!cols.is_empty()); - let mut iter = cols.iter(); - combine_group_hash_column::(iter.next().unwrap(), values); - for col in iter { - combine_group_hash_column::(col, values); - } -} - pub fn combine_group_hash_column(c: &Column, values: &mut [u64]) { match c.data_type() { DataType::Null => {} @@ -82,6 +86,9 @@ pub fn combine_group_hash_column(c: &Column, values: &mut DataType::Bitmap => combine_group_hash_string_column::(c, values), DataType::Variant => combine_group_hash_string_column::(c, values), DataType::Geometry => combine_group_hash_string_column::(c, values), + DataType::Geography => { + combine_group_hash_string_column::(c, values) + } DataType::Nullable(_) => { let col = c.as_nullable().unwrap(); if IS_FIRST { @@ -149,6 +156,199 @@ fn combine_group_hash_string_column( } } +pub fn group_hash_value_spread( + indices: &[I], + value: Value, + first: bool, + target: &mut [u64], +) -> Result<()> { + if first { + let mut v = IndexHashVisitor::::new(indices, target); + v.visit_value(value) + } else { + let mut v = IndexHashVisitor::::new(indices, target); + v.visit_value(value) + } +} + +struct IndexHashVisitor<'a, 'b, const IS_FIRST: bool, I> +where I: Index +{ + indices: &'a [I], + target: &'b mut [u64], +} + +impl<'a, 'b, const IS_FIRST: bool, I> IndexHashVisitor<'a, 'b, IS_FIRST, I> +where I: Index +{ + fn new(indices: &'a [I], target: &'b mut [u64]) -> Self { + Self { indices, target } + } +} + +impl<'a, 'b, const IS_FIRST: bool, I> ValueVisitor for IndexHashVisitor<'a, 'b, IS_FIRST, I> +where I: Index +{ + fn visit_scalar(&mut self, scalar: Scalar) -> Result<()> { + let hash = match scalar { + Scalar::EmptyArray | Scalar::EmptyMap => return Ok(()), + Scalar::Null => NULL_HASH_VAL, + Scalar::Number(v) => with_number_type!(|NUM_TYPE| match v { + NumberScalar::NUM_TYPE(v) => v.agg_hash(), + }), + Scalar::Decimal(v) => match v { + DecimalScalar::Decimal128(v, _) => v.agg_hash(), + DecimalScalar::Decimal256(v, _) => v.agg_hash(), + }, + Scalar::Timestamp(v) => v.agg_hash(), + Scalar::Date(v) => v.agg_hash(), + Scalar::Boolean(v) => v.agg_hash(), + Scalar::Binary(v) => v.agg_hash(), + Scalar::String(v) => v.as_bytes().agg_hash(), + Scalar::Variant(v) => v.agg_hash(), + Scalar::Geometry(v) => v.agg_hash(), + Scalar::Geography(v) => v.0.agg_hash(), + v => v.as_ref().agg_hash(), + }; + self.visit_indices(|_| hash) + } + + fn visit_null(&mut self, _len: usize) -> Result<()> { + Ok(()) + } + + fn visit_empty_array(&mut self, _len: usize) -> Result<()> { + Ok(()) + } + + fn visit_empty_map(&mut self, _len: usize) -> Result<()> { + Ok(()) + } + + fn visit_any_number(&mut self, column: crate::types::NumberColumn) -> Result<()> { + with_number_type!(|NUM_TYPE| match column { + NumberColumn::NUM_TYPE(buffer) => { + let buffer = buffer.as_ref(); + self.visit_indices(|i| buffer[i.to_usize()].agg_hash()) + } + }) + } + + fn visit_timestamp(&mut self, buffer: Buffer) -> Result<()> { + self.visit_number(buffer) + } + + fn visit_date(&mut self, buffer: Buffer) -> Result<()> { + self.visit_number(buffer) + } + + fn visit_any_decimal(&mut self, column: DecimalColumn) -> Result<()> { + with_decimal_type!(|DECIMAL_TYPE| match column { + DecimalColumn::DECIMAL_TYPE(buffer, _) => { + let buffer = buffer.as_ref(); + self.visit_indices(|i| buffer[i.to_usize()].agg_hash()) + } + }) + } + + fn visit_binary(&mut self, column: crate::types::BinaryColumn) -> Result<()> { + self.visit_indices(|i| column.index(i.to_usize()).unwrap().agg_hash()) + } + + fn visit_variant(&mut self, column: crate::types::BinaryColumn) -> Result<()> { + self.visit_binary(column) + } + + fn visit_bitmap(&mut self, column: crate::types::BinaryColumn) -> Result<()> { + self.visit_binary(column) + } + + fn visit_string(&mut self, column: crate::types::StringColumn) -> Result<()> { + self.visit_indices(|i| column.index(i.to_usize()).unwrap().as_bytes().agg_hash()) + } + + fn visit_boolean( + &mut self, + bitmap: databend_common_arrow::arrow::bitmap::Bitmap, + ) -> Result<()> { + self.visit_indices(|i| bitmap.get(i.to_usize()).unwrap().agg_hash()) + } + + fn visit_geometry(&mut self, column: crate::types::BinaryColumn) -> Result<()> { + self.visit_binary(column) + } + + fn visit_geography(&mut self, column: crate::types::GeographyColumn) -> Result<()> { + self.visit_binary(column.0) + } + + fn visit_nullable(&mut self, column: Box>) -> Result<()> { + let indices = self + .indices + .iter() + .cloned() + .filter(|&i| { + let i = i.to_usize(); + let ok = column.validity.get(i).unwrap(); + if !ok { + let val = &mut self.target[i]; + if IS_FIRST { + *val = NULL_HASH_VAL; + } else { + *val = merge_hash(*val, NULL_HASH_VAL); + } + } + ok + }) + .collect::>(); + if IS_FIRST { + let mut v = IndexHashVisitor::::new(&indices, self.target); + v.visit_column(column.column) + } else { + let mut v = IndexHashVisitor::::new(&indices, self.target); + v.visit_column(column.column) + } + } + + fn visit_typed_column(&mut self, column: T::Column) -> Result<()> { + self.visit_indices(|i| { + let x = T::upcast_scalar(T::to_owned_scalar( + T::index_column(&column, i.to_usize()).unwrap(), + )); + x.as_ref().agg_hash() + }) + } +} + +impl<'a, 'b, const IS_FIRST: bool, I> IndexHashVisitor<'a, 'b, IS_FIRST, I> +where I: Index +{ + fn visit_indices(&mut self, do_hash: F) -> Result<()> + where F: Fn(&I) -> u64 { + self.visit_indices_update(|i, val| { + let hash = do_hash(i); + if IS_FIRST { + *val = hash; + } else { + *val = merge_hash(*val, hash); + } + }) + } + + fn visit_indices_update(&mut self, update: F) -> Result<()> + where F: Fn(&I, &mut u64) { + for i in self.indices { + let val = &mut self.target[i.to_usize()]; + update(i, val); + } + Ok(()) + } +} + +fn merge_hash(a: u64, b: u64) -> u64 { + a.wrapping_mul(NULL_HASH_VAL) ^ b +} + pub trait AggHash { fn agg_hash(&self) -> u64; } @@ -263,3 +463,142 @@ impl AggHash for ScalarRef<'_> { self.to_string().as_bytes().agg_hash() } } + +#[cfg(test)] +mod tests { + use databend_common_arrow::arrow::bitmap::Bitmap; + + use super::*; + use crate::types::ArgType; + use crate::types::Int32Type; + use crate::types::NullableColumn; + use crate::types::NullableType; + use crate::types::StringType; + use crate::BlockEntry; + use crate::DataBlock; + use crate::FromData; + use crate::Scalar; + use crate::Value; + + fn merge_hash_slice(ls: &[u64]) -> u64 { + ls.iter().cloned().reduce(merge_hash).unwrap() + } + + #[test] + fn test_value_spread() -> Result<()> { + let data = DataBlock::new( + vec![ + BlockEntry::new( + Int32Type::data_type(), + Value::Column(Int32Type::from_data(vec![3, 1, 2, 2, 4, 3, 7, 0, 3])), + ), + BlockEntry::new( + StringType::data_type(), + Value::Scalar(Scalar::String("a".to_string())), + ), + BlockEntry::new( + Int32Type::data_type(), + Value::Column(Int32Type::from_data(vec![3, 1, 3, 2, 2, 3, 4, 3, 3])), + ), + BlockEntry::new( + StringType::data_type(), + Value::Column(StringType::from_data(vec![ + "a", "b", "c", "d", "e", "f", "g", "h", "i", + ])), + ), + ], + 9, + ); + data.check_valid()?; + + { + let mut target = vec![0; data.num_rows()]; + for (i, entry) in data.columns().iter().enumerate() { + let indices = [0, 3, 8]; + group_hash_value_spread(&indices, entry.value.to_owned(), i == 0, &mut target)?; + } + + assert_eq!( + [ + merge_hash_slice(&[ + 3.agg_hash(), + b"a".agg_hash(), + 3.agg_hash(), + b"a".agg_hash(), + ]), + 0, + 0, + merge_hash_slice(&[ + 2.agg_hash(), + b"a".agg_hash(), + 2.agg_hash(), + b"d".agg_hash(), + ]), + 0, + 0, + 0, + 0, + merge_hash_slice(&[ + 3.agg_hash(), + b"a".agg_hash(), + 3.agg_hash(), + b"i".agg_hash(), + ]), + ] + .as_slice(), + &target + ); + } + + { + let c = Int32Type::from_data(vec![3, 1, 2]); + let c = NullableColumn::::new(c, Bitmap::from([true, true, false])); + let nc = NullableType::::upcast_column(c); + + let indices = [0, 1, 2]; + let mut target = vec![0; 3]; + group_hash_value_spread( + &indices, + Value::::Column(nc.clone()), + true, + &mut target, + )?; + + assert_eq!( + [ + merge_hash_slice(&[3.agg_hash()]), + merge_hash_slice(&[1.agg_hash()]), + merge_hash_slice(&[NULL_HASH_VAL]), + ] + .as_slice(), + &target + ); + + let c = Int32Type::from_data(vec![2, 4, 3]); + group_hash_value_spread(&indices, Value::::Column(c), false, &mut target)?; + + assert_eq!( + [ + merge_hash_slice(&[3.agg_hash(), 2.agg_hash()]), + merge_hash_slice(&[1.agg_hash(), 4.agg_hash()]), + merge_hash_slice(&[NULL_HASH_VAL, 3.agg_hash()]), + ] + .as_slice(), + &target + ); + + group_hash_value_spread(&indices, Value::::Column(nc), false, &mut target)?; + + assert_eq!( + [ + merge_hash_slice(&[3.agg_hash(), 2.agg_hash(), 3.agg_hash()]), + merge_hash_slice(&[1.agg_hash(), 4.agg_hash(), 1.agg_hash()]), + merge_hash_slice(&[NULL_HASH_VAL, 3.agg_hash(), NULL_HASH_VAL]), + ] + .as_slice(), + &target + ); + } + Ok(()) + } +} diff --git a/src/query/expression/src/kernels/sort_compare.rs b/src/query/expression/src/kernels/sort_compare.rs index 9f5078c621c3..210054addf61 100644 --- a/src/query/expression/src/kernels/sort_compare.rs +++ b/src/query/expression/src/kernels/sort_compare.rs @@ -37,6 +37,7 @@ pub struct SortCompare { current_column_index: usize, validity: Option, equality_index: Vec, + force_equality: bool, } macro_rules! do_sorter { @@ -112,12 +113,25 @@ impl SortCompare { current_column_index: 0, validity: None, equality_index, + force_equality: matches!(limit, LimitType::LimitRank(_)), + } + } + + pub fn with_force_equality(ordering_descs: Vec, rows: usize) -> Self { + Self { + rows, + limit: LimitType::None, + permutation: (0..rows as u32).collect(), + ordering_descs, + current_column_index: 0, + validity: None, + equality_index: vec![1; rows as _], + force_equality: true, } } fn need_update_equality_index(&self) -> bool { - self.current_column_index != self.ordering_descs.len() - 1 - || matches!(self.limit, LimitType::LimitRank(_)) + self.force_equality || self.current_column_index != self.ordering_descs.len() - 1 } pub fn increment_column_index(&mut self) { @@ -254,6 +268,11 @@ impl SortCompare { } } } + + pub fn equality_index(&self) -> &[u8] { + debug_assert!(self.force_equality); + &self.equality_index + } } impl ValueVisitor for SortCompare { diff --git a/src/query/expression/src/types.rs b/src/query/expression/src/types.rs index aa6b837bfa7d..06837d55f3b4 100755 --- a/src/query/expression/src/types.rs +++ b/src/query/expression/src/types.rs @@ -57,6 +57,7 @@ pub use self::empty_map::EmptyMapType; pub use self::generic::GenericType; pub use self::geography::GeographyColumn; pub use self::geography::GeographyType; +pub use self::geometry::GeometryType; pub use self::map::MapType; pub use self::null::NullType; pub use self::nullable::NullableColumn; diff --git a/src/query/expression/src/types/geography.rs b/src/query/expression/src/types/geography.rs index b41aad37d9df..1a559eab0895 100644 --- a/src/query/expression/src/types/geography.rs +++ b/src/query/expression/src/types/geography.rs @@ -83,6 +83,12 @@ impl<'a> GeographyRef<'a> { } } +impl<'a> AsRef<[u8]> for GeographyRef<'a> { + fn as_ref(&self) -> &[u8] { + self.0 + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct GeographyType; diff --git a/src/query/expression/src/utils/visitor.rs b/src/query/expression/src/utils/visitor.rs index 535383ce7968..231127ac8254 100755 --- a/src/query/expression/src/utils/visitor.rs +++ b/src/query/expression/src/utils/visitor.rs @@ -36,6 +36,12 @@ pub trait ValueVisitor { self.visit_typed_column::(len) } + fn visit_any_number(&mut self, column: NumberColumn) -> Result<()> { + with_number_type!(|NUM_TYPE| match column { + NumberColumn::NUM_TYPE(b) => self.visit_number(b), + }) + } + fn visit_number( &mut self, column: as ValueType>::Column, @@ -43,6 +49,12 @@ pub trait ValueVisitor { self.visit_typed_column::>(column) } + fn visit_any_decimal(&mut self, column: DecimalColumn) -> Result<()> { + with_decimal_type!(|DECIMAL_TYPE| match column { + DecimalColumn::DECIMAL_TYPE(b, size) => self.visit_decimal(b, size), + }) + } + fn visit_decimal(&mut self, column: Buffer, _size: DecimalSize) -> Result<()> { self.visit_typed_column::>(column) } @@ -113,16 +125,8 @@ pub trait ValueVisitor { Column::Null { len } => self.visit_null(len), Column::EmptyArray { len } => self.visit_empty_array(len), Column::EmptyMap { len } => self.visit_empty_map(len), - Column::Number(column) => { - with_number_type!(|NUM_TYPE| match column { - NumberColumn::NUM_TYPE(b) => self.visit_number(b), - }) - } - Column::Decimal(column) => { - with_decimal_type!(|DECIMAL_TYPE| match column { - DecimalColumn::DECIMAL_TYPE(b, size) => self.visit_decimal(b, size), - }) - } + Column::Number(column) => self.visit_any_number(column), + Column::Decimal(column) => self.visit_any_decimal(column), Column::Boolean(bitmap) => self.visit_boolean(bitmap), Column::Binary(column) => self.visit_binary(column), Column::String(column) => self.visit_string(column), diff --git a/src/query/pipeline/core/src/processors/shuffle_processor.rs b/src/query/pipeline/core/src/processors/shuffle_processor.rs index 6d709988a09f..724e76bc3839 100644 --- a/src/query/pipeline/core/src/processors/shuffle_processor.rs +++ b/src/query/pipeline/core/src/processors/shuffle_processor.rs @@ -31,6 +31,7 @@ pub enum MultiwayStrategy { } pub trait Exchange: Send + Sync + 'static { + const NAME: &'static str; const STRATEGY: MultiwayStrategy = MultiwayStrategy::Random; fn partition(&self, data_block: DataBlock, n: usize) -> Result>; @@ -185,7 +186,7 @@ impl PartitionProcessor { impl Processor for PartitionProcessor { fn name(&self) -> String { - String::from("ShufflePartition") + format!("ShufflePartition({})", T::NAME) } fn as_any(&mut self) -> &mut dyn Any { @@ -287,7 +288,7 @@ impl MergePartitionProcessor { impl Processor for MergePartitionProcessor { fn name(&self) -> String { - String::from("ShuffleMergePartition") + format!("ShuffleMergePartition({})", T::NAME) } fn as_any(&mut self) -> &mut dyn Any { diff --git a/src/query/service/src/pipelines/builders/builder_window.rs b/src/query/service/src/pipelines/builders/builder_window.rs index 252bd50b592d..4205eb35cc56 100644 --- a/src/query/service/src/pipelines/builders/builder_window.rs +++ b/src/query/service/src/pipelines/builders/builder_window.rs @@ -34,6 +34,7 @@ use crate::pipelines::processors::transforms::TransformWindow; use crate::pipelines::processors::transforms::TransformWindowPartitionCollect; use crate::pipelines::processors::transforms::WindowFunctionInfo; use crate::pipelines::processors::transforms::WindowPartitionExchange; +use crate::pipelines::processors::transforms::WindowPartitionTopNExchange; use crate::pipelines::processors::transforms::WindowSortDesc; use crate::pipelines::processors::transforms::WindowSpillSettings; use crate::pipelines::PipelineBuilder; @@ -169,10 +170,23 @@ impl PipelineBuilder { }) .collect::>>()?; - self.main_pipeline.exchange( - num_processors, - WindowPartitionExchange::create(partition_by.clone(), num_partitions), - ); + if let Some(top_n) = &window_partition.top_n { + self.main_pipeline.exchange( + num_processors, + WindowPartitionTopNExchange::create( + partition_by.clone(), + sort_desc.clone(), + top_n.top, + top_n.func, + num_partitions as u64, + ), + ) + } else { + self.main_pipeline.exchange( + num_processors, + WindowPartitionExchange::create(partition_by.clone(), num_partitions), + ); + } let disk_bytes_limit = settings.get_window_partition_spilling_to_disk_bytes_limit()?; let temp_dir_manager = TempDirManager::instance(); diff --git a/src/query/service/src/pipelines/processors/transforms/window/partition/mod.rs b/src/query/service/src/pipelines/processors/transforms/window/partition/mod.rs index b75ff9d671cf..3eebde7955a8 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/partition/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/partition/mod.rs @@ -16,8 +16,10 @@ mod transform_window_partition_collect; mod window_partition_buffer; mod window_partition_exchange; mod window_partition_meta; +mod window_partition_partial_top_n_exchange; pub use transform_window_partition_collect::*; pub use window_partition_buffer::*; pub use window_partition_exchange::*; pub use window_partition_meta::*; +pub use window_partition_partial_top_n_exchange::*; diff --git a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_exchange.rs b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_exchange.rs index a23e6b030c61..bf6ea988acf6 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_exchange.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_exchange.rs @@ -15,10 +15,9 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::group_hash_columns_slice; -use databend_common_expression::ColumnBuilder; +use databend_common_expression::group_hash_columns; use databend_common_expression::DataBlock; -use databend_common_expression::Value; +use databend_common_expression::InputColumns; use databend_common_pipeline_core::processors::Exchange; use super::WindowPartitionMeta; @@ -38,27 +37,17 @@ impl WindowPartitionExchange { } impl Exchange for WindowPartitionExchange { + const NAME: &'static str = "Window"; fn partition(&self, data_block: DataBlock, n: usize) -> Result> { let num_rows = data_block.num_rows(); // Extract the columns used for hash computation. - let hash_cols = self - .hash_keys - .iter() - .map(|&offset| { - let entry = data_block.get_by_offset(offset); - match &entry.value { - Value::Scalar(s) => { - ColumnBuilder::repeat(&s.as_ref(), num_rows, &entry.data_type).build() - } - Value::Column(c) => c.clone(), - } - }) - .collect::>(); + let data_block = data_block.consume_convert_to_full(); + let hash_cols = InputColumns::new_block_proxy(&self.hash_keys, &data_block); // Compute the hash value for each row. let mut hashes = vec![0u64; num_rows]; - group_hash_columns_slice(&hash_cols, &mut hashes); + group_hash_columns(hash_cols, &mut hashes); // Scatter the data block to different partitions. let indices = hashes diff --git a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_partial_top_n_exchange.rs b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_partial_top_n_exchange.rs new file mode 100644 index 000000000000..8601895dcffd --- /dev/null +++ b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_partial_top_n_exchange.rs @@ -0,0 +1,336 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::Result; +use databend_common_expression::group_hash_value_spread; +use databend_common_expression::visitor::ValueVisitor; +use databend_common_expression::DataBlock; +use databend_common_expression::SortColumnDescription; +use databend_common_expression::SortCompare; +use databend_common_pipeline_core::processors::Exchange; + +use super::WindowPartitionMeta; +use crate::sql::executor::physical_plans::WindowPartitionTopNFunc; + +pub struct WindowPartitionTopNExchange { + partition_indices: Box<[usize]>, + top: usize, + func: WindowPartitionTopNFunc, + + sort_desc: Box<[SortColumnDescription]>, + num_partitions: u64, +} + +impl WindowPartitionTopNExchange { + pub fn create( + partition_indices: Vec, + order_by: Vec, + top: usize, + func: WindowPartitionTopNFunc, + num_partitions: u64, + ) -> Arc { + assert!(top > 0); + let partition_indices = partition_indices.into_boxed_slice(); + let sort_desc = partition_indices + .iter() + .map(|&offset| SortColumnDescription { + offset, + asc: true, + nulls_first: false, + }) + .chain(order_by) + .collect::>() + .into(); + + Arc::new(WindowPartitionTopNExchange { + num_partitions, + partition_indices, + top, + func, + sort_desc, + }) + } +} + +impl Exchange for WindowPartitionTopNExchange { + const NAME: &'static str = "WindowTopN"; + fn partition(&self, block: DataBlock, n: usize) -> Result> { + let partition_permutation = self.partition_permutation(&block); + + // Partition the data blocks to different processors. + let mut output_data_blocks = vec![vec![]; n]; + for (partition_id, indices) in partition_permutation.into_iter().enumerate() { + output_data_blocks[partition_id % n].push((partition_id, block.take(&indices)?)); + } + + // Union data blocks for each processor. + Ok(output_data_blocks + .into_iter() + .map(WindowPartitionMeta::create) + .map(DataBlock::empty_with_meta) + .collect()) + } +} + +impl WindowPartitionTopNExchange { + fn partition_permutation(&self, block: &DataBlock) -> Vec> { + let rows = block.num_rows(); + let mut sort_compare = SortCompare::with_force_equality(self.sort_desc.to_vec(), rows); + + for &offset in &self.partition_indices { + let array = block.get_by_offset(offset).value.clone(); + sort_compare.visit_value(array).unwrap(); + sort_compare.increment_column_index(); + } + + let partition_equality = sort_compare.equality_index().to_vec(); + + for desc in self.sort_desc.iter().skip(self.partition_indices.len()) { + let array = block.get_by_offset(desc.offset).value.clone(); + sort_compare.visit_value(array).unwrap(); + sort_compare.increment_column_index(); + } + + let full_equality = sort_compare.equality_index().to_vec(); + let permutation = sort_compare.take_permutation(); + + let hash_indices = std::iter::once(permutation[0]) + .chain( + partition_equality + .iter() + .enumerate() + .filter_map(|(i, &eq)| if eq == 0 { Some(permutation[i]) } else { None }), + ) + .collect::>(); + + let mut hashes = vec![0u64; rows]; + for (i, &offset) in self.partition_indices.iter().enumerate() { + let entry = block.get_by_offset(offset); + group_hash_value_spread(&hash_indices, entry.value.to_owned(), i == 0, &mut hashes) + .unwrap(); + } + + let mut partition_permutation = vec![Vec::new(); self.num_partitions as usize]; + + let mut start = 0; + let mut cur = 0; + while cur < rows { + let partition = &mut partition_permutation + [(hashes[permutation[start] as usize] % self.num_partitions) as usize]; + partition.push(permutation[start]); + + let mut rank = 0; // this first value is rank 0 + cur = start + 1; + while cur < rows { + if partition_equality[cur] == 0 { + start = cur; + break; + } + + match self.func { + WindowPartitionTopNFunc::RowNumber => { + if cur - start < self.top { + partition.push(permutation[cur]); + } + } + WindowPartitionTopNFunc::Rank | WindowPartitionTopNFunc::DenseRank => { + if full_equality[cur] == 0 { + if matches!(self.func, WindowPartitionTopNFunc::Rank) { + rank = cur - start + } else { + rank += 1 + } + } + + if rank < self.top { + partition.push(permutation[cur]); + } + } + } + cur += 1; + } + } + partition_permutation + } +} + +#[cfg(test)] +mod tests { + use databend_common_expression::types::ArgType; + use databend_common_expression::types::Int32Type; + use databend_common_expression::types::StringType; + use databend_common_expression::BlockEntry; + use databend_common_expression::FromData; + use databend_common_expression::Scalar; + use databend_common_expression::Value; + + use super::*; + + #[test] + fn test_row_number() -> Result<()> { + let p = WindowPartitionTopNExchange::create( + vec![1, 2], + vec![SortColumnDescription { + offset: 0, + asc: true, + nulls_first: false, + }], + 3, + WindowPartitionTopNFunc::RowNumber, + 8, + ); + + let data = DataBlock::new( + vec![ + BlockEntry::new( + Int32Type::data_type(), + Value::Column(Int32Type::from_data(vec![3, 1, 2, 2, 4, 3, 7, 0, 3])), + ), + BlockEntry::new( + StringType::data_type(), + Value::Scalar(Scalar::String("a".to_string())), + ), + BlockEntry::new( + Int32Type::data_type(), + Value::Column(Int32Type::from_data(vec![3, 1, 3, 2, 2, 3, 4, 3, 3])), + ), + BlockEntry::new( + StringType::data_type(), + Value::Column(StringType::from_data(vec![ + "a", "b", "c", "d", "e", "f", "g", "h", "i", + ])), + ), + ], + 9, + ); + data.check_valid()?; + + let got = p.partition_permutation(&data); + + let want = vec![ + vec![], + vec![1], + vec![], + vec![3, 4], + vec![], + vec![6], + vec![], + vec![7, 2, 0], + ]; + // if got != want { + // let got = got + // .iter() + // .map(|indices| data.take(indices, &mut None).unwrap()) + // .collect::>(); + // for x in got { + // println!("{}", x) + // } + // } + assert_eq!(&want, &got); + + Ok(()) + } + + #[test] + fn test_rank() -> Result<()> { + let p = WindowPartitionTopNExchange::create( + vec![1], + vec![SortColumnDescription { + offset: 0, + asc: true, + nulls_first: false, + }], + 3, + WindowPartitionTopNFunc::Rank, + 8, + ); + + let data = DataBlock::new( + vec![ + BlockEntry::new( + Int32Type::data_type(), + Value::Column(Int32Type::from_data(vec![7, 7, 7, 6, 5, 5, 4, 1, 3, 1, 1])), + ), + BlockEntry::new( + Int32Type::data_type(), + Value::Column(Int32Type::from_data(vec![7, 6, 5, 5, 5, 4, 3, 3, 2, 3, 3])), + ), + ], + 11, + ); + data.check_valid()?; + + let got = p.partition_permutation(&data); + + let want = vec![ + vec![], + vec![1], + vec![8, 0], + vec![], + vec![5, 4, 3, 2], + vec![], + vec![7, 9, 10], + vec![], + ]; + assert_eq!(&want, &got); + Ok(()) + } + + #[test] + fn test_dense_rank() -> Result<()> { + let p = WindowPartitionTopNExchange::create( + vec![1], + vec![SortColumnDescription { + offset: 0, + asc: true, + nulls_first: false, + }], + 3, + WindowPartitionTopNFunc::DenseRank, + 8, + ); + + let data = DataBlock::new( + vec![ + BlockEntry::new( + Int32Type::data_type(), + Value::Column(Int32Type::from_data(vec![5, 2, 3, 3, 2, 2, 1, 1, 1, 1, 1])), + ), + BlockEntry::new( + Int32Type::data_type(), + Value::Column(Int32Type::from_data(vec![2, 2, 4, 3, 2, 2, 5, 4, 3, 3, 3])), + ), + ], + 11, + ); + data.check_valid()?; + + let got = p.partition_permutation(&data); + + let want = vec![ + vec![], + vec![], + vec![1, 4, 5, 0], + vec![], + vec![7, 2, 6], + vec![], + vec![8, 9, 10, 3], + vec![], + ]; + assert_eq!(&want, &got); + Ok(()) + } +} diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 6dcab582c7f3..dd86b4cda858 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -242,6 +242,7 @@ pub trait PhysicalPlanReplacer { partition_by: plan.partition_by.clone(), order_by: plan.order_by.clone(), after_exchange: plan.after_exchange, + top_n: plan.top_n.clone(), stat_info: plan.stat_info.clone(), })) } diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index f8b1c86702d7..957443396364 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -57,6 +57,7 @@ mod physical_table_scan; mod physical_udf; mod physical_union_all; mod physical_window; +mod physical_window_partition; pub use common::*; pub use physical_add_stream_column::AddStreamColumn; @@ -101,10 +102,9 @@ pub use physical_replace_deduplicate::*; pub use physical_replace_into::ReplaceInto; pub use physical_row_fetch::RowFetch; pub use physical_sort::Sort; -mod physical_window_partition; pub use physical_table_scan::TableScan; pub use physical_udf::Udf; pub use physical_udf::UdfFunctionDesc; pub use physical_union_all::UnionAll; pub use physical_window::*; -pub use physical_window_partition::WindowPartition; +pub use physical_window_partition::*; diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index c535a33a2fcc..8bfc63719f2d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -24,9 +24,12 @@ use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; use crate::executor::physical_plans::common::SortDesc; use crate::executor::physical_plans::WindowPartition; +use crate::executor::physical_plans::WindowPartitionTopN; +use crate::executor::physical_plans::WindowPartitionTopNFunc; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::SExpr; +use crate::plans::WindowFuncType; use crate::ColumnSet; use crate::IndexType; @@ -122,12 +125,6 @@ impl PhysicalPlanBuilder { let input_plan = self.build(s_expr.child(0)?, required).await?; - let window_partition = sort - .window_partition - .iter() - .map(|v| v.index) - .collect::>(); - let order_by = sort .items .iter() @@ -140,16 +137,31 @@ impl PhysicalPlanBuilder { .collect::>(); // Add WindowPartition for parallel sort in window. - if !window_partition.is_empty() { + if let Some(window) = &sort.window_partition { + let window_partition = window + .partition_by + .iter() + .map(|v| v.index) + .collect::>(); + return Ok(PhysicalPlan::WindowPartition(WindowPartition { plan_id: 0, input: Box::new(input_plan.clone()), partition_by: window_partition.clone(), order_by: order_by.clone(), after_exchange: sort.after_exchange, + top_n: window.top.map(|top| WindowPartitionTopN { + func: match window.func { + WindowFuncType::RowNumber => WindowPartitionTopNFunc::RowNumber, + WindowFuncType::Rank => WindowPartitionTopNFunc::Rank, + WindowFuncType::DenseRank => WindowPartitionTopNFunc::DenseRank, + _ => unreachable!(), + }, + top, + }), stat_info: Some(stat_info.clone()), })); - } + }; // 2. Build physical plan. Ok(PhysicalPlan::Sort(Sort { diff --git a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs index 002daa1955f0..b0ff12d3f868 100644 --- a/src/query/sql/src/executor/physical_plans/physical_window_partition.rs +++ b/src/query/sql/src/executor/physical_plans/physical_window_partition.rs @@ -27,10 +27,24 @@ pub struct WindowPartition { pub partition_by: Vec, pub order_by: Vec, pub after_exchange: Option, + pub top_n: Option, pub stat_info: Option, } +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct WindowPartitionTopN { + pub func: WindowPartitionTopNFunc, + pub top: usize, +} + +#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] +pub enum WindowPartitionTopNFunc { + RowNumber, + Rank, + DenseRank, +} + impl WindowPartition { pub fn output_schema(&self) -> Result { self.input.output_schema() diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index e2c0b565f377..f3556bc02170 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -159,7 +159,7 @@ impl Binder { limit: None, after_exchange: None, pre_projection: None, - window_partition: vec![], + window_partition: None, }; Ok(SExpr::create_unary( Arc::new(sort_plan.into()), diff --git a/src/query/sql/src/planner/binder/sort.rs b/src/query/sql/src/planner/binder/sort.rs index 5deb9ed8c1a6..c3d630fce11b 100644 --- a/src/query/sql/src/planner/binder/sort.rs +++ b/src/query/sql/src/planner/binder/sort.rs @@ -217,7 +217,7 @@ impl Binder { limit: None, after_exchange: None, pre_projection: None, - window_partition: vec![], + window_partition: None, }; let new_expr = SExpr::create_unary(Arc::new(sort_plan.into()), Arc::new(child)); Ok(new_expr) diff --git a/src/query/sql/src/planner/binder/window.rs b/src/query/sql/src/planner/binder/window.rs index f10a088859d0..e43511df31b4 100644 --- a/src/query/sql/src/planner/binder/window.rs +++ b/src/query/sql/src/planner/binder/window.rs @@ -42,6 +42,7 @@ use crate::plans::WindowFunc; use crate::plans::WindowFuncFrame; use crate::plans::WindowFuncType; use crate::plans::WindowOrderBy; +use crate::plans::WindowPartition; use crate::BindContext; use crate::Binder; use crate::ColumnEntry; @@ -116,10 +117,18 @@ impl Binder { let child = if !sort_items.is_empty() { let sort_plan = Sort { items: sort_items, - limit: window_plan.limit, + limit: None, after_exchange: None, pre_projection: None, - window_partition: window_plan.partition_by.clone(), + window_partition: if window_plan.partition_by.is_empty() { + None + } else { + Some(WindowPartition { + partition_by: window_plan.partition_by.clone(), + top: None, + func: window_plan.function.clone(), + }) + }, }; SExpr::create_unary(Arc::new(sort_plan.into()), Arc::new(child)) } else { diff --git a/src/query/sql/src/planner/format/display_rel_operator.rs b/src/query/sql/src/planner/format/display_rel_operator.rs index f4d236f9c249..23e7c4329132 100644 --- a/src/query/sql/src/planner/format/display_rel_operator.rs +++ b/src/query/sql/src/planner/format/display_rel_operator.rs @@ -385,10 +385,23 @@ fn sort_to_format_tree .join(", "); let limit = op.limit.map_or("NONE".to_string(), |l| l.to_string()); - FormatTreeNode::with_children("Sort".to_string(), vec![ - FormatTreeNode::new(format!("sort keys: [{}]", scalars)), - FormatTreeNode::new(format!("limit: [{}]", limit)), - ]) + let children = match &op.window_partition { + Some(window) => vec![ + FormatTreeNode::new(format!("sort keys: [{}]", scalars)), + FormatTreeNode::new(format!("limit: [{}]", limit)), + FormatTreeNode::new(format!( + "window top: {}", + window.top.map_or("NONE".to_string(), |n| n.to_string()) + )), + FormatTreeNode::new(format!("window function: {:?}", window.func)), + ], + None => vec![ + FormatTreeNode::new(format!("sort keys: [{}]", scalars)), + FormatTreeNode::new(format!("limit: [{}]", limit)), + ], + }; + + FormatTreeNode::with_children("Sort".to_string(), children) } fn constant_scan_to_format_tree>( diff --git a/src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs b/src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs index d973b91a6944..5a2b34ceec35 100644 --- a/src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs +++ b/src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs @@ -162,10 +162,13 @@ impl SubqueryRewriter { RelOperator::Sort(mut sort) => { let mut input = self.rewrite(s_expr.child(0)?)?; - for item in sort.window_partition.iter_mut() { - let res = self.try_rewrite_subquery(&item.scalar, &input, false)?; - input = res.1; - item.scalar = res.0; + + if let Some(window) = &mut sort.window_partition { + for item in window.partition_by.iter_mut() { + let res = self.try_rewrite_subquery(&item.scalar, &input, false)?; + input = res.1; + item.scalar = res.0; + } } Ok(SExpr::create_unary(Arc::new(sort.into()), Arc::new(input))) diff --git a/src/query/sql/src/planner/optimizer/rule/factory.rs b/src/query/sql/src/planner/optimizer/rule/factory.rs index 98ece251cf6b..50ab1ed44ef9 100644 --- a/src/query/sql/src/planner/optimizer/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/rule/factory.rs @@ -16,41 +16,42 @@ use databend_common_exception::Result; use super::rewrite::RuleCommuteJoin; use super::rewrite::RuleEliminateEvalScalar; +use super::rewrite::RuleEliminateFilter; +use super::rewrite::RuleEliminateSort; use super::rewrite::RuleEliminateUnion; +use super::rewrite::RuleFilterNulls; use super::rewrite::RuleFoldCountAggregate; +use super::rewrite::RuleMergeEvalScalar; +use super::rewrite::RuleMergeFilter; use super::rewrite::RuleNormalizeScalarFilter; use super::rewrite::RulePushDownFilterAggregate; use super::rewrite::RulePushDownFilterEvalScalar; use super::rewrite::RulePushDownFilterJoin; +use super::rewrite::RulePushDownFilterProjectSet; +use super::rewrite::RulePushDownFilterScan; +use super::rewrite::RulePushDownFilterSort; +use super::rewrite::RulePushDownFilterUnion; use super::rewrite::RulePushDownFilterWindow; +use super::rewrite::RulePushDownFilterWindowTopN; +use super::rewrite::RulePushDownLimit; use super::rewrite::RulePushDownLimitEvalScalar; +use super::rewrite::RulePushDownLimitOuterJoin; +use super::rewrite::RulePushDownLimitScan; +use super::rewrite::RulePushDownLimitSort; +use super::rewrite::RulePushDownLimitUnion; +use super::rewrite::RulePushDownLimitWindow; use super::rewrite::RulePushDownPrewhere; use super::rewrite::RulePushDownRankLimitAggregate; use super::rewrite::RulePushDownSortEvalScalar; +use super::rewrite::RulePushDownSortScan; +use super::rewrite::RuleSemiToInnerJoin; +use super::rewrite::RuleSplitAggregate; use super::rewrite::RuleTryApplyAggIndex; -use crate::optimizer::rule::rewrite::RuleEliminateFilter; -use crate::optimizer::rule::rewrite::RuleEliminateSort; -use crate::optimizer::rule::rewrite::RuleFilterNulls; -use crate::optimizer::rule::rewrite::RuleMergeEvalScalar; -use crate::optimizer::rule::rewrite::RuleMergeFilter; -use crate::optimizer::rule::rewrite::RulePushDownFilterProjectSet; -use crate::optimizer::rule::rewrite::RulePushDownFilterScan; -use crate::optimizer::rule::rewrite::RulePushDownFilterSort; -use crate::optimizer::rule::rewrite::RulePushDownFilterUnion; -use crate::optimizer::rule::rewrite::RulePushDownLimit; -use crate::optimizer::rule::rewrite::RulePushDownLimitOuterJoin; -use crate::optimizer::rule::rewrite::RulePushDownLimitScan; -use crate::optimizer::rule::rewrite::RulePushDownLimitSort; -use crate::optimizer::rule::rewrite::RulePushDownLimitUnion; -use crate::optimizer::rule::rewrite::RulePushDownLimitWindow; -use crate::optimizer::rule::rewrite::RulePushDownSortScan; -use crate::optimizer::rule::rewrite::RuleSemiToInnerJoin; -use crate::optimizer::rule::rewrite::RuleSplitAggregate; -use crate::optimizer::rule::transform::RuleCommuteJoinBaseTable; -use crate::optimizer::rule::transform::RuleEagerAggregation; -use crate::optimizer::rule::transform::RuleLeftExchangeJoin; -use crate::optimizer::rule::RuleID; -use crate::optimizer::rule::RulePtr; +use super::transform::RuleCommuteJoinBaseTable; +use super::transform::RuleEagerAggregation; +use super::transform::RuleLeftExchangeJoin; +use super::RuleID; +use super::RulePtr; use crate::optimizer::OptimizerContext; pub struct RuleFactory; @@ -91,6 +92,7 @@ impl RuleFactory { } RuleID::PushDownFilterAggregate => Ok(Box::new(RulePushDownFilterAggregate::new())), RuleID::PushDownFilterWindow => Ok(Box::new(RulePushDownFilterWindow::new())), + RuleID::PushDownFilterWindowRank => Ok(Box::new(RulePushDownFilterWindowTopN::new())), RuleID::EliminateFilter => Ok(Box::new(RuleEliminateFilter::new(ctx.metadata))), RuleID::MergeEvalScalar => Ok(Box::new(RuleMergeEvalScalar::new())), RuleID::MergeFilter => Ok(Box::new(RuleMergeFilter::new())), diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs index d8c306a88bfa..c564477e2f32 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs @@ -32,6 +32,7 @@ mod rule_push_down_filter_scan; mod rule_push_down_filter_sort; mod rule_push_down_filter_union; mod rule_push_down_filter_window; +mod rule_push_down_filter_window_top_n; mod rule_push_down_limit; mod rule_push_down_limit_aggregate; mod rule_push_down_limit_expression; @@ -66,6 +67,7 @@ pub use rule_push_down_filter_scan::RulePushDownFilterScan; pub use rule_push_down_filter_sort::RulePushDownFilterSort; pub use rule_push_down_filter_union::RulePushDownFilterUnion; pub use rule_push_down_filter_window::RulePushDownFilterWindow; +pub use rule_push_down_filter_window_top_n::RulePushDownFilterWindowTopN; pub use rule_push_down_limit::RulePushDownLimit; pub use rule_push_down_limit_aggregate::RulePushDownRankLimitAggregate; pub use rule_push_down_limit_expression::RulePushDownLimitEvalScalar; diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_eliminate_sort.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_eliminate_sort.rs index 6201891de8f5..4aacb621359d 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_eliminate_sort.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_eliminate_sort.rs @@ -55,14 +55,14 @@ impl Rule for RuleEliminateSort { let rel_expr = RelExpr::with_s_expr(input); let prop = rel_expr.derive_relational_prop()?; - if !sort.window_partition.is_empty() { + if let Some(window) = &sort.window_partition { if let Some((partition, ordering)) = &prop.partition_orderings { // must has same partition // if the ordering of the current node is empty, we can eliminate the sort // eg: explain select number, sum(number - 1) over (partition by number % 3 order by number + 1), // avg(number) over (partition by number % 3 order by number + 1) // from numbers(50); - if partition == &sort.window_partition + if partition == &window.partition_by && (ordering == &sort.items || sort.sort_items_exclude_partition().is_empty()) { state.add_result(input.clone()); diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_window_top_n.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_window_top_n.rs new file mode 100644 index 000000000000..136aa8622076 --- /dev/null +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_window_top_n.rs @@ -0,0 +1,185 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::Result; +use databend_common_expression::type_check::check_number; +use databend_common_expression::FunctionContext; +use databend_common_functions::BUILTIN_FUNCTIONS; + +use crate::optimizer::extract::Matcher; +use crate::optimizer::rule::Rule; +use crate::optimizer::rule::TransformResult; +use crate::optimizer::RuleID; +use crate::optimizer::SExpr; +use crate::plans::ComparisonOp; +use crate::plans::Filter; +use crate::plans::RelOp; +use crate::plans::ScalarExpr; +use crate::plans::Sort; +use crate::plans::Window; +use crate::plans::WindowFuncType; + +/// Input: Filter +/// \ +/// Window +/// \ +/// Sort +/// +/// Output: Filter +/// \ +/// Window +/// \ +/// Sort(top n) +pub struct RulePushDownFilterWindowTopN { + id: RuleID, + matchers: Vec, +} + +impl RulePushDownFilterWindowTopN { + pub fn new() -> Self { + Self { + id: RuleID::PushDownFilterWindowRank, + matchers: vec![Matcher::MatchOp { + op_type: RelOp::Filter, + children: vec![Matcher::MatchOp { + op_type: RelOp::Window, + children: vec![Matcher::MatchOp { + op_type: RelOp::Sort, + children: vec![Matcher::Leaf], + }], + }], + }], + } + } +} + +impl Rule for RulePushDownFilterWindowTopN { + fn id(&self) -> RuleID { + self.id + } + + fn apply(&self, s_expr: &SExpr, state: &mut TransformResult) -> Result<()> { + let filter: Filter = s_expr.plan().clone().try_into()?; + let window_expr = s_expr.child(0)?; + let window: Window = window_expr.plan().clone().try_into()?; + let sort_expr = window_expr.child(0)?; + let mut sort: Sort = sort_expr.plan().clone().try_into()?; + + if !is_ranking_function(&window.function) || sort.window_partition.is_none() { + return Ok(()); + } + + let predicates = filter + .predicates + .into_iter() + .filter_map(|predicate| extract_top_n(window.index, predicate)) + .collect::>(); + + let Some(top_n) = predicates.into_iter().min() else { + return Ok(()); + }; + + if top_n == 0 { + // TODO + return Ok(()); + } + + sort.window_partition.as_mut().unwrap().top = Some(top_n); + + let mut result = SExpr::create_unary( + s_expr.plan.clone(), + SExpr::create_unary( + window_expr.plan.clone(), + sort_expr.replace_plan(Arc::new(sort.into())).into(), + ) + .into(), + ); + result.set_applied_rule(&self.id); + + state.add_result(result); + + Ok(()) + } + + fn matchers(&self) -> &[Matcher] { + &self.matchers + } +} + +fn extract_top_n(column: usize, predicate: ScalarExpr) -> Option { + let ScalarExpr::FunctionCall(call) = predicate else { + return None; + }; + + let func_name = &call.func_name; + if func_name == ComparisonOp::Equal.to_func_name() { + return match (&call.arguments[0], &call.arguments[1]) { + (ScalarExpr::BoundColumnRef(col), number) + | (number, ScalarExpr::BoundColumnRef(col)) + if col.column.index == column => + { + extract_i32(number).map(|n| n.max(0) as usize) + } + _ => None, + }; + } + + let (left, right) = match ( + func_name == ComparisonOp::LTE.to_func_name() + || func_name == ComparisonOp::LT.to_func_name(), + func_name == ComparisonOp::GTE.to_func_name() + || func_name == ComparisonOp::GT.to_func_name(), + ) { + (true, _) => (&call.arguments[0], &call.arguments[1]), + (_, true) => (&call.arguments[1], &call.arguments[0]), + _ => return None, + }; + + let ScalarExpr::BoundColumnRef(col) = left else { + return None; + }; + if col.column.index != column { + return None; + } + + let eq = func_name == ComparisonOp::GTE.to_func_name() + || func_name == ComparisonOp::LTE.to_func_name(); + + extract_i32(right).map(|n| { + if eq { + n.max(0) as usize + } else { + n.max(1) as usize - 1 + } + }) +} + +fn extract_i32(expr: &ScalarExpr) -> Option { + check_number( + None, + &FunctionContext::default(), + &expr.as_expr().ok()?, + &BUILTIN_FUNCTIONS, + ) + .ok() +} + +fn is_ranking_function(func: &WindowFuncType) -> bool { + matches!( + func, + WindowFuncType::RowNumber | WindowFuncType::Rank | WindowFuncType::DenseRank + ) +} diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit_aggregate.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit_aggregate.rs index ad1cae65cf99..aaf241cd5e63 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit_aggregate.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit_aggregate.rs @@ -105,7 +105,7 @@ impl RulePushDownRankLimitAggregate { limit: Some(count), after_exchange: None, pre_projection: None, - window_partition: vec![], + window_partition: None, }; let agg = SExpr::create_unary( diff --git a/src/query/sql/src/planner/optimizer/rule/rule.rs b/src/query/sql/src/planner/optimizer/rule/rule.rs index b765aba22c82..f13a67a6b5a5 100644 --- a/src/query/sql/src/planner/optimizer/rule/rule.rs +++ b/src/query/sql/src/planner/optimizer/rule/rule.rs @@ -37,6 +37,7 @@ pub static DEFAULT_REWRITE_RULES: LazyLock> = LazyLock::new(|| { RuleID::PushDownFilterUnion, RuleID::PushDownFilterAggregate, RuleID::PushDownFilterWindow, + RuleID::PushDownFilterWindowRank, RuleID::PushDownFilterSort, RuleID::PushDownFilterEvalScalar, RuleID::PushDownFilterJoin, @@ -90,6 +91,7 @@ pub enum RuleID { PushDownFilterSort, PushDownFilterProjectSet, PushDownFilterWindow, + PushDownFilterWindowRank, PushDownLimit, PushDownLimitUnion, PushDownLimitOuterJoin, @@ -140,6 +142,7 @@ impl Display for RuleID { RuleID::PushDownSortEvalScalar => write!(f, "PushDownSortEvalScalar"), RuleID::PushDownLimitWindow => write!(f, "PushDownLimitWindow"), RuleID::PushDownFilterWindow => write!(f, "PushDownFilterWindow"), + RuleID::PushDownFilterWindowRank => write!(f, "PushDownFilterWindowRank"), RuleID::EliminateEvalScalar => write!(f, "EliminateEvalScalar"), RuleID::EliminateFilter => write!(f, "EliminateFilter"), RuleID::EliminateSort => write!(f, "EliminateSort"), diff --git a/src/query/sql/src/planner/plans/sort.rs b/src/query/sql/src/planner/plans/sort.rs index cc31a7ed5fbe..1f7a1536f287 100644 --- a/src/query/sql/src/planner/plans/sort.rs +++ b/src/query/sql/src/planner/plans/sort.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use super::WindowPartition; use crate::optimizer::Distribution; use crate::optimizer::PhysicalProperty; use crate::optimizer::RelExpr; @@ -25,7 +26,6 @@ use crate::optimizer::RequiredProperty; use crate::optimizer::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; -use crate::plans::ScalarItem; use crate::ColumnSet; use crate::IndexType; @@ -43,7 +43,7 @@ pub struct Sort { pub pre_projection: Option>, /// If sort is for window clause, we need the input to exchange by partitions - pub window_partition: Vec, + pub window_partition: Option, } impl Sort { @@ -54,11 +54,12 @@ impl Sort { pub fn sort_items_exclude_partition(&self) -> Vec { self.items .iter() - .filter(|item| { - !self - .window_partition + .filter(|item| match &self.window_partition { + Some(window) => !window + .partition_by .iter() - .any(|partition| partition.index == item.index) + .any(|partition| partition.index == item.index), + None => true, }) .cloned() .collect() @@ -79,14 +80,15 @@ impl Operator for Sort { fn derive_physical_prop(&self, rel_expr: &RelExpr) -> Result { let input_physical_prop = rel_expr.derive_physical_prop_child(0)?; - if input_physical_prop.distribution == Distribution::Serial - || self.window_partition.is_empty() - { + if input_physical_prop.distribution == Distribution::Serial { return Ok(input_physical_prop); } + let Some(window) = &self.window_partition else { + return Ok(input_physical_prop); + }; - let partition_by = self - .window_partition + let partition_by = window + .partition_by .iter() .map(|s| s.scalar.clone()) .collect(); @@ -105,9 +107,9 @@ impl Operator for Sort { let mut required = required.clone(); required.distribution = Distribution::Serial; - if self.window_partition.is_empty() { + let Some(window) = &self.window_partition else { return Ok(required); - } + }; let child_physical_prop = rel_expr.derive_physical_prop_child(0)?; // Can't merge to shuffle @@ -115,8 +117,8 @@ impl Operator for Sort { return Ok(required); } - let partition_by = self - .window_partition + let partition_by = window + .partition_by .iter() .map(|s| s.scalar.clone()) .collect(); @@ -134,9 +136,9 @@ impl Operator for Sort { let mut required = required.clone(); required.distribution = Distribution::Serial; - if self.window_partition.is_empty() { + let Some(window) = &self.window_partition else { return Ok(vec![vec![required]]); - } + }; // Can't merge to shuffle let child_physical_prop = rel_expr.derive_physical_prop_child(0)?; @@ -144,8 +146,8 @@ impl Operator for Sort { return Ok(vec![vec![required]]); } - let partition_by = self - .window_partition + let partition_by = window + .partition_by .iter() .map(|s| s.scalar.clone()) .collect(); @@ -163,13 +165,13 @@ impl Operator for Sort { // Derive orderings let orderings = self.items.clone(); - let (orderings, partition_orderings) = if !self.window_partition.is_empty() { - ( + + let (orderings, partition_orderings) = match &self.window_partition { + Some(window) => ( vec![], - Some((self.window_partition.clone(), orderings.clone())), - ) - } else { - (self.items.clone(), None) + Some((window.partition_by.clone(), orderings.clone())), + ), + None => (self.items.clone(), None), }; Ok(Arc::new(RelationalProperty { diff --git a/src/query/sql/src/planner/plans/window.rs b/src/query/sql/src/planner/plans/window.rs index da9746d377a8..b3fc44536ad7 100644 --- a/src/query/sql/src/planner/plans/window.rs +++ b/src/query/sql/src/planner/plans/window.rs @@ -283,3 +283,10 @@ impl WindowFuncType { } } } + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct WindowPartition { + pub partition_by: Vec, + pub top: Option, + pub func: WindowFuncType, +} diff --git a/tests/sqllogictests/suites/mode/standalone/explain/window.test b/tests/sqllogictests/suites/mode/standalone/explain/window.test index 971b3dd1dccc..37044436493e 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/window.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/window.test @@ -57,8 +57,8 @@ CompoundBlockOperator(Project) × 1 Merge to Resize × 4 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -79,8 +79,8 @@ CompoundBlockOperator(Project) × 1 Merge to Resize × 4 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -367,8 +367,8 @@ CompoundBlockOperator(Project) × 1 LimitTransform × 1 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -385,8 +385,8 @@ CompoundBlockOperator(Project) × 1 LimitTransform × 1 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -404,8 +404,8 @@ CompoundBlockOperator(Project) × 1 LimitTransform × 1 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -418,8 +418,8 @@ CompoundBlockOperator(Project) × 1 LimitTransform × 1 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -432,8 +432,8 @@ CompoundBlockOperator(Project) × 1 LimitTransform × 1 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -452,8 +452,8 @@ CompoundBlockOperator(Project) × 1 Merge to Resize × 4 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -471,8 +471,8 @@ CompoundBlockOperator(Project) × 1 Merge to Resize × 4 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 TransformFilter × 1 AddInternalColumnsTransform × 1 DeserializeDataTransform × 1 @@ -535,8 +535,8 @@ CompoundBlockOperator(Project) × 1 TransformFilter × 1 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(WindowTopN) × 1 + ShufflePartition(WindowTopN) × 1 DeserializeDataTransform × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 @@ -565,8 +565,8 @@ CompoundBlockOperator(Project) × 1 Transform Window × 1 Transform Window × 1 TransformWindowPartitionCollect × 1 - ShuffleMergePartition × 1 - ShufflePartition × 1 + ShuffleMergePartition(Window) × 1 + ShufflePartition(Window) × 1 CompoundBlockOperator(Map) × 1 NumbersSourceTransform × 1 diff --git a/tests/sqllogictests/suites/tpcds/spill.test b/tests/sqllogictests/suites/tpcds/spill.test index 366c85121b6d..de6c46c8f8cf 100644 --- a/tests/sqllogictests/suites/tpcds/spill.test +++ b/tests/sqllogictests/suites/tpcds/spill.test @@ -47,25 +47,25 @@ statement ok drop table if exists t; statement ok -set max_block_size = 65536; +unset max_block_size; statement ok -set join_spilling_memory_ratio = 60; +unset join_spilling_memory_ratio; statement ok -set join_spilling_bytes_threshold_per_proc = 0; +unset join_spilling_bytes_threshold_per_proc; statement ok -set join_spilling_buffer_threshold_per_proc_mb = 512; +unset join_spilling_buffer_threshold_per_proc_mb; statement ok -set sort_spilling_memory_ratio = 60; +unset sort_spilling_memory_ratio; statement ok -set sort_spilling_bytes_threshold_per_proc = 0; +unset sort_spilling_bytes_threshold_per_proc; statement ok -set window_partition_spilling_memory_ratio = 60; +unset window_partition_spilling_memory_ratio; statement ok -set window_partition_spilling_bytes_threshold_per_proc = 0; \ No newline at end of file +unset window_partition_spilling_bytes_threshold_per_proc; \ No newline at end of file From 65b5b3d673da99b4f162f090fc883a053f7a0d11 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 13 Nov 2024 10:47:37 +0800 Subject: [PATCH 28/92] chore(ci): test with different license (#16815) * chore(ci): test with different license * z * z * z * z --- .github/actions/build_linux/action.yml | 2 +- .github/actions/setup_license/action.yml | 12 ++-- .github/workflows/cloud.yml | 2 + .github/workflows/dev.yml | 1 + .github/workflows/main.yml | 1 + .github/workflows/merge_group.yml | 1 + .github/workflows/release.yml | 3 + .github/workflows/reuse.linux.hive.yml | 10 +-- .github/workflows/reuse.linux.yml | 82 ++++++++++++++---------- .github/workflows/reuse.sqllogic.yml | 7 ++ src/common/building/src/lib.rs | 10 +-- 11 files changed, 84 insertions(+), 47 deletions(-) diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml index b7ec2a862e7c..5b032f29b704 100644 --- a/.github/actions/build_linux/action.yml +++ b/.github/actions/build_linux/action.yml @@ -30,7 +30,7 @@ runs: uses: ./.github/actions/setup_build_tool with: target: ${{ inputs.target }} - bypass_env_vars: RUSTFLAGS,RUST_LOG + bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY - name: Cross setup if: startsWith(inputs.target, 'aarch64-') diff --git a/.github/actions/setup_license/action.yml b/.github/actions/setup_license/action.yml index 6ce6167fc6c4..3e72b31f4b62 100644 --- a/.github/actions/setup_license/action.yml +++ b/.github/actions/setup_license/action.yml @@ -5,6 +5,10 @@ inputs: description: "Self-hosted runner provider, aws or gcp" required: true default: "aws" + type: + description: "License type, enterprise or trial" + required: true + default: "trial" runs: using: "composite" @@ -13,15 +17,15 @@ runs: if: inputs.runner_provider == 'aws' shell: bash run: | - aws s3 cp s3://databend-ci/misc/license.key license.key - aws s3 cp s3://databend-ci/misc/license.json license.json + aws s3 cp s3://databend-ci/misc/license-${{ inputs.type }}.key license.key + aws s3 cp s3://databend-ci/misc/license-${{ inputs.type }}.json license.json - name: Get License from GCS if: inputs.runner_provider == 'gcp' shell: bash run: | - gcloud storage cp gs://databend-ci/misc/license.key license.key - gcloud storage cp gs://databend-ci/misc/license.json license.json + gcloud storage cp gs://databend-ci/misc/license-${{ inputs.type }}.key license.key + gcloud storage cp gs://databend-ci/misc/license-${{ inputs.type }}.json license.json - name: Output License shell: bash diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml index 6ae8f8364bd6..6bb7c37488be 100644 --- a/.github/workflows/cloud.yml +++ b/.github/workflows/cloud.yml @@ -59,6 +59,8 @@ jobs: - name: Build Release uses: ./.github/actions/build_linux timeout-minutes: 60 + env: + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ needs.info.outputs.sha }} target: ${{ matrix.arch }}-unknown-linux-gnu diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 948829c94004..bc8c519d0495 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -52,6 +52,7 @@ jobs: with: build_profile: release runner_provider: aws + license_type: trial linux_hive: needs: changes diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e22d76b029d2..fe1510711c19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,7 @@ jobs: with: build_profile: release runner_provider: aws + license_type: enterprise # macos: # needs: linux diff --git a/.github/workflows/merge_group.yml b/.github/workflows/merge_group.yml index bc69873db068..7fb73638f6df 100644 --- a/.github/workflows/merge_group.yml +++ b/.github/workflows/merge_group.yml @@ -56,6 +56,7 @@ jobs: with: build_profile: release runner_provider: aws + license_type: enterprise ready: if: always() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a08a02c1b58c..b956abaec2a8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -117,6 +117,7 @@ jobs: uses: ./.github/actions/build_linux env: DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.target }} @@ -149,6 +150,7 @@ jobs: uses: ./.github/actions/build_linux env: DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.target }} @@ -173,6 +175,7 @@ jobs: uses: ./.github/actions/build_linux env: DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.target }} diff --git a/.github/workflows/reuse.linux.hive.yml b/.github/workflows/reuse.linux.hive.yml index d91c8133195d..eeae250f4921 100644 --- a/.github/workflows/reuse.linux.hive.yml +++ b/.github/workflows/reuse.linux.hive.yml @@ -4,15 +4,15 @@ on: workflow_call: inputs: build_profile: - description: 'Build profile, debug or release' + description: "Build profile, debug or release" type: string required: true - default: 'debug' + default: "debug" runner_provider: - description: 'Self-hosted runner provider, aws or gcp' + description: "Self-hosted runner provider, aws or gcp" type: string required: true - default: 'aws' + default: "aws" env: BUILD_PROFILE: ${{ inputs.build_profile }} @@ -32,6 +32,8 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/build_linux timeout-minutes: 60 + env: + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }} diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml index 51695f781331..116c6c719995 100644 --- a/.github/workflows/reuse.linux.yml +++ b/.github/workflows/reuse.linux.yml @@ -13,6 +13,11 @@ on: type: string required: true default: "aws" + license_type: + description: "License type, enterprise or trial" + type: string + required: true + default: "trial" env: BUILD_PROFILE: ${{ inputs.build_profile }} @@ -20,7 +25,7 @@ env: jobs: check: - runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 with: @@ -50,6 +55,8 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/build_linux timeout-minutes: 60 + env: + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.arch }}-unknown-linux-gnu @@ -75,6 +82,8 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/build_linux timeout-minutes: 60 + env: + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.arch }}-unknown-linux-gnu @@ -106,7 +115,7 @@ jobs: # artifacts: query test_unit: - runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 with: @@ -116,75 +125,76 @@ jobs: timeout-minutes: 60 test_metactl: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_metactl timeout-minutes: 10 test_compat_meta_query: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_compat_meta_query timeout-minutes: 10 test_compat_fuse: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_compat_fuse timeout-minutes: 20 test_compat_meta_meta: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_compat_meta_meta timeout-minutes: 20 test_logs: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_logs timeout-minutes: 20 test_meta_cluster: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_meta_cluster timeout-minutes: 10 test_stateless_standalone: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateless_standalone_linux timeout-minutes: 15 test_stateless_cluster: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_stateless_cluster_linux timeout-minutes: 15 test_stateful_standalone: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateful_standalone_linux @@ -196,13 +206,14 @@ jobs: name: test-stateful-standalone-linux test_stateful_cluster: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_stateful_cluster_linux timeout-minutes: 15 - name: Upload failure @@ -213,16 +224,16 @@ jobs: test_stateful_large_data: if: contains(github.event.pull_request.labels.*.name, 'ci-largedata') - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateful_large_data timeout-minutes: 60 test_stateful_iceberg_rest: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateful_iceberg_rest_standalone @@ -243,13 +254,14 @@ jobs: # continue-on-error: true test_ee_standalone: - needs: [ build, check ] - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + needs: [build, check] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_ee_standalone_linux timeout-minutes: 10 - name: Upload failure @@ -259,14 +271,15 @@ jobs: name: test-stateful-standalone-linux test_ee_standalone_background: - needs: [ build, check ] - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + needs: [build, check] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_bendsql - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_ee_standalone_background_linux timeout-minutes: 10 - name: Upload failure @@ -288,6 +301,7 @@ jobs: # - uses: ./.github/actions/setup_license # with: # runner_provider: ${{ inputs.runner_provider }} + # type: ${{ inputs.license_type }} # - uses: ./.github/actions/test_ee_standalone_fake_time_linux # timeout-minutes: 10 # - name: Upload failure @@ -297,14 +311,15 @@ jobs: # name: test-stateful-standalone-fake-time-linux test_ee_management_mode: - needs: [ build, check ] - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + needs: [build, check] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_bendsql - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_ee_management_mode_linux timeout-minutes: 10 - name: Upload failure @@ -314,9 +329,10 @@ jobs: name: test-ee-management-mode-linux sqllogic: - needs: [ build, check ] + needs: [build, check] uses: ./.github/workflows/reuse.sqllogic.yml secrets: inherit with: build_profile: ${{ inputs.build_profile }} runner_provider: ${{ inputs.runner_provider }} + license_type: ${{ inputs.license_type }} diff --git a/.github/workflows/reuse.sqllogic.yml b/.github/workflows/reuse.sqllogic.yml index db3697bc5b56..ea56077fad83 100644 --- a/.github/workflows/reuse.sqllogic.yml +++ b/.github/workflows/reuse.sqllogic.yml @@ -13,6 +13,11 @@ on: type: string required: true default: "aws" + license_type: + description: "License type, enterprise or trial" + type: string + required: true + default: "trial" env: BUILD_PROFILE: ${{ inputs.build_profile }} @@ -196,6 +201,7 @@ jobs: - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_sqllogic_cluster_linux timeout-minutes: 15 with: @@ -266,6 +272,7 @@ jobs: - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_ee_sqllogic_standalone_linux timeout-minutes: 15 with: diff --git a/src/common/building/src/lib.rs b/src/common/building/src/lib.rs index 070381a7873c..70e2dacd281a 100644 --- a/src/common/building/src/lib.rs +++ b/src/common/building/src/lib.rs @@ -89,17 +89,17 @@ fn discover_version() -> Result { } } +pub fn add_env_license() { + let v = env::var("DATABEND_ENTERPRISE_LICENSE_EMBEDDED").unwrap_or_default(); + println!("cargo:rustc-env=DATABEND_ENTERPRISE_LICENSE_EMBEDDED={v}"); +} + pub fn add_license_public_key() { let v = env::var("DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY").unwrap_or_default(); let v = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, v.as_bytes()); println!("cargo:rustc-env=DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY={v}"); } -pub fn add_env_license() { - let v = env::var("DATABEND_ENTERPRISE_LICENSE_EMBEDDED").unwrap_or_default(); - println!("cargo:rustc-env=DATABEND_ENTERPRISE_LICENSE_EMBEDDED={v}"); -} - pub fn add_env_commit_authors(repo: &Repository) { match git::get_commit_authors(repo) { Ok(authors) => println!("cargo:rustc-env=DATABEND_COMMIT_AUTHORS={}", authors), From 7abfa9616e84d047bfcb72821c15df46acf06a48 Mon Sep 17 00:00:00 2001 From: TCeason <33082201+TCeason@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:18:33 +0800 Subject: [PATCH 29/92] feat(query): support SHOW DROP DATABASES (#16811) add system.databases_with_history --- src/meta/api/src/schema_api_impl.rs | 1 - src/query/ast/src/ast/statements/database.rs | 20 + src/query/ast/src/ast/statements/statement.rs | 2 + src/query/ast/src/ast/statements/table.rs | 2 +- src/query/ast/src/parser/statement.rs | 14 + src/query/ast/tests/it/parser.rs | 2 + src/query/ast/tests/it/testdata/stmt.txt | 36 +- src/query/catalog/src/catalog/interface.rs | 3 + .../src/catalogs/default/database_catalog.rs | 11 + .../src/catalogs/default/immutable_catalog.rs | 4 + .../src/catalogs/default/mutable_catalog.rs | 27 + .../src/catalogs/default/session_catalog.rs | 5 + .../src/databases/system/system_database.rs | 6 +- .../access/management_mode_access.rs | 1 + .../interpreters/access/privilege_access.rs | 4 +- .../tests/it/sql/exec/get_table_bind_test.rs | 4 + .../it/storages/fuse/operations/commit.rs | 4 + src/query/service/tests/it/storages/system.rs | 23 +- .../it/storages/testdata/columns_table.txt | 1016 +++++++++-------- .../it/storages/testdata/databases_table.txt | 14 +- .../testdata/databases_with_history_table.txt | 12 + src/query/sql/src/planner/binder/binder.rs | 1 + .../sql/src/planner/binder/ddl/database.rs | 42 + src/query/sql/src/planner/plans/plan.rs | 1 + .../storages/hive/hive/src/hive_catalog.rs | 5 + src/query/storages/iceberg/src/catalog.rs | 4 + .../storages/system/src/databases_table.rs | 193 +++- src/query/storages/system/src/lib.rs | 2 + .../base/06_show/06_0009_show_databases.test | 5 + .../18_rbac/18_0003_db_visibility.result | 4 + .../18_rbac/18_0003_db_visibility.sh | 6 + 31 files changed, 906 insertions(+), 568 deletions(-) create mode 100644 src/query/service/tests/it/storages/testdata/databases_with_history_table.txt diff --git a/src/meta/api/src/schema_api_impl.rs b/src/meta/api/src/schema_api_impl.rs index 814ddc85d2ca..6070e4cac9aa 100644 --- a/src/meta/api/src/schema_api_impl.rs +++ b/src/meta/api/src/schema_api_impl.rs @@ -669,7 +669,6 @@ impl + ?Sized> SchemaApi for KV { name_ident: DatabaseNameIdent::new_from(db_id_list_key.clone()), meta: db_meta, }; - dbs.insert(db_id.db_id, Arc::new(db)); } } diff --git a/src/query/ast/src/ast/statements/database.rs b/src/query/ast/src/ast/statements/database.rs index 8541dd95c0fb..9f3808d99dce 100644 --- a/src/query/ast/src/ast/statements/database.rs +++ b/src/query/ast/src/ast/statements/database.rs @@ -49,6 +49,26 @@ impl Display for ShowDatabasesStmt { } } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct ShowDropDatabasesStmt { + pub catalog: Option, + pub limit: Option, +} + +impl Display for ShowDropDatabasesStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "SHOW DROP DATABASES")?; + if let Some(catalog) = &self.catalog { + write!(f, " FROM {catalog}")?; + } + if let Some(limit) = &self.limit { + write!(f, " {limit}")?; + } + + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub struct ShowCreateDatabaseStmt { pub catalog: Option, diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index 6ccf682ddc0f..0c520fbd0b44 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -127,6 +127,7 @@ pub enum Statement { // Databases ShowDatabases(ShowDatabasesStmt), + ShowDropDatabases(ShowDropDatabasesStmt), ShowCreateDatabase(ShowCreateDatabaseStmt), CreateDatabase(CreateDatabaseStmt), DropDatabase(DropDatabaseStmt), @@ -548,6 +549,7 @@ impl Display for Statement { Statement::CreateCatalog(stmt) => write!(f, "{stmt}")?, Statement::DropCatalog(stmt) => write!(f, "{stmt}")?, Statement::ShowDatabases(stmt) => write!(f, "{stmt}")?, + Statement::ShowDropDatabases(stmt) => write!(f, "{stmt}")?, Statement::ShowCreateDatabase(stmt) => write!(f, "{stmt}")?, Statement::CreateDatabase(stmt) => write!(f, "{stmt}")?, Statement::DropDatabase(stmt) => write!(f, "{stmt}")?, diff --git a/src/query/ast/src/ast/statements/table.rs b/src/query/ast/src/ast/statements/table.rs index b37989c27993..57792449eebe 100644 --- a/src/query/ast/src/ast/statements/table.rs +++ b/src/query/ast/src/ast/statements/table.rs @@ -116,7 +116,7 @@ pub struct ShowDropTablesStmt { impl Display for ShowDropTablesStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "SHOW DROP TABLE")?; + write!(f, "SHOW DROP TABLES")?; if let Some(database) = &self.database { write!(f, " FROM {database}")?; } diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 0c581bfe5046..1705f94753fe 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -529,6 +529,19 @@ pub fn statement_body(i: Input) -> IResult { }) }, ); + + let show_drop_databases = map( + rule! { + SHOW ~ DROP ~ ( DATABASES | DATABASES ) ~ ( FROM ~ ^#ident )? ~ #show_limit? + }, + |(_, _, _, opt_catalog, limit)| { + Statement::ShowDropDatabases(ShowDropDatabasesStmt { + catalog: opt_catalog.map(|(_, catalog)| catalog), + limit, + }) + }, + ); + let show_create_database = map( rule! { SHOW ~ CREATE ~ ( DATABASE | SCHEMA ) ~ #dot_separated_idents_1_to_2 @@ -2284,6 +2297,7 @@ pub fn statement_body(i: Input) -> IResult { #show_databases : "`SHOW [FULL] DATABASES [(FROM | IN) ] []`" | #undrop_database : "`UNDROP DATABASE `" | #show_create_database : "`SHOW CREATE DATABASE `" + | #show_drop_databases : "`SHOW DROP DATABASES [FROM ] []`" | #create_database : "`CREATE [OR REPLACE] DATABASE [IF NOT EXISTS] [ENGINE = ]`" | #drop_database : "`DROP DATABASE [IF EXISTS] `" | #alter_database : "`ALTER DATABASE [IF EXISTS] `" diff --git a/src/query/ast/tests/it/parser.rs b/src/query/ast/tests/it/parser.rs index e249ca198fa5..9a536b2c2ee4 100644 --- a/src/query/ast/tests/it/parser.rs +++ b/src/query/ast/tests/it/parser.rs @@ -87,6 +87,8 @@ fn test_statement() { let file = &mut mint.new_goldenfile("stmt.txt").unwrap(); let cases = &[ r#"show databases"#, + r#"show drop databases"#, + r#"show drop databases like 'db%'"#, r#"show databases format TabSeparatedWithNamesAndTypes;"#, r#"show tables"#, r#"show drop tables"#, diff --git a/src/query/ast/tests/it/testdata/stmt.txt b/src/query/ast/tests/it/testdata/stmt.txt index 27869c0c54d2..8bd0d0d7a129 100644 --- a/src/query/ast/tests/it/testdata/stmt.txt +++ b/src/query/ast/tests/it/testdata/stmt.txt @@ -12,6 +12,36 @@ ShowDatabases( ) +---------- Input ---------- +show drop databases +---------- Output --------- +SHOW DROP DATABASES +---------- AST ------------ +ShowDropDatabases( + ShowDropDatabasesStmt { + catalog: None, + limit: None, + }, +) + + +---------- Input ---------- +show drop databases like 'db%' +---------- Output --------- +SHOW DROP DATABASES LIKE 'db%' +---------- AST ------------ +ShowDropDatabases( + ShowDropDatabasesStmt { + catalog: None, + limit: Some( + Like { + pattern: "db%", + }, + ), + }, +) + + ---------- Input ---------- show databases format TabSeparatedWithNamesAndTypes; ---------- Output --------- @@ -49,7 +79,7 @@ ShowTables( ---------- Input ---------- show drop tables ---------- Output --------- -SHOW DROP TABLE +SHOW DROP TABLES ---------- AST ------------ ShowDropTables( ShowDropTablesStmt { @@ -62,7 +92,7 @@ ShowDropTables( ---------- Input ---------- show drop tables like 't%' ---------- Output --------- -SHOW DROP TABLE LIKE 't%' +SHOW DROP TABLES LIKE 't%' ---------- AST ------------ ShowDropTables( ShowDropTablesStmt { @@ -79,7 +109,7 @@ ShowDropTables( ---------- Input ---------- show drop tables where name='t' ---------- Output --------- -SHOW DROP TABLE WHERE name = 't' +SHOW DROP TABLES WHERE name = 't' ---------- AST ------------ ShowDropTables( ShowDropTablesStmt { diff --git a/src/query/catalog/src/catalog/interface.rs b/src/query/catalog/src/catalog/interface.rs index 935c3706a9fc..4b2af1165132 100644 --- a/src/query/catalog/src/catalog/interface.rs +++ b/src/query/catalog/src/catalog/interface.rs @@ -152,6 +152,9 @@ pub trait Catalog: DynClone + Send + Sync + Debug { // Get the database by name. async fn get_database(&self, tenant: &Tenant, db_name: &str) -> Result>; + // List all databases history + async fn list_databases_history(&self, tenant: &Tenant) -> Result>>; + // Get all the databases. async fn list_databases(&self, tenant: &Tenant) -> Result>>; diff --git a/src/query/service/src/catalogs/default/database_catalog.rs b/src/query/service/src/catalogs/default/database_catalog.rs index b1ecc39360e4..b467d9939532 100644 --- a/src/query/service/src/catalogs/default/database_catalog.rs +++ b/src/query/service/src/catalogs/default/database_catalog.rs @@ -187,6 +187,17 @@ impl Catalog for DatabaseCatalog { } } + #[async_backtrace::framed] + async fn list_databases_history(&self, tenant: &Tenant) -> Result>> { + let mut dbs = self + .immutable_catalog + .list_databases_history(tenant) + .await?; + let mut other = self.mutable_catalog.list_databases_history(tenant).await?; + dbs.append(&mut other); + Ok(dbs) + } + #[async_backtrace::framed] async fn list_databases(&self, tenant: &Tenant) -> Result>> { let mut dbs = self.immutable_catalog.list_databases(tenant).await?; diff --git a/src/query/service/src/catalogs/default/immutable_catalog.rs b/src/query/service/src/catalogs/default/immutable_catalog.rs index cfb88afa8ad8..ac3cf23d1b85 100644 --- a/src/query/service/src/catalogs/default/immutable_catalog.rs +++ b/src/query/service/src/catalogs/default/immutable_catalog.rs @@ -170,6 +170,10 @@ impl Catalog for ImmutableCatalog { } } + async fn list_databases_history(&self, _tenant: &Tenant) -> Result>> { + Ok(vec![self.sys_db.clone(), self.info_schema_db.clone()]) + } + #[async_backtrace::framed] async fn list_databases(&self, _tenant: &Tenant) -> Result>> { Ok(vec![self.sys_db.clone(), self.info_schema_db.clone()]) diff --git a/src/query/service/src/catalogs/default/mutable_catalog.rs b/src/query/service/src/catalogs/default/mutable_catalog.rs index 07ae212bbdc2..4db4ef1e160c 100644 --- a/src/query/service/src/catalogs/default/mutable_catalog.rs +++ b/src/query/service/src/catalogs/default/mutable_catalog.rs @@ -243,6 +243,33 @@ impl Catalog for MutableCatalog { self.build_db_instance(&db_info) } + #[async_backtrace::framed] + async fn list_databases_history(&self, tenant: &Tenant) -> Result>> { + let dbs = self + .ctx + .meta + .get_tenant_history_databases( + ListDatabaseReq { + tenant: tenant.clone(), + }, + false, + ) + .await?; + + dbs.iter() + .try_fold(vec![], |mut acc, item: &Arc| { + let db_result = self.build_db_instance(item); + match db_result { + Ok(db) => acc.push(db), + Err(err) => { + // Ignore the error and continue, allow partial failure. + warn!("Failed to build database '{:?}': {:?}", item, err); + } + } + Ok(acc) + }) + } + #[async_backtrace::framed] async fn list_databases(&self, tenant: &Tenant) -> Result>> { let dbs = self diff --git a/src/query/service/src/catalogs/default/session_catalog.rs b/src/query/service/src/catalogs/default/session_catalog.rs index 7565b800c2f5..5ee189017537 100644 --- a/src/query/service/src/catalogs/default/session_catalog.rs +++ b/src/query/service/src/catalogs/default/session_catalog.rs @@ -153,6 +153,11 @@ impl Catalog for SessionCatalog { self.inner.get_database(tenant, db_name).await } + // List all the databases history. + async fn list_databases_history(&self, tenant: &Tenant) -> Result>> { + self.inner.list_databases_history(tenant).await + } + // Get all the databases. async fn list_databases(&self, tenant: &Tenant) -> Result>> { self.inner.list_databases(tenant).await diff --git a/src/query/service/src/databases/system/system_database.rs b/src/query/service/src/databases/system/system_database.rs index a4ad5c7d36d7..ccb6c693662d 100644 --- a/src/query/service/src/databases/system/system_database.rs +++ b/src/query/service/src/databases/system/system_database.rs @@ -34,7 +34,8 @@ use databend_common_storages_system::ColumnsTable; use databend_common_storages_system::ConfigsTable; use databend_common_storages_system::ContributorsTable; use databend_common_storages_system::CreditsTable; -use databend_common_storages_system::DatabasesTable; +use databend_common_storages_system::DatabasesTableWithHistory; +use databend_common_storages_system::DatabasesTableWithoutHistory; use databend_common_storages_system::DictionariesTable; use databend_common_storages_system::EnginesTable; use databend_common_storages_system::FullStreamsTable; @@ -103,7 +104,8 @@ impl SystemDatabase { TablesTableWithoutHistory::create(sys_db_meta.next_table_id()), TablesTableWithHistory::create(sys_db_meta.next_table_id()), ClustersTable::create(sys_db_meta.next_table_id()), - DatabasesTable::create(sys_db_meta.next_table_id()), + DatabasesTableWithHistory::create(sys_db_meta.next_table_id()), + DatabasesTableWithoutHistory::create(sys_db_meta.next_table_id()), FullStreamsTable::create(sys_db_meta.next_table_id()), TerseStreamsTable::create(sys_db_meta.next_table_id()), ProcessesTable::create(sys_db_meta.next_table_id()), diff --git a/src/query/service/src/interpreters/access/management_mode_access.rs b/src/query/service/src/interpreters/access/management_mode_access.rs index 09b6243180ae..22600b0383e9 100644 --- a/src/query/service/src/interpreters/access/management_mode_access.rs +++ b/src/query/service/src/interpreters/access/management_mode_access.rs @@ -45,6 +45,7 @@ impl AccessChecker for ManagementModeAccess { match rewrite_kind { Some(ref v) => matches!(v, RewriteKind::ShowDatabases + | RewriteKind::ShowDropDatabases | RewriteKind::ShowTables(_, _) | RewriteKind::ShowColumns(_, _, _) | RewriteKind::ShowEngines diff --git a/src/query/service/src/interpreters/access/privilege_access.rs b/src/query/service/src/interpreters/access/privilege_access.rs index e5ed82af8dc8..95d538266d3e 100644 --- a/src/query/service/src/interpreters/access/privilege_access.rs +++ b/src/query/service/src/interpreters/access/privilege_access.rs @@ -61,10 +61,11 @@ enum ObjectId { // some statements like `SELECT 1`, `SHOW USERS`, `SHOW ROLES`, `SHOW TABLES` will be // rewritten to the queries on the system tables, we need to skip the privilege check on // these tables. -const SYSTEM_TABLES_ALLOW_LIST: [&str; 20] = [ +const SYSTEM_TABLES_ALLOW_LIST: [&str; 21] = [ "catalogs", "columns", "databases", + "databases_with_history", "dictionaries", "tables", "views", @@ -708,6 +709,7 @@ impl AccessChecker for PrivilegeAccess { } => { match rewrite_kind { Some(RewriteKind::ShowDatabases) + | Some(RewriteKind::ShowDropDatabases) | Some(RewriteKind::ShowEngines) | Some(RewriteKind::ShowFunctions) | Some(RewriteKind::ShowUserFunctions) diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index a1cc8fce6f38..3cb3b651c17a 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -174,6 +174,10 @@ impl Catalog for FakedCatalog { todo!() } + async fn list_databases_history(&self, _tenant: &Tenant) -> Result>> { + todo!() + } + async fn list_databases(&self, _tenant: &Tenant) -> Result>> { todo!() } diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index fe5b5e69a207..ddceed7daac3 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -918,6 +918,10 @@ impl Catalog for FakedCatalog { todo!() } + async fn list_databases_history(&self, _tenant: &Tenant) -> Result>> { + todo!() + } + async fn create_database(&self, _req: CreateDatabaseReq) -> Result { todo!() } diff --git a/src/query/service/tests/it/storages/system.rs b/src/query/service/tests/it/storages/system.rs index b59ad6b13593..f556c12b4fe6 100644 --- a/src/query/service/tests/it/storages/system.rs +++ b/src/query/service/tests/it/storages/system.rs @@ -37,7 +37,8 @@ use databend_common_storages_system::ColumnsTable; use databend_common_storages_system::ConfigsTable; use databend_common_storages_system::ContributorsTable; use databend_common_storages_system::CreditsTable; -use databend_common_storages_system::DatabasesTable; +use databend_common_storages_system::DatabasesTableWithHistory; +use databend_common_storages_system::DatabasesTableWithoutHistory; use databend_common_storages_system::EnginesTable; use databend_common_storages_system::FunctionsTable; use databend_common_storages_system::MetricsTable; @@ -261,7 +262,7 @@ async fn test_databases_table() -> Result<()> { let fixture = TestFixture::setup_with_config(&config).await?; let ctx = fixture.new_query_ctx().await?; - let table = DatabasesTable::create(1); + let table = DatabasesTableWithoutHistory::create(1); let mut mint = Mint::new("tests/it/storages/testdata"); let file = &mut mint.new_goldenfile("databases_table.txt").unwrap(); @@ -270,6 +271,24 @@ async fn test_databases_table() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_databases_history_table() -> Result<()> { + let mut config = ConfigBuilder::create().build(); + config.storage.params = StorageParams::Fs(StorageFsConfig::default()); + let fixture = TestFixture::setup_with_config(&config).await?; + let ctx = fixture.new_query_ctx().await?; + + let table = DatabasesTableWithHistory::create(1); + + let mut mint = Mint::new("tests/it/storages/testdata"); + let file = &mut mint + .new_goldenfile("databases_with_history_table.txt") + .unwrap(); + run_table_tests(file, ctx, table).await?; + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn test_engines_table() -> Result<()> { let fixture = TestFixture::setup().await?; diff --git a/src/query/service/tests/it/storages/testdata/columns_table.txt b/src/query/service/tests/it/storages/testdata/columns_table.txt index e217a3fc0d65..b18e7992fae0 100644 --- a/src/query/service/tests/it/storages/testdata/columns_table.txt +++ b/src/query/service/tests/it/storages/testdata/columns_table.txt @@ -1,510 +1,516 @@ ---------- TABLE INFO ------------ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemColumns -------- TABLE CONTENTS ---------- -+-----------------------------------+----------------------+------------------------+-----------------------+---------------------+----------+----------+----------+----------+ -| Column 0 | Column 1 | Column 2 | Column 3 | Column 4 | Column 5 | Column 6 | Column 7 | Column 8 | -+-----------------------------------+----------------------+------------------------+-----------------------+---------------------+----------+----------+----------+----------+ -| 'Comment' | 'system' | 'engines' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'Engine' | 'system' | 'engines' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'access' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'acquired_on' | 'system' | 'locks' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'active_result_scan' | 'system' | 'query_cache' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | -| 'after' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'agg_spilled_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'agg_spilled_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'arguments' | 'system' | 'procedures' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'arguments' | 'system' | 'user_functions' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | -| 'attempt_number' | 'system' | 'task_history' | 'Int32' | 'INT' | '' | '' | 'NO' | '' | -| 'attribute_names' | 'system' | 'dictionaries' | 'Array(String)' | 'ARRAY(STRING)' | '' | '' | 'NO' | '' | -| 'attribute_types' | 'system' | 'dictionaries' | 'Array(String)' | 'ARRAY(STRING)' | '' | '' | 'NO' | '' | -| 'auth_type' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'auto_increment' | 'information_schema' | 'tables' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'byte_size' | 'system' | 'clustering_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'bytes_from_local_disk' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'bytes_from_memory' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'bytes_from_remote_disk' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'capacity' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'cardinality' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'cargo_features' | 'system' | 'build_options' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'catalog' | 'system' | 'databases' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'catalog' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'catalog' | 'system' | 'streams_terse' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'catalog' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'catalog' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'catalog' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'catalog' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'catalog_name' | 'information_schema' | 'schemata' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'character_maximum_length' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'character_octet_length' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'character_set_catalog' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'character_set_name' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'character_set_schema' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'check_option' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'client_address' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'client_info' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'cluster' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'cluster_by' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'cluster_by' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'cluster_id' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'collation' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'collation_catalog' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'collation_name' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'collation_schema' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'column_comment' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'column_default' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'column_key' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'column_name' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'column_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'column_name' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'column_type' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'columns' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'command' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'dictionaries' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'notifications' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'comment' | 'system' | 'password_policies' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'procedures' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'comment' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'comment' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'comment' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'compaction_stats' | 'system' | 'background_tasks' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | -| 'completed_time' | 'system' | 'task_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'condition_text' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'condition_text' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'constraint_catalog' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'constraint_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'constraint_schema' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'copy_options' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'cpu_usage' | 'system' | 'query_log' | 'UInt32' | 'INT UNSIGNED' | '' | '' | 'NO' | '' | -| 'create_time' | 'information_schema' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'background_jobs' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'background_tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'dictionaries' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'indexes' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'locks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'notification_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'notifications' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'password_policies' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'procedures' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'roles' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'stages' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'streams' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'tables_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'user_functions' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'users' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'created_on' | 'system' | 'views' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'views_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_on' | 'system' | 'virtual_columns' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'created_time' | 'system' | 'processes' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'creator' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'creator' | 'system' | 'background_tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'creator' | 'system' | 'stages' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'current_database' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'current_query_id' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'data_compressed_size' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'data_compressed_size' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'data_free' | 'information_schema' | 'tables' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'data_length' | 'information_schema' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'data_read_bytes' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'data_size' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'data_size' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'data_type' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'data_type' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'data_write_bytes' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'clustering_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'dictionaries' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'streams_terse' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'temporary_tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'database_id' | 'system' | 'background_tasks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'database_id' | 'system' | 'databases' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'databases' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'datetime_precision' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'default' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'default' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'default_character_set_catalog' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'default_character_set_name' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'default_character_set_schema' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'default_collation_name' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'default_expression' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'default_kind' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'default_role' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'definition' | 'system' | 'indexes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'definition' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'definition' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'definition' | 'system' | 'user_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'description' | 'system' | 'configs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'description' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'description' | 'system' | 'procedures' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'description' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'description' | 'system' | 'user_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'disabled' | 'system' | 'users' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | -| 'domain_catalog' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'domain_name' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'domain_schema' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'drop_time' | 'information_schema' | 'tables' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'dropped_on' | 'system' | 'tables' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'dropped_on' | 'system' | 'tables_with_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'dropped_on' | 'system' | 'views' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'dropped_on' | 'system' | 'views_with_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'dummy' | 'system' | 'one' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'enabled' | 'system' | 'notifications' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | -| 'end_time' | 'system' | 'clustering_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'engine' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine' | 'system' | 'temporary_tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine_full' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine_full' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine_full' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'engine_full' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'error_integration' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'error_message' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'errors' | 'system' | 'queries_profiling' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | -| 'event_date' | 'system' | 'query_log' | 'Date' | 'DATE' | '' | '' | 'NO' | '' | -| 'event_time' | 'system' | 'query_log' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'example' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'exception_code' | 'system' | 'query_log' | 'Int32' | 'INT' | '' | '' | 'NO' | '' | -| 'exception_code' | 'system' | 'task_history' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | -| 'exception_text' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'exception_text' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'extra' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'extra' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'extra_info' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'extra_info' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'file_content_length' | 'system' | 'temp_files' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'file_format_options' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'file_last_modified_time' | 'system' | 'temp_files' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'file_name' | 'system' | 'temp_files' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'file_type' | 'system' | 'temp_files' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'group' | 'system' | 'configs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'group_by_spilled_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'group_by_spilled_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'handler_type' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'has_profile' | 'system' | 'query_log' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | -| 'hit' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'host' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'host' | 'system' | 'processes' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'hostname' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'id' | 'system' | 'background_tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'id' | 'system' | 'notifications' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'id' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'id' | 'system' | 'task_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'id' | 'system' | 'tasks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'index_comment' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'index_length' | 'information_schema' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'index_name' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'index_schema' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'index_size' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'index_size' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'index_type' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'inherited_roles' | 'system' | 'roles' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'inherited_roles_name' | 'system' | 'roles' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'integration_name' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'invalid_reason' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'is_aggregate' | 'system' | 'functions' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | -| 'is_aggregate' | 'system' | 'user_functions' | 'Nullable(Boolean)' | 'BOOLEAN' | '' | '' | 'YES' | '' | -| 'is_attach' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'is_attach' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'is_configured' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'is_insertable_into' | 'information_schema' | 'views' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | -| 'is_nullable' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'is_nullable' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'is_transient' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'is_transient' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'is_trigger_deletable' | 'information_schema' | 'views' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'is_trigger_insertable_into' | 'information_schema' | 'views' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'is_trigger_updatable' | 'information_schema' | 'views' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'is_updatable' | 'information_schema' | 'views' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'job_state' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'job_type' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'join_spilled_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'join_spilled_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'key_names' | 'system' | 'dictionaries' | 'Array(String)' | 'ARRAY(STRING)' | '' | '' | 'NO' | '' | -| 'key_types' | 'system' | 'dictionaries' | 'Array(String)' | 'ARRAY(STRING)' | '' | '' | 'NO' | '' | -| 'keywords' | 'information_schema' | 'keywords' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'kind' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'labels' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'language' | 'system' | 'user_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'last_committed_on' | 'system' | 'tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'last_suspended_on' | 'system' | 'tasks' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'last_task_id' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'last_task_run_at' | 'system' | 'background_jobs' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'last_updated' | 'system' | 'background_jobs' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'level' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'license' | 'system' | 'credits' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'location' | 'system' | 'query_cache' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'log_type' | 'system' | 'query_log' | 'Int8' | 'TINYINT' | '' | '' | 'NO' | '' | -| 'log_type_name' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'memory_usage' | 'system' | 'processes' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | -| 'memory_usage' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'message' | 'system' | 'background_jobs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'message' | 'system' | 'background_tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'message' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'message_source' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'metric' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'miss' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'mode' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'mode' | 'system' | 'streams_terse' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'must_change_password' | 'system' | 'users' | 'Nullable(Boolean)' | 'BOOLEAN' | '' | '' | 'YES' | '' | -| 'mysql_connection_id' | 'system' | 'processes' | 'Nullable(UInt32)' | 'INT UNSIGNED' | '' | '' | 'YES' | '' | -| 'name' | 'system' | 'background_jobs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'caches' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'catalogs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'configs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'contributors' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'credits' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'databases' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'dictionaries' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'indexes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'malloc_stats_totals' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'notifications' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'password_policies' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'procedures' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'roles' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'streams_terse' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'table_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'temporary_tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'user_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'name' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'network_policy' | 'system' | 'users' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'next_schedule_time' | 'system' | 'tasks' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'next_task_scheduled_time' | 'system' | 'background_jobs' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'node' | 'system' | 'backtrace' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'node' | 'system' | 'caches' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'node' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'node' | 'system' | 'malloc_stats_totals' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'node' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'node' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'node' | 'system' | 'queries_profiling' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'node_id' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'non_unique' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'nullable' | 'information_schema' | 'columns' | 'Nullable(UInt8)' | 'TINYINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'nullable' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'num_items' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'num_rows' | 'system' | 'query_cache' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'num_rows' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'num_rows' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'number_of_blocks' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'number_of_blocks' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'number_of_files' | 'system' | 'stages' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'number_of_segments' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'number_of_segments' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'numeric_precision' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'numeric_precision_radix' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'numeric_scale' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'options' | 'system' | 'password_policies' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'ordinal_position' | 'information_schema' | 'columns' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'ordinal_position' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'original' | 'system' | 'indexes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'owner' | 'system' | 'databases' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'owner' | 'system' | 'stages' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'owner' | 'system' | 'streams' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'owner' | 'system' | 'tables' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'owner' | 'system' | 'tables_with_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'owner' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'owner' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'owner' | 'system' | 'views' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'owner' | 'system' | 'views_with_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'packed' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'parent_plan_id' | 'system' | 'queries_profiling' | 'Nullable(UInt32)' | 'INT UNSIGNED' | '' | '' | 'YES' | '' | -| 'partitions_sha' | 'system' | 'query_cache' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'password_policy' | 'system' | 'users' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'plan_id' | 'system' | 'queries_profiling' | 'Nullable(UInt32)' | 'INT UNSIGNED' | '' | '' | 'YES' | '' | -| 'plan_name' | 'system' | 'queries_profiling' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'port' | 'system' | 'clusters' | 'UInt16' | 'SMALLINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'position_in_unique_constraint' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'privileges' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'procedure_id' | 'system' | 'procedures' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'processed' | 'system' | 'notification_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'projections' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_duration_ms' | 'system' | 'query_log' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | -| 'query_hash' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_id' | 'system' | 'backtrace' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_id' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_id' | 'system' | 'queries_profiling' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_id' | 'system' | 'query_cache' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_id' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_id' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_kind' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_parameterized_hash' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'query_queued_duration_ms' | 'system' | 'query_log' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | -| 'query_start_time' | 'system' | 'query_log' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'query_text' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'range' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'referenced_column_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'referenced_table_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'referenced_table_schema' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'reserved' | 'information_schema' | 'keywords' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'result_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'result_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'result_size' | 'system' | 'query_cache' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'revision' | 'system' | 'locks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'roles' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'root_task_id' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'row_count' | 'system' | 'clustering_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'run_id' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'scan_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'scan_io_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'scan_io_bytes_cost_ms' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'scan_partitions' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'scan_progress_read_bytes' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'scan_progress_read_rows' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'scan_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'schedule' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'schedule' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'scheduled_job_cron_expression' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'scheduled_job_cron_timezone' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'scheduled_job_interval_secs' | 'system' | 'background_jobs' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'scheduled_time' | 'system' | 'task_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'schema_name' | 'information_schema' | 'schemata' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'schema_owner' | 'information_schema' | 'schemata' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'seq_in_index' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'server_version' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'session_parameters' | 'system' | 'task_history' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | -| 'session_parameters' | 'system' | 'tasks' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | -| 'session_settings' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'size' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'snapshot_location' | 'system' | 'streams' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'source' | 'system' | 'dictionaries' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'sql' | 'system' | 'query_cache' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'sql_path' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'sql_user' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'sql_user_privileges' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'sql_user_quota' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'stack' | 'system' | 'backtrace' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'stack_trace' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'stage_params' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'stage_type' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'start_time' | 'system' | 'clustering_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'state' | 'system' | 'background_tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'state' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'state' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'statistics' | 'system' | 'malloc_stats' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | -| 'statistics' | 'system' | 'queries_profiling' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | -| 'status' | 'system' | 'backtrace' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'status' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'status' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'status' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'stream_id' | 'system' | 'streams' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'sub_part' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'suspend_task_after_num_failures' | 'system' | 'tasks' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'syntax' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table' | 'system' | 'clustering_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_catalog' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_catalog' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'table_catalog' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'table_catalog' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_catalog' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_collation' | 'information_schema' | 'tables' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'table_comment' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_id' | 'system' | 'background_tasks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'table_id' | 'system' | 'locks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'table_id' | 'system' | 'streams' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'table_id' | 'system' | 'tables' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'table_id' | 'system' | 'tables_with_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'table_id' | 'system' | 'temporary_tables' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'table_id' | 'system' | 'views' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'table_id' | 'system' | 'views_with_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'table_name' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'table_name' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'table_name' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_name' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_name' | 'system' | 'streams' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'table_name' | 'system' | 'streams_terse' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'table_rows' | 'information_schema' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'table_schema' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_schema' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'table_schema' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'table_schema' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_schema' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_type' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_type' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_type' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'table_version' | 'system' | 'streams' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'tables' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'target_features' | 'system' | 'build_options' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'task_running_secs' | 'system' | 'background_tasks' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | -| 'task_type' | 'system' | 'background_jobs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'tenant_id' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'time' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'total_columns' | 'system' | 'tables' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'total_columns' | 'system' | 'tables_with_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'total_partitions' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'trigger' | 'system' | 'background_tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'type' | 'system' | 'background_tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'type' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'type' | 'system' | 'indexes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'type' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'type' | 'system' | 'notifications' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'type' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'type' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'unit' | 'system' | 'caches' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'update_on' | 'system' | 'roles' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'update_on' | 'system' | 'users' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'updated_on' | 'system' | 'background_tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'updated_on' | 'system' | 'dictionaries' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'updated_on' | 'system' | 'indexes' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'updated_on' | 'system' | 'password_policies' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'updated_on' | 'system' | 'streams' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'updated_on' | 'system' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'updated_on' | 'system' | 'tables_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'updated_on' | 'system' | 'views' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'updated_on' | 'system' | 'views_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | -| 'updated_on' | 'system' | 'virtual_columns' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | -| 'user' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'user' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'user_agent' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'vacuum_stats' | 'system' | 'background_tasks' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | -| 'value' | 'system' | 'configs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'value' | 'system' | 'malloc_stats_totals' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'value' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'value' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'version' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'version' | 'system' | 'credits' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'view_definition' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'view_query' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'view_query' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'virtual_columns' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | -| 'warehouse' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'warehouse' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | -| 'webhook_options' | 'system' | 'notifications' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | -| 'written_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'written_io_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'written_io_bytes_cost_ms' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -| 'written_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | -+-----------------------------------+----------------------+------------------------+-----------------------+---------------------+----------+----------+----------+----------+ ++-----------------------------------+----------------------+--------------------------+-----------------------+---------------------+----------+----------+----------+----------+ +| Column 0 | Column 1 | Column 2 | Column 3 | Column 4 | Column 5 | Column 6 | Column 7 | Column 8 | ++-----------------------------------+----------------------+--------------------------+-----------------------+---------------------+----------+----------+----------+----------+ +| 'Comment' | 'system' | 'engines' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'Engine' | 'system' | 'engines' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'access' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'acquired_on' | 'system' | 'locks' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'active_result_scan' | 'system' | 'query_cache' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | +| 'after' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'agg_spilled_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'agg_spilled_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'arguments' | 'system' | 'procedures' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'arguments' | 'system' | 'user_functions' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | +| 'attempt_number' | 'system' | 'task_history' | 'Int32' | 'INT' | '' | '' | 'NO' | '' | +| 'attribute_names' | 'system' | 'dictionaries' | 'Array(String)' | 'ARRAY(STRING)' | '' | '' | 'NO' | '' | +| 'attribute_types' | 'system' | 'dictionaries' | 'Array(String)' | 'ARRAY(STRING)' | '' | '' | 'NO' | '' | +| 'auth_type' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'auto_increment' | 'information_schema' | 'tables' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'byte_size' | 'system' | 'clustering_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'bytes_from_local_disk' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'bytes_from_memory' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'bytes_from_remote_disk' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'capacity' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'cardinality' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'cargo_features' | 'system' | 'build_options' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog' | 'system' | 'databases' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog' | 'system' | 'databases_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog' | 'system' | 'streams_terse' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'catalog_name' | 'information_schema' | 'schemata' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'character_maximum_length' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'character_octet_length' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'character_set_catalog' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'character_set_name' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'character_set_schema' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'check_option' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'client_address' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'client_info' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'cluster' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'cluster_by' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'cluster_by' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'cluster_id' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'collation' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'collation_catalog' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'collation_name' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'collation_schema' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'column_comment' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'column_default' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'column_key' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'column_name' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'column_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'column_name' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'column_type' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'columns' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'command' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'dictionaries' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'notifications' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'comment' | 'system' | 'password_policies' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'procedures' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'comment' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'comment' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'comment' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'compaction_stats' | 'system' | 'background_tasks' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | +| 'completed_time' | 'system' | 'task_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'condition_text' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'condition_text' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'constraint_catalog' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'constraint_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'constraint_schema' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'copy_options' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'cpu_usage' | 'system' | 'query_log' | 'UInt32' | 'INT UNSIGNED' | '' | '' | 'NO' | '' | +| 'create_time' | 'information_schema' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'background_jobs' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'background_tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'dictionaries' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'indexes' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'locks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'notification_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'notifications' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'password_policies' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'procedures' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'roles' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'stages' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'streams' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'tables_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'user_functions' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'users' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'created_on' | 'system' | 'views' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'views_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'virtual_columns' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_time' | 'system' | 'processes' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'creator' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'creator' | 'system' | 'background_tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'creator' | 'system' | 'stages' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'current_database' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'current_query_id' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'data_compressed_size' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'data_compressed_size' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'data_free' | 'information_schema' | 'tables' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'data_length' | 'information_schema' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'data_read_bytes' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'data_size' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'data_size' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'data_type' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'data_type' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'data_write_bytes' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'clustering_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'dictionaries' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'streams_terse' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'temporary_tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database_id' | 'system' | 'background_tasks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'database_id' | 'system' | 'databases' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'database_id' | 'system' | 'databases_with_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'databases' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'datetime_precision' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'default' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'default' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'default_character_set_catalog' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'default_character_set_name' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'default_character_set_schema' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'default_collation_name' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'default_expression' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'default_kind' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'default_role' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'definition' | 'system' | 'indexes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'definition' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'definition' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'definition' | 'system' | 'user_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'description' | 'system' | 'configs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'description' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'description' | 'system' | 'procedures' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'description' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'description' | 'system' | 'user_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'disabled' | 'system' | 'users' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | +| 'domain_catalog' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'domain_name' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'domain_schema' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'drop_time' | 'information_schema' | 'tables' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'dropped_on' | 'system' | 'databases' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'dropped_on' | 'system' | 'databases_with_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'dropped_on' | 'system' | 'tables' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'dropped_on' | 'system' | 'tables_with_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'dropped_on' | 'system' | 'views' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'dropped_on' | 'system' | 'views_with_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'dummy' | 'system' | 'one' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'enabled' | 'system' | 'notifications' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | +| 'end_time' | 'system' | 'clustering_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'engine' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine' | 'system' | 'temporary_tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine_full' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine_full' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine_full' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'engine_full' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'error_integration' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'error_message' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'errors' | 'system' | 'queries_profiling' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | +| 'event_date' | 'system' | 'query_log' | 'Date' | 'DATE' | '' | '' | 'NO' | '' | +| 'event_time' | 'system' | 'query_log' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'example' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'exception_code' | 'system' | 'query_log' | 'Int32' | 'INT' | '' | '' | 'NO' | '' | +| 'exception_code' | 'system' | 'task_history' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | +| 'exception_text' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'exception_text' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'extra' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'extra' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'extra_info' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'extra_info' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'file_content_length' | 'system' | 'temp_files' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'file_format_options' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'file_last_modified_time' | 'system' | 'temp_files' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'file_name' | 'system' | 'temp_files' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'file_type' | 'system' | 'temp_files' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'group' | 'system' | 'configs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'group_by_spilled_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'group_by_spilled_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'handler_type' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'has_profile' | 'system' | 'query_log' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | +| 'hit' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'host' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'host' | 'system' | 'processes' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'hostname' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'id' | 'system' | 'background_tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'id' | 'system' | 'notifications' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'id' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'id' | 'system' | 'task_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'id' | 'system' | 'tasks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'index_comment' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'index_length' | 'information_schema' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'index_name' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'index_schema' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'index_size' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'index_size' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'index_type' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'inherited_roles' | 'system' | 'roles' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'inherited_roles_name' | 'system' | 'roles' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'integration_name' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'invalid_reason' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'is_aggregate' | 'system' | 'functions' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | +| 'is_aggregate' | 'system' | 'user_functions' | 'Nullable(Boolean)' | 'BOOLEAN' | '' | '' | 'YES' | '' | +| 'is_attach' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'is_attach' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'is_configured' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'is_insertable_into' | 'information_schema' | 'views' | 'Boolean' | 'BOOLEAN' | '' | '' | 'NO' | '' | +| 'is_nullable' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'is_nullable' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'is_transient' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'is_transient' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'is_trigger_deletable' | 'information_schema' | 'views' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'is_trigger_insertable_into' | 'information_schema' | 'views' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'is_trigger_updatable' | 'information_schema' | 'views' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'is_updatable' | 'information_schema' | 'views' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'job_state' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'job_type' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'join_spilled_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'join_spilled_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'key_names' | 'system' | 'dictionaries' | 'Array(String)' | 'ARRAY(STRING)' | '' | '' | 'NO' | '' | +| 'key_types' | 'system' | 'dictionaries' | 'Array(String)' | 'ARRAY(STRING)' | '' | '' | 'NO' | '' | +| 'keywords' | 'information_schema' | 'keywords' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'kind' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'labels' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'language' | 'system' | 'user_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'last_committed_on' | 'system' | 'tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'last_suspended_on' | 'system' | 'tasks' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'last_task_id' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'last_task_run_at' | 'system' | 'background_jobs' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'last_updated' | 'system' | 'background_jobs' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'level' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'license' | 'system' | 'credits' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'location' | 'system' | 'query_cache' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'log_type' | 'system' | 'query_log' | 'Int8' | 'TINYINT' | '' | '' | 'NO' | '' | +| 'log_type_name' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'memory_usage' | 'system' | 'processes' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | +| 'memory_usage' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'message' | 'system' | 'background_jobs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'message' | 'system' | 'background_tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'message' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'message_source' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'metric' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'miss' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'mode' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'mode' | 'system' | 'streams_terse' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'must_change_password' | 'system' | 'users' | 'Nullable(Boolean)' | 'BOOLEAN' | '' | '' | 'YES' | '' | +| 'mysql_connection_id' | 'system' | 'processes' | 'Nullable(UInt32)' | 'INT UNSIGNED' | '' | '' | 'YES' | '' | +| 'name' | 'system' | 'background_jobs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'caches' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'catalogs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'configs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'contributors' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'credits' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'databases' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'databases_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'dictionaries' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'indexes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'malloc_stats_totals' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'notifications' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'password_policies' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'procedures' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'roles' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'streams_terse' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'table_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'temporary_tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'user_functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'name' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'network_policy' | 'system' | 'users' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'next_schedule_time' | 'system' | 'tasks' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'next_task_scheduled_time' | 'system' | 'background_jobs' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'node' | 'system' | 'backtrace' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'node' | 'system' | 'caches' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'node' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'node' | 'system' | 'malloc_stats_totals' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'node' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'node' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'node' | 'system' | 'queries_profiling' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'node_id' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'non_unique' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'nullable' | 'information_schema' | 'columns' | 'Nullable(UInt8)' | 'TINYINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'nullable' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'num_items' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'num_rows' | 'system' | 'query_cache' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'num_rows' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'num_rows' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'number_of_blocks' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'number_of_blocks' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'number_of_files' | 'system' | 'stages' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'number_of_segments' | 'system' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'number_of_segments' | 'system' | 'tables_with_history' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'numeric_precision' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'numeric_precision_radix' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'numeric_scale' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'options' | 'system' | 'password_policies' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'ordinal_position' | 'information_schema' | 'columns' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'ordinal_position' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'original' | 'system' | 'indexes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'owner' | 'system' | 'databases' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'owner' | 'system' | 'databases_with_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'owner' | 'system' | 'stages' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'owner' | 'system' | 'streams' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'owner' | 'system' | 'tables' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'owner' | 'system' | 'tables_with_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'owner' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'owner' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'owner' | 'system' | 'views' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'owner' | 'system' | 'views_with_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'packed' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'parent_plan_id' | 'system' | 'queries_profiling' | 'Nullable(UInt32)' | 'INT UNSIGNED' | '' | '' | 'YES' | '' | +| 'partitions_sha' | 'system' | 'query_cache' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'password_policy' | 'system' | 'users' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'plan_id' | 'system' | 'queries_profiling' | 'Nullable(UInt32)' | 'INT UNSIGNED' | '' | '' | 'YES' | '' | +| 'plan_name' | 'system' | 'queries_profiling' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'port' | 'system' | 'clusters' | 'UInt16' | 'SMALLINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'position_in_unique_constraint' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'privileges' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'procedure_id' | 'system' | 'procedures' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'processed' | 'system' | 'notification_history' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'projections' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_duration_ms' | 'system' | 'query_log' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | +| 'query_hash' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_id' | 'system' | 'backtrace' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_id' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_id' | 'system' | 'queries_profiling' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_id' | 'system' | 'query_cache' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_id' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_id' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_kind' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_parameterized_hash' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'query_queued_duration_ms' | 'system' | 'query_log' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | +| 'query_start_time' | 'system' | 'query_log' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'query_text' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'range' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'referenced_column_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'referenced_table_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'referenced_table_schema' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'reserved' | 'information_schema' | 'keywords' | 'UInt8' | 'TINYINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'result_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'result_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'result_size' | 'system' | 'query_cache' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'revision' | 'system' | 'locks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'roles' | 'system' | 'users' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'root_task_id' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'row_count' | 'system' | 'clustering_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'run_id' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'scan_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'scan_io_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'scan_io_bytes_cost_ms' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'scan_partitions' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'scan_progress_read_bytes' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'scan_progress_read_rows' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'scan_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'schedule' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'schedule' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'scheduled_job_cron_expression' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'scheduled_job_cron_timezone' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'scheduled_job_interval_secs' | 'system' | 'background_jobs' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'scheduled_time' | 'system' | 'task_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'schema_name' | 'information_schema' | 'schemata' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'schema_owner' | 'information_schema' | 'schemata' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'seq_in_index' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'server_version' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'session_parameters' | 'system' | 'task_history' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | +| 'session_parameters' | 'system' | 'tasks' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | +| 'session_settings' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'size' | 'system' | 'caches' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'snapshot_location' | 'system' | 'streams' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'source' | 'system' | 'dictionaries' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'sql' | 'system' | 'query_cache' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'sql_path' | 'information_schema' | 'schemata' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'sql_user' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'sql_user_privileges' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'sql_user_quota' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'stack' | 'system' | 'backtrace' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'stack_trace' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'stage_params' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'stage_type' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'start_time' | 'system' | 'clustering_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'state' | 'system' | 'background_tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'state' | 'system' | 'task_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'state' | 'system' | 'tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'statistics' | 'system' | 'malloc_stats' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | +| 'statistics' | 'system' | 'queries_profiling' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | +| 'status' | 'system' | 'backtrace' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'status' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'status' | 'system' | 'notification_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'status' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'stream_id' | 'system' | 'streams' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'sub_part' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'suspend_task_after_num_failures' | 'system' | 'tasks' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'syntax' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table' | 'system' | 'clustering_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_catalog' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_catalog' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'table_catalog' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'table_catalog' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_catalog' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_collation' | 'information_schema' | 'tables' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'table_comment' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_id' | 'system' | 'background_tasks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'table_id' | 'system' | 'locks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'table_id' | 'system' | 'streams' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'table_id' | 'system' | 'tables' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'table_id' | 'system' | 'tables_with_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'table_id' | 'system' | 'temporary_tables' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'table_id' | 'system' | 'views' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'table_id' | 'system' | 'views_with_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'table_name' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'table_name' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'table_name' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_name' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_name' | 'system' | 'streams' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'table_name' | 'system' | 'streams_terse' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'table_rows' | 'information_schema' | 'tables' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'table_schema' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_schema' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'table_schema' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'table_schema' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_schema' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_type' | 'information_schema' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_type' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_type' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table_version' | 'system' | 'streams' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'tables' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'target_features' | 'system' | 'build_options' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'task_running_secs' | 'system' | 'background_tasks' | 'Nullable(UInt64)' | 'BIGINT UNSIGNED' | '' | '' | 'YES' | '' | +| 'task_type' | 'system' | 'background_jobs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'tenant_id' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'time' | 'system' | 'processes' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'total_columns' | 'system' | 'tables' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'total_columns' | 'system' | 'tables_with_history' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'total_partitions' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'trigger' | 'system' | 'background_tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'type' | 'system' | 'background_tasks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'type' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'type' | 'system' | 'indexes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'type' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'type' | 'system' | 'notifications' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'type' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'type' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'unit' | 'system' | 'caches' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'update_on' | 'system' | 'roles' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'update_on' | 'system' | 'users' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'updated_on' | 'system' | 'background_tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'updated_on' | 'system' | 'dictionaries' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'updated_on' | 'system' | 'indexes' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'updated_on' | 'system' | 'password_policies' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'updated_on' | 'system' | 'streams' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'updated_on' | 'system' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'updated_on' | 'system' | 'tables_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'updated_on' | 'system' | 'views' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'updated_on' | 'system' | 'views_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'updated_on' | 'system' | 'virtual_columns' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | +| 'user' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'user' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'user_agent' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'vacuum_stats' | 'system' | 'background_tasks' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | +| 'value' | 'system' | 'configs' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'value' | 'system' | 'malloc_stats_totals' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'value' | 'system' | 'metrics' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'value' | 'system' | 'settings' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'version' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'version' | 'system' | 'credits' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'view_definition' | 'information_schema' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'view_query' | 'system' | 'views' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'view_query' | 'system' | 'views_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'virtual_columns' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'warehouse' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'warehouse' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'webhook_options' | 'system' | 'notifications' | 'Nullable(Variant)' | 'VARIANT' | '' | '' | 'YES' | '' | +| 'written_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'written_io_bytes' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'written_io_bytes_cost_ms' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'written_rows' | 'system' | 'query_log' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | ++-----------------------------------+----------------------+--------------------------+-----------------------+---------------------+----------+----------+----------+----------+ diff --git a/src/query/service/tests/it/storages/testdata/databases_table.txt b/src/query/service/tests/it/storages/testdata/databases_table.txt index 286957a4934b..a18103a8b1eb 100644 --- a/src/query/service/tests/it/storages/testdata/databases_table.txt +++ b/src/query/service/tests/it/storages/testdata/databases_table.txt @@ -1,12 +1,12 @@ ---------- TABLE INFO ------------ DB.Table: 'system'.'databases', Table: databases-table_id:1, ver:0, Engine: SystemDatabases -------- TABLE CONTENTS ---------- -+-----------+----------------------+---------------------+----------+ -| Column 0 | Column 1 | Column 2 | Column 3 | -+-----------+----------------------+---------------------+----------+ -| 'default' | 'default' | 1 | NULL | -| 'default' | 'information_schema' | 4611686018427387906 | NULL | -| 'default' | 'system' | 4611686018427387905 | NULL | -+-----------+----------------------+---------------------+----------+ ++-----------+----------------------+---------------------+----------+----------+ +| Column 0 | Column 1 | Column 2 | Column 3 | Column 4 | ++-----------+----------------------+---------------------+----------+----------+ +| 'default' | 'default' | 1 | NULL | NULL | +| 'default' | 'information_schema' | 4611686018427387906 | NULL | NULL | +| 'default' | 'system' | 4611686018427387905 | NULL | NULL | ++-----------+----------------------+---------------------+----------+----------+ diff --git a/src/query/service/tests/it/storages/testdata/databases_with_history_table.txt b/src/query/service/tests/it/storages/testdata/databases_with_history_table.txt new file mode 100644 index 000000000000..d9a4ded4d128 --- /dev/null +++ b/src/query/service/tests/it/storages/testdata/databases_with_history_table.txt @@ -0,0 +1,12 @@ +---------- TABLE INFO ------------ +DB.Table: 'system'.'databases_with_history', Table: databases_with_history-table_id:1, ver:0, Engine: SystemDatabases +-------- TABLE CONTENTS ---------- ++-----------+----------------------+---------------------+----------+----------+ +| Column 0 | Column 1 | Column 2 | Column 3 | Column 4 | ++-----------+----------------------+---------------------+----------+----------+ +| 'default' | 'default' | 1 | NULL | NULL | +| 'default' | 'information_schema' | 4611686018427387906 | NULL | NULL | +| 'default' | 'system' | 4611686018427387905 | NULL | NULL | ++-----------+----------------------+---------------------+----------+----------+ + + diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index c38bdc067f1c..abc243ae2cc0 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -268,6 +268,7 @@ impl<'a> Binder { // Databases Statement::ShowDatabases(stmt) => self.bind_show_databases(bind_context, stmt).await?, + Statement::ShowDropDatabases(stmt) => self.bind_show_drop_databases(bind_context, stmt).await?, Statement::ShowCreateDatabase(stmt) => self.bind_show_create_database(stmt).await?, Statement::CreateDatabase(stmt) => self.bind_create_database(stmt).await?, Statement::DropDatabase(stmt) => self.bind_drop_database(stmt).await?, diff --git a/src/query/sql/src/planner/binder/ddl/database.rs b/src/query/sql/src/planner/binder/ddl/database.rs index ad5028ee3b33..f4991733dd8b 100644 --- a/src/query/sql/src/planner/binder/ddl/database.rs +++ b/src/query/sql/src/planner/binder/ddl/database.rs @@ -23,6 +23,7 @@ use databend_common_ast::ast::DropDatabaseStmt; use databend_common_ast::ast::SQLProperty; use databend_common_ast::ast::ShowCreateDatabaseStmt; use databend_common_ast::ast::ShowDatabasesStmt; +use databend_common_ast::ast::ShowDropDatabasesStmt; use databend_common_ast::ast::ShowLimit; use databend_common_ast::ast::UndropDatabaseStmt; use databend_common_exception::Result; @@ -93,6 +94,47 @@ impl Binder { .await } + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_show_drop_databases( + &mut self, + bind_context: &mut BindContext, + stmt: &ShowDropDatabasesStmt, + ) -> Result { + let ShowDropDatabasesStmt { catalog, limit } = stmt; + let mut select_builder = SelectBuilder::from("system.databases_with_history"); + + let ctl = if let Some(ctl) = catalog { + normalize_identifier(ctl, &self.name_resolution_ctx).name + } else { + self.ctx.get_current_catalog().to_string() + }; + + select_builder.with_filter(format!("catalog = '{ctl}'")); + + select_builder.with_column("catalog"); + select_builder.with_column("name"); + select_builder.with_column("database_id"); + select_builder.with_column("dropped_on"); + + select_builder.with_order_by("catalog"); + select_builder.with_order_by("name"); + + match limit { + Some(ShowLimit::Like { pattern }) => { + select_builder.with_filter(format!("name LIKE '{pattern}'")); + } + Some(ShowLimit::Where { selection }) => { + select_builder.with_filter(format!("({selection})")); + } + None => (), + } + let query = select_builder.build(); + debug!("show databases rewrite to: {:?}", query); + + self.bind_rewrite_to_query(bind_context, query.as_str(), RewriteKind::ShowDropDatabases) + .await + } + #[async_backtrace::framed] pub(in crate::planner::binder) async fn bind_show_create_database( &self, diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index 80556dbbcdd6..da139fa99e92 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -389,6 +389,7 @@ pub enum RewriteKind { ShowCatalogs, ShowDatabases, + ShowDropDatabases, ShowTables(String, String), ShowColumns(String, String, String), ShowTablesStatus, diff --git a/src/query/storages/hive/hive/src/hive_catalog.rs b/src/query/storages/hive/hive/src/hive_catalog.rs index 990dd0607937..3e547a2258dc 100644 --- a/src/query/storages/hive/hive/src/hive_catalog.rs +++ b/src/query/storages/hive/hive/src/hive_catalog.rs @@ -293,6 +293,11 @@ impl Catalog for HiveCatalog { Ok(res) } + async fn list_databases_history(&self, _tenant: &Tenant) -> Result>> { + // TODO: Implement list_databases_history + unimplemented!() + } + // Get all the databases. #[fastrace::trace] #[async_backtrace::framed] diff --git a/src/query/storages/iceberg/src/catalog.rs b/src/query/storages/iceberg/src/catalog.rs index 92ef717b8685..664cec399f21 100644 --- a/src/query/storages/iceberg/src/catalog.rs +++ b/src/query/storages/iceberg/src/catalog.rs @@ -226,6 +226,10 @@ impl Catalog for IcebergCatalog { Ok(Arc::new(IcebergDatabase::create(self.clone(), db_name))) } + async fn list_databases_history(&self, _tenant: &Tenant) -> Result>> { + unimplemented!() + } + #[async_backtrace::framed] async fn list_databases(&self, _tenant: &Tenant) -> Result>> { let db_names = self diff --git a/src/query/storages/system/src/databases_table.rs b/src/query/storages/system/src/databases_table.rs index cfebe2c1b97f..4adbbb880d73 100644 --- a/src/query/storages/system/src/databases_table.rs +++ b/src/query/storages/system/src/databases_table.rs @@ -16,12 +16,14 @@ use std::sync::Arc; use databend_common_catalog::catalog::Catalog; use databend_common_catalog::catalog::CatalogManager; +use databend_common_catalog::database::Database; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; @@ -33,19 +35,60 @@ use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; +use databend_common_meta_app::tenant::Tenant; use databend_common_users::UserApiProvider; use log::warn; use crate::table::AsyncOneBlockSystemTable; use crate::table::AsyncSystemTable; -pub struct DatabasesTable { +pub type DatabasesTableWithHistory = DatabasesTable; +pub type DatabasesTableWithoutHistory = DatabasesTable; + +pub struct DatabasesTable { table_info: TableInfo, } #[async_trait::async_trait] -impl AsyncSystemTable for DatabasesTable { - const NAME: &'static str = "system.databases"; +pub trait HistoryAware { + const TABLE_NAME: &'static str; + async fn list_databases( + catalog: &Arc, + tenant: &Tenant, + with_history: bool, + ) -> Result>>; +} + +macro_rules! impl_history_aware { + ($with_history:expr, $table_name:expr) => { + #[async_trait::async_trait] + impl HistoryAware for DatabasesTable<$with_history> { + const TABLE_NAME: &'static str = $table_name; + + #[async_backtrace::framed] + async fn list_databases( + catalog: &Arc, + tenant: &Tenant, + with_history: bool, + ) -> Result>> { + if with_history { + catalog.list_databases_history(tenant).await + } else { + catalog.list_databases(tenant).await + } + } + } + }; +} + +impl_history_aware!(true, "databases_with_history"); +impl_history_aware!(false, "databases"); + +#[async_trait::async_trait] +impl AsyncSystemTable for DatabasesTable +where DatabasesTable: HistoryAware +{ + const NAME: &'static str = Self::TABLE_NAME; fn get_table_info(&self) -> &TableInfo { &self.table_info @@ -72,57 +115,106 @@ impl AsyncSystemTable for DatabasesTable { let mut db_names = vec![]; let mut db_ids = vec![]; let mut owners: Vec> = vec![]; + let mut dropped_on: Vec> = vec![]; let visibility_checker = ctx.get_visibility_checker().await?; let catalog_dbs = visibility_checker.get_visibility_database(); // None means has global level privileges if let Some(catalog_dbs) = catalog_dbs { - for (catalog, dbs) in catalog_dbs { - let mut catalog_db_ids = vec![]; - let mut catalog_db_names = vec![]; - let ctl = ctx.get_catalog(catalog).await?; - catalog_db_names.extend( - dbs.iter() - .filter_map(|(db_name, _)| *db_name) - .map(|db_name| db_name.to_string()), - ); - catalog_db_ids.extend(dbs.iter().filter_map(|(_, db_id)| *db_id)); - - if let Ok(databases) = ctl - .mget_database_names_by_ids(&tenant, &catalog_db_ids) - .await - { - catalog_db_names.extend(databases.into_iter().flatten()); - } else { - let msg = format!("Failed to get database name by id: {}", ctl.name()); - warn!("{}", msg); + if WITH_HISTORY { + for (ctl_name, dbs) in catalog_dbs { + let catalog = ctx.get_catalog(ctl_name).await?; + let dbs_history = catalog.list_databases_history(&tenant).await?; + for db_history in dbs_history { + let db_name = db_history + .get_db_info() + .name_ident + .database_name() + .to_string(); + let id = db_history.get_db_info().database_id.db_id; + if db_ids.contains(&id) { + continue; + } + if dbs.contains(&(None, Some(&id))) + || db_name.to_lowercase() == "information_schema" + || db_name.to_lowercase() == "system" + { + catalog_names.push(ctl_name.clone()); + db_names.push(db_name); + db_ids.push(id); + owners.push( + user_api + .get_ownership(&tenant, &OwnershipObject::Database { + catalog_name: ctl_name.to_string(), + db_id: id, + }) + .await + .ok() + .and_then(|ownership| ownership.map(|o| o.role.clone())), + ); + dropped_on.push( + db_history + .get_db_info() + .meta + .drop_on + .map(|v| v.timestamp_micros()), + ); + } + } } - - let db_idents = catalog_db_names - .iter() - .map(|name| DatabaseNameIdent::new(&tenant, name)) - .collect::>(); - let dbs = ctl.mget_databases(&tenant, &db_idents).await?; - for db in dbs { - catalog_names.push(catalog.clone()); - db_names.push(db.get_db_info().name_ident.database_name().to_string()); - let db_id = db.get_db_info().database_id.db_id; - db_ids.push(db_id); - owners.push( - user_api - .get_ownership(&tenant, &OwnershipObject::Database { - catalog_name: catalog.to_string(), - db_id, - }) - .await - .ok() - .and_then(|ownership| ownership.map(|o| o.role.clone())), + } else { + for (catalog, dbs) in catalog_dbs { + let mut catalog_db_ids = vec![]; + let mut catalog_db_names = vec![]; + let ctl = ctx.get_catalog(catalog).await?; + catalog_db_names.extend( + dbs.iter() + .filter_map(|(db_name, _)| *db_name) + .map(|db_name| db_name.to_string()), ); + catalog_db_ids.extend(dbs.iter().filter_map(|(_, db_id)| *db_id)); + + if let Ok(databases) = ctl + .mget_database_names_by_ids(&tenant, &catalog_db_ids) + .await + { + catalog_db_names.extend(databases.into_iter().flatten()); + } else { + let msg = format!("Failed to get database name by id: {}", ctl.name()); + warn!("{}", msg); + } + let db_idents = catalog_db_names + .iter() + .map(|name| DatabaseNameIdent::new(&tenant, name)) + .collect::>(); + let dbs = ctl.mget_databases(&tenant, &db_idents).await?; + + for db in dbs { + let db_id = db.get_db_info().database_id.db_id; + if db_ids.contains(&db_id) { + continue; + } + catalog_names.push(catalog.clone()); + db_names.push(db.get_db_info().name_ident.database_name().to_string()); + db_ids.push(db_id); + owners.push( + user_api + .get_ownership(&tenant, &OwnershipObject::Database { + catalog_name: catalog.to_string(), + db_id, + }) + .await + .ok() + .and_then(|ownership| ownership.map(|o| o.role.clone())), + ); + dropped_on + .push(db.get_db_info().meta.drop_on.map(|v| v.timestamp_micros())); + } } } } else { for (ctl_name, catalog) in catalogs.into_iter() { - let databases = catalog.list_databases(&tenant).await?; + let databases = Self::list_databases(&catalog, &tenant, WITH_HISTORY).await?; let final_dbs = databases .into_iter() .filter(|db| { @@ -150,6 +242,7 @@ impl AsyncSystemTable for DatabasesTable { .ok() .and_then(|ownership| ownership.map(|o| o.role.clone())), ); + dropped_on.push(db.get_db_info().meta.drop_on.map(|v| v.timestamp_micros())); } } } @@ -159,11 +252,14 @@ impl AsyncSystemTable for DatabasesTable { StringType::from_data(db_names), UInt64Type::from_data(db_ids), StringType::from_opt_data(owners), + TimestampType::from_opt_data(dropped_on), ])) } } -impl DatabasesTable { +impl DatabasesTable +where DatabasesTable: HistoryAware +{ pub fn create(table_id: u64) -> Arc { let schema = TableSchemaRefExt::create(vec![ TableField::new("catalog", TableDataType::String), @@ -173,11 +269,16 @@ impl DatabasesTable { "owner", TableDataType::Nullable(Box::from(TableDataType::String)), ), + TableField::new( + "dropped_on", + TableDataType::Nullable(Box::new(TableDataType::Timestamp)), + ), ]); + let name = Self::TABLE_NAME; let table_info = TableInfo { - desc: "'system'.'databases'".to_string(), - name: "databases".to_string(), + desc: format!("'system'.'{name}'"), + name: Self::NAME.to_owned(), ident: TableIdent::new(table_id, 0), meta: TableMeta { schema, diff --git a/src/query/storages/system/src/lib.rs b/src/query/storages/system/src/lib.rs index 21210b47148f..8b067b9232e0 100644 --- a/src/query/storages/system/src/lib.rs +++ b/src/query/storages/system/src/lib.rs @@ -84,6 +84,8 @@ pub use configs_table::ConfigsTable; pub use contributors_table::ContributorsTable; pub use credits_table::CreditsTable; pub use databases_table::DatabasesTable; +pub use databases_table::DatabasesTableWithHistory; +pub use databases_table::DatabasesTableWithoutHistory; pub use dictionaries_table::DictionariesTable; pub use engines_table::EnginesTable; pub use functions_table::FunctionsTable; diff --git a/tests/sqllogictests/suites/base/06_show/06_0009_show_databases.test b/tests/sqllogictests/suites/base/06_show/06_0009_show_databases.test index c19550335bba..55ac566855e2 100644 --- a/tests/sqllogictests/suites/base/06_show/06_0009_show_databases.test +++ b/tests/sqllogictests/suites/base/06_show/06_0009_show_databases.test @@ -38,3 +38,8 @@ DROP DATABASE IF EXISTS ss1 statement ok DROP DATABASE IF EXISTS ss2 + +query T +select name, dropped_on is not null from system.databases_with_history where name='ss1' order by dropped_on limit 1; +---- +ss1 1 diff --git a/tests/suites/0_stateless/18_rbac/18_0003_db_visibility.result b/tests/suites/0_stateless/18_rbac/18_0003_db_visibility.result index 1ff68e3a1f7b..c96959197bde 100644 --- a/tests/suites/0_stateless/18_rbac/18_0003_db_visibility.result +++ b/tests/suites/0_stateless/18_rbac/18_0003_db_visibility.result @@ -1,6 +1,10 @@ === test u1 with role1 === information_schema system +true true +db1 true +system +information_schema 1 Error: APIError: ResponseError with 1063: Permission denied: privilege [Select] is required on 'default'.'db_root'.'t1' for user 'u1'@'%' with roles [public,role1] db1 diff --git a/tests/suites/0_stateless/18_rbac/18_0003_db_visibility.sh b/tests/suites/0_stateless/18_rbac/18_0003_db_visibility.sh index ff8970bbfba9..9d0869d9fd99 100755 --- a/tests/suites/0_stateless/18_rbac/18_0003_db_visibility.sh +++ b/tests/suites/0_stateless/18_rbac/18_0003_db_visibility.sh @@ -33,6 +33,12 @@ echo "grant role role1 to u1;" | $BENDSQL_CLIENT_CONNECT export TEST_U1_CONNECT="bendsql --user=u1 --password=123 --host=${QUERY_MYSQL_HANDLER_HOST} --port ${QUERY_HTTP_HANDLER_PORT}" echo "show databases" | $TEST_U1_CONNECT echo "create database db1;" | $TEST_U1_CONNECT +echo "grant delete on db1.* to u1" | $BENDSQL_CLIENT_CONNECT +echo "drop database db1;" | $TEST_U1_CONNECT +echo "select count(name)>0, count(dropped_on is not null)>0 from system.databases_with_history where name='db1'" | $BENDSQL_CLIENT_CONNECT +echo "select name, dropped_on is not null from system.databases_with_history where name='db1'" | $TEST_U1_CONNECT +echo "select name from system.databases_with_history where name!='db1'" | $TEST_U1_CONNECT +echo "create database db1;" | $TEST_U1_CONNECT echo "create table db1.t1(id int);" | $TEST_U1_CONNECT echo "insert into db1.t1 values(1);" | $TEST_U1_CONNECT echo "select * from db1.t1;" | $TEST_U1_CONNECT From 9af8ba3381492726f072041d579b09a02aa184fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Wed, 13 Nov 2024 15:03:52 +0800 Subject: [PATCH 30/92] refactor: align metrics field names with current behavior (#16826) - Rename `db_size` to `raft_log_size`, which is the size of on-disk data of raft-log. - Rename `key_num` to `snapshot_key_count`. This commit updates field names in - the the metrics API, - and the cluster status gRPC API. --- src/meta/binaries/metactl/main.rs | 4 +-- src/meta/service/src/api/grpc/grpc_service.rs | 4 +-- .../service/src/meta_service/meta_node.rs | 19 +++++++------ .../src/meta_service/meta_node_status.rs | 8 +++--- src/meta/service/src/metrics/meta_metrics.rs | 28 +++++++++++-------- src/meta/service/src/store/store_inner.rs | 2 +- src/meta/types/proto/meta.proto | 4 +-- 7 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/meta/binaries/metactl/main.rs b/src/meta/binaries/metactl/main.rs index 04b9aee59816..bffe0aae683c 100644 --- a/src/meta/binaries/metactl/main.rs +++ b/src/meta/binaries/metactl/main.rs @@ -236,8 +236,8 @@ impl App { let res = client.get_cluster_status().await?; println!("BinaryVersion: {}", res.binary_version); println!("DataVersion: {}", res.data_version); - println!("DBSize: {}", res.db_size); - println!("KeyNumber: {}", res.key_num); + println!("RaftLogSize: {}", res.raft_log_size); + println!("SnapshotKeyCount: {}", res.snapshot_key_count); println!("Node: id={} raft={}", res.id, res.endpoint); println!("State: {}", res.state); if let Some(leader) = res.leader { diff --git a/src/meta/service/src/api/grpc/grpc_service.rs b/src/meta/service/src/api/grpc/grpc_service.rs index a10e3171032c..1ca6b2e458ef 100644 --- a/src/meta/service/src/api/grpc/grpc_service.rs +++ b/src/meta/service/src/api/grpc/grpc_service.rs @@ -462,8 +462,8 @@ impl MetaService for MetaServiceImpl { binary_version: status.binary_version, data_version: status.data_version.to_string(), endpoint: status.endpoint, - db_size: status.db_size, - key_num: status.key_num as u64, + raft_log_size: status.raft_log_size, + snapshot_key_count: status.snapshot_key_count as u64, state: status.state, is_leader: status.is_leader, current_term: status.current_term, diff --git a/src/meta/service/src/meta_service/meta_node.rs b/src/meta/service/src/meta_service/meta_node.rs index 81c5d33dca7d..02ea28e34208 100644 --- a/src/meta/service/src/meta_service/meta_node.rs +++ b/src/meta/service/src/meta_service/meta_node.rs @@ -447,8 +447,8 @@ impl MetaNode { server_metrics::set_last_seq(meta_node.get_last_seq().await); // metrics about server storage - server_metrics::set_db_size(meta_node.get_db_size().await); - server_metrics::set_snapshot_key_num(meta_node.get_key_num().await); + server_metrics::set_raft_log_size(meta_node.get_raft_log_size().await); + server_metrics::set_snapshot_key_count(meta_node.get_snapshot_key_count().await); last_leader = mm.current_leader; } @@ -817,13 +817,14 @@ impl MetaNode { nodes } - async fn get_db_size(&self) -> u64 { + /// Get the size in bytes of the on disk files of the raft log storage. + async fn get_raft_log_size(&self) -> u64 { self.sto.log.read().await.on_disk_size() } - async fn get_key_num(&self) -> u64 { + async fn get_snapshot_key_count(&self) -> u64 { self.sto - .try_get_snapshot_key_num() + .try_get_snapshot_key_count() .await .unwrap_or_default() } @@ -841,8 +842,8 @@ impl MetaNode { let endpoint = self.sto.get_node_raft_endpoint(&self.sto.id).await?; - let db_size = self.get_db_size().await; - let key_num = self.get_key_num().await; + let raft_log_size = self.get_raft_log_size().await; + let key_count = self.get_snapshot_key_count().await; let metrics = self.raft.metrics().borrow().clone(); @@ -859,8 +860,8 @@ impl MetaNode { binary_version: METASRV_COMMIT_VERSION.as_str().to_string(), data_version: DATA_VERSION, endpoint: endpoint.to_string(), - db_size, - key_num, + raft_log_size, + snapshot_key_count: key_count, state: format!("{:?}", metrics.state), is_leader: metrics.state == openraft::ServerState::Leader, current_term: metrics.current_term, diff --git a/src/meta/service/src/meta_service/meta_node_status.rs b/src/meta/service/src/meta_service/meta_node_status.rs index 73c084fb77b5..cd995fb3784f 100644 --- a/src/meta/service/src/meta_service/meta_node_status.rs +++ b/src/meta/service/src/meta_service/meta_node_status.rs @@ -32,11 +32,11 @@ pub struct MetaNodeStatus { /// The raft service endpoint for internal communication pub endpoint: String, - /// The size in bytes of the on disk data. - pub db_size: u64, + /// The size in bytes of the raft-log on disk data. + pub raft_log_size: u64, - /// key number of current snapshot - pub key_num: u64, + /// Total number of keys in current snapshot + pub snapshot_key_count: u64, /// Server state, one of "Follower", "Learner", "Candidate", "Leader". pub state: String, diff --git a/src/meta/service/src/metrics/meta_metrics.rs b/src/meta/service/src/metrics/meta_metrics.rs index 6698b0385657..022cb1f5c832 100644 --- a/src/meta/service/src/metrics/meta_metrics.rs +++ b/src/meta/service/src/metrics/meta_metrics.rs @@ -52,8 +52,8 @@ pub mod server_metrics { node_is_health: Gauge, leader_changes: Counter, applying_snapshot: Gauge, - snapshot_key_num: Gauge, - db_size: Gauge, + snapshot_key_count: Gauge, + raft_log_size: Gauge, last_log_index: Gauge, last_seq: Gauge, current_term: Gauge, @@ -73,8 +73,8 @@ pub mod server_metrics { node_is_health: Gauge::default(), leader_changes: Counter::default(), applying_snapshot: Gauge::default(), - snapshot_key_num: Gauge::default(), - db_size: Gauge::default(), + snapshot_key_count: Gauge::default(), + raft_log_size: Gauge::default(), last_log_index: Gauge::default(), last_seq: Gauge::default(), current_term: Gauge::default(), @@ -109,11 +109,15 @@ pub mod server_metrics { metrics.applying_snapshot.clone(), ); registry.register( - key!("snapshot_key_num"), - "snapshot key numbers", - metrics.snapshot_key_num.clone(), + key!("snapshot_key_count"), + "number of keys in the last snapshot", + metrics.snapshot_key_count.clone(), + ); + registry.register( + key!("raft_log_size"), + "the size in bytes of the on disk data of raft log", + metrics.raft_log_size.clone(), ); - registry.register(key!("db_size"), "db size", metrics.db_size.clone()); registry.register( key!("proposals_applied"), "proposals applied", @@ -174,12 +178,12 @@ pub mod server_metrics { SERVER_METRICS.applying_snapshot.inc_by(cnt); } - pub fn set_snapshot_key_num(snapshot_key_num: u64) { - SERVER_METRICS.snapshot_key_num.set(snapshot_key_num as i64); + pub fn set_snapshot_key_count(n: u64) { + SERVER_METRICS.snapshot_key_count.set(n as i64); } - pub fn set_db_size(db_size: u64) { - SERVER_METRICS.db_size.set(db_size as i64); + pub fn set_raft_log_size(raft_log_size: u64) { + SERVER_METRICS.raft_log_size.set(raft_log_size as i64); } pub fn set_proposals_applied(proposals_applied: u64) { diff --git a/src/meta/service/src/store/store_inner.rs b/src/meta/service/src/store/store_inner.rs index 7ba4b33c24f2..2d39defc66fc 100644 --- a/src/meta/service/src/store/store_inner.rs +++ b/src/meta/service/src/store/store_inner.rs @@ -292,7 +292,7 @@ impl StoreInner { /// Return snapshot id and meta of the last snapshot. /// /// It returns None if there is no snapshot or there is an error parsing snapshot meta or id. - pub(crate) async fn try_get_snapshot_key_num(&self) -> Option { + pub(crate) async fn try_get_snapshot_key_count(&self) -> Option { let sm = self.state_machine.read().await; let db = sm.levels().persisted()?; Some(db.stat().key_num) diff --git a/src/meta/types/proto/meta.proto b/src/meta/types/proto/meta.proto index 03fc08f222cf..ab3a304e3aae 100644 --- a/src/meta/types/proto/meta.proto +++ b/src/meta/types/proto/meta.proto @@ -167,7 +167,7 @@ message ClusterStatus { string binary_version = 2; string data_version = 3; string endpoint = 4; - uint64 db_size = 5; + uint64 raft_log_size = 5; string state = 6; bool is_leader = 7; uint64 current_term = 8; @@ -180,7 +180,7 @@ message ClusterStatus { repeated string voters = 15; repeated string non_voters = 16; uint64 last_seq = 17; - uint64 key_num = 18; + uint64 snapshot_key_count = 18; } message ClientInfo { From 80c8cf69fff42521deb4d78e6e26f1320f161754 Mon Sep 17 00:00:00 2001 From: Bohu Date: Wed, 13 Nov 2024 16:35:58 +0800 Subject: [PATCH 31/92] feat: uuid function from v4 to v7 (#16827) --- Cargo.toml | 2 +- src/query/functions/src/scalars/other.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 94a00b803105..04088ab2da64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -485,7 +485,7 @@ typetag = "0.2.3" unicode-segmentation = "1.10.1" unindent = "0.2" url = "2.3.1" -uuid = { version = "1.10.0", features = ["serde", "v4", "v7"] } +uuid = { version = "1.10.0", features = ["std", "serde", "v4", "v7"] } volo-thrift = "0.10" walkdir = "2.3.2" wiremock = "0.6" diff --git a/src/query/functions/src/scalars/other.rs b/src/query/functions/src/scalars/other.rs index 0dd6f34ec2e2..1904b40aa9a7 100644 --- a/src/query/functions/src/scalars/other.rs +++ b/src/query/functions/src/scalars/other.rs @@ -234,7 +234,7 @@ pub fn register(registry: &mut FunctionRegistry) { let mut builder = BinaryColumnBuilder::with_capacity(ctx.num_rows, 0); for _ in 0..ctx.num_rows { - let value = Uuid::new_v4(); + let value = Uuid::now_v7(); write!(&mut builder.data, "{}", value).unwrap(); builder.commit_row(); } From 872627c6185b8f597890bafbaf47841c88fce5b2 Mon Sep 17 00:00:00 2001 From: coldWater Date: Wed, 13 Nov 2024 19:17:41 +0800 Subject: [PATCH 32/92] feat: refine the rule `PushDownFilterWindowTopN` in case top n is equal to 0. (#16830) * top n 0 Signed-off-by: coldWater * test Signed-off-by: coldWater * rename Signed-off-by: coldWater * fix Signed-off-by: coldWater --------- Signed-off-by: coldWater --- .../sql/src/planner/optimizer/rule/factory.rs | 4 ++- .../rule_push_down_filter_window_top_n.rs | 30 +++++++++++++++++-- .../sql/src/planner/optimizer/rule/rule.rs | 6 ++-- .../mode/standalone/explain/window.test | 10 ++++++- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/rule/factory.rs b/src/query/sql/src/planner/optimizer/rule/factory.rs index 50ab1ed44ef9..0aa1fd27b4ff 100644 --- a/src/query/sql/src/planner/optimizer/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/rule/factory.rs @@ -92,7 +92,9 @@ impl RuleFactory { } RuleID::PushDownFilterAggregate => Ok(Box::new(RulePushDownFilterAggregate::new())), RuleID::PushDownFilterWindow => Ok(Box::new(RulePushDownFilterWindow::new())), - RuleID::PushDownFilterWindowRank => Ok(Box::new(RulePushDownFilterWindowTopN::new())), + RuleID::PushDownFilterWindowTopN => { + Ok(Box::new(RulePushDownFilterWindowTopN::new(ctx.metadata))) + } RuleID::EliminateFilter => Ok(Box::new(RuleEliminateFilter::new(ctx.metadata))), RuleID::MergeEvalScalar => Ok(Box::new(RuleMergeEvalScalar::new())), RuleID::MergeFilter => Ok(Box::new(RuleMergeFilter::new())), diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_window_top_n.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_window_top_n.rs index 136aa8622076..981f722ee884 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_window_top_n.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_filter_window_top_n.rs @@ -16,21 +16,28 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_expression::type_check::check_number; +use databend_common_expression::DataField; +use databend_common_expression::DataSchemaRefExt; use databend_common_expression::FunctionContext; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::optimizer::extract::Matcher; use crate::optimizer::rule::Rule; use crate::optimizer::rule::TransformResult; +use crate::optimizer::RelExpr; use crate::optimizer::RuleID; use crate::optimizer::SExpr; use crate::plans::ComparisonOp; +use crate::plans::ConstantTableScan; use crate::plans::Filter; +use crate::plans::Operator; use crate::plans::RelOp; +use crate::plans::RelOperator; use crate::plans::ScalarExpr; use crate::plans::Sort; use crate::plans::Window; use crate::plans::WindowFuncType; +use crate::MetadataRef; /// Input: Filter /// \ @@ -45,13 +52,15 @@ use crate::plans::WindowFuncType; /// Sort(top n) pub struct RulePushDownFilterWindowTopN { id: RuleID, + metadata: MetadataRef, matchers: Vec, } impl RulePushDownFilterWindowTopN { - pub fn new() -> Self { + pub fn new(metadata: MetadataRef) -> Self { Self { - id: RuleID::PushDownFilterWindowRank, + id: RuleID::PushDownFilterWindowTopN, + metadata, matchers: vec![Matcher::MatchOp { op_type: RelOp::Filter, children: vec![Matcher::MatchOp { @@ -93,7 +102,22 @@ impl Rule for RulePushDownFilterWindowTopN { }; if top_n == 0 { - // TODO + let output_columns = s_expr + .plan() + .derive_relational_prop(&RelExpr::with_s_expr(s_expr))? + .output_columns + .clone(); + let metadata = self.metadata.read(); + let mut columns = output_columns.iter().copied().collect::>(); + columns.sort(); + let fields = columns + .into_iter() + .map(|col| DataField::new(&col.to_string(), metadata.column(col).data_type())) + .collect::>(); + let empty_scan = + ConstantTableScan::new_empty_scan(DataSchemaRefExt::create(fields), output_columns); + let result = SExpr::create_leaf(Arc::new(RelOperator::ConstantTableScan(empty_scan))); + state.add_result(result); return Ok(()); } diff --git a/src/query/sql/src/planner/optimizer/rule/rule.rs b/src/query/sql/src/planner/optimizer/rule/rule.rs index f13a67a6b5a5..7ea8ec9ea591 100644 --- a/src/query/sql/src/planner/optimizer/rule/rule.rs +++ b/src/query/sql/src/planner/optimizer/rule/rule.rs @@ -37,7 +37,7 @@ pub static DEFAULT_REWRITE_RULES: LazyLock> = LazyLock::new(|| { RuleID::PushDownFilterUnion, RuleID::PushDownFilterAggregate, RuleID::PushDownFilterWindow, - RuleID::PushDownFilterWindowRank, + RuleID::PushDownFilterWindowTopN, RuleID::PushDownFilterSort, RuleID::PushDownFilterEvalScalar, RuleID::PushDownFilterJoin, @@ -91,7 +91,7 @@ pub enum RuleID { PushDownFilterSort, PushDownFilterProjectSet, PushDownFilterWindow, - PushDownFilterWindowRank, + PushDownFilterWindowTopN, PushDownLimit, PushDownLimitUnion, PushDownLimitOuterJoin, @@ -142,7 +142,7 @@ impl Display for RuleID { RuleID::PushDownSortEvalScalar => write!(f, "PushDownSortEvalScalar"), RuleID::PushDownLimitWindow => write!(f, "PushDownLimitWindow"), RuleID::PushDownFilterWindow => write!(f, "PushDownFilterWindow"), - RuleID::PushDownFilterWindowRank => write!(f, "PushDownFilterWindowRank"), + RuleID::PushDownFilterWindowTopN => write!(f, "PushDownFilterWindowTopN"), RuleID::EliminateEvalScalar => write!(f, "EliminateEvalScalar"), RuleID::EliminateFilter => write!(f, "EliminateFilter"), RuleID::EliminateSort => write!(f, "EliminateSort"), diff --git a/tests/sqllogictests/suites/mode/standalone/explain/window.test b/tests/sqllogictests/suites/mode/standalone/explain/window.test index 37044436493e..454ab3efd904 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/window.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/window.test @@ -541,9 +541,17 @@ CompoundBlockOperator(Project) × 1 SyncReadParquetDataTransform × 1 BlockPartitionSource × 1 +# top n 0 +query T +explain optimized select time, rowkey from (select *, row_number() over(partition by rowkey order by time desc) as rn from table43764_orc) a where rn < 1 +---- +EvalScalar +├── scalars: [a.rowkey (#0) AS (#0), table43764_orc.rowkey (#0) AS (#0), a.time (#1) AS (#1), table43764_orc.time (#1) AS (#1), table43764_orc.sirc_action (#2) AS (#2), table43764_orc.sirc_operation_count (#3) AS (#3), table43764_orc.akc087 (#4) AS (#4), table43764_orc.aae035 (#5) AS (#5), row_number() OVER ( PARTITION BY rowkey ORDER BY time DESC ) (#6) AS (#6), a.rn (#6) AS (#7)] +└── EmptyResultScan + # same order multi window query T -explain pipeline select *,lead(number,1, 42) over (order by number), lead(number,2,44) over (order by number), lead(number,3,44) over (order by number) from numbers(5); +explain pipeline select *,lead(number,1, 42) over (order by number), lead(number,2,44) over (order by number), lead(number,3,44) over (order by number) from numbers(5); ---- CompoundBlockOperator(Project) × 1 Transform Window × 1 From 6debfa29a0d2e849c5eee5c0ab7490031663bc42 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Wed, 13 Nov 2024 20:19:06 +0800 Subject: [PATCH 33/92] chore(base): rollback to kernel signal handler after dump backtrace (#16828) * chore(base): rollback to kernel signal handler after dump backrace * chore(base): avoid use heap memory in signal handler * chore(base): avoid use heap memory in signal handler --- Cargo.lock | 1 - .../exception/src/exception_backtrace.rs | 17 + src/common/tracing/Cargo.toml | 1 - src/common/tracing/src/crash_hook.rs | 308 ++++++++++-------- src/common/tracing/src/panic_hook.rs | 15 +- 5 files changed, 191 insertions(+), 151 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44a3a3fa546a..209c872db89e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4592,7 +4592,6 @@ name = "databend-common-tracing" version = "0.1.0" dependencies = [ "backtrace", - "bincode 2.0.0-rc.3", "chrono", "databend-common-base", "databend-common-exception", diff --git a/src/common/exception/src/exception_backtrace.rs b/src/common/exception/src/exception_backtrace.rs index 7dfdeaa5de3c..7d067d237f6d 100644 --- a/src/common/exception/src/exception_backtrace.rs +++ b/src/common/exception/src/exception_backtrace.rs @@ -157,6 +157,23 @@ impl StackTrace { StackTrace { frames: vec![] } } + #[cfg(target_os = "linux")] + pub fn from_ips(frames: &[u64]) -> StackTrace { + let mut stack_frames = Vec::with_capacity(frames.len()); + for frame in frames { + stack_frames.push(StackFrame::Ip(*frame as usize)); + } + + StackTrace { + frames: stack_frames, + } + } + + #[cfg(not(target_os = "linux"))] + pub fn from_ips(_: &[u64]) -> StackTrace { + StackTrace::no_capture() + } + #[cfg(not(target_os = "linux"))] fn capture_frames(frames: &mut Vec) { unsafe { diff --git a/src/common/tracing/Cargo.toml b/src/common/tracing/Cargo.toml index b00fd7b2e2af..c60d50b63d30 100644 --- a/src/common/tracing/Cargo.toml +++ b/src/common/tracing/Cargo.toml @@ -12,7 +12,6 @@ test = true [dependencies] backtrace = { workspace = true, features = ["std", "serialize-serde"] } -bincode = { workspace = true } chrono = { workspace = true } databend-common-base = { workspace = true } databend-common-exception = { workspace = true } diff --git a/src/common/tracing/src/crash_hook.rs b/src/common/tracing/src/crash_hook.rs index a3646a75950c..5ef6dd42fa84 100644 --- a/src/common/tracing/src/crash_hook.rs +++ b/src/common/tracing/src/crash_hook.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::fs::File; -use std::io::BufRead; +use std::io::Read; use std::io::Write; use std::os::fd::FromRawFd; use std::os::fd::IntoRawFd; @@ -24,12 +24,19 @@ use std::sync::Mutex; use std::sync::PoisonError; use std::time::Duration; -use backtrace::Backtrace; -use backtrace::BacktraceFrame; use databend_common_base::runtime::Thread; use databend_common_base::runtime::ThreadTracker; - -use crate::panic_hook::captures_frames; +use databend_common_exception::StackTrace; + +const BUFFER_SIZE: usize = { + size_of::() // sig + + size_of::() // si_code + + size_of::() // si_addr + + size_of::() // current query id length + + 36 // max query id length, example: a9ef0b3f-b3a1-4759-b129-e1bcf3a551c0 + + size_of::() // frame length + + size_of::() * 50 // max frames ip size +}; struct CrashHandler { write_file: File, @@ -41,61 +48,80 @@ impl CrashHandler { } pub fn recv_signal(&mut self, sig: i32, info: *mut libc::siginfo_t, _uc: *mut libc::c_void) { - let mut writer = std::io::BufWriter::new(&mut self.write_file); - let current_query_id = match ThreadTracker::query_id() { - None => "Unknown", - Some(query_id) => query_id, - }; + let mut buffer = [0_u8; BUFFER_SIZE]; - bincode::serde::encode_into_std_write(sig, &mut writer, bincode::config::standard()) - .unwrap(); + let mut pos = 0; - bincode::serde::encode_into_std_write( - unsafe { (*info).si_code }, - &mut writer, - bincode::config::standard(), - ) - .unwrap(); + fn write_i32(buf: &mut [u8], v: i32, mut pos: usize) -> usize { + for x in v.to_le_bytes() { + buf[pos] = x; + pos += 1; + } - bincode::serde::encode_into_std_write( - unsafe { (*info).si_addr() as usize }, - &mut writer, - bincode::config::standard(), - ) - .unwrap(); + pos + } - bincode::serde::encode_into_std_write( - std::thread::current().id().as_u64(), - &mut writer, - bincode::config::standard(), - ) - .unwrap(); + fn write_u64(buf: &mut [u8], v: u64, mut pos: usize) -> usize { + for x in v.to_le_bytes() { + buf[pos] = x; + pos += 1; + } - bincode::serde::encode_into_std_write( - current_query_id, - &mut writer, - bincode::config::standard(), - ) - .unwrap(); + pos + } - let mut frames = Vec::with_capacity(50); - captures_frames(&mut frames); + fn write_string(buf: &mut [u8], v: &str, mut pos: usize, max_length: usize) -> usize { + let bytes = v.as_bytes(); + let length = std::cmp::min(bytes.len(), max_length); + pos = write_u64(buf, length as u64, pos); - bincode::serde::encode_into_std_write( - frames.len(), - &mut writer, - bincode::config::standard(), - ) - .unwrap(); + #[allow(clippy::needless_range_loop)] + for index in 0..length { + buf[pos] = bytes[index]; + pos += 1; + } + + pos + } + + let current_query_id = match ThreadTracker::query_id() { + None => "Unknown", + Some(query_id) => query_id, + }; + + pos = write_i32(&mut buffer, sig, pos); + pos = write_i32(&mut buffer, unsafe { (*info).si_code }, pos); + pos = write_u64(&mut buffer, unsafe { (*info).si_addr() as u64 }, pos); + pos = write_string(&mut buffer, current_query_id, pos, 36); - writer.flush().unwrap(); + let mut frames_len = 0; + let mut frames = [0_u64; 50]; + unsafe { + backtrace::trace_unsynchronized(|frame| { + frames[frames_len] = frame.ip() as u64; + frames_len += 1; + frames_len < 50 + }); + } + + pos = write_u64(&mut buffer, frames_len as u64, pos); for frame in frames { - bincode::serde::encode_into_std_write(frame, &mut writer, bincode::config::standard()) - .unwrap(); + pos = write_u64(&mut buffer, frame, pos); + } + + if let Err(e) = self.write_file.write(&buffer) { + eprintln!("write signal pipe failure"); + let _ = std::io::stderr().flush(); + eprintln!("cause: {:?}", e); + } + + if let Err(e) = self.write_file.flush() { + eprintln!("flush signal pipe failure"); + let _ = std::io::stderr().flush(); + eprintln!("cause: {:?}", e); } - writer.flush().unwrap(); std::thread::sleep(Duration::from_secs(4)); } } @@ -254,9 +280,13 @@ unsafe extern "C" fn signal_handler(sig: i32, info: *mut libc::siginfo_t, uc: *m #[allow(unreachable_code)] if sig != libc::SIGTRAP { - drop(guard); - let _ = std::io::stderr().flush(); - std::process::exit(1); + match libc::SIG_ERR == libc::signal(sig, libc::SIG_DFL) { + true => std::process::exit(1), + false => match libc::raise(sig) { + 0 => {} + _ => std::process::exit(1), + }, + } } } @@ -329,6 +359,7 @@ pub fn set_crash_hook(output: File) { libc::SIGFPE, libc::SIGSYS, libc::SIGTRAP, + libc::SIGABRT, ]); }; } @@ -376,66 +407,70 @@ pub fn pipe_file() -> std::io::Result<(File, File)> { } } +fn read_i32(buf: &[u8], pos: usize) -> (i32, usize) { + let bytes: [u8; 4] = buf[pos..pos + 4].try_into().expect("expect"); + (i32::from_le_bytes(bytes), pos + 4) +} + +fn read_u64(buf: &[u8], pos: usize) -> (u64, usize) { + let bytes: [u8; 8] = buf[pos..pos + 8].try_into().expect("expect"); + (u64::from_le_bytes(bytes), pos + 8) +} + +fn read_string(buf: &[u8], pos: usize) -> (&str, usize) { + unsafe { + let (len, pos) = read_u64(buf, pos); + ( + std::str::from_utf8_unchecked(&buf[pos..pos + len as usize]), + pos + len as usize, + ) + } +} + pub struct SignalListener; impl SignalListener { - pub fn spawn(file: File, crash_version: String) { + pub fn spawn(mut file: File, crash_version: String) { Thread::named_spawn(Some(String::from("SignalListener")), move || { - let mut reader = std::io::BufReader::new(file); - while let Ok(true) = reader.has_data_left() { - let sig: i32 = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - let si_code: i32 = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - let si_addr: usize = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - - let crash_thread_id: u64 = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - let crash_query_id: String = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - let frame_size: usize = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - - let mut frames = Vec::::with_capacity(frame_size); - for _index in 0..frame_size { - frames.push( - bincode::serde::decode_from_reader( - &mut reader, - bincode::config::standard(), - ) - .unwrap(), - ); + loop { + let mut buffer = [0_u8; BUFFER_SIZE]; + + if file.read_exact(&mut buffer).is_ok() { + let pos = 0; + let (sig, pos) = read_i32(&buffer, pos); + let (si_code, pos) = read_i32(&buffer, pos); + let (si_addr, pos) = read_u64(&buffer, pos); + let (crash_query_id, pos) = read_string(&buffer, pos); + + let (frames_len, mut pos) = read_u64(&buffer, pos); + let mut frames = Vec::with_capacity(50); + + for _ in 0..frames_len { + let (ip, new_pos) = read_u64(&buffer, pos); + frames.push(ip); + pos = new_pos; + } + + let stack_trace = StackTrace::from_ips(&frames); + + eprintln!("{:#^80}", " Crash fault info "); + eprintln!("PID: {}", std::process::id()); + eprintln!("Version: {}", crash_version); + eprintln!("Timestamp(UTC): {}", chrono::Utc::now()); + eprintln!("Timestamp(Local): {}", chrono::Local::now()); + eprintln!("QueryId: {:?}", crash_query_id); + eprintln!("{}", signal_message(sig, si_code, si_addr as usize)); + eprintln!("Backtrace:\n {:?}", stack_trace); + + log::error!("{:#^80}", " Crash fault info "); + log::error!("PID: {}", std::process::id()); + log::error!("Version: {}", crash_version); + log::error!("Timestamp(UTC): {}", chrono::Utc::now()); + log::error!("Timestamp(Local): {}", chrono::Local::now()); + log::error!("QueryId: {:?}", crash_query_id); + log::error!("{}", signal_message(sig, si_code, si_addr as usize)); + log::error!("Backtrace:\n {:?}", stack_trace); } - - let mut backtrace = Backtrace::from(frames); - backtrace.resolve(); - - eprintln!("{:#^80}", " Crash fault info "); - eprintln!("PID: {}", std::process::id()); - eprintln!("TID: {}", crash_thread_id); - eprintln!("Version: {}", crash_version); - eprintln!("Timestamp(UTC): {}", chrono::Utc::now()); - eprintln!("Timestamp(Local): {}", chrono::Local::now()); - eprintln!("QueryId: {:?}", crash_query_id); - eprintln!("{}", signal_message(sig, si_code, si_addr)); - eprintln!("Backtrace:\n {:?}", backtrace); - - log::error!("{:#^80}", " Crash fault info "); - log::error!("PID: {}", std::process::id()); - log::error!("TID: {}", crash_thread_id); - log::error!("Version: {}", crash_version); - log::error!("Timestamp(UTC): {}", chrono::Utc::now()); - log::error!("Timestamp(Local): {}", chrono::Local::now()); - log::error!("QueryId: {:?}", crash_query_id); - log::error!("{}", signal_message(sig, si_code, si_addr)); - log::error!("Backtrace:\n {:?}", backtrace); } }); } @@ -443,13 +478,17 @@ impl SignalListener { #[cfg(test)] mod tests { + use std::io::Read; use std::ptr::addr_of_mut; - use backtrace::BacktraceFrame; use databend_common_base::runtime::ThreadTracker; use crate::crash_hook::pipe_file; + use crate::crash_hook::read_i32; + use crate::crash_hook::read_string; + use crate::crash_hook::read_u64; use crate::crash_hook::sigsetjmp; + use crate::crash_hook::BUFFER_SIZE; use crate::crash_hook::TEST_JMP_BUFFER; use crate::set_crash_hook; @@ -478,40 +517,29 @@ mod tests { libc::raise(signal); } - let sig: i32 = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - let _si_code: i32 = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - let _si_addr: usize = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - - let _crash_thread_id: u64 = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - let crash_query_id: String = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - let frame_size: usize = - bincode::serde::decode_from_reader(&mut reader, bincode::config::standard()) - .unwrap(); - - let mut frames = Vec::::with_capacity(frame_size); - for _index in 0..frame_size { - frames.push( - bincode::serde::decode_from_reader( - &mut reader, - bincode::config::standard(), - ) - .unwrap(), - ); - } + let mut buffer = [0_u8; BUFFER_SIZE]; + + let pos = 0; + if reader.read_exact(&mut buffer).is_ok() { + let (sig, pos) = read_i32(&buffer, pos); + let (_si_code, pos) = read_i32(&buffer, pos); + let (_si_addr, pos) = read_u64(&buffer, pos); + let (crash_query_id, pos) = read_string(&buffer, pos); + + let (frames_len, mut pos) = read_u64(&buffer, pos); + let mut frames = Vec::with_capacity(50); - assert_eq!(sig, signal); - assert_eq!(crash_query_id, query_id); - assert!(!frames.is_empty()); + for _index in 0..frames_len { + let (ip, new_pos) = read_u64(&buffer, pos); + frames.push(ip); + pos = new_pos; + } + + // eprintln!("{:?}", StackTrace::from_ips(&frames)); + assert_eq!(sig, signal); + assert_eq!(crash_query_id, query_id); + assert!(!frames.is_empty()); + } } } } diff --git a/src/common/tracing/src/panic_hook.rs b/src/common/tracing/src/panic_hook.rs index 179ddd36a489..2a6abfb7dcd2 100644 --- a/src/common/tracing/src/panic_hook.rs +++ b/src/common/tracing/src/panic_hook.rs @@ -34,6 +34,7 @@ pub fn set_panic_hook() { })); } +#[allow(dead_code)] fn should_backtrace() -> bool { // if user not specify or user set to enable, we should backtrace match USER_SET_ENABLE_BACKTRACE.load(Ordering::Relaxed) { @@ -70,15 +71,11 @@ pub fn captures_frames(frames: &mut Vec) { } pub fn backtrace(frames: usize) -> String { - if should_backtrace() { - let mut frames = Vec::with_capacity(frames); - captures_frames(&mut frames); - let mut backtrace = Backtrace::from(frames); - backtrace.resolve(); - format!("{:?}", backtrace) - } else { - String::new() - } + let mut frames = Vec::with_capacity(frames); + captures_frames(&mut frames); + let mut backtrace = Backtrace::from(frames); + backtrace.resolve(); + format!("{:?}", backtrace) } #[cfg(test)] From 18bd255cf35242ee42740785475da3e3ceb02373 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Wed, 13 Nov 2024 21:52:04 +0800 Subject: [PATCH 34/92] refactor(base): support stacktrace in cluster mode (#16821) * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node * refactor(base): output physical address for diff version node --- .../exception/src/elf/library_loader.rs | 16 +- .../exception/src/elf/library_manager.rs | 157 +++++++++++++----- .../exception/src/exception_backtrace.rs | 85 +++++++++- src/common/exception/src/exception_into.rs | 3 +- src/query/service/src/global_services.rs | 3 + 5 files changed, 212 insertions(+), 52 deletions(-) diff --git a/src/common/exception/src/elf/library_loader.rs b/src/common/exception/src/elf/library_loader.rs index e356cb71791a..2fbdd9caa35f 100644 --- a/src/common/exception/src/elf/library_loader.rs +++ b/src/common/exception/src/elf/library_loader.rs @@ -211,12 +211,24 @@ impl LibraryLoader { } } - pub fn finalize(mut self) -> (Vec, Vec) { + fn executable_build_id(&mut self) -> std::io::Result>> { + unsafe { + let library_name = OsString::from("/proc/self/exe"); + let binary_path = std::fs::canonicalize(library_name)?.to_path_buf(); + let binary_library = self.mmap_library(binary_path.clone())?; + Ok(binary_library.build_id().map(|x| x.to_vec())) + } + } + + pub fn finalize(mut self) -> (Vec, Vec, Option>) { self.symbols.sort_by(Symbol::sort_begin_address); self.libraries.sort_by(Library::sort_begin_address); self.symbols.dedup_by(Symbol::same_address); - (self.libraries, self.symbols) + match self.executable_build_id() { + Err(_) => (self.libraries, self.symbols, None), + Ok(build_id) => (self.libraries, self.symbols, build_id), + } } } diff --git a/src/common/exception/src/elf/library_manager.rs b/src/common/exception/src/elf/library_manager.rs index daa4bffec7d0..2f8c84c58ca6 100644 --- a/src/common/exception/src/elf/library_manager.rs +++ b/src/common/exception/src/elf/library_manager.rs @@ -25,6 +25,7 @@ use crate::elf::dwarf::Dwarf; use crate::elf::library_loader::LibraryLoader; use crate::elf::library_symbol::Symbol; use crate::elf::ElfFile; +use crate::exception_backtrace::PhysicalAddr; use crate::exception_backtrace::ResolvedStackFrame; use crate::exception_backtrace::StackFrame; @@ -33,16 +34,29 @@ pub struct Library { pub address_begin: usize, pub address_end: usize, pub elf: Option>, + pub build_id: Option>>, library_data: &'static [u8], } impl Library { pub fn create(name: String, data: *const u8, size: usize) -> Library { + let build_id = unsafe { + let data = std::slice::from_raw_parts(data, size); + match ElfFile::parse(data) { + Err(_) => None, + Ok(elf_file) => match elf_file.build_id() { + Ok(None) | Err(_) => None, + Ok(Some(build)) => Some(Arc::new(build.to_vec())), + }, + } + }; + Library { name, - address_begin: 0, - address_end: 0, + build_id, elf: None, + address_end: 0, + address_begin: 0, // Leak memory library_data: unsafe { std::slice::from_raw_parts(data, size) }, } @@ -55,12 +69,8 @@ impl Library { self.library_data } - pub unsafe fn build_id(&self) -> Option<&'static [u8]> { - let elf_file = ElfFile::parse(self.data()).ok()?; - match elf_file.build_id() { - Ok(None) | Err(_) => None, - Ok(Some(build)) => Some(build), - } + pub unsafe fn build_id(&self) -> Option>> { + self.build_id.clone() } } @@ -80,6 +90,7 @@ impl Debug for Library { pub struct LibraryManager { symbols: Vec, libraries: Vec, + executable_build_id: Option>>, } impl Debug for LibraryManager { @@ -98,6 +109,44 @@ impl LibraryManager { .find(|library| library.address_begin <= addr && addr <= library.address_end) } + fn find_library_by_build_id(&self, build_id: &Arc>) -> Option<&Library> { + for library in &self.libraries { + if let Some(v) = &library.build_id { + if v == build_id { + return Some(library); + } + } + } + + None + } + + pub fn to_physical_frames(&self, frames: &[StackFrame]) -> Vec { + let mut res = Vec::with_capacity(frames.len()); + + for frame in frames { + let StackFrame::Ip(addr) = frame else { + res.push(frame.clone()); + continue; + }; + + let Some(library) = self.find_library(*addr) else { + res.push(StackFrame::PhysicalAddr(PhysicalAddr { + physical_addr: 0, + library_build_id: None, + })); + continue; + }; + + res.push(StackFrame::PhysicalAddr(PhysicalAddr { + library_build_id: library.build_id.clone(), + physical_addr: addr - library.address_begin, + })); + } + + res + } + pub fn resolve_frames Result<(), E>>( &self, frames: &[StackFrame], @@ -107,36 +156,56 @@ impl LibraryManager { let mut dwarf_cache = HashMap::with_capacity(self.libraries.len()); for frame in frames { - let StackFrame::Ip(addr) = frame; + let (library, addr) = match frame { + StackFrame::Ip(addr) => { + let Some(library) = self.find_library(*addr) else { + f(ResolvedStackFrame { + virtual_address: *addr, + physical_address: *addr, + symbol: String::from(""), + inlined: false, + file: None, + line: None, + column: None, + })?; - let mut resolved_frame = ResolvedStackFrame { - virtual_address: *addr, - physical_address: *addr, - symbol: String::from(""), - inlined: false, - file: None, - line: None, - column: None, - }; + continue; + }; - if let Some(library) = self.find_library(*addr) { - resolved_frame.physical_address = *addr - library.address_begin; - } - let Some(library) = self.find_library(*addr) else { - f(ResolvedStackFrame { - virtual_address: *addr, - physical_address: *addr, - symbol: String::from(""), - inlined: false, - file: None, - line: None, - column: None, - })?; + (library, *addr - library.address_begin) + } + StackFrame::PhysicalAddr(physical_addr) => { + let Some(build_id) = &physical_addr.library_build_id else { + f(ResolvedStackFrame { + virtual_address: 0, + physical_address: physical_addr.physical_addr, + symbol: String::from(""), + inlined: false, + file: None, + line: None, + column: None, + })?; - continue; - }; + continue; + }; + + let Some(library) = self.find_library_by_build_id(build_id) else { + f(ResolvedStackFrame { + virtual_address: 0, + physical_address: physical_addr.physical_addr, + symbol: String::from(""), + inlined: false, + file: None, + line: None, + column: None, + })?; - let physical_address = *addr - library.address_begin; + continue; + }; + + (library, physical_addr.physical_addr) + } + }; if !only_address { let dwarf = match library.elf.as_ref() { @@ -151,13 +220,13 @@ impl LibraryManager { }; if let Some(dwarf) = dwarf { - let adjusted_addr = (physical_address - 1) as u64; + let adjusted_addr = (addr - 1) as u64; if let Ok(locations) = dwarf.find_frames(adjusted_addr) { for location in locations { f(ResolvedStackFrame { virtual_address: 0, - physical_address, + physical_address: addr, symbol: location.symbol.unwrap_or("".to_string()), inlined: location.is_inlined, file: location.file, @@ -172,8 +241,8 @@ impl LibraryManager { } f(ResolvedStackFrame { - physical_address, - virtual_address: *addr, + physical_address: addr, + virtual_address: 0, inlined: false, symbol: String::from(""), file: None, @@ -185,10 +254,18 @@ impl LibraryManager { Ok(()) } + pub fn executable_build_id(&self) -> Option>> { + self.executable_build_id.clone() + } + pub fn create() -> Arc { let loader = LibraryLoader::load(); - let (libraries, symbols) = loader.finalize(); - Arc::new(LibraryManager { symbols, libraries }) + let (libraries, symbols, build_id) = loader.finalize(); + Arc::new(LibraryManager { + symbols, + libraries, + executable_build_id: build_id.map(Arc::new), + }) } pub fn instance() -> Arc { diff --git a/src/common/exception/src/exception_backtrace.rs b/src/common/exception/src/exception_backtrace.rs index 7d067d237f6d..471931aa44d8 100644 --- a/src/common/exception/src/exception_backtrace.rs +++ b/src/common/exception/src/exception_backtrace.rs @@ -77,10 +77,18 @@ pub struct ResolvedStackFrame { pub column: Option, } +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] +pub struct PhysicalAddr { + pub physical_addr: usize, + pub library_build_id: Option>>, +} + #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub enum StackFrame { #[cfg(target_os = "linux")] Ip(usize), + #[cfg(target_os = "linux")] + PhysicalAddr(PhysicalAddr), #[cfg(not(target_os = "linux"))] Backtrace(backtrace::BacktraceFrame), } @@ -91,9 +99,13 @@ impl PartialEq for StackFrame { fn eq(&self, other: &Self) -> bool { #[cfg(target_os = "linux")] { - let StackFrame::Ip(addr) = &self; - let StackFrame::Ip(other_addr) = &other; - addr == other_addr + match (&self, &other) { + (StackFrame::Ip(addr), StackFrame::Ip(other_addr)) => addr == other_addr, + (StackFrame::PhysicalAddr(addr), StackFrame::PhysicalAddr(other_addr)) => { + addr.physical_addr == other_addr.physical_addr + } + _ => false, + } } #[cfg(not(target_os = "linux"))] @@ -109,8 +121,10 @@ impl Hash for StackFrame { fn hash(&self, state: &mut H) { #[cfg(target_os = "linux")] { - let StackFrame::Ip(addr) = &self; - addr.hash(state); + match &self { + StackFrame::Ip(addr) => addr.hash(state), + StackFrame::PhysicalAddr(addr) => addr.physical_addr.hash(state), + }; } #[cfg(not(target_os = "linux"))] @@ -136,6 +150,7 @@ impl Hash for StackFrame { #[derive(Clone, serde::Serialize, serde::Deserialize)] pub struct StackTrace { pub(crate) frames: Vec, + pub(crate) build_id: Option>>, } impl Eq for StackTrace {} @@ -150,11 +165,54 @@ impl StackTrace { pub fn capture() -> StackTrace { let mut frames = Vec::with_capacity(50); Self::capture_frames(&mut frames); - StackTrace { frames } + StackTrace { + frames, + build_id: Self::executable_build_id(), + } } pub fn no_capture() -> StackTrace { - StackTrace { frames: vec![] } + StackTrace { + frames: vec![], + build_id: Self::executable_build_id(), + } + } + + #[cfg(not(target_os = "linux"))] + pub fn pre_load_symbol() { + static INIT_GUARD: LazyLock = + LazyLock::new(backtrace::Backtrace::new); + let _frames = INIT_GUARD.frames(); + } + + #[cfg(target_os = "linux")] + pub fn pre_load_symbol() { + let _lib_manager = crate::elf::LibraryManager::instance(); + } + + #[cfg(not(target_os = "linux"))] + pub fn to_physical(&self) -> StackTrace { + self.clone() + } + + #[cfg(target_os = "linux")] + pub fn to_physical(&self) -> StackTrace { + let frames = crate::elf::LibraryManager::instance().to_physical_frames(&self.frames); + + StackTrace { + frames, + build_id: Self::executable_build_id(), + } + } + + #[cfg(not(target_os = "linux"))] + fn executable_build_id() -> Option>> { + None + } + + #[cfg(target_os = "linux")] + fn executable_build_id() -> Option>> { + crate::elf::LibraryManager::instance().executable_build_id() } #[cfg(target_os = "linux")] @@ -166,6 +224,7 @@ impl StackTrace { StackTrace { frames: stack_frames, + build_id: Self::executable_build_id(), } } @@ -199,7 +258,7 @@ impl StackTrace { #[cfg(not(target_os = "linux"))] fn fmt_frames(&self, display_text: &mut String, address: bool) -> std::fmt::Result { - let mut frames = std::vec::Vec::with_capacity(self.frames.len()); + let mut frames = Vec::with_capacity(self.frames.len()); for frame in &self.frames { let StackFrame::Backtrace(frame) = frame; frames.push(frame.clone()); @@ -215,7 +274,15 @@ impl StackTrace { } #[cfg(target_os = "linux")] - fn fmt_frames(&self, f: &mut String, address: bool) -> std::fmt::Result { + fn fmt_frames(&self, f: &mut String, mut address: bool) -> std::fmt::Result { + if !address { + let binary_id = crate::elf::LibraryManager::instance().executable_build_id(); + address = match (&binary_id, &self.build_id) { + (Some(binary_id), Some(build_id)) => binary_id != build_id, + _ => true, + }; + } + let mut idx = 0; crate::elf::LibraryManager::instance().resolve_frames(&self.frames, address, |frame| { write!(f, "{:4}: {}", idx, frame.symbol)?; diff --git a/src/common/exception/src/exception_into.rs b/src/common/exception/src/exception_into.rs index 57ec00e482b4..72788198bd35 100644 --- a/src/common/exception/src/exception_into.rs +++ b/src/common/exception/src/exception_into.rs @@ -284,12 +284,13 @@ impl Display for SerializedError { impl From<&ErrorCode> for SerializedError { fn from(e: &ErrorCode) -> Self { + // let binary_version = (*databend_common_config::DATABEND_COMMIT_VERSION).clone(); SerializedError { code: e.code(), name: e.name(), message: e.message(), span: e.span(), - backtrace: e.backtrace.clone(), + backtrace: e.backtrace.to_physical(), stacks: e.stacks().iter().map(|f| f.into()).collect(), } } diff --git a/src/query/service/src/global_services.rs b/src/query/service/src/global_services.rs index c40a4ebb8601..263a7b5b86f3 100644 --- a/src/query/service/src/global_services.rs +++ b/src/query/service/src/global_services.rs @@ -24,6 +24,7 @@ use databend_common_cloud_control::cloud_api::CloudControlApiProvider; use databend_common_config::GlobalConfig; use databend_common_config::InnerConfig; use databend_common_exception::Result; +use databend_common_exception::StackTrace; use databend_common_meta_app::schema::CatalogType; use databend_common_storage::DataOperator; use databend_common_storage::ShareTableConfig; @@ -62,6 +63,8 @@ impl GlobalServices { #[async_backtrace::framed] pub async fn init_with(config: &InnerConfig) -> Result<()> { + StackTrace::pre_load_symbol(); + // app name format: node_id[0..7]@cluster_id let app_name_shuffle = format!("databend-query-{}", config.query.cluster_id); From e909334fd3ade668ada4e64b5e9354491324f1e1 Mon Sep 17 00:00:00 2001 From: zhya Date: Wed, 13 Nov 2024 22:35:51 +0800 Subject: [PATCH 35/92] fix(storage): recluster endless loop (#16831) * fix recluster endless loop * fix * fix test * fix test --- .../transform_compact_no_split_builder.rs | 36 +++++-------------- .../09_0016_remote_alter_recluster.test | 4 +-- .../09_0041_auto_compaction.test | 12 ++++--- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/src/query/pipeline/transforms/src/processors/transforms/transform_compact_no_split_builder.rs b/src/query/pipeline/transforms/src/processors/transforms/transform_compact_no_split_builder.rs index aea6c8b6ec08..13a8786660c2 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/transform_compact_no_split_builder.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/transform_compact_no_split_builder.rs @@ -94,12 +94,12 @@ impl AccumulatingTransform for BlockCompactNoSplitBuilder { res.push(Self::create_output_data(&mut self.staged_blocks)); } - if !self.check_for_compact() && !self.pending_blocks.is_empty() { - // blocks > 2N - res.push(Self::create_output_data(&mut self.pending_blocks)); - } else { + if self.pending_blocks.is_empty() || self.check_for_compact() { // N <= blocks < 2N std::mem::swap(&mut self.staged_blocks, &mut self.pending_blocks); + } else { + // blocks > 2N + res.push(Self::create_output_data(&mut self.pending_blocks)); } self.staged_blocks.push(data); self.reset_accumulated(); @@ -107,29 +107,11 @@ impl AccumulatingTransform for BlockCompactNoSplitBuilder { } fn on_finish(&mut self, _output: bool) -> Result> { - match ( - self.pending_blocks.is_empty(), - self.staged_blocks.is_empty(), - ) { - (true, true) => Ok(vec![]), - (true, false) => Ok(vec![Self::create_output_data(&mut self.staged_blocks)]), - (false, true) => Ok(vec![Self::create_output_data(&mut self.pending_blocks)]), - (false, false) => { - for block in &self.staged_blocks { - self.accumulated_rows += block.num_rows(); - self.accumulated_bytes += block.memory_size(); - } - if self.check_for_compact() { - self.staged_blocks.append(&mut self.pending_blocks); - Ok(vec![Self::create_output_data(&mut self.staged_blocks)]) - } else { - // blocks > 2N - Ok(vec![ - Self::create_output_data(&mut self.staged_blocks), - Self::create_output_data(&mut self.pending_blocks), - ]) - } - } + self.staged_blocks.append(&mut self.pending_blocks); + if self.staged_blocks.is_empty() { + Ok(vec![]) + } else { + Ok(vec![Self::create_output_data(&mut self.staged_blocks)]) } } } diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0016_remote_alter_recluster.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0016_remote_alter_recluster.test index 4d09b40b950b..217db4cbd587 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0016_remote_alter_recluster.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0016_remote_alter_recluster.test @@ -76,9 +76,9 @@ statement ok ALTER TABLE t3 RECLUSTER query FFT -select average_overlaps, average_depth, block_depth_histogram from clustering_information('db_09_0016','t3') +select average_overlaps, average_depth from clustering_information('db_09_0016','t3') ---- -0.0 1.0 {"00001":2} +0.0 1.0 # test trim string statement ok diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test index ddaffdb1a146..db96ae7968df 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test @@ -7,6 +7,8 @@ use i15760; statement ok set auto_compaction_imperfect_blocks_threshold = 3; +statement ok +set enable_parallel_multi_merge_sort = 0; # ISSUE 15760 statement ok @@ -83,17 +85,17 @@ statement ok insert into t1 values(1),(2),(7); statement ok -insert into t1 values(3),(5),(9); +insert into t1 values(3),(5),(8); statement ok -insert into t1 values(4),(6),(8); +insert into t1 values(4),(6); query III select segment_count, block_count, row_count from fuse_snapshot('i15760', 't1') limit 10; ---- -1 2 9 -1 2 9 -3 3 9 +1 1 8 +1 2 8 +3 3 8 2 2 6 1 1 3 From 6c05298b71d39a30dce1b1318e44260ce837b833 Mon Sep 17 00:00:00 2001 From: zhya Date: Thu, 14 Nov 2024 00:33:19 +0800 Subject: [PATCH 36/92] chore: remove unused codes (#16834) remove unused codes --- src/query/catalog/src/table_context.rs | 5 ----- .../builders/builder_column_mutation.rs | 18 ---------------- src/query/service/src/sessions/query_ctx.rs | 21 ------------------- .../tests/it/sql/exec/get_table_bind_test.rs | 17 --------------- .../it/storages/fuse/operations/commit.rs | 16 -------------- 5 files changed, 77 deletions(-) diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index e7a5e74f60e6..6683f176875a 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -61,7 +61,6 @@ use databend_common_users::GrantObjectVisibilityChecker; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; -use databend_storages_common_table_meta::meta::TableSnapshot; use parking_lot::Mutex; use parking_lot::RwLock; use xorf::BinaryFuse16; @@ -181,10 +180,6 @@ pub trait TableContext: Send + Sync { fn get_compaction_num_block_hint(&self, _table_name: &str) -> u64 { unimplemented!() } - fn set_table_snapshot(&self, snapshot: Arc); - fn get_table_snapshot(&self) -> Option>; - fn set_lazy_mutation_delete(&self, lazy: bool); - fn get_lazy_mutation_delete(&self) -> bool; fn attach_query_str(&self, kind: QueryKind, query: String); fn attach_query_hash(&self, text_hash: String, parameterized_hash: String); diff --git a/src/query/service/src/pipelines/builders/builder_column_mutation.rs b/src/query/service/src/pipelines/builders/builder_column_mutation.rs index ae9c4cb54247..225fef28c138 100644 --- a/src/query/service/src/pipelines/builders/builder_column_mutation.rs +++ b/src/query/service/src/pipelines/builders/builder_column_mutation.rs @@ -15,7 +15,6 @@ use std::collections::HashMap; use databend_common_catalog::table::Table; -use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; @@ -25,10 +24,8 @@ use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::evaluator::CompoundBlockOperator; use databend_common_sql::executor::physical_plans::ColumnMutation; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_storages_fuse::operations::TableMutationAggregator; use databend_common_storages_fuse::operations::TransformSerializeBlock; use databend_common_storages_fuse::FuseTable; -use databend_storages_common_table_meta::meta::Statistics; use crate::pipelines::PipelineBuilder; @@ -68,21 +65,6 @@ impl PipelineBuilder { )?; proc.into_processor() })?; - - if self.ctx.get_lazy_mutation_delete() { - self.main_pipeline.try_resize(1)?; - self.main_pipeline.add_async_accumulating_transformer(|| { - TableMutationAggregator::create( - table, - self.ctx.clone(), - self.ctx.get_table_snapshot().unwrap().segments.clone(), - vec![], - vec![], - Statistics::default(), - MutationKind::Delete, - ) - }); - } } else { let block_thresholds = table.get_block_thresholds(); let cluster_stats_gen = table.cluster_gen_for_append( diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index bdf4b7866eda..30f64ea68377 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -113,7 +113,6 @@ use databend_common_users::UserApiProvider; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; -use databend_storages_common_table_meta::meta::TableSnapshot; use log::debug; use log::info; use parking_lot::Mutex; @@ -151,8 +150,6 @@ pub struct QueryContext { fragment_id: Arc, // Used by synchronized generate aggregating indexes when new data written. inserted_segment_locs: Arc>>, - snapshot: Arc>>>, - lazy_mutaion_delete: Arc>, } impl QueryContext { @@ -178,8 +175,6 @@ impl QueryContext { fragment_id: Arc::new(AtomicUsize::new(0)), inserted_segment_locs: Arc::new(RwLock::new(HashSet::new())), block_threshold: Arc::new(RwLock::new(BlockThresholds::default())), - snapshot: Arc::new(RwLock::new(None)), - lazy_mutaion_delete: Arc::new(RwLock::new(false)), }) } @@ -532,22 +527,6 @@ impl TableContext for QueryContext { Ok(()) } - fn set_table_snapshot(&self, snapshot: Arc) { - *self.snapshot.write() = Some(snapshot); - } - - fn get_table_snapshot(&self) -> Option> { - self.snapshot.read().clone() - } - - fn set_lazy_mutation_delete(&self, lazy: bool) { - *self.lazy_mutaion_delete.write() = lazy; - } - - fn get_lazy_mutation_delete(&self) -> bool { - *self.lazy_mutaion_delete.read() - } - fn partition_num(&self) -> usize { self.partition_queue.read().len() } diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 3cb3b651c17a..9bd031cf5598 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -148,7 +148,6 @@ use databend_query::test_kits::*; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; -use databend_storages_common_table_meta::meta::TableSnapshot; use parking_lot::Mutex; use parking_lot::RwLock; use xorf::BinaryFuse16; @@ -584,22 +583,6 @@ impl TableContext for CtxDelegation { todo!() } - fn set_table_snapshot(&self, _snapshot: Arc) { - todo!() - } - - fn get_table_snapshot(&self) -> Option> { - todo!() - } - - fn set_lazy_mutation_delete(&self, _lazy: bool) { - todo!() - } - - fn get_lazy_mutation_delete(&self) -> bool { - todo!() - } - fn add_partitions_sha(&self, _sha: String) { todo!() } diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index ddceed7daac3..3453e4d96647 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -486,22 +486,6 @@ impl TableContext for CtxDelegation { todo!() } - fn set_table_snapshot(&self, _snapshot: Arc) { - todo!() - } - - fn get_table_snapshot(&self) -> Option> { - todo!() - } - - fn set_lazy_mutation_delete(&self, _lazy: bool) { - todo!() - } - - fn get_lazy_mutation_delete(&self) -> bool { - todo!() - } - fn add_partitions_sha(&self, _sha: String) { todo!() } From fb5dbcba73d1871f1c7c2f39ce7198c1cc501104 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Thu, 14 Nov 2024 09:10:51 +0800 Subject: [PATCH 37/92] fix: incorrect table data disk cache key (#16837) * fix: incorrect table data disck cache key should use `offset` and `len` of column meta as corresponding parts of table data cache key * refactor: enforce using same column data key --- .../read/block/block_reader_merge_io_async.rs | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs index 0f2b2bcf8cb7..082cbc6a20ac 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs @@ -48,6 +48,9 @@ impl BlockReader { let column_array_cache = CacheManager::instance().get_table_data_array_cache(); let mut cached_column_data = vec![]; let mut cached_column_array = vec![]; + + let column_cache_key_builder = ColumnCacheKeyBuilder::new(location); + for (_index, (column_id, ..)) in self.project_indices.iter() { if let Some(ignore_column_ids) = ignore_column_ids { if ignore_column_ids.contains(column_id) { @@ -58,7 +61,7 @@ impl BlockReader { if let Some(column_meta) = columns_meta.get(column_id) { let (offset, len) = column_meta.offset_length(); - let column_cache_key = TableDataCacheKey::new(location, *column_id, offset, len); + let column_cache_key = column_cache_key_builder.cache_cache(column_id, column_meta); // first, check in memory table data cache // column_array_cache @@ -91,20 +94,25 @@ impl BlockReader { .await?; if self.put_cache { - let table_data_cache = CacheManager::instance().get_table_data_cache(); // add raw data (compressed raw bytes) to column cache for (column_id, (chunk_idx, range)) in &merge_io_result.columns_chunk_offsets { - let cache_key = TableDataCacheKey::new( - &merge_io_result.block_path, - *column_id, - range.start as u64, - (range.end - range.start) as u64, - ); + // Should NOT use `range.start` as part of the cache key, + // as they are not stable and can vary for the same column depending on the query's projection. + // For instance: + // - `SELECT col1, col2 FROM t;` + // - `SELECT col2 FROM t;` + // may result in different ranges for `col2` + // This can lead to cache missing or INCONSISTENCIES + + // Safe to unwrap here, since this column has been fetched, its meta must be present. + let column_meta = columns_meta.get(column_id).unwrap(); + let column_cache_key = column_cache_key_builder.cache_cache(column_id, column_meta); + let chunk_data = merge_io_result .owner_memory .get_chunk(*chunk_idx, &merge_io_result.block_path)?; let data = chunk_data.slice(range.clone()); - table_data_cache.insert(cache_key.as_ref().to_owned(), data); + column_data_cache.insert(column_cache_key.as_ref().to_owned(), data); } } @@ -116,3 +124,17 @@ impl BlockReader { Ok(block_read_res) } } + +struct ColumnCacheKeyBuilder<'a> { + block_path: &'a str, +} + +impl<'a> ColumnCacheKeyBuilder<'a> { + fn new(block_path: &'a str) -> Self { + Self { block_path } + } + fn cache_cache(&self, column_id: &ColumnId, column_meta: &ColumnMeta) -> TableDataCacheKey { + let (offset, len) = column_meta.offset_length(); + TableDataCacheKey::new(self.block_path, *column_id, offset, len) + } +} From 2ccede319f28099130269656fb7d6a6d9ad609fe Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Thu, 14 Nov 2024 11:01:14 +0800 Subject: [PATCH 38/92] refactor: refactor cte binder and fix materialized cte used in subquery (#16785) * refactor: refactor cte binder * extract cte context and refactor join parts, todo: set op, merge into and r cte * all cte tests pass * cargo fix * make tpcds q95 materialized cte work * remove used count in cte info * fix set_cte_context * fix the distinct number for ctes * try to fix stackoverflow * fix with consume * add test for tpcds95 --- .../src/interpreters/interpreter_replace.rs | 2 - src/query/sql/src/planner/binder/aggregate.rs | 2 - .../sql/src/planner/binder/bind_context.rs | 98 ++++++++++++++++--- .../src/planner/binder/bind_mutation/bind.rs | 3 - .../bind_mutation/mutation_expression.rs | 25 +---- .../sql/src/planner/binder/bind_query/bind.rs | 11 +-- .../planner/binder/bind_query/bind_select.rs | 3 + .../planner/binder/bind_query/bind_value.rs | 3 - .../binder/bind_table_reference/bind_join.rs | 38 +++---- .../bind_table_reference/bind_subquery.rs | 3 + .../binder/bind_table_reference/bind_table.rs | 11 ++- .../bind_table_function.rs | 2 - src/query/sql/src/planner/binder/binder.rs | 38 +------ .../sql/src/planner/binder/copy_into_table.rs | 4 - src/query/sql/src/planner/binder/ddl/table.rs | 4 - .../sql/src/planner/binder/expr_values.rs | 14 +-- src/query/sql/src/planner/binder/having.rs | 2 - .../src/planner/binder/insert_multi_table.rs | 4 - src/query/sql/src/planner/binder/project.rs | 4 - src/query/sql/src/planner/binder/qualify.rs | 2 - src/query/sql/src/planner/binder/scalar.rs | 12 --- src/query/sql/src/planner/binder/select.rs | 34 ++++--- src/query/sql/src/planner/binder/sort.rs | 2 - src/query/sql/src/planner/binder/table.rs | 67 ++++++++----- .../sql/src/planner/semantic/type_check.rs | 22 +---- .../suites/tpcds/materialized_cte.test | 36 +++++++ 26 files changed, 231 insertions(+), 215 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_replace.rs b/src/query/service/src/interpreters/interpreter_replace.rs index b85dc319bab5..0aea9d1f2e8a 100644 --- a/src/query/service/src/interpreters/interpreter_replace.rs +++ b/src/query/service/src/interpreters/interpreter_replace.rs @@ -214,8 +214,6 @@ impl ReplaceInterpreter { &name_resolution_ctx, metadata, &[], - Default::default(), - Default::default(), ); let (scalar, _) = scalar_binder.bind(expr)?; let columns = scalar.used_columns(); diff --git a/src/query/sql/src/planner/binder/aggregate.rs b/src/query/sql/src/planner/binder/aggregate.rs index 4b7e9af2c862..4911f65ab5e7 100644 --- a/src/query/sql/src/planner/binder/aggregate.rs +++ b/src/query/sql/src/planner/binder/aggregate.rs @@ -639,8 +639,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (scalar_expr, _) = scalar_binder .bind(expr) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 8ff684a7e288..1a501500afdb 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -14,7 +14,9 @@ use std::collections::btree_map; use std::collections::BTreeMap; +use std::collections::HashMap; use std::hash::Hash; +use std::sync::Arc; use dashmap::DashMap; use databend_common_ast::ast::Identifier; @@ -40,6 +42,9 @@ use crate::binder::column_binding::ColumnBinding; use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; +use crate::optimizer::SExpr; +use crate::plans::MaterializedCte; +use crate::plans::RelOperator; use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::ColumnSet; @@ -117,10 +122,7 @@ pub struct BindContext { pub windows: WindowInfo, - /// If the `BindContext` is created from a CTE, record the cte name - pub cte_name: Option, - - pub cte_map_ref: Box>, + pub cte_context: CteContext, /// True if there is aggregation in current context, which means /// non-grouping columns cannot be referenced outside aggregation @@ -153,6 +155,83 @@ pub struct BindContext { pub window_definitions: DashMap, } +#[derive(Clone, Debug, Default)] +pub struct CteContext { + /// If the `BindContext` is created from a CTE, record the cte name + pub cte_name: Option, + /// Use `IndexMap` because need to keep the insertion order + /// Then wrap materialized ctes to main plan. + pub cte_map: Box>, + /// Record the bound s_expr of materialized cte + pub m_cte_bound_s_expr: HashMap, + pub m_cte_materialized_indexes: HashMap, +} + +impl CteContext { + pub fn set_m_cte_bound_s_expr(&mut self, cte_idx: IndexType, s_expr: SExpr) { + self.m_cte_bound_s_expr.insert(cte_idx, s_expr); + } + + pub fn set_m_cte_materialized_indexes(&mut self, cte_idx: IndexType, index: IndexType) { + self.m_cte_materialized_indexes.insert(cte_idx, index); + } + + // Check if the materialized cte has been bound. + pub fn has_bound(&self, cte_idx: IndexType) -> bool { + self.m_cte_bound_s_expr.contains_key(&cte_idx) + } + + // Merge two `CteContext` into one. + pub fn merge(&mut self, other: CteContext) { + let mut merged_cte_map = IndexMap::new(); + for (left_key, left_value) in self.cte_map.iter() { + if let Some(right_value) = other.cte_map.get(left_key) { + let mut merged_value = left_value.clone(); + if left_value.columns.is_empty() { + merged_value.columns = right_value.columns.clone() + } + merged_cte_map.insert(left_key.clone(), merged_value); + } + } + self.cte_map = Box::new(merged_cte_map); + self.m_cte_bound_s_expr.extend(other.m_cte_bound_s_expr); + self.m_cte_materialized_indexes + .extend(other.m_cte_materialized_indexes); + } + + // Wrap materialized cte to main plan. + // It will be called at the end of binding. + pub fn wrap_m_cte(&self, mut s_expr: SExpr) -> SExpr { + for (_, cte_info) in self.cte_map.iter().rev() { + if !cte_info.materialized { + continue; + } + if let Some(cte_s_expr) = self.m_cte_bound_s_expr.get(&cte_info.cte_idx) { + let materialized_output_columns = cte_info.columns.clone(); + s_expr = SExpr::create_binary( + Arc::new(RelOperator::MaterializedCte(MaterializedCte { + cte_idx: cte_info.cte_idx, + materialized_output_columns, + materialized_indexes: self.m_cte_materialized_indexes.clone(), + })), + Arc::new(s_expr), + Arc::new(cte_s_expr.clone()), + ); + } + } + s_expr + } + + // Set cte context to current `BindContext`. + // To make sure the last `BindContext` of the whole binding phase contains `cte context` + // Then we can wrap materialized cte to main plan. + pub fn set_cte_context(&mut self, cte_context: CteContext) { + self.cte_map = cte_context.cte_map; + self.m_cte_bound_s_expr = cte_context.m_cte_bound_s_expr; + self.m_cte_materialized_indexes = cte_context.m_cte_materialized_indexes; + } +} + #[derive(Clone, Debug)] pub struct CteInfo { pub columns_alias: Vec, @@ -160,8 +239,6 @@ pub struct CteInfo { pub materialized: bool, pub recursive: bool, pub cte_idx: IndexType, - // Record how many times this cte is used - pub used_count: usize, // If cte is materialized, save its columns pub columns: Vec, } @@ -174,8 +251,7 @@ impl BindContext { bound_internal_columns: BTreeMap::new(), aggregate_info: AggregateInfo::default(), windows: WindowInfo::default(), - cte_name: None, - cte_map_ref: Box::default(), + cte_context: CteContext::default(), in_grouping: false, view_info: None, srfs: Vec::new(), @@ -196,8 +272,7 @@ impl BindContext { bound_internal_columns: BTreeMap::new(), aggregate_info: Default::default(), windows: Default::default(), - cte_name: parent.cte_name, - cte_map_ref: parent.cte_map_ref.clone(), + cte_context: parent.cte_context.clone(), in_grouping: false, view_info: None, srfs: Vec::new(), @@ -215,8 +290,7 @@ impl BindContext { pub fn replace(&self) -> Self { let mut bind_context = BindContext::new(); bind_context.parent = self.parent.clone(); - bind_context.cte_name = self.cte_name.clone(); - bind_context.cte_map_ref = self.cte_map_ref.clone(); + bind_context.cte_context = self.cte_context.clone(); bind_context } diff --git a/src/query/sql/src/planner/binder/bind_mutation/bind.rs b/src/query/sql/src/planner/binder/bind_mutation/bind.rs index ca75491633e4..019b6fa76b0c 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/bind.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/bind.rs @@ -32,7 +32,6 @@ use databend_common_expression::Scalar; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::ROW_VERSION_COL_NAME; -use indexmap::IndexMap; use crate::binder::bind_mutation::mutation_expression::MutationExpression; use crate::binder::bind_mutation::mutation_expression::MutationExpressionBindResult; @@ -258,8 +257,6 @@ impl Binder { &name_resolution_ctx, self.metadata.clone(), &[], - HashMap::new(), - Box::new(IndexMap::new()), ); // Bind matched clause columns and add update fields and exprs diff --git a/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs b/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs index 9634e7ae0cc0..66dddfe8b131 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs @@ -39,7 +39,6 @@ use crate::optimizer::SExpr; use crate::optimizer::SubqueryRewriter; use crate::plans::BoundColumnRef; use crate::plans::Filter; -use crate::plans::MaterializedCte; use crate::plans::MutationSource; use crate::plans::RelOperator; use crate::plans::SubqueryExpr; @@ -121,7 +120,7 @@ impl MutationExpression { } // Wrap `LogicalMaterializedCte` to `source_expr`. - source_s_expr = binder.wrap_cte(source_s_expr); + source_s_expr = source_context.cte_context.wrap_m_cte(source_s_expr); // When there is "update *" or "insert *", prepare all source columns. let all_source_columns = Self::all_source_columns( @@ -443,26 +442,6 @@ impl Binder { Ok(row_id_index) } - fn wrap_cte(&mut self, mut s_expr: SExpr) -> SExpr { - for (_, cte_info) in self.ctes_map.iter().rev() { - if !cte_info.materialized || cte_info.used_count == 0 { - continue; - } - let cte_s_expr = self.m_cte_bound_s_expr.get(&cte_info.cte_idx).unwrap(); - let materialized_output_columns = cte_info.columns.clone(); - s_expr = SExpr::create_binary( - Arc::new(RelOperator::MaterializedCte(MaterializedCte { - cte_idx: cte_info.cte_idx, - materialized_output_columns, - materialized_indexes: self.m_cte_materialized_indexes.clone(), - })), - Arc::new(s_expr), - Arc::new(cte_s_expr.clone()), - ); - } - s_expr - } - // Recursively flatten the AND expressions. pub fn flatten_and_scalar_expr(scalar: &ScalarExpr) -> Vec { if let ScalarExpr::FunctionCall(func) = scalar @@ -489,8 +468,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (scalar, _) = scalar_binder.bind(expr)?; if !self.check_allowed_scalar_expr_with_subquery(&scalar)? { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index f3556bc02170..7c5a1314f869 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -70,7 +70,7 @@ impl Binder { for (idx, cte) in with.ctes.iter().enumerate() { let table_name = self.normalize_identifier(&cte.alias.name).name; - if bind_context.cte_map_ref.contains_key(&table_name) { + if bind_context.cte_context.cte_map.contains_key(&table_name) { return Err(ErrorCode::SemanticError(format!( "Duplicate common table expression: {table_name}" ))); @@ -87,11 +87,12 @@ impl Binder { materialized: cte.materialized, recursive: with.recursive, cte_idx: idx, - used_count: 0, columns: vec![], }; - self.ctes_map.insert(table_name.clone(), cte_info.clone()); - bind_context.cte_map_ref.insert(table_name, cte_info); + bind_context + .cte_context + .cte_map + .insert(table_name, cte_info); } Ok(()) @@ -117,8 +118,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let mut order_by_items = Vec::with_capacity(query.order_by.len()); diff --git a/src/query/sql/src/planner/binder/bind_query/bind_select.rs b/src/query/sql/src/planner/binder/bind_query/bind_select.rs index 55c04b7ff89a..ed21bd7f0b19 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind_select.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind_select.rs @@ -247,6 +247,9 @@ impl Binder { let mut output_context = BindContext::new(); output_context.parent = from_context.parent; + output_context + .cte_context + .set_cte_context(from_context.cte_context.clone()); output_context.columns = from_context.columns; Ok((s_expr, output_context)) diff --git a/src/query/sql/src/planner/binder/bind_query/bind_value.rs b/src/query/sql/src/planner/binder/bind_query/bind_value.rs index 08aee00a7cb5..788a3017c99f 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind_value.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind_value.rs @@ -30,7 +30,6 @@ use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::Evaluator; use databend_common_functions::BUILTIN_FUNCTIONS; -use indexmap::IndexMap; use crate::binder::wrap_cast; use crate::optimizer::ColumnSet; @@ -389,8 +388,6 @@ pub fn bind_values( name_resolution_ctx, metadata.clone(), &[], - HashMap::new(), - Box::new(IndexMap::new()), ); let num_values = values.len(); diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_join.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_join.rs index 87f428aa5591..e0fe1e4fa705 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_join.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_join.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::sync::Arc; use async_recursion::async_recursion; @@ -25,9 +24,7 @@ use databend_common_ast::Span; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use indexmap::IndexMap; -use crate::binder::CteInfo; use crate::binder::Finder; use crate::binder::JoinPredicate; use crate::binder::Visibility; @@ -50,7 +47,6 @@ use crate::plans::ScalarExpr; use crate::plans::Visitor; use crate::BindContext; use crate::ColumnBinding; -use crate::IndexType; use crate::MetadataRef; pub struct JoinConditions { @@ -89,6 +85,10 @@ impl Binder { let (right_child, right_context) = if join.right.is_lateral_subquery() { self.bind_table_reference(&mut left_context, &join.right)? } else { + // Merge cte info from left context to `bind_context` + bind_context + .cte_context + .merge(left_context.cte_context.clone()); self.bind_table_reference(bind_context, &join.right)? }; @@ -122,8 +122,16 @@ impl Binder { build_side_cache_info, )?; - let bind_context = join_bind_context(&join_type, bind_context, left_context, right_context); + let mut bind_context = join_bind_context( + &join_type, + bind_context, + left_context.clone(), + right_context.clone(), + ); + bind_context + .cte_context + .set_cte_context(right_context.cte_context); Ok((s_expr, bind_context)) } @@ -164,8 +172,12 @@ impl Binder { right_child, None, )?; - let bind_context = join_bind_context(&join_type, bind_context, left_context, right_context); - + let bind_context = join_bind_context( + &join_type, + bind_context, + left_context.clone(), + right_context, + ); Ok((s_expr, bind_context)) } @@ -185,8 +197,6 @@ impl Binder { self.ctx.clone(), &self.name_resolution_ctx, self.metadata.clone(), - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), join_op.clone(), left_column_bindings, right_column_bindings, @@ -515,8 +525,6 @@ struct JoinConditionResolver<'a> { ctx: Arc, name_resolution_ctx: &'a NameResolutionContext, metadata: MetadataRef, - m_cte_bound_ctx: HashMap, - ctes_map: Box>, join_op: JoinOperator, left_column_bindings: &'a [ColumnBinding], right_column_bindings: &'a [ColumnBinding], @@ -530,8 +538,6 @@ impl<'a> JoinConditionResolver<'a> { ctx: Arc, name_resolution_ctx: &'a NameResolutionContext, metadata: MetadataRef, - m_cte_bound_ctx: HashMap, - ctes_map: Box>, join_op: JoinOperator, left_column_bindings: &'a [ColumnBinding], right_column_bindings: &'a [ColumnBinding], @@ -542,8 +548,6 @@ impl<'a> JoinConditionResolver<'a> { ctx, name_resolution_ctx, metadata, - m_cte_bound_ctx, - ctes_map, join_op, left_column_bindings, right_column_bindings, @@ -689,8 +693,6 @@ impl<'a> JoinConditionResolver<'a> { self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); // Given two tables: t1(a, b), t2(a, b) // A predicate can be regarded as an equi-predicate iff: @@ -840,8 +842,6 @@ impl<'a> JoinConditionResolver<'a> { self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (predicate, _) = scalar_binder.bind(predicate)?; let predicate_used_columns = predicate.used_columns(); diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_subquery.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_subquery.rs index c4da3833b666..16c1c5810d7c 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_subquery.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_subquery.rs @@ -41,6 +41,9 @@ impl Binder { .clone() .unwrap_or_else(|| Box::new(BindContext::new())), ); + new_bind_context + .cte_context + .set_cte_context(bind_context.cte_context.clone()); self.bind_query(&mut new_bind_context, subquery)? }; diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 99c6f3a3c750..4a029814ad23 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -70,8 +70,8 @@ impl Binder { }; // Check and bind common table expression - let ctes_map = self.ctes_map.clone(); - if let Some(cte_info) = ctes_map.get(&table_name) { + let cte_map = bind_context.cte_context.cte_map.clone(); + if let Some(cte_info) = cte_map.get(&table_name) { if self .metadata .read() @@ -117,8 +117,8 @@ impl Binder { break; } let bind_context = parent.unwrap().as_mut(); - let ctes_map = self.ctes_map.clone(); - if let Some(cte_info) = ctes_map.get(&table_name) { + let cte_map = bind_context.cte_context.cte_map.clone(); + if let Some(cte_info) = cte_map.get(&table_name) { return if !cte_info.materialized { self.bind_cte(*span, bind_context, &table_name, alias, cte_info) } else { @@ -182,6 +182,9 @@ impl Binder { unreachable!() }; let (s_expr, mut new_bind_context) = self.bind_query(&mut new_bind_context, query)?; + bind_context + .cte_context + .set_cte_context(new_bind_context.cte_context.clone()); let cols = table_meta .schema() diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index e48a93b98bd6..2df37771e0b2 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -121,8 +121,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let table_args = bind_table_args(&mut scalar_binder, params, named_params)?; diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index abc243ae2cc0..6b0661f4e3bc 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -42,7 +42,6 @@ use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_app::principal::FileFormatOptionsReader; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::StageFileFormatType; -use indexmap::IndexMap; use log::warn; use super::Finder; @@ -50,7 +49,6 @@ use crate::binder::bind_query::ExpressionScanContext; use crate::binder::util::illegal_ident_name; use crate::binder::wrap_cast; use crate::binder::ColumnBindingBuilder; -use crate::binder::CteInfo; use crate::normalize_identifier; use crate::optimizer::SExpr; use crate::planner::query_executor::QueryExecutor; @@ -63,7 +61,6 @@ use crate::plans::DropFileFormatPlan; use crate::plans::DropRolePlan; use crate::plans::DropStagePlan; use crate::plans::DropUserPlan; -use crate::plans::MaterializedCte; use crate::plans::Plan; use crate::plans::RelOperator; use crate::plans::RewriteKind; @@ -74,7 +71,6 @@ use crate::plans::UseDatabasePlan; use crate::plans::Visitor; use crate::BindContext; use crate::ColumnBinding; -use crate::IndexType; use crate::MetadataRef; use crate::NameResolutionContext; use crate::ScalarExpr; @@ -95,13 +91,6 @@ pub struct Binder { pub catalogs: Arc, pub name_resolution_ctx: NameResolutionContext, pub metadata: MetadataRef, - // Save the bound context for materialized cte, the key is cte_idx - pub m_cte_bound_ctx: HashMap, - pub m_cte_bound_s_expr: HashMap, - pub m_cte_materialized_indexes: HashMap, - /// Use `IndexMap` because need to keep the insertion order - /// Then wrap materialized ctes to main plan. - pub ctes_map: Box>, /// The `ExpressionScanContext` is used to store the information of /// expression scan and hash join build cache. pub expression_scan_context: ExpressionScanContext, @@ -132,10 +121,6 @@ impl<'a> Binder { catalogs, name_resolution_ctx, metadata, - m_cte_bound_ctx: Default::default(), - m_cte_bound_s_expr: Default::default(), - m_cte_materialized_indexes: Default::default(), - ctes_map: Box::default(), expression_scan_context: ExpressionScanContext::new(), bind_recursive_cte: false, enable_result_cache, @@ -177,19 +162,7 @@ impl<'a> Binder { let (mut s_expr, bind_context) = self.bind_query(bind_context, query)?; // Wrap `LogicalMaterializedCte` to `s_expr` - for (_, cte_info) in self.ctes_map.iter().rev() { - if !cte_info.materialized || cte_info.used_count == 0 { - continue; - } - let cte_s_expr = self.m_cte_bound_s_expr.get(&cte_info.cte_idx).unwrap(); - let materialized_output_columns = cte_info.columns.clone(); - s_expr = SExpr::create_binary( - Arc::new(RelOperator::MaterializedCte(MaterializedCte { cte_idx: cte_info.cte_idx, materialized_output_columns, materialized_indexes: self.m_cte_materialized_indexes.clone() })), - Arc::new(s_expr), - Arc::new(cte_s_expr.clone()), - ); - } - + s_expr = bind_context.cte_context.wrap_m_cte(s_expr); // Remove unused cache columns and join conditions and construct ExpressionScan's child. (s_expr, _) = self.construct_expression_scan(&s_expr, self.metadata.clone())?; let formatted_ast = if self.ctx.get_settings().get_enable_query_result_cache()? { @@ -719,15 +692,6 @@ impl<'a> Binder { .set_batch_settings(&hint_settings, true) } - // After the materialized cte was bound, add it to `m_cte_bound_ctx` - pub fn set_m_cte_bound_ctx(&mut self, cte_idx: IndexType, bound_ctx: BindContext) { - self.m_cte_bound_ctx.insert(cte_idx, bound_ctx); - } - - pub fn set_m_cte_bound_s_expr(&mut self, cte_idx: IndexType, s_expr: SExpr) { - self.m_cte_bound_s_expr.insert(cte_idx, s_expr); - } - pub fn set_bind_recursive_cte(&mut self, val: bool) { self.bind_recursive_cte = val; } diff --git a/src/query/sql/src/planner/binder/copy_into_table.rs b/src/query/sql/src/planner/binder/copy_into_table.rs index 78d7d0837b84..84adfb0b719d 100644 --- a/src/query/sql/src/planner/binder/copy_into_table.rs +++ b/src/query/sql/src/planner/binder/copy_into_table.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; @@ -59,7 +58,6 @@ use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_storage::StageFilesInfo; use databend_common_users::UserApiProvider; use derive_visitor::Drive; -use indexmap::IndexMap; use log::debug; use log::warn; use parking_lot::RwLock; @@ -572,8 +570,6 @@ impl<'a> Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - HashMap::new(), - Box::new(IndexMap::new()), ); let mut values = Vec::with_capacity(data_schema.fields.len()); for field in &data_schema.fields { diff --git a/src/query/sql/src/planner/binder/ddl/table.rs b/src/query/sql/src/planner/binder/ddl/table.rs index 0799b19b7186..656b25e74e68 100644 --- a/src/query/sql/src/planner/binder/ddl/table.rs +++ b/src/query/sql/src/planner/binder/ddl/table.rs @@ -1014,8 +1014,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); scalar_binder.forbid_udf(); let (scalar, _) = scalar_binder.bind(expr)?; @@ -1654,8 +1652,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); // cluster keys cannot be a udf expression. scalar_binder.forbid_udf(); diff --git a/src/query/sql/src/planner/binder/expr_values.rs b/src/query/sql/src/planner/binder/expr_values.rs index a2627d0c37e0..d3aab5ebb45c 100644 --- a/src/query/sql/src/planner/binder/expr_values.rs +++ b/src/query/sql/src/planner/binder/expr_values.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::sync::Arc; use databend_common_ast::ast::Expr as AExpr; @@ -28,7 +27,6 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::Scalar; use databend_common_expression::Value; use databend_common_pipeline_transforms::processors::Transform; -use indexmap::IndexMap; use crate::binder::wrap_cast; use crate::evaluator::BlockOperator; @@ -110,15 +108,9 @@ impl BindContext { exprs ))); } - let mut scalar_binder = ScalarBinder::new( - self, - ctx.clone(), - name_resolution_ctx, - metadata.clone(), - &[], - HashMap::new(), - Box::new(IndexMap::new()), - ); + let mut scalar_binder = + ScalarBinder::new(self, ctx.clone(), name_resolution_ctx, metadata.clone(), &[ + ]); let mut map_exprs = Vec::with_capacity(exprs.len()); diff --git a/src/query/sql/src/planner/binder/having.rs b/src/query/sql/src/planner/binder/having.rs index 09774c3b80a0..74dcba5604f5 100644 --- a/src/query/sql/src/planner/binder/having.rs +++ b/src/query/sql/src/planner/binder/having.rs @@ -48,8 +48,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), aliases, - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (mut scalar, _) = scalar_binder.bind(having)?; let mut rewriter = AggregateRewriter::new(bind_context, self.metadata.clone()); diff --git a/src/query/sql/src/planner/binder/insert_multi_table.rs b/src/query/sql/src/planner/binder/insert_multi_table.rs index c8f0ca98a898..b2a701b23619 100644 --- a/src/query/sql/src/planner/binder/insert_multi_table.rs +++ b/src/query/sql/src/planner/binder/insert_multi_table.rs @@ -94,8 +94,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (condition, _) = scalar_binder.bind(&when_clause.condition)?; if !matches!(condition.data_type()?.remove_nullable(), DataType::Boolean) { @@ -244,8 +242,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let mut source_scalar_exprs = vec![]; for source_column in source_columns { diff --git a/src/query/sql/src/planner/binder/project.rs b/src/query/sql/src/planner/binder/project.rs index 795c2a18f92b..d06b058a4ae5 100644 --- a/src/query/sql/src/planner/binder/project.rs +++ b/src/query/sql/src/planner/binder/project.rs @@ -258,8 +258,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &prev_aliases, - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (bound_expr, _) = scalar_binder.bind(expr)?; @@ -316,8 +314,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), &[], - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let sql_tokens = tokenize_sql(virtual_computed_expr.as_str())?; let expr = parse_expr(&sql_tokens, self.dialect)?; diff --git a/src/query/sql/src/planner/binder/qualify.rs b/src/query/sql/src/planner/binder/qualify.rs index ba45a726a7c8..f8e6de7784ce 100644 --- a/src/query/sql/src/planner/binder/qualify.rs +++ b/src/query/sql/src/planner/binder/qualify.rs @@ -53,8 +53,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), aliases, - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (mut scalar, _) = scalar_binder.bind(qualify)?; let mut rewriter = WindowRewriter::new(bind_context, self.metadata.clone()); diff --git a/src/query/sql/src/planner/binder/scalar.rs b/src/query/sql/src/planner/binder/scalar.rs index 8d26372cebd6..e7eab3090e89 100644 --- a/src/query/sql/src/planner/binder/scalar.rs +++ b/src/query/sql/src/planner/binder/scalar.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::sync::Arc; use databend_common_ast::ast::Expr; @@ -26,15 +25,12 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; -use indexmap::IndexMap; use crate::binder::wrap_cast; -use crate::binder::CteInfo; use crate::planner::binder::BindContext; use crate::planner::semantic::NameResolutionContext; use crate::planner::semantic::TypeChecker; use crate::plans::ScalarExpr; -use crate::IndexType; use crate::MetadataRef; /// Helper for binding scalar expression with `BindContext`. @@ -44,8 +40,6 @@ pub struct ScalarBinder<'a> { dialect: Dialect, name_resolution_ctx: &'a NameResolutionContext, metadata: MetadataRef, - m_cte_bound_ctx: HashMap, - ctes_map: Box>, aliases: &'a [(String, ScalarExpr)], forbid_udf: bool, } @@ -57,8 +51,6 @@ impl<'a> ScalarBinder<'a> { name_resolution_ctx: &'a NameResolutionContext, metadata: MetadataRef, aliases: &'a [(String, ScalarExpr)], - m_cte_bound_ctx: HashMap, - ctes_map: Box>, ) -> Self { let dialect = ctx.get_settings().get_sql_dialect().unwrap_or_default(); @@ -68,8 +60,6 @@ impl<'a> ScalarBinder<'a> { dialect, name_resolution_ctx, metadata, - m_cte_bound_ctx, - ctes_map, aliases, forbid_udf: false, } @@ -88,8 +78,6 @@ impl<'a> ScalarBinder<'a> { self.aliases, self.forbid_udf, )?; - type_checker.set_m_cte_bound_ctx(self.m_cte_bound_ctx.clone()); - type_checker.set_ctes_map(self.ctes_map.clone()); Ok(*type_checker.resolve(expr)?) } diff --git a/src/query/sql/src/planner/binder/select.rs b/src/query/sql/src/planner/binder/select.rs index c52f5679a6ce..192e0d31b234 100644 --- a/src/query/sql/src/planner/binder/select.rs +++ b/src/query/sql/src/planner/binder/select.rs @@ -83,8 +83,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), aliases, - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (scalar, _) = scalar_binder.bind(expr)?; @@ -130,7 +128,7 @@ impl Binder { )); } // Add recursive cte's columns to cte info - let mut_cte_info = self.ctes_map.get_mut(cte_name).unwrap(); + let mut_cte_info = bind_context.cte_context.cte_map.get_mut(cte_name).unwrap(); // The recursive cte may be used by multiple times in main query, so clear cte_info's columns mut_cte_info.columns.clear(); for column in left_bind_context.columns.iter() { @@ -145,6 +143,10 @@ impl Binder { mut_cte_info.columns.push(col); } } + // Merge cte info from left context to `bind_context` + bind_context + .cte_context + .merge(left_bind_context.cte_context.clone()); let (right_expr, right_bind_context) = self.bind_set_expr(bind_context, right, &[], None)?; @@ -256,16 +258,19 @@ impl Binder { left_span, right_span, left_context, - right_context, + right_context.clone(), coercion_types, )?; - if let Some(cte_name) = &cte_name { - for (col, cte_col) in new_bind_context - .columns - .iter_mut() - .zip(self.ctes_map.get(cte_name).unwrap().columns.iter()) - { + for (col, cte_col) in new_bind_context.columns.iter_mut().zip( + new_bind_context + .cte_context + .cte_map + .get(cte_name) + .unwrap() + .columns + .iter(), + ) { col.table_name = cte_col.table_name.clone(); col.column_name = cte_col.column_name.clone(); } @@ -340,7 +345,7 @@ impl Binder { &mut self, left_span: Span, right_span: Span, - left_context: BindContext, + mut left_context: BindContext, right_context: BindContext, left_expr: SExpr, right_expr: SExpr, @@ -384,6 +389,9 @@ impl Binder { }; let s_expr = self.bind_join_with_type(join_type, join_conditions, left_expr, right_expr, None)?; + left_context + .cte_context + .set_cte_context(right_context.cte_context); Ok((s_expr, left_context)) } @@ -404,6 +412,10 @@ impl Binder { let mut left_outputs = Vec::with_capacity(left_bind_context.columns.len()); let mut right_outputs = Vec::with_capacity(right_bind_context.columns.len()); let mut new_bind_context = BindContext::new(); + new_bind_context + .cte_context + .set_cte_context(right_bind_context.cte_context); + for (idx, (left_col, right_col)) in left_bind_context .columns .iter() diff --git a/src/query/sql/src/planner/binder/sort.rs b/src/query/sql/src/planner/binder/sort.rs index c3d630fce11b..8af8d17a89cf 100644 --- a/src/query/sql/src/planner/binder/sort.rs +++ b/src/query/sql/src/planner/binder/sort.rs @@ -105,8 +105,6 @@ impl Binder { &self.name_resolution_ctx, self.metadata.clone(), aliases, - self.m_cte_bound_ctx.clone(), - self.ctes_map.clone(), ); let (bound_expr, _) = scalar_binder.bind(&order.expr)?; diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 8ff180b34cf4..36db3599cebb 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -154,8 +154,15 @@ impl Binder { mut bind_context: BindContext, ) -> Result<(SExpr, BindContext)> { let blocks = Arc::new(RwLock::new(vec![])); + let cte_distinct_number = 1 + self + .ctx + .get_materialized_ctes() + .read() + .iter() + .filter(|((idx, _), _)| idx == &cte_info.cte_idx) + .count(); self.ctx - .set_materialized_cte((cte_info.cte_idx, cte_info.used_count), blocks)?; + .set_materialized_cte((cte_info.cte_idx, cte_distinct_number), blocks)?; // Get the fields in the cte let mut fields = vec![]; let mut offsets = vec![]; @@ -168,7 +175,9 @@ impl Binder { *column.data_type.clone(), None, ); - self.m_cte_materialized_indexes + bind_context + .cte_context + .m_cte_materialized_indexes .insert(column.index, materialized_index); fields.push(DataField::new( column.index.to_string().as_str(), @@ -178,10 +187,9 @@ impl Binder { } let cte_scan = SExpr::create_leaf(Arc::new( CteScan { - cte_idx: (cte_info.cte_idx, cte_info.used_count), + cte_idx: (cte_info.cte_idx, cte_distinct_number), fields, materialized_indexes, - // It is safe to unwrap here because we have checked that the cte is materialized. offsets, stat: Arc::new(StatInfo::default()), } @@ -198,7 +206,7 @@ impl Binder { alias: &Option, cte_info: &CteInfo, ) -> Result<(SExpr, BindContext)> { - if let Some(cte_name) = &bind_context.cte_name { + if let Some(cte_name) = &bind_context.cte_context.cte_name { // `cte_name` exists, which means the current cte is a nested cte // If the `cte_name` is the same as the current cte's name, it means the cte is recursive if cte_name == table_name { @@ -214,8 +222,7 @@ impl Binder { columns: vec![], aggregate_info: Default::default(), windows: Default::default(), - cte_name: Some(table_name.to_string()), - cte_map_ref: Box::default(), + cte_context: bind_context.cte_context.clone(), in_grouping: false, view_info: None, srfs: vec![], @@ -228,6 +235,8 @@ impl Binder { window_definitions: DashMap::new(), }; + new_bind_context.cte_context.cte_name = Some(table_name.to_string()); + let (s_expr, mut res_bind_context) = self.bind_query(&mut new_bind_context, &cte_info.query)?; let mut cols_alias = cte_info.columns_alias.clone(); @@ -273,41 +282,41 @@ impl Binder { alias: &Option, span: &Span, ) -> Result<(SExpr, BindContext)> { - let new_bind_context = if cte_info.used_count == 0 { - let (cte_s_expr, cte_bind_ctx) = + let new_bind_context = if !bind_context.cte_context.has_bound(cte_info.cte_idx) { + let (cte_s_expr, mut cte_bind_ctx) = self.bind_cte(*span, bind_context, table_name, alias, cte_info)?; - self.ctes_map + cte_bind_ctx + .cte_context + .cte_map .entry(table_name.clone()) .and_modify(|cte_info| { cte_info.columns = cte_bind_ctx.columns.clone(); }); - self.set_m_cte_bound_ctx(cte_info.cte_idx, cte_bind_ctx.clone()); - self.set_m_cte_bound_s_expr(cte_info.cte_idx, cte_s_expr); + cte_bind_ctx + .cte_context + .set_m_cte_bound_s_expr(cte_info.cte_idx, cte_s_expr); cte_bind_ctx } else { - // If the cte has been bound, get the bound context from `Binder`'s `m_cte_bound_ctx` - let mut bound_ctx = self.m_cte_bound_ctx.get(&cte_info.cte_idx).unwrap().clone(); + let mut bound_ctx = BindContext::with_parent(Box::new(bind_context.clone())); // Resolve the alias name for the bound cte. let alias_table_name = alias .as_ref() .map(|alias| normalize_identifier(&alias.name, &self.name_resolution_ctx).name) .unwrap_or_else(|| table_name.to_string()); - for column in bound_ctx.columns.iter_mut() { + for column in cte_info.columns.iter() { + let mut column = column.clone(); column.database_name = None; column.table_name = Some(alias_table_name.clone()); + bound_ctx.columns.push(column); } - // Pass parent to bound_ctx - bound_ctx.parent = bind_context.parent.clone(); bound_ctx }; - // `bind_context` is the main BindContext for the whole query - // Update the `used_count` which will be used in runtime phase - self.ctes_map - .entry(table_name.clone()) - .and_modify(|cte_info| { - cte_info.used_count += 1; - }); - let cte_info = self.ctes_map.get(table_name).unwrap().clone(); + let cte_info = new_bind_context + .cte_context + .cte_map + .get(table_name) + .unwrap() + .clone(); self.bind_cte_scan(&cte_info, new_bind_context) } @@ -364,7 +373,9 @@ impl Binder { )); } // Update the cte_info of the recursive cte - self.ctes_map + bind_context + .cte_context + .cte_map .entry(cte_name.to_string()) .and_modify(|cte_info| { cte_info.columns = new_bind_ctx.columns.clone(); @@ -418,6 +429,10 @@ impl Binder { self.set_bind_recursive_cte(false); if let Some(alias) = alias { new_bind_ctx.apply_table_alias(alias, &self.name_resolution_ctx)?; + } else { + for (index, column_name) in cte_info.columns_alias.iter().enumerate() { + new_bind_ctx.columns[index].column_name = column_name.clone(); + } } Ok((union_s_expr, new_bind_ctx.clone())) } diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index 5edd35bd756d..cbec287d390b 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -102,7 +102,6 @@ use databend_common_storage::init_stage_operator; use databend_common_users::UserApiProvider; use derive_visitor::Drive; use derive_visitor::Visitor; -use indexmap::IndexMap; use itertools::Itertools; use jsonb::keypath::KeyPath; use jsonb::keypath::KeyPaths; @@ -114,7 +113,6 @@ use crate::binder::bind_values; use crate::binder::resolve_file_location; use crate::binder::wrap_cast; use crate::binder::Binder; -use crate::binder::CteInfo; use crate::binder::ExprContext; use crate::binder::InternalColumnBinding; use crate::binder::NameResolutionResult; @@ -161,7 +159,6 @@ use crate::BaseTableColumn; use crate::BindContext; use crate::ColumnBinding; use crate::ColumnEntry; -use crate::IndexType; use crate::MetadataRef; /// A helper for type checking. @@ -180,8 +177,6 @@ pub struct TypeChecker<'a> { func_ctx: FunctionContext, name_resolution_ctx: &'a NameResolutionContext, metadata: MetadataRef, - ctes_map: Box>, - m_cte_bound_ctx: HashMap, aliases: &'a [(String, ScalarExpr)], @@ -213,8 +208,6 @@ impl<'a> TypeChecker<'a> { func_ctx, name_resolution_ctx, metadata, - ctes_map: Box::default(), - m_cte_bound_ctx: Default::default(), aliases, in_aggregate_function: false, in_window_function: false, @@ -222,14 +215,6 @@ impl<'a> TypeChecker<'a> { }) } - pub fn set_m_cte_bound_ctx(&mut self, m_cte_bound_ctx: HashMap) { - self.m_cte_bound_ctx = m_cte_bound_ctx; - } - - pub fn set_ctes_map(&mut self, ctes_map: Box>) { - self.ctes_map = ctes_map; - } - #[allow(dead_code)] fn post_resolve( &mut self, @@ -3069,14 +3054,13 @@ impl<'a> TypeChecker<'a> { self.name_resolution_ctx.clone(), self.metadata.clone(), ); - for (cte_idx, bound_ctx) in self.m_cte_bound_ctx.iter() { - binder.set_m_cte_bound_ctx(*cte_idx, bound_ctx.clone()); - } - binder.ctes_map = self.ctes_map.clone(); // Create new `BindContext` with current `bind_context` as its parent, so we can resolve outer columns. let mut bind_context = BindContext::with_parent(Box::new(self.bind_context.clone())); let (s_expr, output_context) = binder.bind_query(&mut bind_context, subquery)?; + self.bind_context + .cte_context + .set_cte_context(output_context.cte_context); if (typ == SubqueryType::Scalar || typ == SubqueryType::Any) && output_context.columns.len() > 1 diff --git a/tests/sqllogictests/suites/tpcds/materialized_cte.test b/tests/sqllogictests/suites/tpcds/materialized_cte.test index 5f58ef4b8d6b..cb4c12bff5b3 100644 --- a/tests/sqllogictests/suites/tpcds/materialized_cte.test +++ b/tests/sqllogictests/suites/tpcds/materialized_cte.test @@ -1528,3 +1528,39 @@ WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id ORDER BY 1 NULLS FIRST LIMIT 100; ---- + +# Q95 +query I +WITH ws_wh AS materialized + (SELECT ws1.ws_order_number, + ws1.ws_warehouse_sk wh1, + ws2.ws_warehouse_sk wh2 + FROM web_sales ws1, + web_sales ws2 + WHERE ws1.ws_order_number = ws2.ws_order_number + AND ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +SELECT count(DISTINCT ws_order_number) AS "order count" , + sum(ws_ext_ship_cost) AS "total shipping cost" , + sum(ws_net_profit) AS "total net profit" +FROM web_sales ws1 , + date_dim , + customer_address , + web_site +WHERE d_date BETWEEN '1999-02-01' AND cast('1999-04-02' AS date) + AND ws1.ws_ship_date_sk = d_date_sk + AND ws1.ws_ship_addr_sk = ca_address_sk + AND ca_state = 'IL' + AND ws1.ws_web_site_sk = web_site_sk + AND web_company_name = 'pri' + AND ws1.ws_order_number IN + (SELECT ws_order_number + FROM ws_wh) + AND ws1.ws_order_number IN + (SELECT wr_order_number + FROM web_returns, + ws_wh + WHERE wr_order_number = ws_wh.ws_order_number) +ORDER BY count(DISTINCT ws_order_number) +LIMIT 100; +---- +0 NULL NULL \ No newline at end of file From 6dff8e3dc7638be3ac9c1e08b8e67845a9c0394f Mon Sep 17 00:00:00 2001 From: dantengsky Date: Thu, 14 Nov 2024 12:45:39 +0800 Subject: [PATCH 39/92] chore: fix typo in `ColumnCacheKeyBuilder` (#16838) --- .../fuse/src/io/read/block/block_reader_merge_io_async.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs index 082cbc6a20ac..f9fe420cd131 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs @@ -61,7 +61,7 @@ impl BlockReader { if let Some(column_meta) = columns_meta.get(column_id) { let (offset, len) = column_meta.offset_length(); - let column_cache_key = column_cache_key_builder.cache_cache(column_id, column_meta); + let column_cache_key = column_cache_key_builder.cache_key(column_id, column_meta); // first, check in memory table data cache // column_array_cache @@ -106,7 +106,7 @@ impl BlockReader { // Safe to unwrap here, since this column has been fetched, its meta must be present. let column_meta = columns_meta.get(column_id).unwrap(); - let column_cache_key = column_cache_key_builder.cache_cache(column_id, column_meta); + let column_cache_key = column_cache_key_builder.cache_key(column_id, column_meta); let chunk_data = merge_io_result .owner_memory @@ -133,7 +133,7 @@ impl<'a> ColumnCacheKeyBuilder<'a> { fn new(block_path: &'a str) -> Self { Self { block_path } } - fn cache_cache(&self, column_id: &ColumnId, column_meta: &ColumnMeta) -> TableDataCacheKey { + fn cache_key(&self, column_id: &ColumnId, column_meta: &ColumnMeta) -> TableDataCacheKey { let (offset, len) = column_meta.offset_length(); TableDataCacheKey::new(self.block_path, *column_id, offset, len) } From 7509439cab4dd087d431329a443d27c6d5d35277 Mon Sep 17 00:00:00 2001 From: baishen Date: Thu, 14 Nov 2024 15:02:48 +0800 Subject: [PATCH 40/92] fix(query): fix group by with alias column can't bind the column (#16804) * fix(query): fix group by with alias column * modify mock redis port * fix explain test * fix * fix typos * fix nested agg srf * add tests --- src/query/sql/src/planner/binder/aggregate.rs | 145 ++++++---- .../sql/src/planner/binder/bind_context.rs | 12 +- .../planner/binder/bind_query/bind_select.rs | 20 +- .../bind_table_function.rs | 15 +- .../sql/src/planner/binder/project_set.rs | 248 +++++++++++++++--- src/query/sql/src/planner/binder/select.rs | 10 +- src/query/sql/src/planner/binder/table.rs | 2 +- .../src/planner/semantic/grouping_check.rs | 188 +++++++++---- .../sql/src/planner/semantic/type_check.rs | 25 +- .../sql/src/planner/semantic/window_check.rs | 77 ++++-- .../src/mock_source/redis_source.rs | 2 +- .../03_common/03_0003_select_group_by.test | 37 +++ .../suites/mode/cluster/create_table.test | 18 +- .../suites/mode/cluster/exchange.test | 38 +-- .../mode/standalone/ee/explain_agg_index.test | 62 ++--- .../mode/standalone/explain/clustering.test | 4 +- .../suites/mode/standalone/explain/limit.test | 16 +- .../mode/standalone/explain/project_set.test | 16 +- .../mode/standalone/explain/prune_column.test | 6 +- .../push_down_filter_eval_scalar.test | 26 +- .../push_down_filter_project_set.test | 12 +- .../mode/standalone/explain_native/limit.test | 16 +- .../explain_native/project_set.test | 6 +- .../explain_native/prune_column.test | 6 +- .../push_down_filter_eval_scalar.test | 22 +- .../push_down_filter_project_set.test | 12 +- .../functions/02_0062_function_unnest.test | 13 +- .../functions/02_0077_function_dict_get.test | 4 +- 28 files changed, 720 insertions(+), 338 deletions(-) diff --git a/src/query/sql/src/planner/binder/aggregate.rs b/src/query/sql/src/planner/binder/aggregate.rs index 4911f65ab5e7..306223240b7b 100644 --- a/src/query/sql/src/planner/binder/aggregate.rs +++ b/src/query/sql/src/planner/binder/aggregate.rs @@ -33,12 +33,14 @@ use itertools::Itertools; use super::prune_by_children; use super::ExprContext; use super::Finder; +use crate::binder::project_set::SetReturningRewriter; use crate::binder::scalar::ScalarBinder; use crate::binder::select::SelectList; use crate::binder::Binder; use crate::binder::ColumnBinding; use crate::binder::ColumnBindingBuilder; use crate::binder::Visibility; +use crate::normalize_identifier; use crate::optimizer::SExpr; use crate::plans::walk_expr_mut; use crate::plans::Aggregate; @@ -347,6 +349,13 @@ impl Binder { bind_context: &mut BindContext, select_list: &mut SelectList, ) -> Result<()> { + if !bind_context.srf_info.srfs.is_empty() { + // Rewrite the Set-returning functions in Aggregate function as columns. + let mut srf_rewriter = SetReturningRewriter::new(bind_context, true); + for item in select_list.items.iter_mut() { + srf_rewriter.visit(&mut item.scalar)?; + } + } let mut rewriter = AggregateRewriter::new(bind_context, self.metadata.clone()); for item in select_list.items.iter_mut() { rewriter.visit(&mut item.scalar)?; @@ -373,22 +382,16 @@ impl Binder { // Extract available aliases from `SELECT` clause, for item in select_list.items.iter() { - if let SelectTarget::AliasedExpr { alias: Some(_), .. } = item.select_target { - let column = if let ScalarExpr::BoundColumnRef(column_ref) = &item.scalar { - let mut column = column_ref.column.clone(); - column.column_name = item.alias.clone(); - column - } else { - self.create_derived_column_binding( - item.alias.clone(), - item.scalar.data_type()?, - Some(item.scalar.clone()), - ) - }; - available_aliases.push((column, item.scalar.clone())); + if let SelectTarget::AliasedExpr { + alias: Some(alias), .. + } = item.select_target + { + let column = normalize_identifier(alias, &self.name_resolution_ctx); + available_aliases.push((column.name, item.scalar.clone())); } } + let original_context = bind_context.expr_context.clone(); bind_context.set_expr_context(ExprContext::GroupClaue); match group_by { GroupBy::Normal(exprs) => self.resolve_group_items( @@ -398,7 +401,7 @@ impl Binder { &available_aliases, false, &mut vec![], - ), + )?, GroupBy::All => { let groups = self.resolve_group_all(select_list)?; self.resolve_group_items( @@ -408,10 +411,10 @@ impl Binder { &available_aliases, false, &mut vec![], - ) + )?; } GroupBy::GroupingSets(sets) => { - self.resolve_grouping_sets(bind_context, select_list, sets, &available_aliases) + self.resolve_grouping_sets(bind_context, select_list, sets, &available_aliases)?; } // TODO: avoid too many clones. GroupBy::Rollup(exprs) => { @@ -420,16 +423,18 @@ impl Binder { for i in (0..=exprs.len()).rev() { sets.push(exprs[0..i].to_vec()); } - self.resolve_grouping_sets(bind_context, select_list, &sets, &available_aliases) + self.resolve_grouping_sets(bind_context, select_list, &sets, &available_aliases)?; } GroupBy::Cube(exprs) => { // CUBE (a,b) => GROUPING SETS ((a,b),(a),(b),()) // All subsets let sets = (0..=exprs.len()) .flat_map(|count| exprs.clone().into_iter().combinations(count)) .collect::>(); - self.resolve_grouping_sets(bind_context, select_list, &sets, &available_aliases) + self.resolve_grouping_sets(bind_context, select_list, &sets, &available_aliases)?; } } + bind_context.set_expr_context(original_context); + Ok(()) } pub fn bind_aggregate( @@ -490,7 +495,7 @@ impl Binder { bind_context: &mut BindContext, select_list: &SelectList<'_>, sets: &[Vec], - available_aliases: &[(ColumnBinding, ScalarExpr)], + available_aliases: &[(String, ScalarExpr)], ) -> Result<()> { let mut grouping_sets = Vec::with_capacity(sets.len()); for set in sets { @@ -587,7 +592,7 @@ impl Binder { bind_context: &mut BindContext, select_list: &SelectList<'_>, group_by: &[Expr], - available_aliases: &[(ColumnBinding, ScalarExpr)], + available_aliases: &[(String, ScalarExpr)], collect_grouping_sets: bool, grouping_sets: &mut Vec>, ) -> Result<()> { @@ -640,9 +645,9 @@ impl Binder { self.metadata.clone(), &[], ); - let (scalar_expr, _) = scalar_binder + let (mut scalar_expr, _) = scalar_binder .bind(expr) - .or_else(|e| Self::resolve_alias_item(bind_context, expr, available_aliases, e))?; + .or_else(|e| self.resolve_alias_item(bind_context, expr, available_aliases, e))?; if collect_grouping_sets && !grouping_sets.last().unwrap().contains(&scalar_expr) { grouping_sets.last_mut().unwrap().push(scalar_expr.clone()); @@ -657,6 +662,11 @@ impl Binder { continue; } + if !bind_context.srf_info.srfs.is_empty() { + let mut srf_rewriter = SetReturningRewriter::new(bind_context, false); + srf_rewriter.visit(&mut scalar_expr)?; + } + let group_item_name = format!("{:#}", expr); let index = if let ScalarExpr::BoundColumnRef(BoundColumnRef { column: ColumnBinding { index, .. }, @@ -766,17 +776,18 @@ impl Binder { Ok((scalar, alias)) } + fn resolve_alias_item( + &mut self, bind_context: &mut BindContext, expr: &Expr, - available_aliases: &[(ColumnBinding, ScalarExpr)], + available_aliases: &[(String, ScalarExpr)], original_error: ErrorCode, ) -> Result<(ScalarExpr, DataType)> { let mut result: Vec = vec![]; // If cannot resolve group item, then try to find an available alias - for (i, (column_binding, _)) in available_aliases.iter().enumerate() { + for (i, (alias, _)) in available_aliases.iter().enumerate() { // Alias of the select item - let col_name = column_binding.column_name.as_str(); if let Expr::ColumnRef { column: ColumnRef { @@ -787,7 +798,7 @@ impl Binder { .. } = expr { - if col_name.eq_ignore_ascii_case(column.name()) { + if alias.eq_ignore_ascii_case(column.name()) { result.push(i); } } @@ -801,31 +812,75 @@ impl Binder { .set_span(expr.span()), ) } else { - let (column_binding, scalar) = available_aliases[result[0]].clone(); - // We will add the alias to BindContext, so we can reference it - // in `HAVING` and `ORDER BY` clause. - bind_context.add_column_binding(column_binding.clone()); + let (alias, mut scalar) = available_aliases[result[0]].clone(); - let index = column_binding.index; - bind_context.aggregate_info.group_items.push(ScalarItem { - scalar: scalar.clone(), - index, - }); - bind_context.aggregate_info.group_items_map.insert( - scalar.clone(), - bind_context.aggregate_info.group_items.len() - 1, - ); + if !bind_context.srf_info.srfs.is_empty() { + let mut srf_rewriter = SetReturningRewriter::new(bind_context, false); + srf_rewriter.visit(&mut scalar)?; + } + + // check scalar first, avoid duplicate create column. + let mut scalar_column_index = None; + let column_binding = if let Some(column_index) = + bind_context.aggregate_info.group_items_map.get(&scalar) + { + scalar_column_index = Some(*column_index); + + let group_item = &bind_context.aggregate_info.group_items[*column_index]; + ColumnBindingBuilder::new( + alias.clone(), + group_item.index, + Box::new(group_item.scalar.data_type()?), + Visibility::Visible, + ) + .build() + } else if let ScalarExpr::BoundColumnRef(column_ref) = &scalar { + let mut column = column_ref.column.clone(); + column.column_name = alias.clone(); + column + } else { + self.create_derived_column_binding( + alias.clone(), + scalar.data_type()?, + Some(scalar.clone()), + ) + }; + + if scalar_column_index.is_none() { + let index = column_binding.index; + bind_context.aggregate_info.group_items.push(ScalarItem { + scalar: scalar.clone(), + index, + }); + bind_context.aggregate_info.group_items_map.insert( + scalar.clone(), + bind_context.aggregate_info.group_items.len() - 1, + ); + scalar_column_index = Some(bind_context.aggregate_info.group_items.len() - 1); + } + + let scalar_column_index = scalar_column_index.unwrap(); - // Add a mapping (alias -> scalar), so we can resolve the alias later let column_ref: ScalarExpr = BoundColumnRef { span: scalar.span(), - column: column_binding, + column: column_binding.clone(), } .into(); - bind_context.aggregate_info.group_items_map.insert( - column_ref, - bind_context.aggregate_info.group_items.len() - 1, - ); + let has_column = bind_context + .aggregate_info + .group_items_map + .contains_key(&column_ref); + if !has_column { + // We will add the alias to BindContext, so we can reference it + // in `HAVING` and `ORDER BY` clause. + bind_context.add_column_binding(column_binding.clone()); + + // Add a mapping (alias -> scalar), so we can resolve the alias later + bind_context + .aggregate_info + .group_items_map + .insert(column_ref, scalar_column_index); + } Ok((scalar.clone(), scalar.data_type()?)) } diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 1a501500afdb..738816402d47 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -39,6 +39,7 @@ use itertools::Itertools; use super::AggregateInfo; use super::INTERNAL_COLUMN_FACTORY; use crate::binder::column_binding::ColumnBinding; +use crate::binder::project_set::SetReturningInfo; use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; @@ -46,7 +47,6 @@ use crate::optimizer::SExpr; use crate::plans::MaterializedCte; use crate::plans::RelOperator; use crate::plans::ScalarExpr; -use crate::plans::ScalarItem; use crate::ColumnSet; use crate::IndexType; use crate::MetadataRef; @@ -122,6 +122,9 @@ pub struct BindContext { pub windows: WindowInfo, + /// Set-returning functions info in current context. + pub srf_info: SetReturningInfo, + pub cte_context: CteContext, /// True if there is aggregation in current context, which means @@ -134,9 +137,6 @@ pub struct BindContext { /// It's used to check if the view has a loop dependency. pub view_info: Option<(String, String)>, - /// Set-returning functions in current context. - pub srfs: Vec, - /// True if there is async function in current context, need rewrite. pub have_async_func: bool, /// True if there is udf script in current context, need rewrite. @@ -251,10 +251,10 @@ impl BindContext { bound_internal_columns: BTreeMap::new(), aggregate_info: AggregateInfo::default(), windows: WindowInfo::default(), + srf_info: SetReturningInfo::default(), cte_context: CteContext::default(), in_grouping: false, view_info: None, - srfs: Vec::new(), have_async_func: false, have_udf_script: false, have_udf_server: false, @@ -272,10 +272,10 @@ impl BindContext { bound_internal_columns: BTreeMap::new(), aggregate_info: Default::default(), windows: Default::default(), + srf_info: Default::default(), cte_context: parent.cte_context.clone(), in_grouping: false, view_info: None, - srfs: Vec::new(), have_async_func: false, have_udf_script: false, have_udf_server: false, diff --git a/src/query/sql/src/planner/binder/bind_query/bind_select.rs b/src/query/sql/src/planner/binder/bind_query/bind_select.rs index ed21bd7f0b19..82907fff517b 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind_select.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind_select.rs @@ -134,10 +134,15 @@ impl Binder { .map(|item| (item.alias.clone(), item.scalar.clone())) .collect::>(); - let have_srfs = !from_context.srfs.is_empty(); - if have_srfs { - // Bind set returning functions first. - s_expr = self.bind_project_set(&mut from_context, s_expr)?; + // Check Set-returning functions, if the argument contains aggregation function or group item, + // set as lazy Set-returning functions. + if !from_context.srf_info.srfs.is_empty() { + self.check_project_set_select(&mut from_context)?; + } + + // Bind Set-returning functions before filter plan and aggregate plan. + if !from_context.srf_info.srfs.is_empty() { + s_expr = self.bind_project_set(&mut from_context, s_expr, false)?; } // To support using aliased column in `WHERE` clause, @@ -179,7 +184,7 @@ impl Binder { )?; // After all analysis is done. - if !have_srfs { + if from_context.srf_info.srfs.is_empty() { // Ignore SRFs. self.analyze_lazy_materialization( &from_context, @@ -208,6 +213,11 @@ impl Binder { s_expr = self.bind_window_function(window_info, s_expr)?; } + // Bind lazy Set-returning functions after aggregate plan. + if !from_context.srf_info.lazy_srf_set.is_empty() { + s_expr = self.bind_project_set(&mut from_context, s_expr, true)?; + } + if let Some(qualify) = qualify { s_expr = self.bind_qualify(&mut from_context, qualify, s_expr)?; } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index 2df37771e0b2..886627804681 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -38,6 +38,7 @@ use databend_common_storages_result_cache::ResultCacheMetaManager; use databend_common_storages_result_cache::ResultScan; use databend_common_users::UserApiProvider; +use crate::binder::project_set::SetReturningRewriter; use crate::binder::scalar::ScalarBinder; use crate::binder::table_args::bind_table_args; use crate::binder::Binder; @@ -49,6 +50,7 @@ use crate::plans::EvalScalar; use crate::plans::FunctionCall; use crate::plans::RelOperator; use crate::plans::ScalarItem; +use crate::plans::VisitorMut; use crate::BindContext; use crate::ScalarExpr; @@ -352,10 +354,17 @@ impl Binder { }]; let mut select_list = self.normalize_select_list(&mut bind_context, &select_list)?; - // analyze set returning functions + // analyze Set-returning functions. self.analyze_project_set_select(&mut bind_context, &mut select_list)?; - // bind set returning functions - let srf_expr = self.bind_project_set(&mut bind_context, child)?; + // rewrite Set-returning functions as columns. + let mut srf_rewriter = SetReturningRewriter::new(&mut bind_context, false); + for item in select_list.items.iter_mut() { + srf_rewriter.visit(&mut item.scalar)?; + } + // bind Set-returning functions. + let srf_expr = self.bind_project_set(&mut bind_context, child, false)?; + // clear Set-returning functions, avoid duplicate bind. + bind_context.srf_info = Default::default(); if let Some(item) = select_list.items.pop() { let srf_result = item.scalar; diff --git a/src/query/sql/src/planner/binder/project_set.rs b/src/query/sql/src/planner/binder/project_set.rs index 0c3913efab84..c507953b394e 100644 --- a/src/query/sql/src/planner/binder/project_set.rs +++ b/src/query/sql/src/planner/binder/project_set.rs @@ -12,19 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; +use std::collections::HashSet; use std::mem; use std::sync::Arc; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::FunctionKind; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::select::SelectList; use crate::binder::ColumnBindingBuilder; +use crate::format_scalar; use crate::optimizer::SExpr; use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; -use crate::plans::FunctionCall; use crate::plans::ProjectSet; use crate::plans::ScalarItem; use crate::plans::VisitorMut; @@ -34,52 +37,99 @@ use crate::MetadataRef; use crate::ScalarExpr; use crate::Visibility; -struct SetReturningRewriter<'a> { - bind_context: &'a mut BindContext, - metadata: MetadataRef, +#[derive(Default, Clone, PartialEq, Eq, Debug)] +pub struct SetReturningInfo { + /// Set-returning functions. + pub srfs: Vec, + /// Mapping: (Set-returning function display name) -> (index of Set-returning function in `srfs`) + /// This is used to find a Set-returning function in current context. + pub srfs_map: HashMap, + /// The lazy index of Set-returning functions in `srfs`. + /// Those set-returning function's argument contains aggregate functions or group by items. + /// Build a lazy `ProjectSet` plan after the `Aggregate` plan. + pub lazy_srf_set: HashSet, +} + +/// Rewrite Set-returning functions as a BoundColumnRef. +pub(crate) struct SetReturningRewriter<'a> { + pub(crate) bind_context: &'a mut BindContext, + // Whether only rewrite the argument of aggregate function. + only_agg: bool, + // Whether current function is the argument of aggregate function. + is_agg_arg: bool, } impl<'a> SetReturningRewriter<'a> { - fn new(bind_context: &'a mut BindContext, metadata: MetadataRef) -> Self { + pub(crate) fn new(bind_context: &'a mut BindContext, only_agg: bool) -> Self { Self { bind_context, - metadata, + only_agg, + is_agg_arg: false, } } +} + +impl<'a> VisitorMut<'a> for SetReturningRewriter<'a> { + fn visit(&mut self, expr: &'a mut ScalarExpr) -> Result<()> { + match expr { + ScalarExpr::FunctionCall(func) => { + if (self.is_agg_arg || !self.only_agg) + && BUILTIN_FUNCTIONS + .get_property(&func.func_name) + .map(|property| property.kind == FunctionKind::SRF) + .unwrap_or(false) + { + let srf_display_name = format_scalar(expr); + if let Some(index) = self.bind_context.srf_info.srfs_map.get(&srf_display_name) + { + let srf_item = &self.bind_context.srf_info.srfs[*index]; - /// Replace the set returning function with a BoundColumnRef. - fn replace_set_returning_function(&mut self, func: &FunctionCall) -> Result { - let srf_func = ScalarExpr::FunctionCall(func.clone()); - let data_type = srf_func.data_type()?; - - let column_index = self.metadata.write().add_derived_column( - func.func_name.clone(), - data_type.clone(), - Some(srf_func.clone()), - ); - let column = ColumnBindingBuilder::new( - func.func_name.clone(), - column_index, - Box::new(data_type), - Visibility::InVisible, - ) - .build(); - - // Add the srf to bind context, build ProjectSet plan later. - self.bind_context.srfs.push(ScalarItem { - index: column_index, - scalar: srf_func, - }); - - Ok(BoundColumnRef { - span: func.span, - column, + let column_binding = ColumnBindingBuilder::new( + srf_display_name, + srf_item.index, + Box::new(srf_item.scalar.data_type()?), + Visibility::InVisible, + ) + .build(); + *expr = BoundColumnRef { + span: None, + column: column_binding, + } + .into(); + + return Ok(()); + } + return Err(ErrorCode::Internal("Invalid Set-returning function")); + } + } + ScalarExpr::AggregateFunction(_) => { + self.is_agg_arg = true; + } + _ => {} } - .into()) + walk_expr_mut(self, expr)?; + + self.is_agg_arg = false; + Ok(()) } } -impl<'a> VisitorMut<'a> for SetReturningRewriter<'a> { +/// Analyze Set-returning functions and create derived columns. +struct SetReturningAnalyzer<'a> { + bind_context: &'a mut BindContext, + metadata: MetadataRef, +} + +impl<'a> SetReturningAnalyzer<'a> { + fn new(bind_context: &'a mut BindContext, metadata: MetadataRef) -> Self { + Self { + bind_context, + metadata, + } + } +} + +impl<'a> VisitorMut<'a> for SetReturningAnalyzer<'a> { fn visit(&mut self, expr: &'a mut ScalarExpr) -> Result<()> { if let ScalarExpr::FunctionCall(func) = expr { if BUILTIN_FUNCTIONS @@ -87,7 +137,22 @@ impl<'a> VisitorMut<'a> for SetReturningRewriter<'a> { .map(|property| property.kind == FunctionKind::SRF) .unwrap_or(false) { - *expr = self.replace_set_returning_function(func)?; + let srf_display_name = format_scalar(expr); + let index = self.metadata.write().add_derived_column( + srf_display_name.clone(), + expr.data_type()?, + Some(expr.clone()), + ); + + // Add the srf to bind context, build ProjectSet plan later. + self.bind_context.srf_info.srfs.push(ScalarItem { + index, + scalar: expr.clone(), + }); + self.bind_context + .srf_info + .srfs_map + .insert(srf_display_name, self.bind_context.srf_info.srfs.len() - 1); return Ok(()); } } @@ -96,18 +161,103 @@ impl<'a> VisitorMut<'a> for SetReturningRewriter<'a> { } } +/// Check whether the argument of Set-returning functions contains aggregation function or group item. +/// If true, we need to lazy build `ProjectSet` plan +struct SetReturningChecker<'a> { + bind_context: &'a mut BindContext, + has_aggregate_argument: bool, +} + +impl<'a> SetReturningChecker<'a> { + fn new(bind_context: &'a mut BindContext) -> Self { + Self { + bind_context, + has_aggregate_argument: false, + } + } +} + +impl<'a> VisitorMut<'a> for SetReturningChecker<'a> { + fn visit(&mut self, expr: &'a mut ScalarExpr) -> Result<()> { + if self + .bind_context + .aggregate_info + .group_items_map + .contains_key(expr) + { + self.has_aggregate_argument = true; + } + + if let ScalarExpr::AggregateFunction(agg_func) = expr { + self.has_aggregate_argument = true; + if let Some(index) = self + .bind_context + .aggregate_info + .aggregate_functions_map + .get(&agg_func.display_name) + { + let agg_item = &self.bind_context.aggregate_info.aggregate_functions[*index]; + let column_binding = ColumnBindingBuilder::new( + agg_func.display_name.clone(), + agg_item.index, + Box::new(agg_item.scalar.data_type()?), + Visibility::InVisible, + ) + .build(); + + let column_ref: ScalarExpr = BoundColumnRef { + span: expr.span(), + column: column_binding.clone(), + } + .into(); + *expr = column_ref; + } + return Ok(()); + } + + walk_expr_mut(self, expr) + } +} + impl Binder { - /// Analyze project sets in select clause, this will rewrite project set functions. - /// See [`SetReturningRewriter`] for more details. + /// Analyze project sets in select clause. + /// See [`SetReturningAnalyzer`] for more details. pub(crate) fn analyze_project_set_select( &mut self, bind_context: &mut BindContext, select_list: &mut SelectList, ) -> Result<()> { - let mut rewriter = SetReturningRewriter::new(bind_context, self.metadata.clone()); + let mut analyzer = SetReturningAnalyzer::new(bind_context, self.metadata.clone()); for item in select_list.items.iter_mut() { - rewriter.visit(&mut item.scalar)?; + analyzer.visit(&mut item.scalar)?; + } + + Ok(()) + } + + pub(crate) fn check_project_set_select( + &mut self, + bind_context: &mut BindContext, + ) -> Result<()> { + let mut srf_info = mem::take(&mut bind_context.srf_info); + let mut checker = SetReturningChecker::new(bind_context); + for srf_item in srf_info.srfs.iter_mut() { + let srf_display_name = format_scalar(&srf_item.scalar); + checker.has_aggregate_argument = false; + checker.visit(&mut srf_item.scalar)?; + + // If the argument contains aggregation function or group item. + // add the srf index to lazy set. + if checker.has_aggregate_argument { + // srf_display_names.push(srf_display_name); + // if let Some(index) = bind_context.srf_info.srfs_map.get(&srf_display_name) { + // bind_context.srf_info.lazy_srf_set.insert(*index); + if let Some(index) = srf_info.srfs_map.get(&srf_display_name) { + srf_info.lazy_srf_set.insert(*index); + } + } } + bind_context.srf_info = srf_info; Ok(()) } @@ -116,15 +266,27 @@ impl Binder { &mut self, bind_context: &mut BindContext, child: SExpr, + is_lazy: bool, ) -> Result { - if bind_context.srfs.is_empty() { + let srf_len = if is_lazy { + bind_context.srf_info.lazy_srf_set.len() + } else { + bind_context.srf_info.srfs.len() - bind_context.srf_info.lazy_srf_set.len() + }; + if srf_len == 0 { return Ok(child); } // Build a ProjectSet Plan. - let srfs = mem::take(&mut bind_context.srfs); - let project_set = ProjectSet { srfs }; + let mut srfs = Vec::with_capacity(srf_len); + for (i, srf) in bind_context.srf_info.srfs.iter().enumerate() { + let is_lazy_srf = bind_context.srf_info.lazy_srf_set.contains(&i); + if (is_lazy && is_lazy_srf) || (!is_lazy && !is_lazy_srf) { + srfs.push(srf.clone()); + } + } + let project_set = ProjectSet { srfs }; let new_expr = SExpr::create_unary(Arc::new(project_set.into()), Arc::new(child)); Ok(new_expr) diff --git a/src/query/sql/src/planner/binder/select.rs b/src/query/sql/src/planner/binder/select.rs index 192e0d31b234..2b8df21fc3fb 100644 --- a/src/query/sql/src/planner/binder/select.rs +++ b/src/query/sql/src/planner/binder/select.rs @@ -33,6 +33,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use super::sort::OrderItem; use super::Finder; use crate::binder::bind_table_reference::JoinConditions; +use crate::binder::project_set::SetReturningRewriter; use crate::binder::scalar_common::split_conjunctions; use crate::binder::ColumnBindingBuilder; use crate::binder::ExprContext; @@ -49,6 +50,7 @@ use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::UnionAll; use crate::plans::Visitor as _; +use crate::plans::VisitorMut; use crate::ColumnEntry; use crate::IndexType; use crate::Visibility; @@ -84,7 +86,13 @@ impl Binder { self.metadata.clone(), aliases, ); - let (scalar, _) = scalar_binder.bind(expr)?; + let (mut scalar, _) = scalar_binder.bind(expr)?; + + // rewrite Set-returning functions as columns. + if !bind_context.srf_info.srfs.is_empty() { + let mut srf_rewriter = SetReturningRewriter::new(bind_context, false); + srf_rewriter.visit(&mut scalar)?; + } let f = |scalar: &ScalarExpr| { matches!( diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 36db3599cebb..c8136d07d089 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -222,10 +222,10 @@ impl Binder { columns: vec![], aggregate_info: Default::default(), windows: Default::default(), + srf_info: Default::default(), cte_context: bind_context.cte_context.clone(), in_grouping: false, view_info: None, - srfs: vec![], have_async_func: false, have_udf_script: false, have_udf_server: false, diff --git a/src/query/sql/src/planner/semantic/grouping_check.rs b/src/query/sql/src/planner/semantic/grouping_check.rs index 2e3ee6c7f8d9..f5b3cce783ac 100644 --- a/src/query/sql/src/planner/semantic/grouping_check.rs +++ b/src/query/sql/src/planner/semantic/grouping_check.rs @@ -14,9 +14,12 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::FunctionKind; +use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::ColumnBindingBuilder; use crate::binder::Visibility; +use crate::format_scalar; use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; use crate::plans::ScalarExpr; @@ -41,6 +44,25 @@ impl<'a> GroupingChecker<'a> { impl<'a> VisitorMut<'_> for GroupingChecker<'a> { fn visit(&mut self, expr: &mut ScalarExpr) -> Result<()> { + if !self + .bind_context + .aggregate_info + .group_items_map + .contains_key(expr) + { + if let ScalarExpr::FunctionCall(func) = expr { + // The srf returns a tuple type column, and a `get` function + // is always used to extract the inner column. + // Try rewrite the srf to a column first, so that we can + // check if the expr is in the `group_items_map`. + if func.func_name == "get" { + for arg in &mut func.arguments { + self.visit(arg)?; + } + } + } + } + if let Some(index) = self.bind_context.aggregate_info.group_items_map.get(expr) { let column = &self.bind_context.aggregate_info.group_items[*index]; let mut column_binding = if let ScalarExpr::BoundColumnRef(column_ref) = &column.scalar @@ -70,71 +92,121 @@ impl<'a> VisitorMut<'_> for GroupingChecker<'a> { return Ok(()); } - if let ScalarExpr::WindowFunction(window) = expr { - if let Some(column) = self - .bind_context - .windows - .window_functions_map - .get(&window.display_name) - { - // The exprs in `win` has already been rewrittern to `BoundColumnRef` in `WindowRewriter`. - // So we need to check the exprs in `bind_context.windows` - let mut window_info = self.bind_context.windows.window_functions[*column].clone(); - // Just check if the exprs are in grouping items. - for part in window_info.partition_by_items.iter_mut() { - self.visit(&mut part.scalar)?; - } - // Just check if the exprs are in grouping items. - for order in window_info.order_by_items.iter_mut() { - self.visit(&mut order.order_by_item.scalar)?; + match expr { + ScalarExpr::WindowFunction(window) => { + if let Some(column) = self + .bind_context + .windows + .window_functions_map + .get(&window.display_name) + { + // The exprs in `win` has already been rewrittern to `BoundColumnRef` in `WindowRewriter`. + // So we need to check the exprs in `bind_context.windows` + let mut window_info = + self.bind_context.windows.window_functions[*column].clone(); + // Just check if the exprs are in grouping items. + for part in window_info.partition_by_items.iter_mut() { + self.visit(&mut part.scalar)?; + } + // Just check if the exprs are in grouping items. + for order in window_info.order_by_items.iter_mut() { + self.visit(&mut order.order_by_item.scalar)?; + } + // Just check if the exprs are in grouping items. + for arg in window_info.arguments.iter_mut() { + self.visit(&mut arg.scalar)?; + } + + let column_binding = ColumnBindingBuilder::new( + window.display_name.clone(), + window_info.index, + Box::new(window_info.func.return_type()), + Visibility::Visible, + ) + .build(); + *expr = BoundColumnRef { + span: None, + column: column_binding, + } + .into(); + return Ok(()); } - // Just check if the exprs are in grouping items. - for arg in window_info.arguments.iter_mut() { - self.visit(&mut arg.scalar)?; + + return Err(ErrorCode::Internal("Group Check: Invalid window function")); + } + ScalarExpr::AggregateFunction(agg) => { + if let Some(column) = self + .bind_context + .aggregate_info + .aggregate_functions_map + .get(&agg.display_name) + { + let agg_func = &self.bind_context.aggregate_info.aggregate_functions[*column]; + let column_binding = ColumnBindingBuilder::new( + agg.display_name.clone(), + agg_func.index, + Box::new(agg_func.scalar.data_type()?), + Visibility::Visible, + ) + .build(); + *expr = BoundColumnRef { + span: None, + column: column_binding, + } + .into(); + return Ok(()); } - let column_binding = ColumnBindingBuilder::new( - window.display_name.clone(), - window_info.index, - Box::new(window_info.func.return_type()), - Visibility::Visible, - ) - .build(); - *expr = BoundColumnRef { - span: None, - column: column_binding, + return Err(ErrorCode::Internal("Invalid aggregate function")); + } + ScalarExpr::FunctionCall(func) => { + if BUILTIN_FUNCTIONS + .get_property(&func.func_name) + .map(|property| property.kind == FunctionKind::SRF) + .unwrap_or(false) + { + let srf_display_name = format_scalar(expr); + if let Some(index) = self.bind_context.srf_info.srfs_map.get(&srf_display_name) + { + // Rewrite srf function as a column. + let srf_item = &self.bind_context.srf_info.srfs[*index]; + + let column_binding = ColumnBindingBuilder::new( + srf_display_name, + srf_item.index, + Box::new(srf_item.scalar.data_type()?), + Visibility::Visible, + ) + .build(); + *expr = BoundColumnRef { + span: None, + column: column_binding, + } + .into(); + return Ok(()); + } + return Err(ErrorCode::Internal("Invalid Set-returning function")); } - .into(); - return Ok(()); } - - return Err(ErrorCode::Internal("Group Check: Invalid window function")); - } - - if let ScalarExpr::AggregateFunction(agg) = expr { - if let Some(column) = self - .bind_context - .aggregate_info - .aggregate_functions_map - .get(&agg.display_name) - { - let agg_func = &self.bind_context.aggregate_info.aggregate_functions[*column]; - let column_binding = ColumnBindingBuilder::new( - agg.display_name.clone(), - agg_func.index, - Box::new(agg_func.scalar.data_type()?), - Visibility::Visible, - ) - .build(); - *expr = BoundColumnRef { - span: None, - column: column_binding, + ScalarExpr::BoundColumnRef(column_ref) => { + if let Some(index) = self + .bind_context + .srf_info + .srfs_map + .get(&column_ref.column.column_name) + { + // If the srf has been rewrote as a column, + // check whether the srf arguments are group item. + let srf_item = &self.bind_context.srf_info.srfs[*index]; + if let ScalarExpr::FunctionCall(func) = &srf_item.scalar { + for mut arg in func.arguments.clone() { + walk_expr_mut(self, &mut arg)?; + } + } + return Ok(()); } - .into(); - return Ok(()); } - - return Err(ErrorCode::Internal("Invalid aggregate function")); + _ => {} } walk_expr_mut(self, expr) diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index cbec287d390b..ea049fe045b4 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -1087,10 +1087,10 @@ impl<'a> TypeChecker<'a> { // TODO: remove this function fn rewrite_substring(args: &mut [ScalarExpr]) { if let ScalarExpr::ConstantExpr(expr) = &args[1] { - if let databend_common_expression::Scalar::Number(NumberScalar::UInt8(0)) = expr.value { + if let Scalar::Number(NumberScalar::UInt8(0)) = expr.value { args[1] = ConstantExpr { span: expr.span, - value: databend_common_expression::Scalar::Number(1i64.into()), + value: Scalar::Number(1i64.into()), } .into(); } @@ -1266,7 +1266,7 @@ impl<'a> TypeChecker<'a> { let box (expr, _) = self.resolve(expr)?; let (expr, _) = ConstantFolder::fold(&expr.as_expr()?, &self.func_ctx, &BUILTIN_FUNCTIONS); - if let databend_common_expression::Expr::Constant { scalar, .. } = expr { + if let EExpr::Constant { scalar, .. } = expr { Ok(Some(scalar)) } else { Err(ErrorCode::SemanticError( @@ -1630,7 +1630,6 @@ impl<'a> TypeChecker<'a> { } /// Resolve aggregation function call. - fn resolve_aggregate_function( &mut self, span: Span, @@ -1650,16 +1649,6 @@ impl<'a> TypeChecker<'a> { .set_span(span)); } - if matches!( - self.bind_context.expr_context, - ExprContext::InSetReturningFunction - ) { - return Err(ErrorCode::SemanticError( - "aggregate functions can not be used in set-returning function".to_string(), - ) - .set_span(span)); - } - if self.in_aggregate_function { if self.in_window_function { // The aggregate function can be in window function call, @@ -2756,7 +2745,7 @@ impl<'a> TypeChecker<'a> { // Note: check function may reorder the args let mut folded_args = match &expr { - databend_common_expression::Expr::FunctionCall { + EExpr::FunctionCall { args: checked_args, .. } => { let mut folded_args = Vec::with_capacity(args.len()); @@ -3563,7 +3552,7 @@ impl<'a> TypeChecker<'a> { } else { let trim_scalar = ConstantExpr { span, - value: databend_common_expression::Scalar::String(" ".to_string()), + value: Scalar::String(" ".to_string()), } .into(); ("trim_both", trim_scalar, DataType::String) @@ -4862,10 +4851,10 @@ impl<'a> TypeChecker<'a> { fn try_fold_constant( &self, - expr: &databend_common_expression::Expr, + expr: &EExpr, ) -> Option> { if expr.is_deterministic(&BUILTIN_FUNCTIONS) { - if let (databend_common_expression::Expr::Constant { scalar, .. }, _) = + if let (EExpr::Constant { scalar, .. }, _) = ConstantFolder::fold(expr, &self.func_ctx, &BUILTIN_FUNCTIONS) { let scalar = shrink_scalar(scalar); diff --git a/src/query/sql/src/planner/semantic/window_check.rs b/src/query/sql/src/planner/semantic/window_check.rs index 803f02e7121d..8ccc5787c2d6 100644 --- a/src/query/sql/src/planner/semantic/window_check.rs +++ b/src/query/sql/src/planner/semantic/window_check.rs @@ -14,8 +14,11 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::FunctionKind; +use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::ColumnBindingBuilder; +use crate::format_scalar; use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; use crate::plans::SubqueryExpr; @@ -36,30 +39,62 @@ impl<'a> WindowChecker<'a> { impl<'a> VisitorMut<'_> for WindowChecker<'a> { fn visit(&mut self, expr: &mut ScalarExpr) -> Result<()> { - if let ScalarExpr::WindowFunction(win) = expr { - if let Some(column) = self - .bind_context - .windows - .window_functions_map - .get(&win.display_name) - { - let window_info = &self.bind_context.windows.window_functions[*column]; - let column_binding = ColumnBindingBuilder::new( - win.display_name.clone(), - window_info.index, - Box::new(window_info.func.return_type()), - Visibility::Visible, - ) - .build(); - *expr = BoundColumnRef { - span: None, - column: column_binding, + match expr { + ScalarExpr::WindowFunction(win) => { + if let Some(column) = self + .bind_context + .windows + .window_functions_map + .get(&win.display_name) + { + let window_info = &self.bind_context.windows.window_functions[*column]; + let column_binding = ColumnBindingBuilder::new( + win.display_name.clone(), + window_info.index, + Box::new(window_info.func.return_type()), + Visibility::Visible, + ) + .build(); + *expr = BoundColumnRef { + span: None, + column: column_binding, + } + .into(); + return Ok(()); } - .into(); - return Ok(()); + + return Err(ErrorCode::Internal("Window Check: Invalid window function")); } + ScalarExpr::FunctionCall(func) => { + if BUILTIN_FUNCTIONS + .get_property(&func.func_name) + .map(|property| property.kind == FunctionKind::SRF) + .unwrap_or(false) + { + let srf_display_name = format_scalar(expr); + if let Some(index) = self.bind_context.srf_info.srfs_map.get(&srf_display_name) + { + // Rewrite srf function as a column. + let srf_item = &self.bind_context.srf_info.srfs[*index]; - return Err(ErrorCode::Internal("Window Check: Invalid window function")); + let column_binding = ColumnBindingBuilder::new( + srf_display_name, + srf_item.index, + Box::new(srf_item.scalar.data_type()?), + Visibility::Visible, + ) + .build(); + *expr = BoundColumnRef { + span: None, + column: column_binding, + } + .into(); + return Ok(()); + } + return Err(ErrorCode::Internal("Invalid Set-returning function")); + } + } + _ => {} } walk_expr_mut(self, expr) diff --git a/tests/sqllogictests/src/mock_source/redis_source.rs b/tests/sqllogictests/src/mock_source/redis_source.rs index e502b55a2909..a36b13e21f06 100644 --- a/tests/sqllogictests/src/mock_source/redis_source.rs +++ b/tests/sqllogictests/src/mock_source/redis_source.rs @@ -19,7 +19,7 @@ use tokio::net::TcpStream; pub async fn run_redis_source() { // Bind the listener to the address - let listener = TcpListener::bind("0.0.0.0:6379").await.unwrap(); + let listener = TcpListener::bind("0.0.0.0:6179").await.unwrap(); loop { let (socket, _) = listener.accept().await.unwrap(); diff --git a/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test b/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test index 5a57592d690f..e1c545fad573 100644 --- a/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test +++ b/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test @@ -227,6 +227,43 @@ select number % 3 a, max(number) - 3, number % 2 b, sum(number) + 4 from number 2 995 0 83504 2 992 1 83004 +statement ok +CREATE OR REPLACE TABLE t_str (col1 string, col2 string); + +statement ok +INSERT INTO t_str VALUES ('test', 'a1,a2'),('test', 'a1,a2,a3'); + +query TT +SELECT t.col1 AS col1, unnest(split(t.col2, ',')) AS col2 FROM t_str AS t GROUP BY col1, col2 ORDER BY col2; +---- +test a1 +test a1 +test a2 +test a2 +test a3 + +query TT +SELECT t.col1 AS col1, unnest(split(t.col2, ',')) AS col3 FROM t_str AS t GROUP BY col1, col3 ORDER BY col3; +---- +test a1 +test a2 +test a3 + +query T +SELECT max(unnest(split(t.col2, ','))) FROM t_str AS t; +---- +a3 + +query T +SELECT unnest(split(max(t.col2), ',')) FROM t_str AS t; +---- +a1 +a2 +a3 + +statement ok +DROP TABLE t_str + query TTT rowsort SELECT ( null, to_hour(to_timestamp(3501857592331)), number::Date) from numbers(3) group by all ---- diff --git a/tests/sqllogictests/suites/mode/cluster/create_table.test b/tests/sqllogictests/suites/mode/cluster/create_table.test index 71a9d6078291..f96f973831b6 100644 --- a/tests/sqllogictests/suites/mode/cluster/create_table.test +++ b/tests/sqllogictests/suites/mode/cluster/create_table.test @@ -4,32 +4,32 @@ explain create or replace table t2 as select number % 400 d, max(number) from n CreateTableAsSelect: (empty) EvalScalar -├── output columns: [max(number) (#6), d (#7)] -├── expressions: [numbers.number (#4) % 400] +├── output columns: [max(number) (#4), d (#5)] +├── expressions: [numbers.number (#3) % 400] ├── estimated rows: 3.00 └── Limit - ├── output columns: [max(number) (#6), numbers.number (#4)] + ├── output columns: [max(number) (#4), numbers.number (#3)] ├── limit: 3 ├── offset: 0 ├── estimated rows: 3.00 └── Sort - ├── output columns: [max(number) (#6), numbers.number (#4)] + ├── output columns: [max(number) (#4), numbers.number (#3)] ├── sort keys: [number ASC NULLS LAST] ├── estimated rows: 10000000.00 └── Exchange - ├── output columns: [max(number) (#6), numbers.number (#4), #_order_col] + ├── output columns: [max(number) (#4), numbers.number (#3), #_order_col] ├── exchange type: Merge └── Sort - ├── output columns: [max(number) (#6), numbers.number (#4), #_order_col] + ├── output columns: [max(number) (#4), numbers.number (#3), #_order_col] ├── sort keys: [number ASC NULLS LAST] ├── estimated rows: 10000000.00 └── AggregateFinal - ├── output columns: [max(number) (#6), numbers.number (#4)] + ├── output columns: [max(number) (#4), numbers.number (#3)] ├── group by: [number] ├── aggregate functions: [max(number)] ├── estimated rows: 10000000.00 └── Exchange - ├── output columns: [max(number) (#6), numbers.number (#4)] + ├── output columns: [max(number) (#4), numbers.number (#3)] ├── exchange type: Hash(0) └── AggregatePartial ├── group by: [number] @@ -38,7 +38,7 @@ EvalScalar ├── rank limit: 3 └── TableScan ├── table: default.system.numbers - ├── output columns: [number (#4)] + ├── output columns: [number (#3)] ├── read rows: 10000000 ├── read size: 76.29 MiB ├── partitions total: 153 diff --git a/tests/sqllogictests/suites/mode/cluster/exchange.test b/tests/sqllogictests/suites/mode/cluster/exchange.test index 964a9207e0fe..1b3a0595456a 100644 --- a/tests/sqllogictests/suites/mode/cluster/exchange.test +++ b/tests/sqllogictests/suites/mode/cluster/exchange.test @@ -149,25 +149,25 @@ query T explain select * from (select sum(number) as number from numbers(1) group by number) t, numbers(2) t1 where t.number = t1.number ---- Exchange -├── output columns: [t1.number (#3), sum(number) (#2)] +├── output columns: [t1.number (#2), sum(number) (#1)] ├── exchange type: Merge └── HashJoin - ├── output columns: [t1.number (#3), sum(number) (#2)] + ├── output columns: [t1.number (#2), sum(number) (#1)] ├── join type: INNER - ├── build keys: [t.number (#2)] - ├── probe keys: [CAST(t1.number (#3) AS UInt64 NULL)] + ├── build keys: [t.number (#1)] + ├── probe keys: [CAST(t1.number (#2) AS UInt64 NULL)] ├── filters: [] ├── estimated rows: 2.00 ├── Exchange(Build) - │ ├── output columns: [sum(number) (#2), numbers.number (#0)] + │ ├── output columns: [sum(number) (#1), numbers.number (#0)] │ ├── exchange type: Broadcast │ └── AggregateFinal - │ ├── output columns: [sum(number) (#2), numbers.number (#0)] + │ ├── output columns: [sum(number) (#1), numbers.number (#0)] │ ├── group by: [number] │ ├── aggregate functions: [sum(number)] │ ├── estimated rows: 1.00 │ └── Exchange - │ ├── output columns: [sum(number) (#2), numbers.number (#0)] + │ ├── output columns: [sum(number) (#1), numbers.number (#0)] │ ├── exchange type: Hash(0) │ └── AggregatePartial │ ├── group by: [number] @@ -184,7 +184,7 @@ Exchange │ └── estimated rows: 1.00 └── TableScan(Probe) ├── table: default.system.numbers - ├── output columns: [number (#3)] + ├── output columns: [number (#2)] ├── read rows: 2 ├── read size: < 1 KiB ├── partitions total: 1 @@ -198,7 +198,7 @@ explain fragments select * from (select sum(number) as number from numbers(1) gr Fragment 0: DataExchange: Shuffle ExchangeSink - ├── output columns: [sum(number) (#2), numbers.number (#0)] + ├── output columns: [sum(number) (#1), numbers.number (#0)] ├── destination fragment: [1] └── AggregatePartial ├── group by: [number] @@ -218,36 +218,36 @@ Fragment 0: Fragment 1: DataExchange: Broadcast ExchangeSink - ├── output columns: [sum(number) (#2), numbers.number (#0)] + ├── output columns: [sum(number) (#1), numbers.number (#0)] ├── destination fragment: [2] └── AggregateFinal - ├── output columns: [sum(number) (#2), numbers.number (#0)] + ├── output columns: [sum(number) (#1), numbers.number (#0)] ├── group by: [number] ├── aggregate functions: [sum(number)] ├── estimated rows: 1.00 └── ExchangeSource - ├── output columns: [sum(number) (#2), numbers.number (#0)] + ├── output columns: [sum(number) (#1), numbers.number (#0)] └── source fragment: [0] (empty) (empty) Fragment 2: DataExchange: Merge ExchangeSink - ├── output columns: [t1.number (#3), sum(number) (#2)] + ├── output columns: [t1.number (#2), sum(number) (#1)] ├── destination fragment: [3] └── HashJoin - ├── output columns: [t1.number (#3), sum(number) (#2)] + ├── output columns: [t1.number (#2), sum(number) (#1)] ├── join type: INNER - ├── build keys: [t.number (#2)] - ├── probe keys: [CAST(t1.number (#3) AS UInt64 NULL)] + ├── build keys: [t.number (#1)] + ├── probe keys: [CAST(t1.number (#2) AS UInt64 NULL)] ├── filters: [] ├── estimated rows: 2.00 ├── ExchangeSource(Build) - │ ├── output columns: [sum(number) (#2), numbers.number (#0)] + │ ├── output columns: [sum(number) (#1), numbers.number (#0)] │ └── source fragment: [1] └── TableScan(Probe) ├── table: default.system.numbers - ├── output columns: [number (#3)] + ├── output columns: [number (#2)] ├── read rows: 2 ├── read size: < 1 KiB ├── partitions total: 1 @@ -258,7 +258,7 @@ Fragment 2: (empty) Fragment 3: ExchangeSource - ├── output columns: [t1.number (#3), sum(number) (#2)] + ├── output columns: [t1.number (#2), sum(number) (#1)] └── source fragment: [2] (empty) diff --git a/tests/sqllogictests/suites/mode/standalone/ee/explain_agg_index.test b/tests/sqllogictests/suites/mode/standalone/ee/explain_agg_index.test index 41159c03b801..32ddcfae4009 100644 --- a/tests/sqllogictests/suites/mode/standalone/ee/explain_agg_index.test +++ b/tests/sqllogictests/suites/mode/standalone/ee/explain_agg_index.test @@ -324,11 +324,11 @@ query T EXPLAIN SELECT station_name, MIN(measurement) AS min_measurement, AVG(measurement) AS mean_measurement, MAX(measurement) AS max_measurement FROM onebrc GROUP BY station_name ---- EvalScalar -├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] -├── expressions: [sum(measurement) (#6) / CAST(if(CAST(count(measurement) (#7) = 0 AS Boolean NULL), 1, count(measurement) (#7)) AS UInt64 NULL)] +├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] +├── expressions: [sum(measurement) (#3) / CAST(if(CAST(count(measurement) (#4) = 0 AS Boolean NULL), 1, count(measurement) (#4)) AS UInt64 NULL)] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [MIN(measurement) (#5), sum(measurement) (#6), count(measurement) (#7), MAX(measurement) (#8), onebrc.station_name (#0)] + ├── output columns: [MIN(measurement) (#2), sum(measurement) (#3), count(measurement) (#4), MAX(measurement) (#5), onebrc.station_name (#0)] ├── group by: [station_name] ├── aggregate functions: [min(measurement), sum(measurement), count(measurement), max(measurement)] ├── estimated rows: 0.00 @@ -358,15 +358,15 @@ query T EXPLAIN SELECT station_name, MIN(measurement) AS min_measurement, AVG(measurement) AS mean_measurement, MAX(measurement) AS max_measurement FROM onebrc GROUP BY station_name ORDER BY station_name ---- Sort -├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] +├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] ├── sort keys: [station_name ASC NULLS LAST] ├── estimated rows: 0.00 └── EvalScalar - ├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] - ├── expressions: [sum(measurement) (#6) / CAST(if(CAST(count(measurement) (#7) = 0 AS Boolean NULL), 1, count(measurement) (#7)) AS UInt64 NULL)] + ├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] + ├── expressions: [sum(measurement) (#3) / CAST(if(CAST(count(measurement) (#4) = 0 AS Boolean NULL), 1, count(measurement) (#4)) AS UInt64 NULL)] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [MIN(measurement) (#5), sum(measurement) (#6), count(measurement) (#7), MAX(measurement) (#8), onebrc.station_name (#0)] + ├── output columns: [MIN(measurement) (#2), sum(measurement) (#3), count(measurement) (#4), MAX(measurement) (#5), onebrc.station_name (#0)] ├── group by: [station_name] ├── aggregate functions: [min(measurement), sum(measurement), count(measurement), max(measurement)] ├── estimated rows: 0.00 @@ -396,15 +396,15 @@ query T EXPLAIN SELECT * FROM (SELECT station_name, MIN(measurement) AS min_measurement, AVG(measurement) AS mean_measurement, MAX(measurement) AS max_measurement FROM onebrc GROUP BY station_name) ORDER BY station_name ---- Sort -├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] +├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] ├── sort keys: [station_name ASC NULLS LAST] ├── estimated rows: 0.00 └── EvalScalar - ├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] - ├── expressions: [sum(measurement) (#6) / CAST(if(CAST(count(measurement) (#7) = 0 AS Boolean NULL), 1, count(measurement) (#7)) AS UInt64 NULL)] + ├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] + ├── expressions: [sum(measurement) (#3) / CAST(if(CAST(count(measurement) (#4) = 0 AS Boolean NULL), 1, count(measurement) (#4)) AS UInt64 NULL)] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [MIN(measurement) (#5), sum(measurement) (#6), count(measurement) (#7), MAX(measurement) (#8), onebrc.station_name (#0)] + ├── output columns: [MIN(measurement) (#2), sum(measurement) (#3), count(measurement) (#4), MAX(measurement) (#5), onebrc.station_name (#0)] ├── group by: [station_name] ├── aggregate functions: [min(measurement), sum(measurement), count(measurement), max(measurement)] ├── estimated rows: 0.00 @@ -434,15 +434,15 @@ query T EXPLAIN WITH aggi AS (SELECT station_name, MIN(measurement) AS min_measurement, AVG(measurement) AS mean_measurement, MAX(measurement) AS max_measurement FROM onebrc GROUP BY station_name) SELECT * FROM aggi ORDER BY station_name ---- Sort -├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] +├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] ├── sort keys: [station_name ASC NULLS LAST] ├── estimated rows: 0.00 └── EvalScalar - ├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] - ├── expressions: [sum(measurement) (#6) / CAST(if(CAST(count(measurement) (#7) = 0 AS Boolean NULL), 1, count(measurement) (#7)) AS UInt64 NULL)] + ├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] + ├── expressions: [sum(measurement) (#3) / CAST(if(CAST(count(measurement) (#4) = 0 AS Boolean NULL), 1, count(measurement) (#4)) AS UInt64 NULL)] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [MIN(measurement) (#5), sum(measurement) (#6), count(measurement) (#7), MAX(measurement) (#8), onebrc.station_name (#0)] + ├── output columns: [MIN(measurement) (#2), sum(measurement) (#3), count(measurement) (#4), MAX(measurement) (#5), onebrc.station_name (#0)] ├── group by: [station_name] ├── aggregate functions: [min(measurement), sum(measurement), count(measurement), max(measurement)] ├── estimated rows: 0.00 @@ -473,15 +473,15 @@ query T EXPLAIN WITH aggi AS (SELECT station_name, measurement, MIN(measurement) AS min_measurement, AVG(measurement) AS mean_measurement, MAX(measurement) AS max_measurement FROM onebrc WHERE station_name='Beijing' AND measurement > 0 AND measurement IN (1, 2) GROUP BY station_name, measurement) SELECT * FROM aggi ORDER BY station_name ---- Sort -├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), onebrc.measurement (#1), mean_measurement (#9)] +├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), onebrc.measurement (#1), mean_measurement (#6)] ├── sort keys: [station_name ASC NULLS LAST] ├── estimated rows: 0.00 └── EvalScalar - ├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), onebrc.measurement (#1), mean_measurement (#9)] - ├── expressions: [sum(measurement) (#6) / CAST(if(CAST(count(measurement) (#7) = 0 AS Boolean NULL), 1, count(measurement) (#7)) AS UInt64 NULL)] + ├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), onebrc.measurement (#1), mean_measurement (#6)] + ├── expressions: [sum(measurement) (#3) / CAST(if(CAST(count(measurement) (#4) = 0 AS Boolean NULL), 1, count(measurement) (#4)) AS UInt64 NULL)] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [MIN(measurement) (#5), sum(measurement) (#6), count(measurement) (#7), MAX(measurement) (#8), onebrc.station_name (#0), onebrc.measurement (#1)] + ├── output columns: [MIN(measurement) (#2), sum(measurement) (#3), count(measurement) (#4), MAX(measurement) (#5), onebrc.station_name (#0), onebrc.measurement (#1)] ├── group by: [station_name, measurement] ├── aggregate functions: [min(measurement), sum(measurement), count(measurement), max(measurement)] ├── estimated rows: 0.00 @@ -516,15 +516,15 @@ query T EXPLAIN WITH aggi AS (SELECT station_name, MIN(measurement) AS min_measurement, AVG(measurement) AS mean_measurement, MAX(measurement) AS max_measurement FROM onebrc WHERE station_name='Beijing' GROUP BY station_name) SELECT * FROM aggi ORDER BY station_name ---- Sort -├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] +├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] ├── sort keys: [station_name ASC NULLS LAST] ├── estimated rows: 0.00 └── EvalScalar - ├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] - ├── expressions: [sum(measurement) (#6) / CAST(if(CAST(count(measurement) (#7) = 0 AS Boolean NULL), 1, count(measurement) (#7)) AS UInt64 NULL)] + ├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] + ├── expressions: [sum(measurement) (#3) / CAST(if(CAST(count(measurement) (#4) = 0 AS Boolean NULL), 1, count(measurement) (#4)) AS UInt64 NULL)] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [MIN(measurement) (#5), sum(measurement) (#6), count(measurement) (#7), MAX(measurement) (#8), onebrc.station_name (#0)] + ├── output columns: [MIN(measurement) (#2), sum(measurement) (#3), count(measurement) (#4), MAX(measurement) (#5), onebrc.station_name (#0)] ├── group by: [station_name] ├── aggregate functions: [min(measurement), sum(measurement), count(measurement), max(measurement)] ├── estimated rows: 0.00 @@ -559,15 +559,15 @@ query T EXPLAIN WITH aggi AS (SELECT station_name, measurement, MIN(measurement) AS min_measurement, AVG(measurement) AS mean_measurement, MAX(measurement) AS max_measurement FROM onebrc where measurement > 2 and measurement < 5 GROUP BY station_name, measurement) SELECT * FROM aggi ORDER BY station_name ---- Sort -├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), onebrc.measurement (#1), mean_measurement (#9)] +├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), onebrc.measurement (#1), mean_measurement (#6)] ├── sort keys: [station_name ASC NULLS LAST] ├── estimated rows: 0.00 └── EvalScalar - ├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), onebrc.measurement (#1), mean_measurement (#9)] - ├── expressions: [sum(measurement) (#6) / CAST(if(CAST(count(measurement) (#7) = 0 AS Boolean NULL), 1, count(measurement) (#7)) AS UInt64 NULL)] + ├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), onebrc.measurement (#1), mean_measurement (#6)] + ├── expressions: [sum(measurement) (#3) / CAST(if(CAST(count(measurement) (#4) = 0 AS Boolean NULL), 1, count(measurement) (#4)) AS UInt64 NULL)] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [MIN(measurement) (#5), sum(measurement) (#6), count(measurement) (#7), MAX(measurement) (#8), onebrc.station_name (#0), onebrc.measurement (#1)] + ├── output columns: [MIN(measurement) (#2), sum(measurement) (#3), count(measurement) (#4), MAX(measurement) (#5), onebrc.station_name (#0), onebrc.measurement (#1)] ├── group by: [station_name, measurement] ├── aggregate functions: [min(measurement), sum(measurement), count(measurement), max(measurement)] ├── estimated rows: 0.00 @@ -602,15 +602,15 @@ query T EXPLAIN WITH aggi AS (SELECT station_name, MIN(measurement) AS min_measurement, AVG(measurement) AS mean_measurement, MAX(measurement) AS max_measurement FROM onebrc WHERE station_name IN ('Paris', 'Beijing') GROUP BY station_name) SELECT * FROM aggi ORDER BY station_name ---- Sort -├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] +├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] ├── sort keys: [station_name ASC NULLS LAST] ├── estimated rows: 0.00 └── EvalScalar - ├── output columns: [MIN(measurement) (#5), MAX(measurement) (#8), onebrc.station_name (#0), mean_measurement (#9)] - ├── expressions: [sum(measurement) (#6) / CAST(if(CAST(count(measurement) (#7) = 0 AS Boolean NULL), 1, count(measurement) (#7)) AS UInt64 NULL)] + ├── output columns: [MIN(measurement) (#2), MAX(measurement) (#5), onebrc.station_name (#0), mean_measurement (#6)] + ├── expressions: [sum(measurement) (#3) / CAST(if(CAST(count(measurement) (#4) = 0 AS Boolean NULL), 1, count(measurement) (#4)) AS UInt64 NULL)] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [MIN(measurement) (#5), sum(measurement) (#6), count(measurement) (#7), MAX(measurement) (#8), onebrc.station_name (#0)] + ├── output columns: [MIN(measurement) (#2), sum(measurement) (#3), count(measurement) (#4), MAX(measurement) (#5), onebrc.station_name (#0)] ├── group by: [station_name] ├── aggregate functions: [min(measurement), sum(measurement), count(measurement), max(measurement)] ├── estimated rows: 0.00 diff --git a/tests/sqllogictests/suites/mode/standalone/explain/clustering.test b/tests/sqllogictests/suites/mode/standalone/explain/clustering.test index 527fbf885880..0a20fe6dc65a 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/clustering.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/clustering.test @@ -2,7 +2,7 @@ statement ok set enable_parallel_multi_merge_sort = 0; statement ok -CREATE TABLE test_linear(a int, b int) cluster by(a,b) row_per_block = 2; +CREATE OR REPLACE TABLE test_linear(a int, b int) cluster by(a,b) row_per_block = 2; statement ok INSERT INTO test_linear VALUES(1, 1), (1, 2); @@ -55,7 +55,7 @@ Filter └── estimated rows: 4.00 statement ok -CREATE TABLE test_hilbert(a int, b int) cluster by hilbert(a,b) row_per_block = 2; +CREATE OR REPLACE TABLE test_hilbert(a int, b int) cluster by hilbert(a,b) row_per_block = 2; statement ok INSERT INTO test_hilbert VALUES(1, 1), (1, 2); diff --git a/tests/sqllogictests/suites/mode/standalone/explain/limit.test b/tests/sqllogictests/suites/mode/standalone/explain/limit.test index 3b1ee5bb12cf..58304597fb2c 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/limit.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/limit.test @@ -166,23 +166,23 @@ query T explain select * from (select count(t1.number) as c1 from numbers(1) as t1 group by number) as t3 left join (select count(t.number) as c from numbers(2) as t group by number) as t4 on t3.c1=t4.c order by t3.c1 limit 1 ---- Limit -├── output columns: [count(t.number) (#5), count(t1.number) (#2)] +├── output columns: [count(t.number) (#3), count(t1.number) (#1)] ├── limit: 1 ├── offset: 0 ├── estimated rows: 1.00 └── Sort - ├── output columns: [count(t.number) (#5), count(t1.number) (#2)] + ├── output columns: [count(t.number) (#3), count(t1.number) (#1)] ├── sort keys: [count(t1.number) ASC NULLS LAST] ├── estimated rows: 2.00 └── HashJoin - ├── output columns: [count(t.number) (#5), count(t1.number) (#2)] + ├── output columns: [count(t.number) (#3), count(t1.number) (#1)] ├── join type: RIGHT OUTER - ├── build keys: [CAST(t3.c1 (#2) AS UInt64 NULL)] - ├── probe keys: [t4.c (#5)] + ├── build keys: [CAST(t3.c1 (#1) AS UInt64 NULL)] + ├── probe keys: [t4.c (#3)] ├── filters: [] ├── estimated rows: 2.00 ├── AggregateFinal(Build) - │ ├── output columns: [count(t1.number) (#2), t1.number (#0)] + │ ├── output columns: [count(t1.number) (#1), t1.number (#0)] │ ├── group by: [number] │ ├── aggregate functions: [count()] │ ├── estimated rows: 1.00 @@ -200,7 +200,7 @@ Limit │ ├── push downs: [filters: [], limit: NONE] │ └── estimated rows: 1.00 └── AggregateFinal(Probe) - ├── output columns: [count(t.number) (#5), t.number (#3)] + ├── output columns: [count(t.number) (#3), t.number (#2)] ├── group by: [number] ├── aggregate functions: [count()] ├── estimated rows: 2.00 @@ -210,7 +210,7 @@ Limit ├── estimated rows: 2.00 └── TableScan ├── table: default.system.numbers - ├── output columns: [number (#3)] + ├── output columns: [number (#2)] ├── read rows: 2 ├── read size: < 1 KiB ├── partitions total: 1 diff --git a/tests/sqllogictests/suites/mode/standalone/explain/project_set.test b/tests/sqllogictests/suites/mode/standalone/explain/project_set.test index f01c2cd3d183..05d50fdef719 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/project_set.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/project_set.test @@ -23,7 +23,7 @@ AggregateFinal ├── aggregate functions: [count()] ├── estimated rows: 1.00 └── ProjectSet - ├── output columns: [unnest (#2)] + ├── output columns: [unnest(fold_count.id (#0)) (#2)] ├── estimated rows: 1.00 ├── set returning functions: unnest(fold_count.id (#0)) └── TableScan @@ -53,7 +53,7 @@ query T explain select number from (select unnest([1,2,3]), number from numbers(10)) t ---- ProjectSet -├── output columns: [numbers.number (#0), unnest (#1)] +├── output columns: [numbers.number (#0), unnest([1, 2, 3]) (#1)] ├── estimated rows: 10.00 ├── set returning functions: unnest([1, 2, 3]) └── TableScan @@ -70,7 +70,7 @@ query T explain select number from (select unnest([1,2,3,number]), number from numbers(10)) t ---- ProjectSet -├── output columns: [numbers.number (#0), unnest (#1)] +├── output columns: [numbers.number (#0), unnest(array(1, 2, 3, numbers.number (#0))) (#1)] ├── estimated rows: 10.00 ├── set returning functions: unnest(CAST(array(1, 2, 3, numbers.number (#0)) AS Array(UInt64 NULL))) └── TableScan @@ -94,10 +94,10 @@ EXPLAIN SELECT t.a, f.seq, f.value FROM t, LATERAL FLATTEN(input => t.b) f ---- EvalScalar ├── output columns: [t.a (#0), seq (#3), value (#7)] -├── expressions: [get(1)(flatten (#2)), get(5)(flatten (#2))] +├── expressions: [get(1)(flatten(t.b (#1)) (#2)), get(5)(flatten(t.b (#1)) (#2))] ├── estimated rows: 0.00 └── ProjectSet - ├── output columns: [t.a (#0), flatten (#2)] + ├── output columns: [t.a (#0), flatten(t.b (#1)) (#2)] ├── estimated rows: 0.00 ├── set returning functions: flatten(1, 5)(t.b (#1)) └── TableScan @@ -114,11 +114,11 @@ query T EXPLAIN SELECT json_each(t.b), unnest(t.b) FROM t ---- EvalScalar -├── output columns: [json_each (#2), unnest(t.b) (#4)] -├── expressions: [get(1)(unnest (#3))] +├── output columns: [json_each(t.b) (#4), unnest(t.b) (#5)] +├── expressions: [json_each(t.b (#1)) (#2), get(1)(unnest(t.b (#1)) (#3))] ├── estimated rows: 0.00 └── ProjectSet - ├── output columns: [json_each (#2), unnest (#3)] + ├── output columns: [json_each(t.b (#1)) (#2), unnest(t.b (#1)) (#3)] ├── estimated rows: 0.00 ├── set returning functions: json_each(t.b (#1)), unnest(t.b (#1)) └── TableScan diff --git a/tests/sqllogictests/suites/mode/standalone/explain/prune_column.test b/tests/sqllogictests/suites/mode/standalone/explain/prune_column.test index 80f7b07f2777..993f13273cc8 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/prune_column.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/prune_column.test @@ -46,12 +46,12 @@ Limit ├── sort keys: [number ASC NULLS LAST] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [numbers.number (#0), numbers.number (#0), numbers.number (#0), numbers.number (#0)] - ├── group by: [number, number, number, number] + ├── output columns: [numbers.number (#0)] + ├── group by: [number] ├── aggregate functions: [] ├── estimated rows: 0.00 └── AggregatePartial - ├── group by: [number, number, number, number] + ├── group by: [number] ├── aggregate functions: [] ├── estimated rows: 0.00 ├── rank limit: 1 diff --git a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_eval_scalar.test b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_eval_scalar.test index 7f4884e75df3..7bba98bae78a 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_eval_scalar.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_eval_scalar.test @@ -61,14 +61,14 @@ AggregateFinal ├── aggregate functions: [] ├── estimated rows: 0.00 └── UnionAll - ├── output columns: [t.id (#0), de (#8)] + ├── output columns: [t.id (#0), de (#6)] ├── estimated rows: 0.00 ├── EvalScalar - │ ├── output columns: [t.id (#0), de (#8)] - │ ├── expressions: [if(CAST(is_not_null(sum(tb.de) (#7)) AS Boolean NULL), CAST(assume_not_null(sum(tb.de) (#7)) AS Int64 NULL), true, 0, NULL)] + │ ├── output columns: [t.id (#0), de (#6)] + │ ├── expressions: [if(CAST(is_not_null(sum(tb.de) (#5)) AS Boolean NULL), CAST(assume_not_null(sum(tb.de) (#5)) AS Int64 NULL), true, 0, NULL)] │ ├── estimated rows: 0.00 │ └── AggregateFinal - │ ├── output columns: [sum(tb.de) (#7), t.id (#0)] + │ ├── output columns: [sum(tb.de) (#5), t.id (#0)] │ ├── group by: [id] │ ├── aggregate functions: [sum(sum(coalesce(t3.val, 0)))] │ ├── estimated rows: 0.00 @@ -77,14 +77,14 @@ AggregateFinal │ ├── aggregate functions: [sum(sum(coalesce(t3.val, 0)))] │ ├── estimated rows: 0.00 │ └── HashJoin - │ ├── output columns: [t.id (#0), sum(coalesce(t3.val, 0)) (#5)] + │ ├── output columns: [t.id (#0), sum(coalesce(t3.val, 0)) (#4)] │ ├── join type: LEFT OUTER │ ├── build keys: [tb.sid (#1)] │ ├── probe keys: [t.id (#0)] │ ├── filters: [] │ ├── estimated rows: 0.00 │ ├── AggregateFinal(Build) - │ │ ├── output columns: [sum(coalesce(t3.val, 0)) (#5), t2.sid (#1)] + │ │ ├── output columns: [sum(coalesce(t3.val, 0)) (#4), t2.sid (#1)] │ │ ├── group by: [sid] │ │ ├── aggregate functions: [sum(sum_arg_0)] │ │ ├── estimated rows: 0.00 @@ -93,7 +93,7 @@ AggregateFinal │ │ ├── aggregate functions: [sum(sum_arg_0)] │ │ ├── estimated rows: 0.00 │ │ └── EvalScalar - │ │ ├── output columns: [t2.sid (#1), sum_arg_0 (#4)] + │ │ ├── output columns: [t2.sid (#1), sum_arg_0 (#3)] │ │ ├── expressions: [if(CAST(is_not_null(t3.val (#2)) AS Boolean NULL), CAST(assume_not_null(t3.val (#2)) AS Int32 NULL), true, 0, NULL)] │ │ ├── estimated rows: 0.00 │ │ └── Filter @@ -123,11 +123,11 @@ AggregateFinal │ ├── push downs: [filters: [is_true(t1.id (#0) = 1)], limit: NONE] │ └── estimated rows: 0.00 └── EvalScalar - ├── output columns: [t.id (#9), de (#11)] + ├── output columns: [t.id (#7), de (#8)] ├── expressions: [0] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [t.id (#9)] + ├── output columns: [t.id (#7)] ├── group by: [id] ├── aggregate functions: [] ├── estimated rows: 0.00 @@ -136,17 +136,17 @@ AggregateFinal ├── aggregate functions: [] ├── estimated rows: 0.00 └── Filter - ├── output columns: [t.id (#9)] - ├── filters: [is_true(t.id (#9) = 1)] + ├── output columns: [t.id (#7)] + ├── filters: [is_true(t.id (#7) = 1)] ├── estimated rows: 0.00 └── TableScan ├── table: default.default.t1 - ├── output columns: [id (#9)] + ├── output columns: [id (#7)] ├── read rows: 0 ├── read size: 0 ├── partitions total: 0 ├── partitions scanned: 0 - ├── push downs: [filters: [is_true(t1.id (#9) = 1)], limit: NONE] + ├── push downs: [filters: [is_true(t1.id (#7) = 1)], limit: NONE] └── estimated rows: 0.00 statement ok diff --git a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test index 2a48fcaf2427..8d685b913538 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test @@ -13,14 +13,14 @@ explain select name, json_path_query(details, '$.features.*') as all_features, j ---- EvalScalar ├── output columns: [products.name (#0), all_features (#3), first_feature (#4)] -├── expressions: [get(1)(json_path_query (#2)), json_path_query_first(products.details (#1), '$.features.*')] +├── expressions: [get(1)(json_path_query(products.details (#1), '$.features.*') (#2)), json_path_query_first(products.details (#1), '$.features.*')] ├── estimated rows: 0.12 └── Filter - ├── output columns: [products.name (#0), products.details (#1), json_path_query (#2)] - ├── filters: [is_true(TRY_CAST(get(1)(json_path_query (#2)) AS String NULL) = '512GB')] + ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] + ├── filters: [is_true(TRY_CAST(get(1)(json_path_query(products.details (#1), '$.features.*') (#2)) AS String NULL) = '512GB')] ├── estimated rows: 0.12 └── ProjectSet - ├── output columns: [products.name (#0), products.details (#1), json_path_query (#2)] + ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] ├── estimated rows: 0.60 ├── set returning functions: json_path_query(products.details (#1), '$.features.*') └── Filter @@ -48,10 +48,10 @@ explain select name, json_path_query(details, '$.features.*') as all_features, j ---- EvalScalar ├── output columns: [products.name (#0), all_features (#3), first_feature (#4)] -├── expressions: [get(1)(json_path_query (#2)), json_path_query_first(products.details (#1), '$.features.*')] +├── expressions: [get(1)(json_path_query(products.details (#1), '$.features.*') (#2)), json_path_query_first(products.details (#1), '$.features.*')] ├── estimated rows: 0.60 └── ProjectSet - ├── output columns: [products.name (#0), products.details (#1), json_path_query (#2)] + ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] ├── estimated rows: 0.60 ├── set returning functions: json_path_query(products.details (#1), '$.features.*') └── Filter diff --git a/tests/sqllogictests/suites/mode/standalone/explain_native/limit.test b/tests/sqllogictests/suites/mode/standalone/explain_native/limit.test index 341a70a18a06..7b7e1840eb1f 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain_native/limit.test +++ b/tests/sqllogictests/suites/mode/standalone/explain_native/limit.test @@ -166,23 +166,23 @@ query T explain select * from (select count(t1.number) as c1 from numbers(1) as t1 group by number) as t3 left join (select count(t.number) as c from numbers(2) as t group by number) as t4 on t3.c1=t4.c order by t3.c1 limit 1 ---- Limit -├── output columns: [count(t.number) (#5), count(t1.number) (#2)] +├── output columns: [count(t.number) (#3), count(t1.number) (#1)] ├── limit: 1 ├── offset: 0 ├── estimated rows: 1.00 └── Sort - ├── output columns: [count(t.number) (#5), count(t1.number) (#2)] + ├── output columns: [count(t.number) (#3), count(t1.number) (#1)] ├── sort keys: [count(t1.number) ASC NULLS LAST] ├── estimated rows: 2.00 └── HashJoin - ├── output columns: [count(t.number) (#5), count(t1.number) (#2)] + ├── output columns: [count(t.number) (#3), count(t1.number) (#1)] ├── join type: RIGHT OUTER - ├── build keys: [CAST(t3.c1 (#2) AS UInt64 NULL)] - ├── probe keys: [t4.c (#5)] + ├── build keys: [CAST(t3.c1 (#1) AS UInt64 NULL)] + ├── probe keys: [t4.c (#3)] ├── filters: [] ├── estimated rows: 2.00 ├── AggregateFinal(Build) - │ ├── output columns: [count(t1.number) (#2), t1.number (#0)] + │ ├── output columns: [count(t1.number) (#1), t1.number (#0)] │ ├── group by: [number] │ ├── aggregate functions: [count()] │ ├── estimated rows: 1.00 @@ -200,7 +200,7 @@ Limit │ ├── push downs: [filters: [], limit: NONE] │ └── estimated rows: 1.00 └── AggregateFinal(Probe) - ├── output columns: [count(t.number) (#5), t.number (#3)] + ├── output columns: [count(t.number) (#3), t.number (#2)] ├── group by: [number] ├── aggregate functions: [count()] ├── estimated rows: 2.00 @@ -210,7 +210,7 @@ Limit ├── estimated rows: 2.00 └── TableScan ├── table: default.system.numbers - ├── output columns: [number (#3)] + ├── output columns: [number (#2)] ├── read rows: 2 ├── read size: < 1 KiB ├── partitions total: 1 diff --git a/tests/sqllogictests/suites/mode/standalone/explain_native/project_set.test b/tests/sqllogictests/suites/mode/standalone/explain_native/project_set.test index d1b91373d45a..40f87aae06df 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain_native/project_set.test +++ b/tests/sqllogictests/suites/mode/standalone/explain_native/project_set.test @@ -23,7 +23,7 @@ AggregateFinal ├── aggregate functions: [count()] ├── estimated rows: 1.00 └── ProjectSet - ├── output columns: [unnest (#2)] + ├── output columns: [unnest(fold_count.id (#0)) (#2)] ├── estimated rows: 1.00 ├── set returning functions: unnest(fold_count.id (#0)) └── TableScan @@ -53,7 +53,7 @@ query T explain select number from (select unnest([1,2,3]), number from numbers(10)) t ---- ProjectSet -├── output columns: [numbers.number (#0), unnest (#1)] +├── output columns: [numbers.number (#0), unnest([1, 2, 3]) (#1)] ├── estimated rows: 10.00 ├── set returning functions: unnest([1, 2, 3]) └── TableScan @@ -70,7 +70,7 @@ query T explain select number from (select unnest([1,2,3,number]), number from numbers(10)) t ---- ProjectSet -├── output columns: [numbers.number (#0), unnest (#1)] +├── output columns: [numbers.number (#0), unnest(array(1, 2, 3, numbers.number (#0))) (#1)] ├── estimated rows: 10.00 ├── set returning functions: unnest(CAST(array(1, 2, 3, numbers.number (#0)) AS Array(UInt64 NULL))) └── TableScan diff --git a/tests/sqllogictests/suites/mode/standalone/explain_native/prune_column.test b/tests/sqllogictests/suites/mode/standalone/explain_native/prune_column.test index f0f749081574..fb3068a8a247 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain_native/prune_column.test +++ b/tests/sqllogictests/suites/mode/standalone/explain_native/prune_column.test @@ -46,12 +46,12 @@ Limit ├── sort keys: [number ASC NULLS LAST] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [numbers.number (#0), numbers.number (#0), numbers.number (#0), numbers.number (#0)] - ├── group by: [number, number, number, number] + ├── output columns: [numbers.number (#0)] + ├── group by: [number] ├── aggregate functions: [] ├── estimated rows: 0.00 └── AggregatePartial - ├── group by: [number, number, number, number] + ├── group by: [number] ├── aggregate functions: [] ├── estimated rows: 0.00 ├── rank limit: 1 diff --git a/tests/sqllogictests/suites/mode/standalone/explain_native/push_down_filter/push_down_filter_eval_scalar.test b/tests/sqllogictests/suites/mode/standalone/explain_native/push_down_filter/push_down_filter_eval_scalar.test index b3915f7fbe08..f7252d926e1e 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain_native/push_down_filter/push_down_filter_eval_scalar.test +++ b/tests/sqllogictests/suites/mode/standalone/explain_native/push_down_filter/push_down_filter_eval_scalar.test @@ -61,14 +61,14 @@ AggregateFinal ├── aggregate functions: [] ├── estimated rows: 0.00 └── UnionAll - ├── output columns: [t.id (#0), de (#8)] + ├── output columns: [t.id (#0), de (#6)] ├── estimated rows: 0.00 ├── EvalScalar - │ ├── output columns: [t.id (#0), de (#8)] - │ ├── expressions: [if(CAST(is_not_null(sum(tb.de) (#7)) AS Boolean NULL), CAST(assume_not_null(sum(tb.de) (#7)) AS Int64 NULL), true, 0, NULL)] + │ ├── output columns: [t.id (#0), de (#6)] + │ ├── expressions: [if(CAST(is_not_null(sum(tb.de) (#5)) AS Boolean NULL), CAST(assume_not_null(sum(tb.de) (#5)) AS Int64 NULL), true, 0, NULL)] │ ├── estimated rows: 0.00 │ └── AggregateFinal - │ ├── output columns: [sum(tb.de) (#7), t.id (#0)] + │ ├── output columns: [sum(tb.de) (#5), t.id (#0)] │ ├── group by: [id] │ ├── aggregate functions: [sum(sum(coalesce(t3.val, 0)))] │ ├── estimated rows: 0.00 @@ -77,14 +77,14 @@ AggregateFinal │ ├── aggregate functions: [sum(sum(coalesce(t3.val, 0)))] │ ├── estimated rows: 0.00 │ └── HashJoin - │ ├── output columns: [t.id (#0), sum(coalesce(t3.val, 0)) (#5)] + │ ├── output columns: [t.id (#0), sum(coalesce(t3.val, 0)) (#4)] │ ├── join type: LEFT OUTER │ ├── build keys: [tb.sid (#1)] │ ├── probe keys: [t.id (#0)] │ ├── filters: [] │ ├── estimated rows: 0.00 │ ├── AggregateFinal(Build) - │ │ ├── output columns: [sum(coalesce(t3.val, 0)) (#5), t2.sid (#1)] + │ │ ├── output columns: [sum(coalesce(t3.val, 0)) (#4), t2.sid (#1)] │ │ ├── group by: [sid] │ │ ├── aggregate functions: [sum(sum_arg_0)] │ │ ├── estimated rows: 0.00 @@ -93,7 +93,7 @@ AggregateFinal │ │ ├── aggregate functions: [sum(sum_arg_0)] │ │ ├── estimated rows: 0.00 │ │ └── EvalScalar - │ │ ├── output columns: [t2.sid (#1), sum_arg_0 (#4)] + │ │ ├── output columns: [t2.sid (#1), sum_arg_0 (#3)] │ │ ├── expressions: [if(CAST(is_not_null(t3.val (#2)) AS Boolean NULL), CAST(assume_not_null(t3.val (#2)) AS Int32 NULL), true, 0, NULL)] │ │ ├── estimated rows: 0.00 │ │ └── TableScan @@ -115,11 +115,11 @@ AggregateFinal │ ├── push downs: [filters: [is_true(t1.id (#0) = 1)], limit: NONE] │ └── estimated rows: 0.00 └── EvalScalar - ├── output columns: [t.id (#9), de (#11)] + ├── output columns: [t.id (#7), de (#8)] ├── expressions: [0] ├── estimated rows: 0.00 └── AggregateFinal - ├── output columns: [t.id (#9)] + ├── output columns: [t.id (#7)] ├── group by: [id] ├── aggregate functions: [] ├── estimated rows: 0.00 @@ -129,12 +129,12 @@ AggregateFinal ├── estimated rows: 0.00 └── TableScan ├── table: default.default.t1 - ├── output columns: [id (#9)] + ├── output columns: [id (#7)] ├── read rows: 0 ├── read size: 0 ├── partitions total: 0 ├── partitions scanned: 0 - ├── push downs: [filters: [is_true(t1.id (#9) = 1)], limit: NONE] + ├── push downs: [filters: [is_true(t1.id (#7) = 1)], limit: NONE] └── estimated rows: 0.00 statement ok diff --git a/tests/sqllogictests/suites/mode/standalone/explain_native/push_down_filter/push_down_filter_project_set.test b/tests/sqllogictests/suites/mode/standalone/explain_native/push_down_filter/push_down_filter_project_set.test index 36ab9dcb98e1..ba0fbc296a06 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain_native/push_down_filter/push_down_filter_project_set.test +++ b/tests/sqllogictests/suites/mode/standalone/explain_native/push_down_filter/push_down_filter_project_set.test @@ -13,14 +13,14 @@ explain select name, json_path_query(details, '$.features.*') as all_features, j ---- EvalScalar ├── output columns: [products.name (#0), all_features (#3), first_feature (#4)] -├── expressions: [get(1)(json_path_query (#2)), json_path_query_first(products.details (#1), '$.features.*')] +├── expressions: [get(1)(json_path_query(products.details (#1), '$.features.*') (#2)), json_path_query_first(products.details (#1), '$.features.*')] ├── estimated rows: 0.12 └── Filter - ├── output columns: [products.name (#0), products.details (#1), json_path_query (#2)] - ├── filters: [is_true(TRY_CAST(get(1)(json_path_query (#2)) AS String NULL) = '512GB')] + ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] + ├── filters: [is_true(TRY_CAST(get(1)(json_path_query(products.details (#1), '$.features.*') (#2)) AS String NULL) = '512GB')] ├── estimated rows: 0.12 └── ProjectSet - ├── output columns: [products.name (#0), products.details (#1), json_path_query (#2)] + ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] ├── estimated rows: 0.60 ├── set returning functions: json_path_query(products.details (#1), '$.features.*') └── TableScan @@ -44,10 +44,10 @@ explain select name, json_path_query(details, '$.features.*') as all_features, j ---- EvalScalar ├── output columns: [products.name (#0), all_features (#3), first_feature (#4)] -├── expressions: [get(1)(json_path_query (#2)), json_path_query_first(products.details (#1), '$.features.*')] +├── expressions: [get(1)(json_path_query(products.details (#1), '$.features.*') (#2)), json_path_query_first(products.details (#1), '$.features.*')] ├── estimated rows: 0.60 └── ProjectSet - ├── output columns: [products.name (#0), products.details (#1), json_path_query (#2)] + ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] ├── estimated rows: 0.60 ├── set returning functions: json_path_query(products.details (#1), '$.features.*') └── TableScan diff --git a/tests/sqllogictests/suites/query/functions/02_0062_function_unnest.test b/tests/sqllogictests/suites/query/functions/02_0062_function_unnest.test index 7fb1478c42a7..52b1b5fecec2 100644 --- a/tests/sqllogictests/suites/query/functions/02_0062_function_unnest.test +++ b/tests/sqllogictests/suites/query/functions/02_0062_function_unnest.test @@ -492,11 +492,16 @@ select unnest([[parse_json('[1,2,3]'), parse_json('false')],[parse_json('{"k":"v false {"k":"v"} -statement error 1065 -select unnest(max(11)) +query T +select unnest(max([11,12])) +---- +11 +12 -statement error 1065 -select unnest(min(11),max(12)) +query T +select max(unnest([11,12])) +---- +12 statement error 1065 select unnest(first_value('aa') OVER (PARTITION BY 'bb')) diff --git a/tests/sqllogictests/suites/query/functions/02_0077_function_dict_get.test b/tests/sqllogictests/suites/query/functions/02_0077_function_dict_get.test index 26decde87ef3..9c5585786fad 100644 --- a/tests/sqllogictests/suites/query/functions/02_0077_function_dict_get.test +++ b/tests/sqllogictests/suites/query/functions/02_0077_function_dict_get.test @@ -5,7 +5,7 @@ statement ok insert into t values('a'),('b'),('%c') statement ok -CREATE OR REPLACE DICTIONARY d(key string not null, value string not null) PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6379')) +CREATE OR REPLACE DICTIONARY d(key string not null, value string not null) PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) query T select a, dict_get(d, 'value', a) from t @@ -84,4 +84,4 @@ Alice 1 24 100.0 1 Bob 2 35 200.1 0 Lily 3 41 1000.2 1 Tom 4 55 3000.55 0 -Tim NULL NULL NULL NULL \ No newline at end of file +Tim NULL NULL NULL NULL From fea44094df9d1606ac8b2e29769b8c19071947fd Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Thu, 14 Nov 2024 20:05:30 +0800 Subject: [PATCH 41/92] fix(cluster): fix the possibility of connection leak when the cluster state is broken. (#16842) * fix(cluster): fix connect leak if server is broken status * fix(cluster): fix connect leak if server is broken status --- src/query/service/src/schedulers/scheduler.rs | 19 +++++++++---- .../flight/v1/actions/init_query_env.rs | 2 +- .../flight/v1/actions/init_query_fragments.rs | 2 +- .../flight/v1/actions/start_prepared_query.rs | 2 +- .../flight/v1/exchange/exchange_manager.rs | 28 ++++++++++++------- .../flight/v1/exchange/statistics_sender.rs | 5 ++-- 6 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/query/service/src/schedulers/scheduler.rs b/src/query/service/src/schedulers/scheduler.rs index e60a5f88736c..5ab76f18a47e 100644 --- a/src/query/service/src/schedulers/scheduler.rs +++ b/src/query/service/src/schedulers/scheduler.rs @@ -114,13 +114,20 @@ pub async fn build_distributed_pipeline( let exchange_manager = ctx.get_exchange_manager(); - let mut build_res = exchange_manager + match exchange_manager .commit_actions(ctx.clone(), fragments_actions) - .await?; - - let settings = ctx.get_settings(); - build_res.set_max_threads(settings.get_max_threads()? as usize); - Ok(build_res) + .await + { + Ok(mut build_res) => { + let settings = ctx.get_settings(); + build_res.set_max_threads(settings.get_max_threads()? as usize); + Ok(build_res) + } + Err(error) => { + exchange_manager.on_finished_query(&ctx.get_id(), Some(error.clone())); + Err(error) + } + } } pub struct ServiceQueryExecutor { diff --git a/src/query/service/src/servers/flight/v1/actions/init_query_env.rs b/src/query/service/src/servers/flight/v1/actions/init_query_env.rs index e4c6aba14761..236f192af217 100644 --- a/src/query/service/src/servers/flight/v1/actions/init_query_env.rs +++ b/src/query/service/src/servers/flight/v1/actions/init_query_env.rs @@ -38,7 +38,7 @@ pub async fn init_query_env(env: QueryEnv) -> Result<()> { .init_query_env(&env, ctx) .await { - DataExchangeManager::instance().on_finished_query(&env.query_id); + DataExchangeManager::instance().on_finished_query(&env.query_id, Some(e.clone())); return Err(e); } diff --git a/src/query/service/src/servers/flight/v1/actions/init_query_fragments.rs b/src/query/service/src/servers/flight/v1/actions/init_query_fragments.rs index 50c01df66448..a90927496b97 100644 --- a/src/query/service/src/servers/flight/v1/actions/init_query_fragments.rs +++ b/src/query/service/src/servers/flight/v1/actions/init_query_fragments.rs @@ -37,7 +37,7 @@ pub async fn init_query_fragments(fragments: QueryFragments) -> Result<()> { })); if let Err(cause) = join_handler.await.flatten() { - DataExchangeManager::instance().on_finished_query(&query_id); + DataExchangeManager::instance().on_finished_query(&query_id, Some(cause.clone())); return Err(cause); } diff --git a/src/query/service/src/servers/flight/v1/actions/start_prepared_query.rs b/src/query/service/src/servers/flight/v1/actions/start_prepared_query.rs index aa79a625711e..10ae27f56f44 100644 --- a/src/query/service/src/servers/flight/v1/actions/start_prepared_query.rs +++ b/src/query/service/src/servers/flight/v1/actions/start_prepared_query.rs @@ -27,7 +27,7 @@ pub async fn start_prepared_query(id: String) -> Result<()> { debug!("start prepared query {}", id); if let Err(cause) = DataExchangeManager::instance().execute_partial_query(&id) { - DataExchangeManager::instance().on_finished_query(&id); + DataExchangeManager::instance().on_finished_query(&id, Some(cause.clone())); return Err(cause); } Ok(()) diff --git a/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs b/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs index a05b9a48c306..3f929c99f594 100644 --- a/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs +++ b/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs @@ -242,7 +242,13 @@ impl DataExchangeManager { "Query {} cannot start command while in 180 seconds", query_id ); - self.on_finished_query(&query_id); + self.on_finished_query( + &query_id, + Some(ErrorCode::Internal(format!( + "Query {} cannot start command while in 180 seconds", + query_id + ))), + ); } } @@ -379,17 +385,17 @@ impl DataExchangeManager { } } - pub fn shutdown_query(&self, query_id: &str) { + pub fn shutdown_query(&self, query_id: &str, cause: Option) { let queries_coordinator_guard = self.queries_coordinator.lock(); let queries_coordinator = unsafe { &mut *queries_coordinator_guard.deref().get() }; if let Some(query_coordinator) = queries_coordinator.get_mut(query_id) { - query_coordinator.shutdown_query(); + query_coordinator.shutdown_query(cause); } } #[fastrace::trace] - pub fn on_finished_query(&self, query_id: &str) { + pub fn on_finished_query(&self, query_id: &str, cause: Option) { let queries_coordinator_guard = self.queries_coordinator.lock(); let queries_coordinator = unsafe { &mut *queries_coordinator_guard.deref().get() }; @@ -397,7 +403,7 @@ impl DataExchangeManager { // Drop mutex guard to avoid deadlock during shutdown, drop(queries_coordinator_guard); - query_coordinator.shutdown_query(); + query_coordinator.shutdown_query(cause); query_coordinator.on_finished(); } } @@ -476,7 +482,8 @@ impl DataExchangeManager { let mut statistics_receiver = statistics_receiver.lock(); statistics_receiver.shutdown(info.res.is_err()); - ctx.get_exchange_manager().on_finished_query(&query_id); + ctx.get_exchange_manager() + .on_finished_query(&query_id, info.res.clone().err()); statistics_receiver.wait_shutdown() }, )); @@ -768,10 +775,10 @@ impl QueryCoordinator { Err(ErrorCode::Unimplemented("ExchangeSource is unimplemented")) } - pub fn shutdown_query(&mut self) { + pub fn shutdown_query(&mut self, cause: Option) { if let Some(query_info) = &mut self.info { if let Some(query_executor) = &query_info.query_executor { - query_executor.finish(None); + query_executor.finish(cause); } if let Some(worker) = query_info.remove_leak_query_worker.take() { @@ -871,10 +878,11 @@ impl QueryCoordinator { Thread::named_spawn(Some(String::from("Distributed-Executor")), move || { let _g = span.set_local_parent(); - statistics_sender.shutdown(executor.execute().err()); + let error = executor.execute().err(); + statistics_sender.shutdown(error.clone()); query_ctx .get_exchange_manager() - .on_finished_query(&query_id); + .on_finished_query(&query_id, error); }); Ok(()) diff --git a/src/query/service/src/servers/flight/v1/exchange/statistics_sender.rs b/src/query/service/src/servers/flight/v1/exchange/statistics_sender.rs index b26e8f4d2854..b370793c7a60 100644 --- a/src/query/service/src/servers/flight/v1/exchange/statistics_sender.rs +++ b/src/query/service/src/servers/flight/v1/exchange/statistics_sender.rs @@ -81,8 +81,9 @@ impl StatisticsSender { notified = right; sleep_future = Box::pin(sleep(Duration::from_millis(100))); - if let Err(_cause) = Self::send_progress(&ctx, &tx).await { - ctx.get_exchange_manager().shutdown_query(&query_id); + if let Err(cause) = Self::send_progress(&ctx, &tx).await { + ctx.get_exchange_manager() + .shutdown_query(&query_id, Some(cause)); return; } From fb79f135c5b220c887f9587cb2f5fce1e90197fc Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Thu, 14 Nov 2024 21:54:35 +0800 Subject: [PATCH 42/92] fix(query): fix aggregator_groups_builder to work with string view (#16843) --- .../transforms/group_by/aggregator_groups_builder.rs | 6 +++++- .../suites/base/03_common/03_0003_select_group_by.test | 7 +++++++ .../suites/base/03_common/03_0003_select_group_by_old.test | 7 +++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/sqllogictests/suites/base/03_common/03_0003_select_group_by_old.test diff --git a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_groups_builder.rs b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_groups_builder.rs index d0d5d0c99330..981e90ab63e3 100644 --- a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_groups_builder.rs +++ b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_groups_builder.rs @@ -92,12 +92,16 @@ impl<'a> SerializedKeysGroupColumnsBuilder<'a> { Some(StringColumnBuilder::with_capacity(capacity)), vec![], ) - } else { + } else if params.group_data_types[0].is_binary() + || params.group_data_types[0].is_variant() + { ( Some(BinaryColumnBuilder::with_capacity(capacity, data_capacity)), None, vec![], ) + } else { + (None, None, Vec::with_capacity(capacity)) } } else { (None, None, Vec::with_capacity(capacity)) diff --git a/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test b/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test index e1c545fad573..a5a5e685429d 100644 --- a/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test +++ b/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test @@ -21,6 +21,13 @@ SELECT number%3 as c1 FROM numbers_mt(10) where number > 2 group by number%3 ord 1 2 +query I +SELECT try_cast(number % 3 as string) c1, count() FROM numbers_mt(10) where number > 2 group by c1 order by c1 +---- +0 3 +1 2 +2 2 + query III SELECT a,b,count() from (SELECT cast((number%4) AS bigint) as a, cast((number%20) AS bigint) as b from numbers(100)) group by a,b order by a,b limit 3 ---- diff --git a/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by_old.test b/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by_old.test new file mode 100644 index 000000000000..d099c541b6c1 --- /dev/null +++ b/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by_old.test @@ -0,0 +1,7 @@ +statement ok +set enable_experimental_aggregate_hashtable = 0; + +include ./03_0003_select_group_by.test + +statement ok +unset enable_experimental_aggregate_hashtable; From 8409f90a22a0de310dc53c3eee04dc8ba4e07c87 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Fri, 15 Nov 2024 14:22:59 +0800 Subject: [PATCH 43/92] chore(cluster): disable parallel commit of cluster tasks (#16851) * chore(cluster): disable parallel commit of cluster tasks * chore(cluster): disable parallel commit of cluster tasks --- src/query/service/src/clusters/cluster.rs | 29 ++++++++++------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/query/service/src/clusters/cluster.rs b/src/query/service/src/clusters/cluster.rs index d4b46ee4cee4..039ce494402b 100644 --- a/src/query/service/src/clusters/cluster.rs +++ b/src/query/service/src/clusters/cluster.rs @@ -137,28 +137,23 @@ impl ClusterHelper for Cluster { ))) } - let mut futures = Vec::with_capacity(message.len()); + let mut response = HashMap::with_capacity(message.len()); for (id, message) in message { let node = get_node(&self.nodes, &id)?; - futures.push({ - let config = GlobalConfig::instance(); - let flight_address = node.flight_address.clone(); - let node_secret = node.secret.clone(); - - async move { - let mut conn = create_client(&config, &flight_address).await?; - Ok::<_, ErrorCode>(( - id, - conn.do_action::<_, Res>(path, node_secret, message, timeout) - .await?, - )) - } - }); + let config = GlobalConfig::instance(); + let flight_address = node.flight_address.clone(); + let node_secret = node.secret.clone(); + + let mut conn = create_client(&config, &flight_address).await?; + response.insert( + id, + conn.do_action::<_, Res>(path, node_secret, message, timeout) + .await?, + ); } - let responses: Vec<(String, Res)> = futures::future::try_join_all(futures).await?; - Ok(responses.into_iter().collect::>()) + Ok(response) } } From 2da2c8e6b6b4a1ad1ef973c99b5a2778486906df Mon Sep 17 00:00:00 2001 From: everpcpc Date: Fri, 15 Nov 2024 14:24:24 +0800 Subject: [PATCH 44/92] fix(query): check storage request method with arg list-type (#16849) * z * fix(query): check storage request method with arg list-type * z --- Cargo.lock | 1 + src/common/storage/Cargo.toml | 1 + src/common/storage/src/http_client.rs | 13 ++++++++----- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 209c872db89e..1bc046b90335 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4111,6 +4111,7 @@ dependencies = [ "reqwest", "serde", "thiserror", + "url", ] [[package]] diff --git a/src/common/storage/Cargo.toml b/src/common/storage/Cargo.toml index a25e3702c3ef..0f197f4dddd2 100644 --- a/src/common/storage/Cargo.toml +++ b/src/common/storage/Cargo.toml @@ -34,6 +34,7 @@ regex = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } +url = { workspace = true } [dev-dependencies] diff --git a/src/common/storage/src/http_client.rs b/src/common/storage/src/http_client.rs index 7c2c99f2b955..d3d81ab171aa 100644 --- a/src/common/storage/src/http_client.rs +++ b/src/common/storage/src/http_client.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::future; use std::mem; use std::str::FromStr; @@ -26,6 +27,7 @@ use opendal::raw::parse_content_length; use opendal::raw::HttpBody; use opendal::raw::HttpFetch; use opendal::Buffer; +use url::Url; pub struct StorageHttpClient { client: reqwest::Client, @@ -46,13 +48,14 @@ impl HttpFetch for StorageHttpClient { let uri = req.uri().clone(); let is_head = req.method() == http::Method::HEAD; - let host = uri.host().unwrap_or_default(); + let url = Url::parse(uri.to_string().as_str()).expect("input request url must be valid"); + let host = url.host_str().unwrap_or_default(); let method = match req.method() { &http::Method::GET => { - if uri.path() == "/" { - "LIST" - } else { - "GET" + let query: HashMap<_, _> = url.query_pairs().collect(); + match query.get("list-type") { + Some(_) => "LIST", + None => "GET", } } m => m.as_str(), From ceb2d03a6afce21b1d0106c5f88fd5b43348e74c Mon Sep 17 00:00:00 2001 From: coldWater Date: Fri, 15 Nov 2024 17:41:32 +0800 Subject: [PATCH 45/92] fix: replace risky `dma_buffer_as_vec` implementations (#16829) * dma_buffer_to_bytes Signed-off-by: coldWater * fix Signed-off-by: coldWater * Memory fitting Signed-off-by: coldWater --------- Signed-off-by: coldWater --- Cargo.lock | 1 + src/common/base/Cargo.toml | 1 + src/common/base/src/base/dma.rs | 69 ++++++++++++++++++----- src/common/base/src/base/mod.rs | 2 +- src/common/base/src/lib.rs | 1 + src/query/service/src/spillers/spiller.rs | 11 ++-- 6 files changed, 65 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1bc046b90335..d92f39b04a84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3100,6 +3100,7 @@ dependencies = [ "async-backtrace", "async-trait", "borsh", + "bytes", "bytesize", "chrono", "ctrlc", diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 81ddc3e8e9b2..0560b19d3a25 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -26,6 +26,7 @@ databend-common-exception = { workspace = true } async-backtrace = { workspace = true } async-trait = { workspace = true } borsh = { workspace = true } +bytes = { workspace = true } bytesize = { workspace = true } chrono = { workspace = true } ctrlc = { workspace = true } diff --git a/src/common/base/src/base/dma.rs b/src/common/base/src/base/dma.rs index 7aeda6307fdd..8851e2acab9b 100644 --- a/src/common/base/src/base/dma.rs +++ b/src/common/base/src/base/dma.rs @@ -28,6 +28,7 @@ use std::path::Path; use std::ptr; use std::ptr::NonNull; +use bytes::Bytes; use rustix::fs::OFlags; use tokio::fs::File; use tokio::io::AsyncSeekExt; @@ -116,10 +117,6 @@ impl DmaAllocator { Layout::from_size_align(layout.size(), self.0.as_usize()).unwrap() } } - - fn real_cap(&self, cap: usize) -> usize { - self.0.align_up(cap) - } } unsafe impl Allocator for DmaAllocator { @@ -131,6 +128,10 @@ unsafe impl Allocator for DmaAllocator { Global {}.allocate_zeroed(self.real_layout(layout)) } + unsafe fn deallocate(&self, ptr: std::ptr::NonNull, layout: Layout) { + Global {}.deallocate(ptr, self.real_layout(layout)) + } + unsafe fn grow( &self, ptr: NonNull, @@ -157,20 +158,38 @@ unsafe impl Allocator for DmaAllocator { ) } - unsafe fn deallocate(&self, ptr: std::ptr::NonNull, layout: Layout) { - Global {}.deallocate(ptr, self.real_layout(layout)) + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + Global {}.shrink( + ptr, + self.real_layout(old_layout), + self.real_layout(new_layout), + ) } } type DmaBuffer = Vec; -pub fn dma_buffer_as_vec(mut buf: DmaBuffer) -> Vec { - let ptr = buf.as_mut_ptr(); - let len = buf.len(); - let cap = buf.allocator().real_cap(buf.capacity()); - std::mem::forget(buf); - - unsafe { Vec::from_raw_parts(ptr, len, cap) } +pub fn dma_buffer_to_bytes(buf: DmaBuffer) -> Bytes { + if buf.is_empty() { + return Bytes::new(); + } + let (ptr, len, cap, alloc) = buf.into_raw_parts_with_alloc(); + // Memory fitting + let old_layout = Layout::from_size_align(cap, alloc.0.as_usize()).unwrap(); + let new_layout = Layout::from_size_align(len, std::mem::align_of::()).unwrap(); + let data = unsafe { + let p = Global {} + .shrink(NonNull::new(ptr).unwrap(), old_layout, new_layout) + .unwrap(); + let cap = p.len(); + Vec::from_raw_parts(p.cast().as_mut(), len, cap) + }; + Bytes::from(data) } /// A `DmaFile` is similar to a `File`, but it is opened with the `O_DIRECT` file in order to @@ -697,4 +716,28 @@ mod tests { let _ = std::fs::remove_file(filename); } + + #[test] + fn test_dma_buffer_to_bytes() { + let want = (0..10_u8).collect::>(); + let alloc = DmaAllocator::new(Alignment::new(4096).unwrap()); + let mut buf = DmaBuffer::with_capacity_in(3000, alloc); + buf.extend_from_slice(&want); + + println!("{:?} {}", buf.as_ptr(), buf.capacity()); + buf.shrink_to_fit(); + println!("{:?} {}", buf.as_ptr(), buf.capacity()); + buf.reserve(3000 - buf.capacity()); + println!("{:?} {}", buf.as_ptr(), buf.capacity()); + + // let slice = buf.into_boxed_slice(); + // println!("{:?}", slice.as_ptr()); + + let got = dma_buffer_to_bytes(buf); + println!("{:?}", got.as_ptr()); + assert_eq!(&want, &got); + + let buf = got.to_vec(); + println!("{:?} {}", buf.as_ptr(), buf.capacity()); + } } diff --git a/src/common/base/src/base/mod.rs b/src/common/base/src/base/mod.rs index 72e96459220c..e77671ec3cb4 100644 --- a/src/common/base/src/base/mod.rs +++ b/src/common/base/src/base/mod.rs @@ -28,7 +28,7 @@ mod take_mut; mod uniq_id; mod watch_notify; -pub use dma::dma_buffer_as_vec; +pub use dma::dma_buffer_to_bytes; pub use dma::dma_read_file; pub use dma::dma_read_file_range; pub use dma::dma_write_file_vectored; diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs index 790a43c49dd0..ab7cec609dbb 100644 --- a/src/common/base/src/lib.rs +++ b/src/common/base/src/lib.rs @@ -25,6 +25,7 @@ #![feature(slice_swap_unchecked)] #![feature(variant_count)] #![feature(ptr_alignment_type)] +#![feature(vec_into_raw_parts)] pub mod base; pub mod containers; diff --git a/src/query/service/src/spillers/spiller.rs b/src/query/service/src/spillers/spiller.rs index 6246d86a2ffc..e545d6c89ce0 100644 --- a/src/query/service/src/spillers/spiller.rs +++ b/src/query/service/src/spillers/spiller.rs @@ -20,8 +20,7 @@ use std::ops::Range; use std::sync::Arc; use std::time::Instant; -use bytes::Bytes; -use databend_common_base::base::dma_buffer_as_vec; +use databend_common_base::base::dma_buffer_to_bytes; use databend_common_base::base::dma_read_file_range; use databend_common_base::base::Alignment; use databend_common_base::base::DmaWriteBuf; @@ -277,7 +276,7 @@ impl Spiller { None => { let file_size = path.size(); let (buf, range) = dma_read_file_range(path, 0..file_size as u64).await?; - Buffer::from(dma_buffer_as_vec(buf)).slice(range) + Buffer::from(dma_buffer_to_bytes(buf)).slice(range) } } } @@ -330,7 +329,7 @@ impl Spiller { ); let (buf, range) = dma_read_file_range(path, 0..file_size as u64).await?; - Buffer::from(dma_buffer_as_vec(buf)).slice(range) + Buffer::from(dma_buffer_to_bytes(buf)).slice(range) } (Location::Local(path), Some(ref local)) => { local @@ -371,7 +370,7 @@ impl Spiller { } None => { let (buf, range) = dma_read_file_range(path, data_range).await?; - Buffer::from(dma_buffer_as_vec(buf)).slice(range) + Buffer::from(dma_buffer_to_bytes(buf)).slice(range) } }, Location::Remote(loc) => self.operator.read_with(loc).range(data_range).await?, @@ -410,7 +409,7 @@ impl Spiller { let buf = buf .into_data() .into_iter() - .map(|x| Bytes::from(dma_buffer_as_vec(x))) + .map(dma_buffer_to_bytes) .collect::(); let written = buf.len(); writer.write(buf).await?; From 100bf4790989d9884331819bec1934373234534d Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Fri, 15 Nov 2024 19:36:22 +0800 Subject: [PATCH 46/92] chore(query): remove SliceExt (#16853) * chore(query): remove SliceExt * chore(query): remove SliceExt --- src/common/base/src/lib.rs | 1 - src/common/base/src/slice_ext.rs | 62 ------------------------ src/query/expression/src/kernels/take.rs | 3 +- src/query/expression/src/types/string.rs | 7 ++- 4 files changed, 4 insertions(+), 69 deletions(-) delete mode 100644 src/common/base/src/slice_ext.rs diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs index ab7cec609dbb..73e20267daa4 100644 --- a/src/common/base/src/lib.rs +++ b/src/common/base/src/lib.rs @@ -36,7 +36,6 @@ pub mod http_client; pub mod mem_allocator; pub mod rangemap; pub mod runtime; -pub mod slice_ext; pub mod vec_ext; pub mod version; diff --git a/src/common/base/src/slice_ext.rs b/src/common/base/src/slice_ext.rs deleted file mode 100644 index a2ff8e30cea2..000000000000 --- a/src/common/base/src/slice_ext.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::slice::SliceIndex; - -pub trait GetSaferUnchecked { - /// # Safety - /// - /// Calling this method with an out-of-bounds index is *[undefined behavior]* - /// even if the resulting reference is not used. - unsafe fn get_unchecked_release(&self, index: I) -> &>::Output - where I: SliceIndex<[T]>; - - /// # Safety - /// - /// Calling this method with an out-of-bounds index is *[undefined behavior]* - /// even if the resulting reference is not used. - unsafe fn get_unchecked_release_mut( - &mut self, - index: I, - ) -> &mut >::Output - where - I: SliceIndex<[T]>; -} - -impl GetSaferUnchecked for [T] { - #[inline(always)] - unsafe fn get_unchecked_release(&self, index: I) -> &>::Output - where I: SliceIndex<[T]> { - if cfg!(debug_assertions) { - &self[index] - } else { - unsafe { self.get_unchecked(index) } - } - } - - #[inline(always)] - unsafe fn get_unchecked_release_mut( - &mut self, - index: I, - ) -> &mut >::Output - where - I: SliceIndex<[T]>, - { - if cfg!(debug_assertions) { - &mut self[index] - } else { - unsafe { self.get_unchecked_mut(index) } - } - } -} diff --git a/src/query/expression/src/kernels/take.rs b/src/query/expression/src/kernels/take.rs index f4957a1c19f3..e142be5b4140 100644 --- a/src/query/expression/src/kernels/take.rs +++ b/src/query/expression/src/kernels/take.rs @@ -19,7 +19,6 @@ use databend_common_arrow::arrow::array::Array; use databend_common_arrow::arrow::array::Utf8ViewArray; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_base::slice_ext::GetSaferUnchecked; use databend_common_exception::Result; use string::StringColumnBuilder; @@ -258,7 +257,7 @@ where I: databend_common_arrow::arrow::types::Index let result: Vec = self .indices .iter() - .map(|index| unsafe { *col.get_unchecked_release(index.to_usize()) }) + .map(|index| unsafe { *col.get_unchecked(index.to_usize()) }) .collect(); result.into() } diff --git a/src/query/expression/src/types/string.rs b/src/query/expression/src/types/string.rs index bc06218b358f..185e2a991fbc 100644 --- a/src/query/expression/src/types/string.rs +++ b/src/query/expression/src/types/string.rs @@ -18,7 +18,6 @@ use std::ops::Range; use databend_common_arrow::arrow::array::MutableBinaryViewArray; use databend_common_arrow::arrow::array::Utf8ViewArray; use databend_common_arrow::arrow::trusted_len::TrustedLen; -use databend_common_base::slice_ext::GetSaferUnchecked; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -293,8 +292,8 @@ impl StringColumn { } pub fn compare(col_i: &Self, i: usize, col_j: &Self, j: usize) -> Ordering { - let view_i = unsafe { col_i.data.views().as_slice().get_unchecked_release(i) }; - let view_j = unsafe { col_j.data.views().as_slice().get_unchecked_release(j) }; + let view_i = unsafe { col_i.data.views().as_slice().get_unchecked(i) }; + let view_j = unsafe { col_j.data.views().as_slice().get_unchecked(j) }; if view_i.prefix == view_j.prefix { unsafe { @@ -311,7 +310,7 @@ impl StringColumn { } pub fn compare_str(col: &Self, i: usize, value: &str) -> Ordering { - let view = unsafe { col.data.views().as_slice().get_unchecked_release(i) }; + let view = unsafe { col.data.views().as_slice().get_unchecked(i) }; let prefix = load_prefix(value.as_bytes()); if view.prefix == prefix { From 182b7448fd5711b5bd48ca7257036728ab94a9fb Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Fri, 15 Nov 2024 20:39:26 +0800 Subject: [PATCH 47/92] fix(query): fix and check total_buffer_len and total_bytes_len (#16854) * chore(ci): enable debug_assertions in release build ci * fix(query): fix and check total_buffer_len and total_bytes_len * fix(query): fix and check total_buffer_len and total_bytes_len * reset ci build * update * update * update --- src/common/arrow/src/arrow/array/binview/mod.rs | 12 ++++++++++++ .../arrow/src/arrow/array/binview/mutable.rs | 10 ++++++++-- .../arrow/src/arrow/array/growable/binview.rs | 6 +----- .../tests/it/arrow/array/binview/mutable_values.rs | 14 ++++++++++++++ src/query/expression/src/kernels/filter.rs | 2 +- src/query/expression/src/kernels/take.rs | 2 +- src/query/expression/src/kernels/take_compact.rs | 2 +- src/query/expression/src/kernels/take_ranges.rs | 2 +- .../base/01_system/01_0001_system_tables.test | 6 ++++++ .../suites/query/join/left_outer.test | 12 +++++++++++- 10 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/common/arrow/src/arrow/array/binview/mod.rs b/src/common/arrow/src/arrow/array/binview/mod.rs index 69bda66794b5..2d0ebc49e5b9 100644 --- a/src/common/arrow/src/arrow/array/binview/mod.rs +++ b/src/common/arrow/src/arrow/array/binview/mod.rs @@ -166,6 +166,18 @@ impl BinaryViewArrayGeneric { total_bytes_len: usize, total_buffer_len: usize, ) -> Self { + #[cfg(debug_assertions)] + { + if total_bytes_len != UNKNOWN_LEN as usize { + let total = views.iter().map(|v| v.length as usize).sum::(); + assert_eq!(total, total_bytes_len); + } + + if total_buffer_len != UNKNOWN_LEN as usize { + let total = buffers.iter().map(|v| v.len()).sum::(); + assert_eq!(total, total_buffer_len); + } + } // # Safety // The caller must ensure // - the data is valid utf8 (if required) diff --git a/src/common/arrow/src/arrow/array/binview/mutable.rs b/src/common/arrow/src/arrow/array/binview/mutable.rs index abf2530b6a38..119fa085bd08 100644 --- a/src/common/arrow/src/arrow/array/binview/mutable.rs +++ b/src/common/arrow/src/arrow/array/binview/mutable.rs @@ -155,12 +155,11 @@ impl MutableBinaryViewArray { #[inline] pub(crate) unsafe fn push_view_unchecked(&mut self, v: View, buffers: &[Buffer]) { let len = v.length; - self.total_bytes_len += len as usize; if len <= 12 { + self.total_bytes_len += len as usize; debug_assert!(self.views.capacity() > self.views.len()); self.views.push(v) } else { - self.total_buffer_len += len as usize; let data = buffers.get_unchecked(v.buffer_idx as usize); let offset = v.offset as usize; let bytes = data.get_unchecked(offset..offset + len as usize); @@ -263,12 +262,19 @@ impl MutableBinaryViewArray { // Push and pop to get the properly encoded value. // For long string this leads to a dictionary encoding, // as we push the string only once in the buffers + + let old_bytes_len = self.total_bytes_len; + let view_value = value .map(|v| { self.push_value_ignore_validity(v); self.views.pop().unwrap() }) .unwrap_or_default(); + + self.total_bytes_len += + (self.total_bytes_len - old_bytes_len) * additional.saturating_sub(1); + self.views .extend(std::iter::repeat(view_value).take(additional)); } diff --git a/src/common/arrow/src/arrow/array/growable/binview.rs b/src/common/arrow/src/arrow/array/growable/binview.rs index a23ba22bffe4..6fa525471e82 100644 --- a/src/common/arrow/src/arrow/array/growable/binview.rs +++ b/src/common/arrow/src/arrow/array/growable/binview.rs @@ -74,7 +74,6 @@ impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> { capacity: usize, ) -> Self { let data_type = arrays[0].data_type().clone(); - // if any of the arrays has nulls, insertions from any array requires setting bits // as there is at least one array with nulls. if !use_validity & arrays.iter().any(|array| array.null_count() > 0) { @@ -91,11 +90,8 @@ impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> { .map(|buf| BufferKey { inner: buf }) }) .collect::>(); - let total_buffer_len = arrays - .iter() - .map(|arr| arr.data_buffers().len()) - .sum::(); + let total_buffer_len = buffers.iter().map(|v| v.inner.len()).sum(); Self { arrays, data_type, diff --git a/src/common/arrow/tests/it/arrow/array/binview/mutable_values.rs b/src/common/arrow/tests/it/arrow/array/binview/mutable_values.rs index 0c23a157f65c..e0384ad7bd39 100644 --- a/src/common/arrow/tests/it/arrow/array/binview/mutable_values.rs +++ b/src/common/arrow/tests/it/arrow/array/binview/mutable_values.rs @@ -29,3 +29,17 @@ fn extend_from_iter() { .as_box() ) } + +#[test] +fn extend_from_repeats() { + let mut b = MutableBinaryViewArray::::new(); + b.extend_constant(4, Some("databend")); + + let a = b.clone(); + b.extend_trusted_len_values(a.values_iter()); + + assert_eq!( + b.as_box(), + MutableBinaryViewArray::::from_values_iter(vec!["databend"; 8].into_iter()).as_box() + ) +} diff --git a/src/query/expression/src/kernels/filter.rs b/src/query/expression/src/kernels/filter.rs index 11bfffe08812..1af12f7b046f 100644 --- a/src/query/expression/src/kernels/filter.rs +++ b/src/query/expression/src/kernels/filter.rs @@ -358,7 +358,7 @@ impl<'a> FilterVisitor<'a> { new_views, values.data.data_buffers().clone(), None, - Some(values.data.total_buffer_len()), + None, ) }; StringColumn::new(new_col) diff --git a/src/query/expression/src/kernels/take.rs b/src/query/expression/src/kernels/take.rs index e142be5b4140..0828c27c9643 100644 --- a/src/query/expression/src/kernels/take.rs +++ b/src/query/expression/src/kernels/take.rs @@ -291,7 +291,7 @@ where I: databend_common_arrow::arrow::types::Index new_views, col.data.data_buffers().clone(), None, - Some(col.data.total_buffer_len()), + None, ) }; StringColumn::new(new_col) diff --git a/src/query/expression/src/kernels/take_compact.rs b/src/query/expression/src/kernels/take_compact.rs index 2cf400264b6d..3353f467c5fe 100644 --- a/src/query/expression/src/kernels/take_compact.rs +++ b/src/query/expression/src/kernels/take_compact.rs @@ -238,7 +238,7 @@ impl<'a> TakeCompactVisitor<'a> { new_views, col.data.data_buffers().clone(), None, - Some(col.data.total_buffer_len()), + None, ) }; StringColumn::new(new_col) diff --git a/src/query/expression/src/kernels/take_ranges.rs b/src/query/expression/src/kernels/take_ranges.rs index 872f3f5829ef..f8ed6e1d84f5 100644 --- a/src/query/expression/src/kernels/take_ranges.rs +++ b/src/query/expression/src/kernels/take_ranges.rs @@ -239,7 +239,7 @@ impl<'a> TakeRangeVisitor<'a> { new_views, col.data.data_buffers().clone(), None, - Some(col.data.total_buffer_len()), + None, ) }; StringColumn::new(new_col) diff --git a/tests/sqllogictests/suites/base/01_system/01_0001_system_tables.test b/tests/sqllogictests/suites/base/01_system/01_0001_system_tables.test index f07c0150dfcb..8a0dfda36bfd 100644 --- a/tests/sqllogictests/suites/base/01_system/01_0001_system_tables.test +++ b/tests/sqllogictests/suites/base/01_system/01_0001_system_tables.test @@ -71,6 +71,12 @@ select name, database, owner from system.tables where database='c' and name ='t1 ---- t100 c account_admin +statement ok +select * from system.malloc_stats_totals; + +statement ok +select * from system.malloc_stats; + statement ok drop database if exists a; diff --git a/tests/sqllogictests/suites/query/join/left_outer.test b/tests/sqllogictests/suites/query/join/left_outer.test index ec53a1887605..d8edb290f8da 100644 --- a/tests/sqllogictests/suites/query/join/left_outer.test +++ b/tests/sqllogictests/suites/query/join/left_outer.test @@ -298,8 +298,18 @@ SELECT * FROM t1 LEFT JOIN t2 ON t1.a = t2.a; 4 5 NULL NULL 5 6 NULL NULL + +statement ok +create or replace table t1 (a string) as select number from numbers(100); + +statement ok +create or replace table t2 (a string) as select number from numbers(100); + +## just check it works or not statement ok -set max_block_size = 65536; +select * from ( + select 'SN0LL' as k from t1 +) as a1 left join (select * from t2) as a2 on a1.k = a2.a; statement ok DROP TABLE IF EXISTS t1; From f9a004d8d3f618e9539464dcaa207b80d48329d4 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Fri, 15 Nov 2024 22:45:33 +0800 Subject: [PATCH 48/92] chore(ci): separate build profile (#16855) * z * z * z * z * z * z * Update mutable.rs --------- Co-authored-by: sundyli <543950155@qq.com> --- .github/actions/build_linux/action.yml | 32 +++---------------- .github/actions/build_macos/action.yml | 25 ++------------- .github/scripts/bump_version.js | 4 +-- .github/workflows/dev.yml | 4 +-- .github/workflows/merge_group.yml | 2 +- Cargo.toml | 6 ++++ .../arrow/src/arrow/array/binview/mutable.rs | 2 +- 7 files changed, 19 insertions(+), 56 deletions(-) diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml index 5b032f29b704..c974a814354b 100644 --- a/.github/actions/build_linux/action.yml +++ b/.github/actions/build_linux/action.yml @@ -64,49 +64,25 @@ runs: target=${{ inputs.target }} echo "BUILD_ARCH=${target/-unknown-linux-*}" >> $GITHUB_ENV - # build all binaries for debug - - name: Build Debug - if: env.BUILD_PROFILE == 'debug' && inputs.artifacts == 'all' - shell: bash - run: | - artifacts="meta,metactl,metabench,query,sqllogictests" - for artifact in ${artifacts//,/ }; do - echo "==> building databend-$artifact ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact - done - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} - ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-* - - - name: Build Debug for specific artifacts - if: env.BUILD_PROFILE == 'debug' && inputs.artifacts != 'all' - shell: bash - run: | - artifacts="${{ inputs.artifacts }}" - for artifact in ${artifacts//,/ }; do - echo "==> building databend-$artifact ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact - done - ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact - - name: Build Release - if: env.BUILD_PROFILE == 'release' && inputs.artifacts == 'all' + if: inputs.artifacts == 'all' shell: bash run: | artifacts="meta,metactl,metabench,query,sqllogictests" for artifact in ${artifacts//,/ }; do echo "==> building databend-$artifact ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact + cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact done ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact - name: Build Release for specific artifacts - if: env.BUILD_PROFILE == 'release' && inputs.artifacts != 'all' + if: inputs.artifacts != 'all' shell: bash run: | artifacts="${{ inputs.artifacts }}" for artifact in ${artifacts//,/ }; do echo "==> building databend-$artifact ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact + cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact done ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact diff --git a/.github/actions/build_macos/action.yml b/.github/actions/build_macos/action.yml index 838e7e97db95..5dc207cfa2c7 100644 --- a/.github/actions/build_macos/action.yml +++ b/.github/actions/build_macos/action.yml @@ -32,7 +32,6 @@ runs: echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> $GITHUB_ENV echo "JEMALLOC_SYS_WITH_MALLOC_CONF=oversize_threshold:0,dirty_decay_ms:5000,muzzy_decay_ms:5000" >> $GITHUB_ENV - - name: Setup build env shell: bash run: | @@ -41,33 +40,15 @@ runs: echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - - name: Build Debug for all artifacts - if: env.BUILD_PROFILE == 'debug' && inputs.artifacts == 'all' - shell: bash - run: cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} - - - name: Build Debug for specific artifacts - if: env.BUILD_PROFILE == 'debug' && inputs.artifacts != 'all' - shell: bash - run: | - artifacts="${{ inputs.artifacts }}" - echo "==> building libs ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --lib - for artifact in ${artifacts//,/ }; do - echo "==> building databend-$artifact ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --bin databend-$artifact - done - - - name: Build Release - if: env.BUILD_PROFILE == 'release' + - name: Build shell: bash run: | artifacts="${{ inputs.artifacts }}" echo "==> building libs ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --release --lib + cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --profile ${{ env.BUILD_PROFILE }} --lib for artifact in ${artifacts//,/ }; do echo "==> building databend-$artifact ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --release --bin databend-$artifact + cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact done - name: Upload artifact diff --git a/.github/scripts/bump_version.js b/.github/scripts/bump_version.js index d4983e78e7c9..e8e6f7052341 100644 --- a/.github/scripts/bump_version.js +++ b/.github/scripts/bump_version.js @@ -32,9 +32,10 @@ module.exports = async ({ github, context, core }) => { core.setFailed("Stable release must be triggered with a nightly tag"); } } else { + core.setOutput("sha", context.sha); if (TAG) { core.setOutput("tag", TAG); - core.info(`Release create manually with tag ${TAG}`); + core.info(`Release create manually with tag ${TAG} (${context.sha})`); } else { let releases = await github.rest.repos.listReleases({ owner: context.repo.owner, @@ -52,7 +53,6 @@ module.exports = async ({ github, context, core }) => { let patch = (parseInt(result[3]) + 1).toString(); let next_tag = `v${major}.${minor}.${patch}-nightly`; core.setOutput("tag", next_tag); - core.setOutput("sha", context.sha); core.info(`Nightly release ${next_tag} from ${tag} (${context.sha})`); } } diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index bc8c519d0495..6d9b30c79651 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -50,7 +50,7 @@ jobs: uses: ./.github/workflows/reuse.linux.yml secrets: inherit with: - build_profile: release + build_profile: ci runner_provider: aws license_type: trial @@ -60,7 +60,7 @@ jobs: uses: ./.github/workflows/reuse.linux.hive.yml secrets: inherit with: - build_profile: release + build_profile: ci runner_provider: aws ready: diff --git a/.github/workflows/merge_group.yml b/.github/workflows/merge_group.yml index 7fb73638f6df..4793fbae31ae 100644 --- a/.github/workflows/merge_group.yml +++ b/.github/workflows/merge_group.yml @@ -54,7 +54,7 @@ jobs: uses: ./.github/workflows/reuse.linux.yml secrets: inherit with: - build_profile: release + build_profile: ci runner_provider: aws license_type: enterprise diff --git a/Cargo.toml b/Cargo.toml index 04088ab2da64..002f075cca23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -569,6 +569,12 @@ overflow-checks = false opt-level = "s" ## defaults to be 3 incremental = true +[profile.ci] +inherits = "release" +overflow-checks = false +incremental = false +debug-assertions = true + # [profile.release.package] # databend-common-arrow = { codegen-units = 16 } # databend-query = { codegen-units = 4 } diff --git a/src/common/arrow/src/arrow/array/binview/mutable.rs b/src/common/arrow/src/arrow/array/binview/mutable.rs index 119fa085bd08..c4a4f7a3c866 100644 --- a/src/common/arrow/src/arrow/array/binview/mutable.rs +++ b/src/common/arrow/src/arrow/array/binview/mutable.rs @@ -424,7 +424,7 @@ impl MutableBinaryViewArray { let value = unsafe { self.value_unchecked(self.len() - 1).to_string() }; self.views.pop(); - + self.total_bytes_len -= value.len(); Some(value) } } From ec9b07741bdeda3d5c3663d5959df4d0846afb7e Mon Sep 17 00:00:00 2001 From: Chase Grainger Date: Fri, 15 Nov 2024 20:16:28 -0500 Subject: [PATCH 49/92] feat: add aws glue as an iceberg connection type (#16824) * added aws glue catalog as option for metastore * linted * removed unused dependencies * ran lint * added test * put toml files in order * added proto test * changed test to use glue as a catalog option * added props for test * got rid of extra block_in_place * linted --------- Co-authored-by: chase.grainger Co-authored-by: Ubuntu --- Cargo.lock | 43 +++++++++++- Cargo.toml | 1 + src/meta/app/src/schema/catalog.rs | 9 +++ .../src/catalog_from_to_protobuf_impl.rs | 39 +++++++++++ src/meta/proto-conv/src/util.rs | 1 + ...v111_add_glue_as_iceberg_catalog_option.rs | 65 +++++++++++++++++++ src/meta/protos/proto/catalog.proto | 9 +++ src/query/service/Cargo.toml | 1 - .../interpreter_catalog_show_create.rs | 3 + src/query/sql/Cargo.toml | 1 - .../sql/src/planner/binder/ddl/catalog.rs | 5 ++ src/query/storages/common/cache/Cargo.toml | 1 - src/query/storages/iceberg/Cargo.toml | 1 + src/query/storages/iceberg/src/catalog.rs | 24 +++++++ tests/sqllogictests/suites/glue/queries.test | 9 +++ 15 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 src/meta/proto-conv/tests/it/v111_add_glue_as_iceberg_catalog_option.rs create mode 100644 tests/sqllogictests/suites/glue/queries.test diff --git a/Cargo.lock b/Cargo.lock index d92f39b04a84..eb1bf21c2636 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -878,6 +878,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "aws-sdk-glue" +version = "1.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf27e013b137b3f441372300d540bf35960db4a87b123ccafe4a36e314502550" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-s3" version = "1.43.0" @@ -4079,7 +4101,6 @@ dependencies = [ "serde", "sha2", "simsearch", - "time", "tokio", "url", ] @@ -4296,6 +4317,7 @@ dependencies = [ "fastrace", "futures", "iceberg", + "iceberg-catalog-glue", "iceberg-catalog-hms", "iceberg-catalog-rest", "match-template", @@ -5160,7 +5182,6 @@ dependencies = [ "sysinfo", "temp-env", "tempfile", - "time", "tokio", "tokio-stream", "toml 0.8.19", @@ -5241,7 +5262,6 @@ dependencies = [ name = "databend-storages-common-cache" version = "0.1.0" dependencies = [ - "arrow", "async-backtrace", "async-trait", "bytes", @@ -8504,6 +8524,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "iceberg-catalog-glue" +version = "0.3.0" +source = "git+https://github.com/Xuanwo/iceberg-rust/?rev=fe5df3f#fe5df3fc432f6f3c0e235bca710a3c7db0c5f369" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-sdk-glue", + "iceberg", + "log", + "serde_json", + "tokio", + "typed-builder 0.20.0", + "uuid", +] + [[package]] name = "iceberg-catalog-hms" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 002f075cca23..d7618d0f5c19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -308,6 +308,7 @@ humantime = "2.1.0" hyper = "1" hyper-util = { version = "0.1.9", features = ["client", "client-legacy", "tokio", "service"] } iceberg = { version = "0.3.0", git = "https://github.com/Xuanwo/iceberg-rust/", rev = "fe5df3f" } +iceberg-catalog-glue = { version = "0.3.0", git = "https://github.com/Xuanwo/iceberg-rust/", rev = "fe5df3f" } iceberg-catalog-hms = { version = "0.3.0", git = "https://github.com/Xuanwo/iceberg-rust/", rev = "fe5df3f" } iceberg-catalog-rest = { version = "0.3.0", git = "https://github.com/Xuanwo/iceberg-rust/", rev = "fe5df3f" } indexmap = "2.0.0" diff --git a/src/meta/app/src/schema/catalog.rs b/src/meta/app/src/schema/catalog.rs index 7031c96be3a3..79d2510566df 100644 --- a/src/meta/app/src/schema/catalog.rs +++ b/src/meta/app/src/schema/catalog.rs @@ -77,6 +77,7 @@ pub struct HiveCatalogOption { pub enum IcebergCatalogType { Rest = 1, Hms = 2, + Glue = 3, } /// Option for creating a iceberg catalog @@ -84,6 +85,7 @@ pub enum IcebergCatalogType { pub enum IcebergCatalogOption { Rest(IcebergRestCatalogOption), Hms(IcebergHmsCatalogOption), + Glue(IcebergGlueCatalogOption), } impl IcebergCatalogOption { @@ -92,6 +94,7 @@ impl IcebergCatalogOption { match self { IcebergCatalogOption::Rest(_) => IcebergCatalogType::Rest, IcebergCatalogOption::Hms(_) => IcebergCatalogType::Hms, + IcebergCatalogOption::Glue(_) => IcebergCatalogType::Glue, } } } @@ -117,6 +120,12 @@ pub struct IcebergHmsCatalogOption { pub props: HashMap, } +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct IcebergGlueCatalogOption { + pub warehouse: String, + pub props: HashMap, +} + /// Same as `CatalogNameIdent`, but with `serde` support, /// and can be used a s part of a value. // #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/src/meta/proto-conv/src/catalog_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/catalog_from_to_protobuf_impl.rs index f0258a28d0da..4f6086ffa348 100644 --- a/src/meta/proto-conv/src/catalog_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/catalog_from_to_protobuf_impl.rs @@ -126,6 +126,9 @@ impl FromToProto for mt::IcebergCatalogOption { pb::iceberg_catalog_option::IcebergCatalogOption::HmsCatalog(v) => { mt::IcebergCatalogOption::Hms(mt::IcebergHmsCatalogOption::from_pb(v)?) } + pb::iceberg_catalog_option::IcebergCatalogOption::GlueCatalog(v) => { + mt::IcebergCatalogOption::Glue(mt::IcebergGlueCatalogOption::from_pb(v)?) + } }) } @@ -140,6 +143,9 @@ impl FromToProto for mt::IcebergCatalogOption { mt::IcebergCatalogOption::Hms(v) => { pb::iceberg_catalog_option::IcebergCatalogOption::HmsCatalog(v.to_pb()?) } + mt::IcebergCatalogOption::Glue(v) => { + pb::iceberg_catalog_option::IcebergCatalogOption::GlueCatalog(v.to_pb()?) + } }), }) } @@ -215,6 +221,39 @@ impl FromToProto for mt::IcebergHmsCatalogOption { } } +impl FromToProto for mt::IcebergGlueCatalogOption { + type PB = pb::IcebergGlueCatalogOption; + + fn get_pb_ver(p: &Self::PB) -> u64 { + p.ver + } + + fn from_pb(p: Self::PB) -> Result + where Self: Sized { + Ok(Self { + warehouse: p.warehouse, + props: p + .props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + }) + } + + fn to_pb(&self) -> Result { + Ok(pb::IcebergGlueCatalogOption { + ver: VER, + min_reader_ver: MIN_READER_VER, + warehouse: self.warehouse.clone(), + props: self + .props + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + }) + } +} + impl FromToProto for mt::HiveCatalogOption { type PB = pb::HiveCatalogOption; diff --git a/src/meta/proto-conv/src/util.rs b/src/meta/proto-conv/src/util.rs index b86e0b8a37a6..1bfefc708568 100644 --- a/src/meta/proto-conv/src/util.rs +++ b/src/meta/proto-conv/src/util.rs @@ -140,6 +140,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[ (108, "2024-08-29: Add: procedure.proto: ProcedureMeta and ProcedureIdentity"), (109, "2024-08-29: Refactor: ProcedureMeta add arg_names"), (110, "2024-09-18: Add: database.proto: DatabaseMeta.gc_in_progress"), + (111, "2024-11-13: Add: Enable AWS Glue as an Apache Iceberg type when creating a catalog."), // Dear developer: // If you're gonna add a new metadata version, you'll have to add a test for it. // You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`) diff --git a/src/meta/proto-conv/tests/it/v111_add_glue_as_iceberg_catalog_option.rs b/src/meta/proto-conv/tests/it/v111_add_glue_as_iceberg_catalog_option.rs new file mode 100644 index 000000000000..478175a338b5 --- /dev/null +++ b/src/meta/proto-conv/tests/it/v111_add_glue_as_iceberg_catalog_option.rs @@ -0,0 +1,65 @@ +// Copyright 2023 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use chrono::TimeZone; +use chrono::Utc; +use databend_common_meta_app::schema::CatalogOption; +use databend_common_meta_app::schema::IcebergCatalogOption; +use databend_common_meta_app::schema::IcebergGlueCatalogOption; +use fastrace::func_name; +use std::collections::HashMap; + +use crate::common; + +// These bytes are built when a new version in introduced, +// and are kept for backward compatibility test. +// +// ************************************************************* +// * These messages should never be updated, * +// * only be added when a new version is added, * +// * or be removed when an old version is no longer supported. * +// ************************************************************* +// +// The message bytes are built from the output of `test_pb_from_to()` +#[test] +fn test_v111_add_glue_as_iceberg_catalog_option() -> anyhow::Result<()> { + let catalog_meta_v111 = vec![ + 18, 55, 26, 53, 18, 45, 10, 21, 104, 116, 116, 112, 58, 47, 47, 49, 50, 55, 46, 48, 46, 48, + 46, 49, 58, 57, 57, 48, 48, 18, 14, 115, 51, 58, 47, 47, 109, 121, 95, 98, 117, 99, 107, + 101, 116, 160, 6, 98, 168, 6, 24, 160, 6, 98, 168, 6, 24, 162, 1, 23, 50, 48, 49, 52, 45, + 49, 49, 45, 50, 56, 32, 49, 50, 58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 160, 6, 98, 168, 6, + 24, + ]; + + let mut props = HashMap::new(); + props.insert("AWS_KEY_ID".to_string(), "super secure access key".to_string()); + props.insert("AWS_SECRET_KEY".to_string(), "even more secure secret key".to_string()); + props.insert("REGION".to_string(), "us-east-1 aka anti-multi-availability".to_string()); + + let want = || databend_common_meta_app::schema::CatalogMeta { + catalog_option: CatalogOption::Iceberg(IcebergGlueCatalogOption::Rest( + IcebergGlueCatalogOption { + address: "http://127.0.0.1:9900".to_string(), + warehouse: "s3://my_bucket".to_string(), + props, + }, + )), + created_on: Utc.with_ymd_and_hms(2014, 11, 28, 12, 0, 9).unwrap(), + }; + + common::test_pb_from_to(func_name!(), want())?; + common::test_load_old(func_name!(), catalog_meta_v111.as_slice(), 111, want())?; + + Ok(()) +} \ No newline at end of file diff --git a/src/meta/protos/proto/catalog.proto b/src/meta/protos/proto/catalog.proto index c69768eb4c84..e601abe65832 100644 --- a/src/meta/protos/proto/catalog.proto +++ b/src/meta/protos/proto/catalog.proto @@ -57,6 +57,7 @@ message IcebergCatalogOption { oneof iceberg_catalog_option { IcebergRestCatalogOption rest_catalog = 2; IcebergHmsCatalogOption hms_catalog = 3; + IcebergGlueCatalogOption glue_catalog = 4; } } @@ -78,6 +79,14 @@ message IcebergHmsCatalogOption { map props = 3; } +message IcebergGlueCatalogOption { + uint64 ver = 100; + uint64 min_reader_ver = 101; + + string warehouse = 1; + map props = 2; +} + message ShareCatalogOption { uint64 ver = 100; uint64 min_reader_ver = 101; diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index e8fdc694cf64..5825ca3ebd7d 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -168,7 +168,6 @@ sqlx = { workspace = true } strength_reduce = { workspace = true } sysinfo = { workspace = true } tempfile = { workspace = true } -time = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true, features = ["net"] } toml = { workspace = true, default-features = false } diff --git a/src/query/service/src/interpreters/interpreter_catalog_show_create.rs b/src/query/service/src/interpreters/interpreter_catalog_show_create.rs index 2f3df63ee33d..4c1d4528208b 100644 --- a/src/query/service/src/interpreters/interpreter_catalog_show_create.rs +++ b/src/query/service/src/interpreters/interpreter_catalog_show_create.rs @@ -78,6 +78,9 @@ impl Interpreter for ShowCreateCatalogInterpreter { IcebergCatalogOption::Hms(cfg) => { format!("ADDRESS\n{}\nWAREHOUSE\n{}", cfg.address, cfg.warehouse) } + IcebergCatalogOption::Glue(cfg) => { + format!("WAREHOUSE\n{}", cfg.warehouse) + } }), }; diff --git a/src/query/sql/Cargo.toml b/src/query/sql/Cargo.toml index a3b7e96abf56..b0453ffd87ea 100644 --- a/src/query/sql/Cargo.toml +++ b/src/query/sql/Cargo.toml @@ -71,7 +71,6 @@ roaring = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } simsearch = { workspace = true } -time = { workspace = true } tokio = { workspace = true } url = { workspace = true } diff --git a/src/query/sql/src/planner/binder/ddl/catalog.rs b/src/query/sql/src/planner/binder/ddl/catalog.rs index 9d2b9290e453..2f89bbadfd6d 100644 --- a/src/query/sql/src/planner/binder/ddl/catalog.rs +++ b/src/query/sql/src/planner/binder/ddl/catalog.rs @@ -35,6 +35,7 @@ use databend_common_meta_app::schema::CatalogOption; use databend_common_meta_app::schema::CatalogType; use databend_common_meta_app::schema::HiveCatalogOption; use databend_common_meta_app::schema::IcebergCatalogOption; +use databend_common_meta_app::schema::IcebergGlueCatalogOption; use databend_common_meta_app::schema::IcebergHmsCatalogOption; use databend_common_meta_app::schema::IcebergRestCatalogOption; use databend_common_meta_app::storage::StorageParams; @@ -234,6 +235,10 @@ fn parse_iceberg_rest_catalog( warehouse, props: HashMap::from_iter(options), }), + "glue" => IcebergCatalogOption::Glue(IcebergGlueCatalogOption { + warehouse, + props: HashMap::from_iter(options), + }), v => { return Err(ErrorCode::InvalidArgument(format!( "iceberg catalog with type {v} is not supported" diff --git a/src/query/storages/common/cache/Cargo.toml b/src/query/storages/common/cache/Cargo.toml index 9448e8f19977..4973c6eec93c 100644 --- a/src/query/storages/common/cache/Cargo.toml +++ b/src/query/storages/common/cache/Cargo.toml @@ -11,7 +11,6 @@ doctest = false test = true [dependencies] -arrow = { workspace = true } databend-common-arrow = { workspace = true } databend-common-base = { workspace = true } databend-common-cache = { workspace = true } diff --git a/src/query/storages/iceberg/Cargo.toml b/src/query/storages/iceberg/Cargo.toml index 13d3089b5690..add1c755faf9 100644 --- a/src/query/storages/iceberg/Cargo.toml +++ b/src/query/storages/iceberg/Cargo.toml @@ -25,6 +25,7 @@ databend-storages-common-table-meta = { workspace = true } fastrace = { workspace = true } futures = { workspace = true } iceberg = { workspace = true } +iceberg-catalog-glue = { workspace = true } iceberg-catalog-hms = { workspace = true } iceberg-catalog-rest = { workspace = true } match-template = { workspace = true } diff --git a/src/query/storages/iceberg/src/catalog.rs b/src/query/storages/iceberg/src/catalog.rs index 664cec399f21..520fd7c012a5 100644 --- a/src/query/storages/iceberg/src/catalog.rs +++ b/src/query/storages/iceberg/src/catalog.rs @@ -103,6 +103,8 @@ use databend_common_meta_app::tenant::Tenant; use databend_common_meta_store::MetaStore; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MetaId; +use iceberg_catalog_glue::GlueCatalog; +use iceberg_catalog_glue::GlueCatalogConfig; use iceberg_catalog_hms::HmsCatalog; use iceberg_catalog_hms::HmsCatalogConfig; use iceberg_catalog_hms::HmsThriftTransport; @@ -196,6 +198,28 @@ impl IcebergCatalog { let ctl = RestCatalog::new(cfg); Arc::new(ctl) } + IcebergCatalogOption::Glue(glue) => { + let cfg = GlueCatalogConfig::builder() + .warehouse(glue.warehouse.clone()) + .props( + glue.props + .clone() + .into_iter() + .map(|(k, v)| (k.trim_matches('"').to_string(), v)) + .collect(), + ) + .build(); + + // Due to the AWS Glue catalog creation being asynchronous, forced to run it a bit different way, so we don't have to make the outer function asynchronous. + let ctl = databend_common_base::runtime::block_on(GlueCatalog::new(cfg)).map_err( + |err| { + ErrorCode::BadArguments(format!( + "There was an error building the AWS Glue catalog: {err:?}" + )) + }, + )?; + Arc::new(ctl) + } }; Ok(Self { info, ctl }) diff --git a/tests/sqllogictests/suites/glue/queries.test b/tests/sqllogictests/suites/glue/queries.test new file mode 100644 index 000000000000..b6ca021d1569 --- /dev/null +++ b/tests/sqllogictests/suites/glue/queries.test @@ -0,0 +1,9 @@ +statement ok +CREATE CATALOG ctl +TYPE = ICEBERG +CONNECTION = ( + TYPE = 'glue', + WAREHOUSE = 's3://bucket' +); + +SHOW CREATE CATALOG ctl; \ No newline at end of file From 20f1fde22291fcc23f8132e760fb6fa4d3aa8941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Sat, 16 Nov 2024 10:14:21 +0800 Subject: [PATCH 50/92] refactor: remove EmbeddedMeta (#16847) * refactor: add InMemoryMeta for testing * refactor: remove EmbeddedMeta EmbeddedMeta is no longer used and should be removed. Replace EmbeddedMeta with InMemoryMeta which is design for testing purpose only. * test: InMemoryMeeta * M src/meta/embedded/src/meta_embedded.rs * M src/meta/embedded/tests/it/kv_api_impl.rs * M src/meta/store/src/lib.rs * M src/meta/raft-store/src/mem_sm.rs * M src/meta/embedded/src/meta_embedded.rs * M Cargo.lock * M Cargo.lock * M src/meta/embedded/src/lib.rs * D src/meta/embedded/src/kv_api_impl.rs * M src/meta/embedded/src/lib.rs * M Cargo.lock * M Cargo.lock --- Cargo.lock | 9 - src/meta/embedded/Cargo.toml | 7 +- src/meta/embedded/src/kv_api_impl.rs | 53 -- src/meta/embedded/src/lib.rs | 15 +- src/meta/embedded/src/meta_embedded.rs | 128 ---- src/meta/embedded/tests/it/kv_api_impl.rs | 66 +- src/meta/embedded/tests/it/schema_api_impl.rs | 27 +- src/meta/embedded/tests/it/testing.rs | 5 +- src/meta/kvapi/src/kvapi/api.rs | 4 +- src/meta/kvapi/src/kvapi/test_suite.rs | 38 +- src/meta/process/src/examples.rs | 23 +- src/meta/raft-store/Cargo.toml | 2 - src/meta/raft-store/src/applier.rs | 25 +- src/meta/raft-store/src/lib.rs | 4 + src/meta/raft-store/src/mem_meta.rs | 130 ++++ src/meta/raft-store/src/mem_state_machine.rs | 54 ++ .../sm_v003/compact_immutable_levels_test.rs | 2 +- .../src/sm_v003/compact_with_db_test.rs | 2 +- src/meta/raft-store/src/sm_v003/mod.rs | 1 + src/meta/raft-store/src/sm_v003/sm_v003.rs | 277 ++------ .../raft-store/src/sm_v003/sm_v003_kv_api.rs | 86 +++ .../raft-store/src/sm_v003/sm_v003_test.rs | 16 +- src/meta/raft-store/src/state_machine_api.rs | 38 ++ .../raft-store/src/state_machine_api_ext.rs | 203 ++++++ src/meta/raft-store/tests/it/main.rs | 2 - .../tests/it/state_machine/expire.rs | 162 ----- .../raft-store/tests/it/state_machine/mod.rs | 476 -------------- .../tests/it/state_machine/schema_api_impl.rs | 63 -- src/meta/raft-store/tests/it/testing.rs | 74 --- .../it/meta_node/meta_node_replication.rs | 1 + src/meta/service/tests/it/testing.rs | 3 - src/meta/sled-store/Cargo.toml | 1 - src/meta/sled-store/src/db.rs | 33 - src/meta/sled-store/src/lib.rs | 3 - src/meta/sled-store/src/sled_iter.rs | 120 ---- src/meta/sled-store/tests/it/main.rs | 20 - src/meta/sled-store/tests/it/sled_iter.rs | 139 ---- src/meta/sled-store/tests/it/sled_tree.rs | 620 ------------------ src/meta/sled-store/tests/it/sled_txn_tree.rs | 101 --- .../tests/it/testing/fake_key_spaces.rs | 69 -- .../it/testing/fake_state_machine_meta.rs | 119 ---- src/meta/sled-store/tests/it/testing/mod.rs | 77 --- src/meta/store/Cargo.toml | 2 +- src/meta/store/src/lib.rs | 8 +- src/meta/types/src/errors/meta_errors.rs | 9 + src/query/management/tests/it/cluster.rs | 4 +- src/query/management/tests/it/quota.rs | 6 +- src/query/management/tests/it/role.rs | 6 +- src/query/management/tests/it/setting.rs | 6 +- src/query/management/tests/it/stage.rs | 6 +- src/query/management/tests/it/udf.rs | 6 +- src/query/service/Cargo.toml | 4 - src/query/service/src/local/mod.rs | 6 - 53 files changed, 695 insertions(+), 2666 deletions(-) delete mode 100644 src/meta/embedded/src/kv_api_impl.rs delete mode 100644 src/meta/embedded/src/meta_embedded.rs create mode 100644 src/meta/raft-store/src/mem_meta.rs create mode 100644 src/meta/raft-store/src/mem_state_machine.rs create mode 100644 src/meta/raft-store/src/sm_v003/sm_v003_kv_api.rs create mode 100644 src/meta/raft-store/src/state_machine_api.rs create mode 100644 src/meta/raft-store/src/state_machine_api_ext.rs delete mode 100644 src/meta/raft-store/tests/it/state_machine/expire.rs delete mode 100644 src/meta/raft-store/tests/it/state_machine/mod.rs delete mode 100644 src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs delete mode 100644 src/meta/raft-store/tests/it/testing.rs delete mode 100644 src/meta/sled-store/src/sled_iter.rs delete mode 100644 src/meta/sled-store/tests/it/main.rs delete mode 100644 src/meta/sled-store/tests/it/sled_iter.rs delete mode 100644 src/meta/sled-store/tests/it/sled_tree.rs delete mode 100644 src/meta/sled-store/tests/it/sled_txn_tree.rs delete mode 100644 src/meta/sled-store/tests/it/testing/fake_key_spaces.rs delete mode 100644 src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs delete mode 100644 src/meta/sled-store/tests/it/testing/mod.rs diff --git a/Cargo.lock b/Cargo.lock index eb1bf21c2636..e91d4d9ca32f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3709,13 +3709,8 @@ dependencies = [ "databend-common-meta-api", "databend-common-meta-kvapi", "databend-common-meta-raft-store", - "databend-common-meta-sled-store", - "databend-common-meta-stoerr", - "databend-common-meta-types", "databend-common-tracing", "fastrace", - "log", - "tempfile", "test-harness", ] @@ -3766,12 +3761,10 @@ dependencies = [ "databend-common-base", "databend-common-exception", "databend-common-grpc", - "databend-common-meta-api", "databend-common-meta-kvapi", "databend-common-meta-sled-store", "databend-common-meta-stoerr", "databend-common-meta-types", - "databend-common-tracing", "deepsize", "derive_more", "fastrace", @@ -3810,7 +3803,6 @@ dependencies = [ "databend-common-base", "databend-common-meta-stoerr", "databend-common-meta-types", - "databend-common-tracing", "fastrace", "log", "openraft", @@ -5077,7 +5069,6 @@ dependencies = [ "databend-common-management", "databend-common-meta-api", "databend-common-meta-app", - "databend-common-meta-embedded", "databend-common-meta-kvapi", "databend-common-meta-store", "databend-common-meta-types", diff --git a/src/meta/embedded/Cargo.toml b/src/meta/embedded/Cargo.toml index 4c85fe9a874d..ccaafe27eafe 100644 --- a/src/meta/embedded/Cargo.toml +++ b/src/meta/embedded/Cargo.toml @@ -12,7 +12,7 @@ doctest = false test = true [features] -io-uring = ["databend-common-meta-sled-store/io-uring", "databend-common-meta-raft-store/io-uring"] +io-uring = ["databend-common-meta-raft-store/io-uring"] [dependencies] async-trait = { workspace = true } @@ -20,13 +20,8 @@ databend-common-base = { workspace = true } databend-common-meta-api = { workspace = true } databend-common-meta-kvapi = { workspace = true } databend-common-meta-raft-store = { workspace = true } -databend-common-meta-sled-store = { workspace = true } -databend-common-meta-stoerr = { workspace = true } -databend-common-meta-types = { workspace = true } databend-common-tracing = { workspace = true } fastrace = { workspace = true } -log = { workspace = true } -tempfile = { workspace = true } [dev-dependencies] anyhow = { workspace = true } diff --git a/src/meta/embedded/src/kv_api_impl.rs b/src/meta/embedded/src/kv_api_impl.rs deleted file mode 100644 index 384654d031a1..000000000000 --- a/src/meta/embedded/src/kv_api_impl.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use async_trait::async_trait; -use databend_common_meta_kvapi::kvapi; -use databend_common_meta_kvapi::kvapi::KVStream; -use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; -use databend_common_meta_types::MetaError; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; - -use crate::MetaEmbedded; - -#[async_trait] -impl kvapi::KVApi for MetaEmbedded { - type Error = MetaError; - - #[fastrace::trace] - async fn upsert_kv(&self, act: UpsertKVReq) -> Result { - let sm = self.inner.lock().await; - sm.upsert_kv(act).await - } - - #[fastrace::trace] - async fn get_kv_stream(&self, keys: &[String]) -> Result, Self::Error> { - let sm = self.inner.lock().await; - sm.get_kv_stream(keys).await - } - - #[fastrace::trace] - async fn list_kv(&self, prefix: &str) -> Result, Self::Error> { - let sm = self.inner.lock().await; - sm.list_kv(prefix).await - } - - #[fastrace::trace] - async fn transaction(&self, txn: TxnRequest) -> Result { - let sm = self.inner.lock().await; - sm.transaction(txn).await - } -} diff --git a/src/meta/embedded/src/lib.rs b/src/meta/embedded/src/lib.rs index fcc7fb3cee21..e396d77855d1 100644 --- a/src/meta/embedded/src/lib.rs +++ b/src/meta/embedded/src/lib.rs @@ -12,16 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Meta store backed with a local sled::Tree. -//! -//! `MetaEmbedded` talks the same API defined in `kvapi::KVApi`. -//! -//! `MetaEmbedded` behave exactly the same as a metasrv without distributed logs(raft), since it is driven by -//! a embedded raft `StateMachine`. - -#![allow(clippy::uninlined_format_args)] - -mod kv_api_impl; -mod meta_embedded; - -pub use meta_embedded::MetaEmbedded; +pub use databend_common_meta_raft_store::mem_meta::MemMeta; +pub use databend_common_meta_raft_store::mem_state_machine::MemStateMachine; diff --git a/src/meta/embedded/src/meta_embedded.rs b/src/meta/embedded/src/meta_embedded.rs deleted file mode 100644 index a1103760581a..000000000000 --- a/src/meta/embedded/src/meta_embedded.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering; -use std::sync::Arc; -use std::sync::LazyLock; - -use databend_common_base::base::tokio::sync::Mutex; -use databend_common_meta_raft_store::config::RaftConfig; -use databend_common_meta_raft_store::state_machine::StateMachine; -pub use databend_common_meta_sled_store::init_temp_sled_db; -use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::anyerror::AnyError; -use log::warn; - -/// Local storage that provides the API defined by `kvapi::KVApi+SchemaApi`. -/// -/// It is just a wrapped `StateMachine`, which is the same one used by raft driven metasrv. -/// For a local kv, there is no distributed WAL involved, -/// thus it just bypasses the raft log and operate directly on the `StateMachine`. -/// -/// Since `StateMachine` is backed with sled::Tree, this impl has the same limitation as metasrv: -/// - A sled::Db has to be a singleton, according to sled doc. -/// - Every unit test has to generate a unique sled::Tree name to create a `MetaEmbedded`. -#[derive(Clone)] -pub struct MetaEmbedded { - pub(crate) inner: Arc>, -} - -static GLOBAL_META_EMBEDDED: LazyLock>>>> = - LazyLock::new(|| Arc::new(Mutex::new(None))); - -impl MetaEmbedded { - /// Creates a kvapi::KVApi impl backed with a `StateMachine`. - /// - /// A MetaEmbedded is identified by the `name`. - /// Caveat: Two instances with the same `name` reference to the same underlying sled::Tree. - /// - /// One of the following has to be called to initialize a process-wise sled::Db, - /// before using `MetaEmbedded`: - /// - `common_meta_sled_store::init_sled_db` - /// - `common_meta_sled_store::init_temp_sled_db` - pub async fn new(name: &str) -> Result { - let mut config = RaftConfig { - sled_tree_prefix: format!("{}-local-kv", name), - ..Default::default() - }; - - if cfg!(target_os = "macos") { - warn!("Disabled fsync for meta data tests. fsync on mac is quite slow"); - config.no_sync = true; - } - - let sm = StateMachine::open(&config, 0).await?; - Ok(MetaEmbedded { - inner: Arc::new(Mutex::new(sm)), - }) - } - - /// Creates a kvapi::KVApi impl with a random and unique name. - pub async fn new_temp() -> Result { - let temp_dir = - tempfile::tempdir().map_err(|e| MetaStorageError::Damaged(AnyError::new(&e)))?; - - init_temp_sled_db(temp_dir); - - // generate a unique id as part of the name of sled::Tree - - static GLOBAL_SEQ: AtomicUsize = AtomicUsize::new(0); - let x = GLOBAL_SEQ.fetch_add(1, Ordering::SeqCst); - let id = 29000_u64 + (x as u64); - - let name = format!("temp-{}", id); - - let m = Self::new(&name).await?; - Ok(m) - } - - /// Initialize a sled db to store embedded meta data. - /// Initialize a global embedded meta store. - /// The data in `path` won't be removed after program exit. - pub async fn init_global_meta_store(path: String) -> Result<(), MetaStorageError> { - databend_common_meta_sled_store::init_sled_db(path, 64 * 1024 * 1024 * 1024); - - { - let mut m = GLOBAL_META_EMBEDDED.as_ref().lock().await; - let r = m.as_ref(); - - if r.is_none() { - let meta = MetaEmbedded::new("global").await?; - let meta = Arc::new(meta); - *m = Some(meta); - return Ok(()); - } - } - - panic!("global meta store can not init twice"); - } - - /// If global meta store is initialized, return it(production use). - /// Otherwise, return a meta store backed with temp dir for test. - pub async fn get_meta() -> Result, MetaStorageError> { - { - let m = GLOBAL_META_EMBEDDED.as_ref().lock().await; - let r = m.as_ref(); - - if let Some(x) = r { - return Ok(x.clone()); - } - } - - let meta = MetaEmbedded::new_temp().await?; - let meta = Arc::new(meta); - Ok(meta) - } -} diff --git a/src/meta/embedded/tests/it/kv_api_impl.rs b/src/meta/embedded/tests/it/kv_api_impl.rs index 32806e2ae458..01b470858b13 100644 --- a/src/meta/embedded/tests/it/kv_api_impl.rs +++ b/src/meta/embedded/tests/it/kv_api_impl.rs @@ -12,48 +12,42 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; +use std::sync::Once; + use databend_common_base::base::tokio; -use databend_common_meta_embedded::MetaEmbedded; use databend_common_meta_kvapi::kvapi; - -#[tokio::test(flavor = "multi_thread")] -async fn test_kv_write_read() -> anyhow::Result<()> { - let kv = MetaEmbedded::new_temp().await?; - kvapi::TestSuite {}.kv_write_read(&kv).await -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_kv_delete() -> anyhow::Result<()> { - let kv = MetaEmbedded::new_temp().await?; - kvapi::TestSuite {}.kv_delete(&kv).await +use databend_common_meta_raft_store::mem_meta::MemMeta; +use databend_common_tracing::init_logging; +use databend_common_tracing::Config; + +#[derive(Clone)] +struct Builder; + +#[async_trait::async_trait] +impl kvapi::ApiBuilder for Builder { + async fn build(&self) -> MemMeta { + MemMeta::default() + } + + async fn build_cluster(&self) -> Vec { + unreachable!("InMemoryMeta does not support cluster") + } } #[tokio::test(flavor = "multi_thread")] -async fn test_kv_update() -> anyhow::Result<()> { - let kv = MetaEmbedded::new_temp().await?; - kvapi::TestSuite {}.kv_update(&kv).await +async fn test_mem_meta_kv_api() -> anyhow::Result<()> { + setup_test(); + kvapi::TestSuite {}.test_single_node(&Builder).await } -#[tokio::test(flavor = "multi_thread")] -async fn test_kv_timeout() -> anyhow::Result<()> { - let kv = MetaEmbedded::new_temp().await?; - kvapi::TestSuite {}.kv_timeout(&kv).await -} +fn setup_test() { + static INIT: Once = Once::new(); + INIT.call_once(|| { + let mut config = Config::new_testing(); + config.file.prefix_filter = "".to_string(); -#[tokio::test(flavor = "multi_thread")] -async fn test_kv_meta() -> anyhow::Result<()> { - let kv = MetaEmbedded::new_temp().await?; - kvapi::TestSuite {}.kv_meta(&kv).await -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_kv_list() -> anyhow::Result<()> { - let kv = MetaEmbedded::new_temp().await?; - kvapi::TestSuite {}.kv_list(&kv).await -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_kv_mget() -> anyhow::Result<()> { - let kv = MetaEmbedded::new_temp().await?; - kvapi::TestSuite {}.kv_mget(&kv).await + let guards = init_logging("meta_unittests", &config, BTreeMap::new()); + Box::leak(Box::new(guards)); + }); } diff --git a/src/meta/embedded/tests/it/schema_api_impl.rs b/src/meta/embedded/tests/it/schema_api_impl.rs index 5a7c35468320..202dcaee92ab 100644 --- a/src/meta/embedded/tests/it/schema_api_impl.rs +++ b/src/meta/embedded/tests/it/schema_api_impl.rs @@ -15,31 +15,36 @@ use async_trait::async_trait; use databend_common_meta_api::BackgroundApiTestSuite; use databend_common_meta_api::SchemaApiTestSuite; -use databend_common_meta_embedded::MetaEmbedded; use databend_common_meta_kvapi::kvapi; +use databend_common_meta_raft_store::mem_meta::MemMeta; use test_harness::test; -use crate::testing::embedded_meta_test_harness; +use crate::testing::mem_meta_test_harness; #[derive(Clone)] -pub struct MetaEmbeddedBuilder {} +pub struct MemMetaBuilder {} #[async_trait] -impl kvapi::ApiBuilder for MetaEmbeddedBuilder { - async fn build(&self) -> MetaEmbedded { - MetaEmbedded::new_temp().await.unwrap() +impl kvapi::ApiBuilder for MemMetaBuilder { + async fn build(&self) -> MemMeta { + MemMeta::default() } - async fn build_cluster(&self) -> Vec { + async fn build_cluster(&self) -> Vec { unimplemented!("embedded meta does not support cluster mode") } } -#[test(harness = embedded_meta_test_harness)] +#[test(harness = mem_meta_test_harness)] #[fastrace::trace] -async fn test_meta_embedded() -> anyhow::Result<()> { - SchemaApiTestSuite::test_single_node(MetaEmbeddedBuilder {}).await?; - BackgroundApiTestSuite::test_single_node(MetaEmbeddedBuilder {}).await?; +async fn test_mem_meta_schema_api() -> anyhow::Result<()> { + SchemaApiTestSuite::test_single_node(MemMetaBuilder {}).await?; + Ok(()) +} +#[test(harness = mem_meta_test_harness)] +#[fastrace::trace] +async fn test_mem_meta_background_api() -> anyhow::Result<()> { + BackgroundApiTestSuite::test_single_node(MemMetaBuilder {}).await?; Ok(()) } diff --git a/src/meta/embedded/tests/it/testing.rs b/src/meta/embedded/tests/it/testing.rs index 789d531c889d..4672afcee496 100644 --- a/src/meta/embedded/tests/it/testing.rs +++ b/src/meta/embedded/tests/it/testing.rs @@ -21,7 +21,7 @@ use databend_common_tracing::init_logging; use databend_common_tracing::Config; use fastrace::prelude::*; -pub fn embedded_meta_test_harness(test: F) +pub fn mem_meta_test_harness(test: F) where F: FnOnce() -> Fut + 'static, Fut: std::future::Future> + Send + 'static, @@ -43,9 +43,6 @@ where fn setup_test() { static INIT: Once = Once::new(); INIT.call_once(|| { - let t = tempfile::tempdir().expect("create temp dir to sled db"); - databend_common_meta_sled_store::init_temp_sled_db(t); - let mut config = Config::new_testing(); config.file.prefix_filter = "".to_string(); diff --git a/src/meta/kvapi/src/kvapi/api.rs b/src/meta/kvapi/src/kvapi/api.rs index 45aff27edeb5..62c62c9f6a05 100644 --- a/src/meta/kvapi/src/kvapi/api.rs +++ b/src/meta/kvapi/src/kvapi/api.rs @@ -18,8 +18,10 @@ use async_trait::async_trait; use databend_common_meta_types::errors; use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::seq_value::SeqV; +use databend_common_meta_types::Change; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use futures_util::stream::BoxStream; use futures_util::StreamExt; use futures_util::TryStreamExt; @@ -56,7 +58,7 @@ pub trait KVApi: Send + Sync { type Error: std::error::Error + From + Send + Sync + 'static; /// Update or insert a key-value record. - async fn upsert_kv(&self, req: UpsertKVReq) -> Result; + async fn upsert_kv(&self, req: UpsertKV) -> Result>, Self::Error>; /// Get a key-value record by key. // TODO: #[deprecated(note = "use get_kv_stream() instead")] diff --git a/src/meta/kvapi/src/kvapi/test_suite.rs b/src/meta/kvapi/src/kvapi/test_suite.rs index 0a0b3a4d2353..bfc8c9918dd0 100644 --- a/src/meta/kvapi/src/kvapi/test_suite.rs +++ b/src/meta/kvapi/src/kvapi/test_suite.rs @@ -50,6 +50,30 @@ pub struct TestSuite {} impl kvapi::TestSuite { #[fastrace::trace] pub async fn test_all(&self, builder: B) -> anyhow::Result<()> + where + KV: kvapi::KVApi, + B: kvapi::ApiBuilder, + { + self.test_single_node(&builder).await?; + + // Run cross node test on every 2 adjacent nodes + let mut i = 0; + loop { + let cluster = builder.build_cluster().await; + self.kv_write_read_across_nodes(&cluster[i], &cluster[i + 1]) + .await?; + + if i + 1 == cluster.len() - 1 { + break; + } + i += 1; + } + + Ok(()) + } + + #[fastrace::trace] + pub async fn test_single_node(&self, builder: &B) -> anyhow::Result<()> where KV: kvapi::KVApi, B: kvapi::ApiBuilder, @@ -74,19 +98,6 @@ impl kvapi::TestSuite { self.kv_delete_by_prefix_transaction(&builder.build().await) .await?; - // Run cross node test on every 2 adjacent nodes - let mut i = 0; - loop { - let cluster = builder.build_cluster().await; - self.kv_write_read_across_nodes(&cluster[i], &cluster[i + 1]) - .await?; - - if i + 1 == cluster.len() - 1 { - break; - } - i += 1; - } - Ok(()) } } @@ -260,6 +271,7 @@ impl kvapi::TestSuite { info!("---get unexpired"); { let res = kv.get_kv("k1").await?; + // dbg!("got k1:{:?}", &res); // dbg!("got non expired k1", &res); assert!(res.is_some(), "got unexpired"); } diff --git a/src/meta/process/src/examples.rs b/src/meta/process/src/examples.rs index e0bf05ffbfed..32d087fe0a0b 100644 --- a/src/meta/process/src/examples.rs +++ b/src/meta/process/src/examples.rs @@ -15,8 +15,6 @@ use std::collections::BTreeMap; use clap::Parser; -use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; -use databend_common_meta_sled_store::init_sled_db; use databend_common_tracing::init_logging; use databend_common_tracing::Config as LogConfig; use serde::Deserialize; @@ -89,23 +87,12 @@ async fn upgrade_09() -> anyhow::Result<()> { /// It does not update any data but just print TableMeta in protobuf message format /// that are found in log or state machine. -pub fn print_table_meta(config: &Config) -> anyhow::Result<()> { - let p = GenericKVProcessor::new(rewrite_kv::print_table_meta); +pub fn print_table_meta(_config: &Config) -> anyhow::Result<()> { + let _p = GenericKVProcessor::new(rewrite_kv::print_table_meta); - let raft_config = &config.raft_config; - - init_sled_db(raft_config.raft_dir.clone(), 64 * 1024 * 1024 * 1024); - - for tree_iter_res in databend_common_meta_sled_store::iter::>() { - let (_tree_name, item_iter) = tree_iter_res?; - - for item_res in item_iter { - let (k, v) = item_res?; - let v1_ent = RaftStoreEntry::deserialize(&k, &v)?; - - p.process(v1_ent)?; - } - } + // TODO: load key values + // let v1_ent = RaftStoreEntry::deserialize(&k, &v)?; + // _p.process(v1_ent)?; Ok(()) } diff --git a/src/meta/raft-store/Cargo.toml b/src/meta/raft-store/Cargo.toml index 29d50e79c610..3cc6e3bae966 100644 --- a/src/meta/raft-store/Cargo.toml +++ b/src/meta/raft-store/Cargo.toml @@ -23,12 +23,10 @@ chrono = { workspace = true } databend-common-base = { workspace = true } databend-common-exception = { workspace = true } databend-common-grpc = { workspace = true } -databend-common-meta-api = { workspace = true } databend-common-meta-kvapi = { workspace = true } databend-common-meta-sled-store = { workspace = true } databend-common-meta-stoerr = { workspace = true } databend-common-meta-types = { workspace = true } -databend-common-tracing = { workspace = true } deepsize = { workspace = true } derive_more = { workspace = true } fastrace = { workspace = true } diff --git a/src/meta/raft-store/src/applier.rs b/src/meta/raft-store/src/applier.rs index be70fd45f460..df6bae352380 100644 --- a/src/meta/raft-store/src/applier.rs +++ b/src/meta/raft-store/src/applier.rs @@ -54,21 +54,26 @@ use log::error; use log::info; use num::FromPrimitive; -use crate::sm_v003::SMV003; +use crate::state_machine_api::StateMachineApi; +use crate::state_machine_api_ext::StateMachineApiExt; /// A helper that applies raft log `Entry` to the state machine. -pub struct Applier<'a> { - sm: &'a mut SMV003, +pub struct Applier<'a, SM> +where SM: StateMachineApi + 'static +{ + sm: &'a mut SM, /// The context of the current applying log. - cmd_ctx: CmdContext, + pub(crate) cmd_ctx: CmdContext, - /// The changes has been made by the applying one log entry + /// The changes have been made by the applying one log entry changes: Vec, String>>, } -impl<'a> Applier<'a> { - pub fn new(sm: &'a mut SMV003) -> Self { +impl<'a, SM> Applier<'a, SM> +where SM: StateMachineApi + 'static +{ + pub fn new(sm: &'a mut SM) -> Self { Self { sm, cmd_ctx: CmdContext::from_millis(0), @@ -114,7 +119,7 @@ impl<'a> Applier<'a> { }; // Send queued change events to subscriber - if let Some(subscriber) = &self.sm.subscriber { + if let Some(subscriber) = self.sm.get_subscriber() { for event in self.changes.drain(..) { subscriber.kv_changed(event); } @@ -236,7 +241,7 @@ impl<'a> Applier<'a> { } #[fastrace::trace] - async fn apply_txn(&mut self, req: &TxnRequest) -> Result { + pub(crate) async fn apply_txn(&mut self, req: &TxnRequest) -> Result { debug!(txn :% =(req); "apply txn cmd"); let success = self.eval_txn_conditions(&req.condition).await?; @@ -461,7 +466,7 @@ impl<'a> Applier<'a> { /// All expired keys will be removed before applying a log. /// This is different from the sled based implementation. #[fastrace::trace] - async fn clean_expired_kvs(&mut self, log_time_ms: u64) -> Result<(), io::Error> { + pub(crate) async fn clean_expired_kvs(&mut self, log_time_ms: u64) -> Result<(), io::Error> { if log_time_ms == 0 { return Ok(()); } diff --git a/src/meta/raft-store/src/lib.rs b/src/meta/raft-store/src/lib.rs index 8e0e9d50f8e7..f58c703fea71 100644 --- a/src/meta/raft-store/src/lib.rs +++ b/src/meta/raft-store/src/lib.rs @@ -23,10 +23,14 @@ pub mod config; pub mod key_spaces; pub mod leveled_store; pub(crate) mod marked; +pub mod mem_meta; +pub mod mem_state_machine; pub mod ondisk; pub mod raft_log_v004; pub mod sm_v003; pub mod snapshot_config; pub mod state; pub mod state_machine; +pub mod state_machine_api; +pub mod state_machine_api_ext; pub mod utils; diff --git a/src/meta/raft-store/src/mem_meta.rs b/src/meta/raft-store/src/mem_meta.rs new file mode 100644 index 000000000000..d02eb1dec7b1 --- /dev/null +++ b/src/meta/raft-store/src/mem_meta.rs @@ -0,0 +1,130 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::future; +use std::io; +use std::sync::Arc; + +use databend_common_meta_kvapi::kvapi::KVApi; +use databend_common_meta_kvapi::kvapi::KVStream; +use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::AppliedState; +use databend_common_meta_types::Change; +use databend_common_meta_types::CmdContext; +use databend_common_meta_types::MetaError; +use databend_common_meta_types::SeqV; +use databend_common_meta_types::SeqValue; +use databend_common_meta_types::TxnReply; +use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; +use futures_util::StreamExt; +use futures_util::TryStreamExt; +use log::debug; +use tokio::sync::Mutex; + +use crate::applier::Applier; +use crate::mem_state_machine::MemStateMachine; +use crate::state_machine_api_ext::StateMachineApiExt; + +#[derive(Clone, Default)] +pub struct MemMeta { + sm: Arc>, +} + +impl MemMeta { + async fn init_applier<'a>( + &self, + a: &mut Applier<'a, MemStateMachine>, + ) -> Result<(), io::Error> { + let now = SeqV::<()>::now_ms(); + a.cmd_ctx = CmdContext::from_millis(now); + a.clean_expired_kvs(now).await?; + Ok(()) + } + + fn non_expired(seq_value: Option>, now_ms: u64) -> Option> { + if seq_value.is_expired(now_ms) { + None + } else { + seq_value + } + } +} + +#[async_trait::async_trait] +impl KVApi for MemMeta { + type Error = MetaError; + + async fn upsert_kv(&self, upsert_kv: UpsertKV) -> Result>, Self::Error> { + debug!("InMemoryStateMachine::upsert_kv({})", upsert_kv); + + let mut sm = self.sm.lock().await; + let mut applier = Applier::new(&mut *sm); + self.init_applier(&mut applier).await?; + + let (prev, result) = applier.upsert_kv(&upsert_kv).await?; + + let st = Change::new(prev, result); + Ok(st) + } + + async fn get_kv_stream(&self, keys: &[String]) -> Result, Self::Error> { + debug!("InMemoryStateMachine::get_kv_stream({:?})", keys); + + let local_now_ms = SeqV::<()>::now_ms(); + + let mut items = Vec::with_capacity(keys.len()); + + let sm = self.sm.lock().await; + + for k in keys { + let got = sm.get_maybe_expired_kv(k.as_str()).await?; + let v = Self::non_expired(got, local_now_ms); + items.push(Ok(StreamItem::from((k.clone(), v)))); + } + + Ok(futures::stream::iter(items).boxed()) + } + + async fn list_kv(&self, prefix: &str) -> Result, Self::Error> { + debug!("InMemoryStateMachine::list_kv({})", prefix); + + let local_now_ms = SeqV::<()>::now_ms(); + + let sm = self.sm.lock().await; + + let strm = sm + .list_kv(prefix) + .await? + .try_filter(move |(_k, v)| future::ready(!v.is_expired(local_now_ms))) + .map_ok(StreamItem::from) + .map_err(|e| e.into()); + + Ok(strm.boxed()) + } + + async fn transaction(&self, txn: TxnRequest) -> Result { + debug!("InMemoryStateMachine::transaction({})", txn); + + let mut sm = self.sm.lock().await; + let mut applier = Applier::new(&mut *sm); + self.init_applier(&mut applier).await?; + + let applied_state = applier.apply_txn(&txn).await?; + match applied_state { + AppliedState::TxnReply(txn_reply) => Ok(txn_reply), + _ => unreachable!("expect TxnReply, got {:?}", applied_state), + } + } +} diff --git a/src/meta/raft-store/src/mem_state_machine.rs b/src/meta/raft-store/src/mem_state_machine.rs new file mode 100644 index 000000000000..709198caf4da --- /dev/null +++ b/src/meta/raft-store/src/mem_state_machine.rs @@ -0,0 +1,54 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_meta_types::sys_data::SysData; + +use crate::leveled_store::level::Level; +use crate::state_machine::ExpireKey; +use crate::state_machine::StateMachineSubscriber; +use crate::state_machine_api::StateMachineApi; +/// A pure in-memory state machine as mock for testing. +#[derive(Debug, Default)] +pub struct MemStateMachine { + level: Level, + expire_cursor: ExpireKey, +} + +impl StateMachineApi for MemStateMachine { + type Map = Level; + + fn get_expire_cursor(&self) -> ExpireKey { + self.expire_cursor + } + + fn set_expire_cursor(&mut self, cursor: ExpireKey) { + self.expire_cursor = cursor; + } + + fn map_ref(&self) -> &Self::Map { + &self.level + } + + fn map_mut(&mut self) -> &mut Self::Map { + &mut self.level + } + + fn sys_data_mut(&mut self) -> &mut SysData { + &mut self.level.sys_data + } + + fn get_subscriber(&self) -> Option<&dyn StateMachineSubscriber> { + None + } +} diff --git a/src/meta/raft-store/src/sm_v003/compact_immutable_levels_test.rs b/src/meta/raft-store/src/sm_v003/compact_immutable_levels_test.rs index 48c602a637aa..4ae45a3f9545 100644 --- a/src/meta/raft-store/src/sm_v003/compact_immutable_levels_test.rs +++ b/src/meta/raft-store/src/sm_v003/compact_immutable_levels_test.rs @@ -258,7 +258,7 @@ async fn build_sm_with_expire() -> anyhow::Result { a.upsert_kv(&UpsertKV::update("b", b"b0").with_expire_sec(5)) .await?; - sm.levels.freeze_writable(); + sm.map_mut().freeze_writable(); let mut a = sm.new_applier(); a.upsert_kv(&UpsertKV::update("c", b"c0").with_expire_sec(20)) diff --git a/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs b/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs index aa868c338f70..a91f06f23b58 100644 --- a/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs +++ b/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs @@ -365,7 +365,7 @@ async fn build_sm_with_expire() -> anyhow::Result<(SMV003, impl Drop)> { a.upsert_kv(&UpsertKV::update("b", b"b0").with_expire_sec(5)) .await?; - sm.levels.freeze_writable(); + sm.map_mut().freeze_writable(); let mut a = sm.new_applier(); a.upsert_kv(&UpsertKV::update("c", b"c0").with_expire_sec(20)) diff --git a/src/meta/raft-store/src/sm_v003/mod.rs b/src/meta/raft-store/src/sm_v003/mod.rs index a7b35b4fdf3b..f215e15b7d21 100644 --- a/src/meta/raft-store/src/sm_v003/mod.rs +++ b/src/meta/raft-store/src/sm_v003/mod.rs @@ -14,6 +14,7 @@ #[allow(clippy::module_inception)] mod sm_v003; +mod sm_v003_kv_api; mod snapshot_store_v002; mod snapshot_store_v003; mod writer_v003; diff --git a/src/meta/raft-store/src/sm_v003/sm_v003.rs b/src/meta/raft-store/src/sm_v003/sm_v003.rs index b310a69df595..a2471eee258b 100644 --- a/src/meta/raft-store/src/sm_v003/sm_v003.rs +++ b/src/meta/raft-store/src/sm_v003/sm_v003.rs @@ -13,116 +13,74 @@ // limitations under the License. use std::fmt::Debug; -use std::future; use std::io; -use databend_common_meta_kvapi::kvapi; -use databend_common_meta_kvapi::kvapi::KVStream; -use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; -use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::StorageError; -use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::seq_value::SeqValue; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::sys_data::SysData; use databend_common_meta_types::AppliedState; -use databend_common_meta_types::CmdContext; -use databend_common_meta_types::EvalExpireTime; -use databend_common_meta_types::MatchSeqExt; -use databend_common_meta_types::Operation; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; -use databend_common_meta_types::UpsertKV; -use futures::Stream; -use futures_util::StreamExt; -use futures_util::TryStreamExt; -use log::debug; use log::info; -use log::warn; use openraft::RaftLogId; use crate::applier::Applier; use crate::leveled_store::leveled_map::compactor::Compactor; use crate::leveled_store::leveled_map::LeveledMap; -use crate::leveled_store::map_api::AsMap; -use crate::leveled_store::map_api::IOResultStream; -use crate::leveled_store::map_api::MapApi; -use crate::leveled_store::map_api::MapApiExt; -use crate::leveled_store::map_api::MapApiRO; use crate::leveled_store::sys_data_api::SysDataApiRO; -use crate::marked::Marked; +use crate::sm_v003::sm_v003_kv_api::SMV003KVApi; use crate::state_machine::ExpireKey; use crate::state_machine::StateMachineSubscriber; -use crate::utils::prefix_right_bound; +use crate::state_machine_api::StateMachineApi; -/// A wrapper that implements KVApi **readonly** methods for the state machine. -pub struct SMV003KVApi<'a> { - sm: &'a SMV003, -} +#[derive(Debug, Default)] +pub struct SMV003 { + levels: LeveledMap, -#[async_trait::async_trait] -impl<'a> kvapi::KVApi for SMV003KVApi<'a> { - type Error = io::Error; + /// The expiration key since which for next clean. + expire_cursor: ExpireKey, - async fn upsert_kv(&self, _req: UpsertKVReq) -> Result { - unreachable!("write operation SM2KVApi::upsert_kv is disabled") - } + /// subscriber of state machine data + pub(crate) subscriber: Option>, +} - async fn get_kv_stream(&self, keys: &[String]) -> Result, Self::Error> { - let local_now_ms = SeqV::<()>::now_ms(); +impl StateMachineApi for SMV003 { + type Map = LeveledMap; - let mut items = Vec::with_capacity(keys.len()); + fn get_expire_cursor(&self) -> ExpireKey { + self.expire_cursor + } - for k in keys { - let got = self.sm.get_maybe_expired_kv(k.as_str()).await?; - let v = Self::non_expired(got, local_now_ms); - items.push(Ok(StreamItem::from((k.clone(), v)))); - } + fn set_expire_cursor(&mut self, cursor: ExpireKey) { + self.expire_cursor = cursor; + } - Ok(futures::stream::iter(items).boxed()) + fn map_ref(&self) -> &Self::Map { + &self.levels } - async fn list_kv(&self, prefix: &str) -> Result, Self::Error> { - let local_now_ms = SeqV::<()>::now_ms(); + fn map_mut(&mut self) -> &mut Self::Map { + &mut self.levels + } - let strm = self - .sm - .list_kv(prefix) - .await? - .try_filter(move |(_k, v)| future::ready(!v.is_expired(local_now_ms))) - .map_ok(StreamItem::from); + // fn sys_data_ref(&self) -> &SysData { + // self.levels.writable_ref().sys_data_ref() + // } - Ok(strm.boxed()) + fn sys_data_mut(&mut self) -> &mut SysData { + self.levels.sys_data_mut() } - async fn transaction(&self, _txn: TxnRequest) -> Result { - unreachable!("write operation SM2KVApi::transaction is disabled") + fn get_subscriber(&self) -> Option<&dyn StateMachineSubscriber> { + self.subscriber.as_ref().map(|x| x.as_ref()) } } - -impl<'a> SMV003KVApi<'a> { - fn non_expired(seq_value: Option>, now_ms: u64) -> Option> { - if seq_value.is_expired(now_ms) { - None - } else { - seq_value - } +impl SMV003 { + /// Return a mutable reference to the map that stores app data. + pub(in crate::sm_v003) fn map_mut(&mut self) -> &mut LeveledMap { + &mut self.levels } } -#[derive(Debug, Default)] -pub struct SMV003 { - pub(in crate::sm_v003) levels: LeveledMap, - - /// The expiration key since which for next clean. - pub(in crate::sm_v003) expire_cursor: ExpireKey, - - /// subscriber of state machine data - pub(crate) subscriber: Option>, -} - impl SMV003 { pub fn kv_api(&self) -> SMV003KVApi { SMV003KVApi { sm: self } @@ -170,7 +128,7 @@ impl SMV003 { } #[allow(dead_code)] - pub(crate) fn new_applier(&mut self) -> Applier { + pub(crate) fn new_applier(&mut self) -> Applier<'_, Self> { Applier::new(self) } @@ -193,88 +151,6 @@ impl SMV003 { Ok(res) } - /// Get a cloned value by key. - /// - /// It does not check expiration of the returned entry. - pub async fn get_maybe_expired_kv(&self, key: &str) -> Result, io::Error> { - let got = self.levels.str_map().get(key).await?; - let seqv = Into::>::into(got); - Ok(seqv) - } - - /// List kv entries by prefix. - /// - /// If a value is expired, it is not returned. - pub async fn list_kv(&self, prefix: &str) -> Result, io::Error> { - let p = prefix.to_string(); - - let strm = if let Some(right) = prefix_right_bound(&p) { - self.levels.str_map().range(p.clone()..right).await? - } else { - self.levels.str_map().range(p.clone()..).await? - }; - - let strm = strm - // Return only keys with the expected prefix - .try_take_while(move |(k, _)| future::ready(Ok(k.starts_with(&p)))) - // Skip tombstone - .try_filter_map(|(k, marked)| { - let seqv = Into::>::into(marked); - let res = seqv.map(|x| (k, x)); - future::ready(Ok(res)) - }); - - // Make it static - - let vs = strm.collect::>().await; - let strm = futures::stream::iter(vs); - - Ok(strm.boxed()) - } - - pub(crate) fn update_expire_cursor(&mut self, log_time_ms: u64) { - if log_time_ms < self.expire_cursor.time_ms { - warn!( - "update_last_cleaned: log_time_ms {} < last_cleaned_expire.time_ms {}", - log_time_ms, self.expire_cursor.time_ms - ); - return; - } - - self.expire_cursor = ExpireKey::new(log_time_ms, 0); - } - - /// List expiration index by expiration time, - /// upto current time(exclusive) in milliseconds. - /// - /// Only records with expire time less than current time will be returned. - /// Expire time that equals to current time is not considered expired. - pub(crate) async fn list_expire_index( - &self, - curr_time_ms: u64, - ) -> Result> + '_, io::Error> { - let start = self.expire_cursor; - - // curr_time > expire_at => expired - let end = ExpireKey::new(curr_time_ms, 0); - - // There is chance the raft leader produce smaller timestamp for later logs. - if start >= end { - return Ok(futures::stream::empty().boxed()); - } - - let strm = self.levels.expire_map().range(start..end).await?; - - let strm = strm - // Return only non-deleted records - .try_filter_map(|(k, marked)| { - let expire_entry = marked.unpack().map(|(v, _v_meta)| (k, v)); - future::ready(Ok(expire_entry)) - }); - - Ok(strm.boxed()) - } - pub fn sys_data_ref(&self) -> &SysData { self.levels.writable_ref().sys_data_ref() } @@ -292,7 +168,7 @@ impl SMV003 { } pub fn levels_mut(&mut self) -> &mut LeveledMap { - &mut self.levels + self.map_mut() } pub fn set_subscriber(&mut self, subscriber: Box) { @@ -304,7 +180,7 @@ impl SMV003 { } pub fn try_acquire_compactor(&mut self) -> Option { - self.levels.try_acquire_compactor() + self.map_mut().try_acquire_compactor() } pub async fn acquire_compactor(&mut self) -> Compactor { @@ -314,7 +190,7 @@ impl SMV003 { /// Replace all the state machine data with the given one. /// The input is a multi-level data. pub fn replace(&mut self, level: LeveledMap) { - let applied = self.levels.writable_ref().last_applied_ref(); + let applied = self.map_mut().writable_ref().last_applied_ref(); let new_applied = level.writable_ref().last_applied_ref(); assert!( @@ -330,81 +206,4 @@ impl SMV003 { // So we need to reset the cursor then the next time applying a log it will clean up all expired. self.expire_cursor = ExpireKey::new(0, 0); } - - /// It returns 2 entries: the previous one and the new one after upsert. - pub(crate) async fn upsert_kv_primary_index( - &mut self, - upsert_kv: &UpsertKV, - cmd_ctx: &CmdContext, - ) -> Result<(Marked>, Marked>), io::Error> { - let kv_meta = upsert_kv.value_meta.as_ref().map(|m| m.to_kv_meta(cmd_ctx)); - - let prev = self.levels.str_map().get(&upsert_kv.key).await?.clone(); - - if upsert_kv.seq.match_seq(prev.seq()).is_err() { - return Ok((prev.clone(), prev)); - } - - let (prev, mut result) = match &upsert_kv.value { - Operation::Update(v) => { - self.levels - .set(upsert_kv.key.clone(), Some((v.clone(), kv_meta.clone()))) - .await? - } - Operation::Delete => self.levels.set(upsert_kv.key.clone(), None).await?, - Operation::AsIs => { - MapApiExt::update_meta(&mut self.levels, upsert_kv.key.clone(), kv_meta.clone()) - .await? - } - }; - - let expire_ms = kv_meta.eval_expire_at_ms(); - if expire_ms < self.expire_cursor.time_ms { - // The record has expired, delete it at once. - // - // Note that it must update first then delete, - // in order to keep compatibility with the old state machine. - // Old SM will just insert an expired record, and that causes the system seq increase by 1. - let (_p, r) = self.levels.set(upsert_kv.key.clone(), None).await?; - result = r; - }; - - debug!( - "applied upsert: {:?}; prev: {:?}; res: {:?}", - upsert_kv, prev, result - ); - - Ok((prev, result)) - } - - /// Update the secondary index for speeding up expiration operation. - /// - /// Remove the expiration index for the removed record, and add a new one for the new record. - pub(crate) async fn update_expire_index( - &mut self, - key: impl ToString, - removed: &Marked>, - added: &Marked>, - ) -> Result<(), io::Error> { - // No change, no need to update expiration index - if removed == added { - return Ok(()); - } - - // Remove previous expiration index, add a new one. - - if let Some(exp_ms) = removed.get_expire_at_ms() { - self.levels - .set(ExpireKey::new(exp_ms, removed.order_key().seq()), None) - .await?; - } - - if let Some(exp_ms) = added.get_expire_at_ms() { - let k = ExpireKey::new(exp_ms, added.order_key().seq()); - let v = key.to_string(); - self.levels.set(k, Some((v, None))).await?; - } - - Ok(()) - } } diff --git a/src/meta/raft-store/src/sm_v003/sm_v003_kv_api.rs b/src/meta/raft-store/src/sm_v003/sm_v003_kv_api.rs new file mode 100644 index 000000000000..1516e6096f25 --- /dev/null +++ b/src/meta/raft-store/src/sm_v003/sm_v003_kv_api.rs @@ -0,0 +1,86 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::future; +use std::io; + +use databend_common_meta_kvapi::kvapi; +use databend_common_meta_kvapi::kvapi::KVStream; +use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::Change; +use databend_common_meta_types::SeqV; +use databend_common_meta_types::SeqValue; +use databend_common_meta_types::TxnReply; +use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; +use futures_util::StreamExt; +use futures_util::TryStreamExt; + +use crate::sm_v003::SMV003; +use crate::state_machine_api_ext::StateMachineApiExt; + +/// A wrapper that implements KVApi **readonly** methods for the state machine. +pub struct SMV003KVApi<'a> { + pub(crate) sm: &'a SMV003, +} + +#[async_trait::async_trait] +impl<'a> kvapi::KVApi for SMV003KVApi<'a> { + type Error = io::Error; + + async fn upsert_kv(&self, _req: UpsertKV) -> Result>, Self::Error> { + unreachable!("write operation SM2KVApi::upsert_kv is disabled") + } + + async fn get_kv_stream(&self, keys: &[String]) -> Result, Self::Error> { + let local_now_ms = SeqV::<()>::now_ms(); + + let mut items = Vec::with_capacity(keys.len()); + + for k in keys { + let got = self.sm.get_maybe_expired_kv(k.as_str()).await?; + let v = Self::non_expired(got, local_now_ms); + items.push(Ok(StreamItem::from((k.clone(), v)))); + } + + Ok(futures::stream::iter(items).boxed()) + } + + async fn list_kv(&self, prefix: &str) -> Result, Self::Error> { + let local_now_ms = SeqV::<()>::now_ms(); + + let strm = self + .sm + .list_kv(prefix) + .await? + .try_filter(move |(_k, v)| future::ready(!v.is_expired(local_now_ms))) + .map_ok(StreamItem::from); + + Ok(strm.boxed()) + } + + async fn transaction(&self, _txn: TxnRequest) -> Result { + unreachable!("write operation SM2KVApi::transaction is disabled") + } +} + +impl<'a> SMV003KVApi<'a> { + fn non_expired(seq_value: Option>, now_ms: u64) -> Option> { + if seq_value.is_expired(now_ms) { + None + } else { + seq_value + } + } +} diff --git a/src/meta/raft-store/src/sm_v003/sm_v003_test.rs b/src/meta/raft-store/src/sm_v003/sm_v003_test.rs index c9418521e87c..546e882c8639 100644 --- a/src/meta/raft-store/src/sm_v003/sm_v003_test.rs +++ b/src/meta/raft-store/src/sm_v003/sm_v003_test.rs @@ -23,6 +23,8 @@ use crate::leveled_store::map_api::MapApiRO; use crate::marked::Marked; use crate::sm_v003::SMV003; use crate::state_machine::ExpireKey; +use crate::state_machine_api::StateMachineApi; +use crate::state_machine_api_ext::StateMachineApiExt; #[tokio::test] async fn test_one_level_upsert_get_range() -> anyhow::Result<()> { @@ -83,7 +85,7 @@ async fn test_two_level_upsert_get_range() -> anyhow::Result<()> { a.upsert_kv(&UpsertKV::update("a/b", b"b0")).await?; a.upsert_kv(&UpsertKV::update("c", b"c0")).await?; - sm.levels.freeze_writable(); + sm.map_mut().freeze_writable(); let mut a = sm.new_applier(); // internal_seq = 3 @@ -140,14 +142,14 @@ async fn test_update_expire_index() -> anyhow::Result<()> { let mut sm = SMV003::default(); sm.update_expire_cursor(1); - assert_eq!(sm.expire_cursor, ExpireKey::new(1, 0)); + assert_eq!(sm.get_expire_cursor(), ExpireKey::new(1, 0)); sm.update_expire_cursor(2); - assert_eq!(sm.expire_cursor, ExpireKey::new(2, 0)); + assert_eq!(sm.get_expire_cursor(), ExpireKey::new(2, 0)); sm.update_expire_cursor(1); assert_eq!( - sm.expire_cursor, + sm.get_expire_cursor(), ExpireKey::new(2, 0), "expire cursor can not go back" ); @@ -171,7 +173,7 @@ async fn build_sm_with_expire() -> anyhow::Result { a.upsert_kv(&UpsertKV::update("b", b"b0").with_expire_sec(5)) .await?; - sm.levels.freeze_writable(); + sm.map_mut().freeze_writable(); let mut a = sm.new_applier(); a.upsert_kv(&UpsertKV::update("c", b"c0").with_expire_sec(20)) @@ -192,7 +194,7 @@ async fn test_internal_expire_index() -> anyhow::Result<()> { // Check internal expire index let got = sm - .levels + .map_ref() .expire_map() .range(..) .await? @@ -298,7 +300,7 @@ async fn test_inserting_expired_becomes_deleting() -> anyhow::Result<()> { // Check expire store let got = sm - .levels + .map_ref() .expire_map() .range(..) .await? diff --git a/src/meta/raft-store/src/state_machine_api.rs b/src/meta/raft-store/src/state_machine_api.rs new file mode 100644 index 000000000000..0d4ff7e776f8 --- /dev/null +++ b/src/meta/raft-store/src/state_machine_api.rs @@ -0,0 +1,38 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_meta_types::sys_data::SysData; + +use crate::leveled_store::map_api::MapApi; +use crate::state_machine::ExpireKey; +use crate::state_machine::StateMachineSubscriber; + +/// The API a state machine implements +pub trait StateMachineApi: Send + Sync { + type Map: MapApi + MapApi + 'static; + + fn get_expire_cursor(&self) -> ExpireKey; + + fn set_expire_cursor(&mut self, cursor: ExpireKey); + + /// Return a reference to the map that stores app data. + fn map_ref(&self) -> &Self::Map; + + /// Return a mutable reference to the map that stores app data. + fn map_mut(&mut self) -> &mut Self::Map; + + fn sys_data_mut(&mut self) -> &mut SysData; + + fn get_subscriber(&self) -> Option<&dyn StateMachineSubscriber>; +} diff --git a/src/meta/raft-store/src/state_machine_api_ext.rs b/src/meta/raft-store/src/state_machine_api_ext.rs new file mode 100644 index 000000000000..359af9a21681 --- /dev/null +++ b/src/meta/raft-store/src/state_machine_api_ext.rs @@ -0,0 +1,203 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::future; +use std::io; + +use databend_common_meta_types::CmdContext; +use databend_common_meta_types::EvalExpireTime; +use databend_common_meta_types::MatchSeqExt; +use databend_common_meta_types::Operation; +use databend_common_meta_types::SeqV; +use databend_common_meta_types::SeqValue; +use databend_common_meta_types::UpsertKV; +use futures_util::StreamExt; +use futures_util::TryStreamExt; +use log::debug; +use log::warn; + +use crate::leveled_store::map_api::AsMap; +use crate::leveled_store::map_api::IOResultStream; +use crate::leveled_store::map_api::MapApi; +use crate::leveled_store::map_api::MapApiExt; +use crate::leveled_store::map_api::MapApiRO; +use crate::marked::Marked; +use crate::state_machine::ExpireKey; +use crate::state_machine_api::StateMachineApi; +use crate::utils::prefix_right_bound; + +#[async_trait::async_trait] +pub trait StateMachineApiExt: StateMachineApi { + /// It returns 2 entries: the previous one and the new one after upsert. + async fn upsert_kv_primary_index( + &mut self, + upsert_kv: &UpsertKV, + cmd_ctx: &CmdContext, + ) -> Result<(Marked>, Marked>), io::Error> { + let kv_meta = upsert_kv.value_meta.as_ref().map(|m| m.to_kv_meta(cmd_ctx)); + + let prev = self.map_ref().str_map().get(&upsert_kv.key).await?.clone(); + + if upsert_kv.seq.match_seq(prev.seq()).is_err() { + return Ok((prev.clone(), prev)); + } + + let (prev, mut result) = match &upsert_kv.value { + Operation::Update(v) => { + self.map_mut() + .set(upsert_kv.key.clone(), Some((v.clone(), kv_meta.clone()))) + .await? + } + Operation::Delete => self.map_mut().set(upsert_kv.key.clone(), None).await?, + Operation::AsIs => { + MapApiExt::update_meta(self.map_mut(), upsert_kv.key.clone(), kv_meta.clone()) + .await? + } + }; + + let expire_ms = kv_meta.eval_expire_at_ms(); + if expire_ms < self.get_expire_cursor().time_ms { + // The record has expired, delete it at once. + // + // Note that it must update first then delete, + // in order to keep compatibility with the old state machine. + // Old SM will just insert an expired record, and that causes the system seq increase by 1. + let (_p, r) = self.map_mut().set(upsert_kv.key.clone(), None).await?; + result = r; + }; + + debug!( + "applied upsert: {:?}; prev: {:?}; res: {:?}", + upsert_kv, prev, result + ); + + Ok((prev, result)) + } + + /// List kv entries by prefix. + /// + /// If a value is expired, it is not returned. + async fn list_kv(&self, prefix: &str) -> Result, io::Error> { + let p = prefix.to_string(); + + let strm = if let Some(right) = prefix_right_bound(&p) { + self.map_ref().str_map().range(p.clone()..right).await? + } else { + self.map_ref().str_map().range(p.clone()..).await? + }; + + let strm = strm + // Return only keys with the expected prefix + .try_take_while(move |(k, _)| future::ready(Ok(k.starts_with(&p)))) + // Skip tombstone + .try_filter_map(|(k, marked)| { + let seqv = Into::>::into(marked); + let res = seqv.map(|x| (k, x)); + future::ready(Ok(res)) + }); + + // Make it static + + let vs = strm.collect::>().await; + let strm = futures::stream::iter(vs); + + Ok(strm.boxed()) + } + + /// Update the secondary index for speeding up expiration operation. + /// + /// Remove the expiration index for the removed record, and add a new one for the new record. + async fn update_expire_index( + &mut self, + key: impl ToString + Send, + removed: &Marked>, + added: &Marked>, + ) -> Result<(), io::Error> { + // No change, no need to update expiration index + if removed == added { + return Ok(()); + } + + // Remove previous expiration index, add a new one. + + if let Some(exp_ms) = removed.get_expire_at_ms() { + self.map_mut() + .set(ExpireKey::new(exp_ms, removed.order_key().seq()), None) + .await?; + } + + if let Some(exp_ms) = added.get_expire_at_ms() { + let k = ExpireKey::new(exp_ms, added.order_key().seq()); + let v = key.to_string(); + self.map_mut().set(k, Some((v, None))).await?; + } + + Ok(()) + } + + /// List expiration index by expiration time, + /// upto current time(exclusive) in milliseconds. + /// + /// Only records with expire time less than current time will be returned. + /// Expire time that equals to current time is not considered expired. + async fn list_expire_index( + &self, + curr_time_ms: u64, + ) -> Result, io::Error> { + let start = self.get_expire_cursor(); + + // curr_time > expire_at => expired + let end = ExpireKey::new(curr_time_ms, 0); + + // There is chance the raft leader produce smaller timestamp for later logs. + if start >= end { + return Ok(futures::stream::empty().boxed()); + } + + let strm = self.map_ref().expire_map().range(start..end).await?; + + let strm = strm + // Return only non-deleted records + .try_filter_map(|(k, marked)| { + let expire_entry = marked.unpack().map(|(v, _v_meta)| (k, v)); + future::ready(Ok(expire_entry)) + }); + + Ok(strm.boxed()) + } + + fn update_expire_cursor(&mut self, log_time_ms: u64) { + if log_time_ms < self.get_expire_cursor().time_ms { + warn!( + "update_last_cleaned: log_time_ms {} < last_cleaned_expire.time_ms {}", + log_time_ms, + self.get_expire_cursor().time_ms + ); + return; + } + + self.set_expire_cursor(ExpireKey::new(log_time_ms, 0)); + } + + /// Get a cloned value by key. + /// + /// It does not check expiration of the returned entry. + async fn get_maybe_expired_kv(&self, key: &str) -> Result, io::Error> { + let got = self.map_ref().str_map().get(key).await?; + let seqv = Into::>::into(got); + Ok(seqv) + } +} + +impl StateMachineApiExt for T where T: StateMachineApi {} diff --git a/src/meta/raft-store/tests/it/main.rs b/src/meta/raft-store/tests/it/main.rs index 47f9570c5dea..6ceafd56f165 100644 --- a/src/meta/raft-store/tests/it/main.rs +++ b/src/meta/raft-store/tests/it/main.rs @@ -16,6 +16,4 @@ #![allow(clippy::diverging_sub_expression)] mod config; -mod state_machine; -mod testing; mod types; diff --git a/src/meta/raft-store/tests/it/state_machine/expire.rs b/src/meta/raft-store/tests/it/state_machine/expire.rs deleted file mode 100644 index 72dd93853531..000000000000 --- a/src/meta/raft-store/tests/it/state_machine/expire.rs +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2022 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::time::SystemTime; -use std::time::UNIX_EPOCH; - -use databend_common_meta_raft_store::key_spaces::Expire; -use databend_common_meta_raft_store::key_spaces::GenericKV; -use databend_common_meta_raft_store::state_machine::ExpireKey; -use databend_common_meta_raft_store::state_machine::StateMachine; -use databend_common_meta_sled_store::AsKeySpace; -use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::EntryPayload; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::LogEntry; -use databend_common_meta_types::MetaSpec; -use databend_common_meta_types::UpsertKV; -use databend_common_meta_types::With; -use test_harness::test; - -use crate::testing::new_raft_test_context; -use crate::testing::raft_store_test_harness; - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_state_machine_update_expiration_index() -> anyhow::Result<()> { - // - Update expiration index when upsert. - // - Remove from expiration index when overriding - // - Remove from expiration index when log-time is provided. - - let tc = new_raft_test_context(); - let sm = StateMachine::open(&tc.raft_config, 0).await?; - let expires = sm.sm_tree.key_space::(); - - let ls_expire = |expire_key_space: &AsKeySpace| { - expire_key_space - .range(..) - .unwrap() - .map(|item_res| { - let item = item_res.unwrap(); - (item.key().unwrap(), item.value().unwrap().key) - }) - .collect::>() - }; - - let s = |x: &str| x.to_string(); - - let now = now(); - - sm.apply(&ent(3, "a", Some(now - 1), None)).await?; - assert_eq!( - vec![(ExpireKey::new((now - 1) * 1000, 1), s("a"))], - ls_expire(&expires) - ); - - // override - sm.apply(&ent(4, "a", Some(now - 2), None)).await?; - assert_eq!( - vec![(ExpireKey::new((now - 2) * 1000, 2), s("a"))], - ls_expire(&expires) - ); - - // Add another expiration index - sm.apply(&ent(5, "b", Some(now - 2), None)).await?; - assert_eq!( - vec![ - // - (ExpireKey::new((now - 2) * 1000, 2), s("a")), - (ExpireKey::new((now - 2) * 1000, 3), s("b")), - ], - ls_expire(&expires) - ); - - // Clean expired - sm.apply(&ent(6, "a", Some(now + 1), Some(now * 1000))) - .await?; - assert_eq!( - vec![ - // - (ExpireKey::new((now + 1) * 1000, 4), s("a")), - ], - ls_expire(&expires) - ); - let kvs = sm.sm_tree.key_space::(); - assert!(kvs.get(&s("b"))?.is_none()); - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_state_machine_list_expired() -> anyhow::Result<()> { - // - Feed logs into state machine. - // - List expired keys - - let tc = new_raft_test_context(); - let sm = StateMachine::open(&tc.raft_config, 0).await?; - - let now = now(); - - let logs = vec![ - ent(3, "a", Some(now - 1), None), // - ent(4, "b", Some(now - 2), None), - ent(5, "c", Some(now + 20), None), - ent(6, "d", Some(now - 3), None), - ]; - - for l in logs.iter() { - sm.apply(l).await?; - } - - let ls = sm.list_expired_kvs(now * 1000)?; - - assert_eq!( - vec![ - expired_item("d", now - 3, 4), // - expired_item("b", now - 2, 2), - expired_item("a", now - 1, 1), - ], - ls - ); - - Ok(()) -} - -fn now() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs() -} - -/// Build an item returned by `list_expired_kvs()`. -fn expired_item(key: &str, time_sec: u64, seq: u64) -> (String, ExpireKey) { - (key.to_string(), ExpireKey::new(time_sec * 1000, seq)) -} - -/// Build a raft log entry with expire time -fn ent(index: u64, key: &str, expire: Option, time_ms: Option) -> Entry { - Entry { - log_id: new_log_id(1, 0, index), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms, - cmd: Cmd::UpsertKV( - UpsertKV::update(key, key.as_bytes()).with(MetaSpec::new(expire, None)), - ), - }), - } -} diff --git a/src/meta/raft-store/tests/it/state_machine/mod.rs b/src/meta/raft-store/tests/it/state_machine/mod.rs deleted file mode 100644 index 27a07649a2ee..000000000000 --- a/src/meta/raft-store/tests/it/state_machine/mod.rs +++ /dev/null @@ -1,476 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::time::Duration; -use std::time::SystemTime; -use std::time::UNIX_EPOCH; - -use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_raft_store::state_machine::StateMachine; -use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::EntryPayload; -use databend_common_meta_types::seq_value::KVMeta; -use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::seq_value::SeqValue; -use databend_common_meta_types::AppliedState; -use databend_common_meta_types::Change; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::CmdContext; -use databend_common_meta_types::Endpoint; -use databend_common_meta_types::LogEntry; -use databend_common_meta_types::MatchSeq; -use databend_common_meta_types::MetaSpec; -use databend_common_meta_types::Node; -use databend_common_meta_types::Operation; -use databend_common_meta_types::UpsertKV; -use databend_common_meta_types::With; -use log::info; -use pretty_assertions::assert_eq; -use test_harness::test; - -use crate::testing::new_raft_test_context; -use crate::testing::raft_store_test_harness; - -mod expire; -mod schema_api_impl; - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_state_machine_apply_add_node() -> anyhow::Result<()> { - let tc = new_raft_test_context(); - let sm = StateMachine::open(&tc.raft_config, 1).await?; - - let apply = |index: u64, n: Node, overriding| { - // - let ss = &sm; - async move { - ss.apply(&Entry { - log_id: new_log_id(1, 0, index), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - cmd: Cmd::AddNode { - node_id: 1, - node: n, - overriding, - }, - }), - }) - .await - } - }; - - let n1 = || Node::new("a", Endpoint::new("1", 1)); - let n2 = || Node::new("b", Endpoint::new("1", 2)); - - // Add node without overriding - { - let resp = apply(5, n1(), false).await?; - assert_eq!( - AppliedState::Node { - prev: None, - result: Some(n1()) - }, - resp - ); - - assert_eq!(Some(n1()), sm.get_node(&1)?); - } - - // Add node without overriding, no update - { - let resp = apply(6, n2(), false).await?; - assert_eq!( - AppliedState::Node { - prev: Some(n1()), - result: Some(n1()) - }, - resp - ); - assert_eq!(Some(n1()), sm.get_node(&1)?); - } - - // Add node with overriding, updated - { - let resp = apply(7, n2(), true).await?; - assert_eq!( - AppliedState::Node { - prev: Some(n1()), - result: Some(n2()) - }, - resp - ); - assert_eq!(Some(n2()), sm.get_node(&1)?); - } - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_state_machine_apply_non_dup_generic_kv_upsert_get() -> anyhow::Result<()> { - let tc = new_raft_test_context(); - let sm = StateMachine::open(&tc.raft_config, 1).await?; - - #[derive(Debug)] - struct T { - // input: - key: String, - seq: MatchSeq, - value: Vec, - value_meta: Option, - // want: - prev: Option>>, - result: Option>>, - } - - fn case( - name: &'static str, - seq: MatchSeq, - value: &'static str, - meta: Option, - prev: Option<(u64, &'static str)>, - result: Option<(u64, &'static str)>, - // The time when it is applied to sm. - now_ms: u64, - ) -> T { - let m = meta.map(|x| MetaSpec::new_ttl(Duration::from_secs(x))); - T { - key: name.to_string(), - seq, - value: value.to_string().into_bytes(), - value_meta: m.clone(), - prev: prev.map(|(a, b)| SeqV::new(a, b.into())), - result: result.map(|(a, b)| { - SeqV::with_meta( - a, - m.map(|m| m.to_kv_meta(&CmdContext::from_millis(now_ms))), - b.into(), - ) - }), - } - } - - let now_ms = SeqV::<()>::now_ms(); - - let cases: Vec = vec![ - case("foo", MatchSeq::Exact(5), "b", None, None, None, now_ms), - case( - "foo", - MatchSeq::GE(0), - "a", - None, - None, - Some((1, "a")), - now_ms, - ), - case( - "foo", - MatchSeq::GE(0), - "b", - None, - Some((1, "a")), - Some((2, "b")), - now_ms, - ), - case( - "foo", - MatchSeq::Exact(5), - "b", - None, - Some((2, "b")), - Some((2, "b")), - now_ms, - ), - case( - "bar", - MatchSeq::Exact(0), - "x", - None, - None, - Some((3, "x")), - now_ms, - ), - case( - "bar", - MatchSeq::Exact(0), - "y", - None, - Some((3, "x")), - Some((3, "x")), - now_ms, - ), - case( - "bar", - MatchSeq::GE(1), - "y", - None, - Some((3, "x")), - Some((4, "y")), - now_ms, - ), - // expired at once - case( - "wow", - MatchSeq::GE(0), - "y", - Some(0), - None, - Some((5, "y")), - now_ms, - ), - // expired value does not exist - case( - "wow", - MatchSeq::GE(0), - "y", - Some(5_000), - None, - Some((6, "y")), - now_ms, - ), - ]; - - for (i, c) in cases.iter().enumerate() { - let mes = format!("{}-th: {}({:?})={:?}", i, c.key, c.seq, c.value); - - // write - let resp = sm.sm_tree.txn(true, |mut t| { - Ok(sm - .apply_cmd( - &Cmd::UpsertKV(UpsertKV { - key: c.key.clone(), - seq: c.seq, - value: Operation::Update(c.value.clone()), - value_meta: c.value_meta.clone(), - }), - &mut t, - None, - now_ms, - ) - .unwrap()) - })?; - assert_eq!( - AppliedState::KV(Change::new(c.prev.clone(), c.result.clone())), - resp, - "write: {}", - mes, - ); - - // get - - let want = match (&c.prev, &c.result) { - (_, Some(ref b)) => Some(b.clone()), - (Some(ref a), _) => Some(a.clone()), - _ => None, - }; - let want = match want { - None => None, - Some(ref w) => { - // trick: in this test all expired timestamps are all 0 - if w.eval_expire_at_ms() < now_ms { - None - } else { - want - } - } - }; - - let got = sm.get_kv(&c.key).await?; - assert_eq!(want, got, "get: {}", mes,); - } - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_state_machine_apply_non_dup_generic_kv_value_meta() -> anyhow::Result<()> { - // - Update a value-meta of None does nothing. - // - Update a value-meta of Some() only updates the value-meta. - - let tc = new_raft_test_context(); - let sm = StateMachine::open(&tc.raft_config, 1).await?; - - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); - - let now_ms = SeqV::<()>::now_ms(); - - let key = "value_meta_foo".to_string(); - - info!("--- update meta of a nonexistent record"); - - let resp = sm.sm_tree.txn(true, |mut t| { - Ok(sm - .apply_cmd( - &Cmd::UpsertKV(UpsertKV { - key: key.clone(), - seq: MatchSeq::GE(0), - value: Operation::AsIs, - value_meta: Some(MetaSpec::new_ttl(Duration::from_secs(10))), - }), - &mut t, - None, - now_ms, - ) - .unwrap()) - })?; - - assert_eq!( - AppliedState::KV(Change::new(None, None)), - resp, - "update meta of None does nothing", - ); - - info!("--- update meta of a existent record"); - - // add a record - sm.sm_tree.txn(true, |mut t| { - Ok(sm - .apply_cmd( - &Cmd::UpsertKV(UpsertKV { - key: key.clone(), - seq: MatchSeq::GE(0), - value: Operation::Update(b"value_meta_bar".to_vec()), - value_meta: Some(MetaSpec::new_ttl(Duration::from_secs(10))), - }), - &mut t, - None, - now_ms, - ) - .unwrap()) - })?; - - let got = sm.get_kv(&key).await?; - println!("after first upsert: got: {:?}", got); - - // update the meta of the record - sm.sm_tree.txn(true, |mut t| { - Ok(sm - .apply_cmd( - &Cmd::UpsertKV(UpsertKV { - key: key.clone(), - seq: MatchSeq::GE(0), - value: Operation::AsIs, - value_meta: Some(MetaSpec::new_ttl(Duration::from_secs(20))), - }), - &mut t, - None, - now_ms, - ) - .unwrap()) - })?; - - info!("--- read the original value and updated meta"); - - let got = sm.get_kv(&key).await?; - let got = got.unwrap(); - - assert_eq!( - SeqV { - seq: got.seq, - meta: Some(KVMeta::new_expire(now + 20)), - data: b"value_meta_bar".to_vec() - }, - got, - "update meta of None does nothing", - ); - - Ok(()) -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_state_machine_apply_non_dup_generic_kv_delete() -> anyhow::Result<()> { - struct T { - // input: - key: String, - seq: MatchSeq, - // want: - prev: Option>>, - result: Option>>, - } - - fn case( - name: &'static str, - seq: MatchSeq, - prev: Option<(u64, &'static str)>, - result: Option<(u64, &'static str)>, - ) -> T { - T { - key: name.to_string(), - seq, - prev: prev.map(|(a, b)| SeqV::new(a, b.into())), - result: result.map(|(a, b)| SeqV::new(a, b.into())), - } - } - - let prev = Some((1u64, "x")); - - let cases: Vec = vec![ - case("foo", MatchSeq::GE(0), prev, None), - case("foo", MatchSeq::Exact(1), prev, None), - case("foo", MatchSeq::Exact(0), prev, prev), - case("foo", MatchSeq::GE(1), prev, None), - case("foo", MatchSeq::GE(2), prev, prev), - ]; - - for (i, c) in cases.iter().enumerate() { - let mes = format!("{}-th: {}({})", i, c.key, c.seq); - - let tc = new_raft_test_context(); - let sm = StateMachine::open(&tc.raft_config, 1).await?; - - // prepare an record - sm.sm_tree.txn(true, |mut t| { - Ok(sm - .apply_cmd( - &Cmd::UpsertKV(UpsertKV::update("foo", b"x")), - &mut t, - None, - 0, - ) - .unwrap()) - })?; - - // delete - let resp = sm.sm_tree.txn(true, |mut t| { - Ok(sm - .apply_cmd( - &Cmd::UpsertKV(UpsertKV::delete(&c.key).with(c.seq)), - &mut t, - None, - 0, - ) - .unwrap()) - })?; - assert_eq!( - AppliedState::KV(Change::new(c.prev.clone(), c.result.clone())), - resp, - "delete: {}", - mes, - ); - - // read it to ensure the modified state. - let want = &c.result; - let got = sm.get_kv(&c.key).await?; - assert_eq!(want, &got, "get: {}", mes,); - } - - Ok(()) -} diff --git a/src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs b/src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs deleted file mode 100644 index 808b301307d3..000000000000 --- a/src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; -use std::sync::Mutex; - -use async_trait::async_trait; -use databend_common_meta_api::BackgroundApiTestSuite; -use databend_common_meta_api::SchemaApiTestSuite; -use databend_common_meta_kvapi::kvapi; -use databend_common_meta_raft_store::state_machine::StateMachine; -use test_harness::test; - -use crate::testing::new_raft_test_context; -use crate::testing::raft_store_test_harness; -use crate::testing::RaftTestContext; - -#[derive(Clone)] -struct StateMachineBuilder { - test_context: Arc>>, -} - -#[async_trait] -impl kvapi::ApiBuilder for StateMachineBuilder { - async fn build(&self) -> StateMachine { - let tc = new_raft_test_context(); - let sm = StateMachine::open(&tc.raft_config, 1).await.unwrap(); - { - let mut x = self.test_context.lock().unwrap(); - *x = Some(tc); - } - - sm - } - - async fn build_cluster(&self) -> Vec { - unimplemented!("StateMachine does not support cluster mode") - } -} - -#[test(harness = raft_store_test_harness)] -#[fastrace::trace] -async fn test_meta_embedded_single() -> anyhow::Result<()> { - let builder = StateMachineBuilder { - test_context: Default::default(), - }; - - SchemaApiTestSuite::test_single_node(builder.clone()).await?; - BackgroundApiTestSuite::test_single_node(builder).await?; - - Ok(()) -} diff --git a/src/meta/raft-store/tests/it/testing.rs b/src/meta/raft-store/tests/it/testing.rs deleted file mode 100644 index 926b3d6816e5..000000000000 --- a/src/meta/raft-store/tests/it/testing.rs +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::BTreeMap; -use std::sync::Once; - -use databend_common_base::base::GlobalSequence; -use databend_common_meta_raft_store::config::RaftConfig; -use databend_common_tracing::closure_name; -use databend_common_tracing::init_logging; -use databend_common_tracing::Config; -use fastrace::prelude::*; - -pub struct RaftTestContext { - pub raft_config: RaftConfig, -} - -/// Create a new context for testing sled -pub fn new_raft_test_context() -> RaftTestContext { - // config for unit test of sled db, meta_sync() is true by default. - let config = RaftConfig { - sled_tree_prefix: format!("test-{}-", 30900 + GlobalSequence::next()), - ..Default::default() - }; - - RaftTestContext { - raft_config: config, - } -} - -pub fn raft_store_test_harness(test: F) -where - F: FnOnce() -> Fut + 'static, - Fut: std::future::Future> + Send + 'static, -{ - setup_test(); - - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(3) - .enable_all() - .build() - .unwrap(); - let root = Span::root(closure_name::(), SpanContext::random()); - let test = test().in_span(root); - rt.block_on(test).unwrap(); - - shutdown_test(); -} - -fn setup_test() { - static INIT: Once = Once::new(); - INIT.call_once(|| { - let t = tempfile::tempdir().expect("create temp dir to sled db"); - databend_common_meta_sled_store::init_temp_sled_db(t); - - let guards = init_logging("meta_unittests", &Config::new_testing(), BTreeMap::new()); - Box::leak(Box::new(guards)); - }); -} - -fn shutdown_test() { - fastrace::flush(); -} diff --git a/src/meta/service/tests/it/meta_node/meta_node_replication.rs b/src/meta/service/tests/it/meta_node/meta_node_replication.rs index 68e752edc9de..f4794de0f201 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_replication.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_replication.rs @@ -18,6 +18,7 @@ use std::io::Read; use databend_common_arrow::arrow::array::ViewType; use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; +use databend_common_meta_raft_store::state_machine_api_ext::StateMachineApiExt; use databend_common_meta_sled_store::openraft::error::SnapshotMismatch; use databend_common_meta_sled_store::openraft::testing::log_id; use databend_common_meta_sled_store::openraft::LogIdOptionExt; diff --git a/src/meta/service/tests/it/testing.rs b/src/meta/service/tests/it/testing.rs index 365d9cd4af0d..26a4f1690306 100644 --- a/src/meta/service/tests/it/testing.rs +++ b/src/meta/service/tests/it/testing.rs @@ -56,9 +56,6 @@ where F: FnOnce() -> anyhow::Result<()> + 'static { fn setup_test() { static INIT: Once = Once::new(); INIT.call_once(|| { - let t = tempfile::tempdir().expect("create temp dir to sled db"); - databend_common_meta_sled_store::init_temp_sled_db(t); - let mut config = Config::new_testing(); config.file.prefix_filter = "".to_string(); diff --git a/src/meta/sled-store/Cargo.toml b/src/meta/sled-store/Cargo.toml index d8d601a43e33..8eb0a9c794d0 100644 --- a/src/meta/sled-store/Cargo.toml +++ b/src/meta/sled-store/Cargo.toml @@ -19,7 +19,6 @@ anyerror = { workspace = true } byteorder = { workspace = true } databend-common-meta-stoerr = { workspace = true } databend-common-meta-types = { workspace = true } -databend-common-tracing = { workspace = true } fastrace = { workspace = true } log = { workspace = true } openraft = { workspace = true } diff --git a/src/meta/sled-store/src/db.rs b/src/meta/sled-store/src/db.rs index 976581630ebf..36ca0e91c40c 100644 --- a/src/meta/sled-store/src/db.rs +++ b/src/meta/sled-store/src/db.rs @@ -31,17 +31,6 @@ pub(crate) struct GlobalSledDb { } impl GlobalSledDb { - pub(crate) fn new_temp(temp_dir: TempDir) -> Self { - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - - GlobalSledDb { - temp_dir: Some(temp_dir), - path: temp_path.clone(), - db: sled::open(temp_path.clone()) - .unwrap_or_else(|e| panic!("open global sled::Db(path: {}): {}", temp_path, e)), - } - } - pub(crate) fn new(path: String, cache_size: u64) -> Self { let db = sled::Config::default() .path(&path) @@ -61,28 +50,6 @@ impl GlobalSledDb { static GLOBAL_SLED: LazyLock>>> = LazyLock::new(|| Arc::new(Mutex::new(None))); -/// Open a db at a temp dir. For test purpose only. -pub fn init_temp_sled_db(temp_dir: TempDir) { - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - - let (inited_as_temp, curr_path) = { - let mut g = GLOBAL_SLED.as_ref().lock().unwrap(); - if let Some(gdb) = g.as_ref() { - (gdb.temp_dir.is_some(), gdb.path.clone()) - } else { - *g = Some(GlobalSledDb::new_temp(temp_dir)); - return; - } - }; - - if !inited_as_temp { - panic!( - "sled db is already initialized with specified path: {}, can not re-init with temp path {}", - curr_path, temp_path - ); - } -} - pub fn init_sled_db(path: String, cache_size: u64) { let (inited_as_temp, curr_path) = { let mut g = GLOBAL_SLED.as_ref().lock().unwrap(); diff --git a/src/meta/sled-store/src/lib.rs b/src/meta/sled-store/src/lib.rs index af488668f521..fe7bce06d972 100644 --- a/src/meta/sled-store/src/lib.rs +++ b/src/meta/sled-store/src/lib.rs @@ -22,10 +22,8 @@ pub use db::drop_sled_db; pub use db::get_sled_db; pub use db::init_get_sled_db; pub use db::init_sled_db; -pub use db::init_temp_sled_db; pub use openraft; pub use sled; -pub use sled_iter::iter; pub use sled_key_space::SledKeySpace; pub use sled_serde::SledOrderedSerde; pub use sled_serde::SledRangeSerde; @@ -40,7 +38,6 @@ pub use store::Store; mod bytes_error; mod db; -mod sled_iter; mod sled_key_space; mod sled_serde; mod sled_serde_impl; diff --git a/src/meta/sled-store/src/sled_iter.rs b/src/meta/sled-store/src/sled_iter.rs deleted file mode 100644 index 955109c4aa08..000000000000 --- a/src/meta/sled-store/src/sled_iter.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::marker::PhantomData; - -use sled::Db; -use sled::IVec; - -use crate::get_sled_db; - -pub struct TreeIter { - db: Db, - tree_names: Vec, - _p: PhantomData, -} - -impl Iterator for TreeIter { - type Item = Result<(String, ItemIter), sled::Error>; - - fn next(&mut self) -> Option { - if self.tree_names.is_empty() { - return None; - } - - let name = self.tree_names.remove(0); - let res = self.db.open_tree(&name); - let tree = match res { - Ok(x) => x, - Err(e) => return Some(Err(e)), - }; - - let it = tree.iter(); - Some(Ok((name, ItemIter:: { - it, - _p: PhantomData, - }))) - } -} - -pub struct ItemIter { - it: sled::Iter, - _p: PhantomData, -} - -impl Iterator for ItemIter { - type Item = Result<(IVec, IVec), sled::Error>; - - fn next(&mut self) -> Option { - let next = self.it.next(); - let res = match next { - Some(x) => x, - None => return None, - }; - - match res { - Ok((k, v)) => Some(Ok((k, v))), - Err(e) => Some(Err(e)), - } - } -} - -/// Iterator that output key and value in `Vec`. -impl Iterator for ItemIter> { - type Item = Result<(Vec, Vec), sled::Error>; - - fn next(&mut self) -> Option { - let next = self.it.next(); - let res = match next { - Some(x) => x, - None => return None, - }; - - match res { - Ok((k, v)) => Some(Ok((k.to_vec(), v.to_vec()))), - Err(e) => Some(Err(e)), - } - } -} - -/// Iterate every record in every tree. -/// -/// It returns an Iterator of record Iterators. -/// -/// Usage: -/// ```ignore -/// for name_tree_res in iter::() { -/// let (tree_name, item_iter) = name_tree_res?; -/// for item_res in item_iter { -/// let (k, v) = item_res?; -/// println!("{:?}: {:?}", k, v); -/// } -/// } -/// ``` -pub fn iter() -> TreeIter { - let db = get_sled_db(); - - let mut tree_names = db - .tree_names() - .iter() - .map(|x: &IVec| String::from_utf8(x.to_vec()).unwrap()) - .collect::>(); - tree_names.sort(); - - TreeIter { - db, - tree_names, - _p: PhantomData, - } -} diff --git a/src/meta/sled-store/tests/it/main.rs b/src/meta/sled-store/tests/it/main.rs deleted file mode 100644 index 9235723bf1be..000000000000 --- a/src/meta/sled-store/tests/it/main.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2021 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![allow(clippy::uninlined_format_args)] - -mod sled_iter; -mod sled_tree; -mod sled_txn_tree; -mod testing; diff --git a/src/meta/sled-store/tests/it/sled_iter.rs b/src/meta/sled-store/tests/it/sled_iter.rs deleted file mode 100644 index dd6804f5bffd..000000000000 --- a/src/meta/sled-store/tests/it/sled_iter.rs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2023 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_sled_store::SledItem; -use databend_common_meta_sled_store::SledTree; -use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::EntryPayload; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::LogEntry; -use databend_common_meta_types::UpsertKV; -use log::info; -use pretty_assertions::assert_eq; -use sled::IVec; -use test_harness::test; - -use crate::testing::fake_key_spaces::Logs; -use crate::testing::new_sled_test_context; -use crate::testing::sled_test_harness; - -/// Feed some data to two trees, iterate them and check output. -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_sled_iter() -> anyhow::Result<()> { - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - info!("--- init some data"); - let t1 = { - let tc = new_sled_test_context(); - - let tree = SledTree::open(&tc.db, tc.tree_name.clone(), true)?; - let log_tree = tree.key_space::(); - - log_tree.append(logs.clone()).await?; - tc.tree_name - }; - - let t2 = { - let tc = new_sled_test_context(); - - let tree = SledTree::open(&tc.db, tc.tree_name.clone(), true)?; - let log_tree = tree.key_space::(); - - log_tree.append(logs.clone()).await?; - tc.tree_name - }; - - // Iterator output IVec - - let mut trees = vec![t1.clone(), t2.clone()]; - - for tree_iter in databend_common_meta_sled_store::iter::() { - let (tree_name, item_iter) = tree_iter?; - - if tree_name == "__sled__default" { - continue; - } - - if !trees.contains(&tree_name) { - // When tests run concurrently, there are other trees created by other test case. - continue; - } - - assert_eq!(trees.remove(0), tree_name); - - let mut got = vec![]; - for item in item_iter { - let (k, v) = item?; - - let item = SledItem::::new(k, v); - let line = format!("{}, {:?}", item.key()?, item.value()?); - got.push(line); - } - - let want = vec![ - "2, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }, payload: blank }".to_string(), - "4, Entry { log_id: LogId { leader_id: LeaderId { term: 3, node_id: 0 }, index: 4 }, payload: normal }".to_string(), - ]; - - assert_eq!(want, got); - } - - // Iterator outputs Vec - - let trees = [t1, t2]; - - let mut got = vec![]; - for tree_iter in databend_common_meta_sled_store::iter::>() { - let (tree_name, item_iter) = tree_iter?; - - if !trees.contains(&tree_name) { - // When tests run concurrently, there are other trees created by other test case. - continue; - } - - for item in item_iter { - let (k, v) = item?; - - let line = format!("{:?}, {}", k, String::from_utf8(v).unwrap()); - got.push(line); - } - } - - let want = vec![ - r#"[1, 0, 0, 0, 0, 0, 0, 0, 2], {"log_id":{"leader_id":{"term":1,"node_id":0},"index":2},"payload":"Blank"}"#, - r#"[1, 0, 0, 0, 0, 0, 0, 0, 4], {"log_id":{"leader_id":{"term":3,"node_id":0},"index":4},"payload":{"Normal":{"txid":null,"cmd":{"UpsertKV":{"key":"foo","seq":{"Exact":0},"value":{"Update":[102,111,111]},"value_meta":null}}}}}"#, - r#"[1, 0, 0, 0, 0, 0, 0, 0, 2], {"log_id":{"leader_id":{"term":1,"node_id":0},"index":2},"payload":"Blank"}"#, - r#"[1, 0, 0, 0, 0, 0, 0, 0, 4], {"log_id":{"leader_id":{"term":3,"node_id":0},"index":4},"payload":{"Normal":{"txid":null,"cmd":{"UpsertKV":{"key":"foo","seq":{"Exact":0},"value":{"Update":[102,111,111]},"value_meta":null}}}}}"#, - ]; - - assert_eq!(want, got); - - Ok(()) -} diff --git a/src/meta/sled-store/tests/it/sled_tree.rs b/src/meta/sled-store/tests/it/sled_tree.rs deleted file mode 100644 index b2ed93a298b1..000000000000 --- a/src/meta/sled-store/tests/it/sled_tree.rs +++ /dev/null @@ -1,620 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_sled_store::SledTree; -use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::EntryPayload; -use databend_common_meta_types::raft_types::LogIndex; -use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::LogEntry; -use databend_common_meta_types::UpsertKV; -use test_harness::test; - -use crate::testing::fake_key_spaces::Files; -use crate::testing::fake_key_spaces::GenericKV; -use crate::testing::fake_key_spaces::Logs; -use crate::testing::fake_key_spaces::StateMachineMeta; -use crate::testing::fake_state_machine_meta::StateMachineMetaKey::Initialized; -use crate::testing::fake_state_machine_meta::StateMachineMetaKey::LastApplied; -use crate::testing::fake_state_machine_meta::StateMachineMetaValue; -use crate::testing::new_sled_test_context; -use crate::testing::sled_test_harness; - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_sled_tree_open() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - SledTree::open(db, tc.tree_name, true)?; - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_as_range() -> anyhow::Result<()> { - // This test assumes the following order. - // to check the range boundary. - // assert!(Logs::PREFIX < StateMachineMeta::PREFIX); - - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - let meta_tree = tree.key_space::(); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - log_tree.append(logs.clone()).await?; - - let metas = vec![ - ( - LastApplied, - StateMachineMetaValue::LogId(new_log_id(1, 0, 2)), - ), - (Initialized, StateMachineMetaValue::Bool(true)), - ]; - - meta_tree.append(metas.clone()).await?; - - // key space Logs - - let mut it = log_tree.range(..)?; - assert_eq!((2, logs[0].clone()), it.next().unwrap()?.kv()?); - assert_eq!((4, logs[1].clone()), it.next().unwrap()?.kv()?); - assert!(it.next().is_none()); - - // key space Logs reversed - - let mut it = log_tree.range(..)?.rev(); - assert_eq!((4, logs[1].clone()), it.next().unwrap()?.kv()?); - assert_eq!((2, logs[0].clone()), it.next().unwrap()?.kv()?); - assert!(it.next().is_none()); - - // key space StateMachineMeta - - let mut it = meta_tree.range(..)?; - assert_eq!(metas[0], it.next().unwrap()?.kv()?); - assert_eq!(metas[1], it.next().unwrap()?.kv()?); - assert!(it.next().is_none()); - - // key space StateMachineMeta reversed - - let mut it = meta_tree.range(..)?.rev(); - assert_eq!(metas[1], it.next().unwrap()?.kv()?); - assert_eq!(metas[0], it.next().unwrap()?.kv()?); - assert!(it.next().is_none()); - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_last() -> anyhow::Result<()> { - // This test assumes the following order. - // To ensure a last() does not returns item from another key space with smaller prefix - // assert!(Logs::PREFIX < StateMachineMeta::PREFIX); - - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - - assert_eq!(None, log_tree.last()?); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - log_tree.append(logs.clone()).await?; - assert_eq!(Some((4, logs[1].clone())), log_tree.last()?); - assert_eq!(None, tree.key_space::().last()?); - - let metas = vec![ - ( - LastApplied, - StateMachineMetaValue::LogId(new_log_id(1, 0, 2)), - ), - (Initialized, StateMachineMetaValue::Bool(true)), - ]; - - tree.key_space::() - .append(metas.clone()) - .await?; - - assert_eq!(Some((4, logs[1].clone())), log_tree.last()?); - assert_eq!( - Some((Initialized, StateMachineMetaValue::Bool(true))), - tree.key_space::().last()? - ); - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_append() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - - let logs: Vec<(LogIndex, Entry)> = vec![ - (8, Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }), - (5, Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }), - ]; - - log_tree.append(logs.clone()).await?; - - let want: Vec = vec![ - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - ]; - - let got = log_tree.range_values(0..)?; - assert_eq!(want, got); - - let got = log_tree.range_values(0..=5)?; - assert_eq!(want[0..1], got); - - let got = log_tree.range_values(6..9)?; - assert_eq!(want[1..], got); - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_append_and_range_get() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - Entry { - log_id: new_log_id(1, 0, 9), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 10), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 256), - payload: EntryPayload::Blank, - }, - ]; - - log_tree.append(logs.clone()).await?; - - let got = log_tree.range_values(0..)?; - assert_eq!(logs, got); - - let got = log_tree.range_values(0..=2)?; - assert_eq!(logs[0..1], got); - - let got = log_tree.range_values(0..3)?; - assert_eq!(logs[0..1], got); - - let got = log_tree.range_values(0..5)?; - assert_eq!(logs[0..2], got); - - let got = log_tree.range_values(0..10)?; - assert_eq!(logs[0..3], got); - - let got = log_tree.range_values(0..11)?; - assert_eq!(logs[0..4], got); - - let got = log_tree.range_values(9..11)?; - assert_eq!(logs[2..4], got); - - let got = log_tree.range_values(10..256)?; - assert_eq!(logs[3..4], got); - - let got = log_tree.range_values(10..257)?; - assert_eq!(logs[3..5], got); - - let got = log_tree.range_values(257..)?; - assert_eq!(logs[5..], got); - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_range_kvs() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 9), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 10), - payload: EntryPayload::Blank, - }, - ]; - - log_tree.append(logs.clone()).await?; - - let got = log_tree - .range(9..)? - .map(|x| x.unwrap().kv().unwrap()) - .collect::>(); - - assert_eq!(vec![(9, logs[1].clone()), (10, logs[2].clone()),], got); - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_scan_prefix() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let file_tree = tree.key_space::(); - let kv_tree = tree.key_space::(); - - let files: Vec<(String, String)> = vec![ - ("a".to_string(), "x".to_string()), - ("ab".to_string(), "xy".to_string()), - ("abc".to_string(), "xyz".to_string()), - ("abd".to_string(), "xyZ".to_string()), - ("b".to_string(), "y".to_string()), - ]; - - file_tree.append(files.clone()).await?; - - let kvs = vec![ - ("a".to_string(), SeqV::new(1, vec![])), - ("ab".to_string(), SeqV::new(2, vec![])), - ("b".to_string(), SeqV::new(3, vec![])), - ]; - - kv_tree.append(kvs.clone()).await?; - - let got = file_tree.scan_prefix(&"".to_string())?; - assert_eq!(files, got); - - let got = file_tree.scan_prefix(&"ab".to_string())?; - assert_eq!(files[1..4], got); - - let got = kv_tree.scan_prefix(&"".to_string())?; - assert_eq!(kvs, got); - - let got = kv_tree.scan_prefix(&"ab".to_string())?; - assert_eq!(kvs[1..2], got); - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_insert() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - - assert_eq!(None, log_tree.get(&5)?); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - log_tree.append(logs.clone()).await?; - - assert_eq!(logs, log_tree.range_values(..)?); - - // insert and override - - let override_2 = Entry { - log_id: new_log_id(10, 0, 2), - payload: EntryPayload::Blank, - }; - - let prev = log_tree - .insert(&override_2.log_id.index, &override_2) - .await?; - assert_eq!(Some(logs[0].clone()), prev); - - // insert and override nothing - - let override_nothing = Entry { - log_id: new_log_id(10, 0, 100), - payload: EntryPayload::Blank, - }; - - let prev = log_tree - .insert(&override_nothing.log_id.index, &override_nothing) - .await?; - assert_eq!(None, prev); - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_get() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - - assert_eq!(None, log_tree.get(&5)?); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - log_tree.append(logs.clone()).await?; - - assert_eq!(None, log_tree.get(&1)?); - assert_eq!(Some(logs[0].clone()), log_tree.get(&2)?); - assert_eq!(None, log_tree.get(&3)?); - assert_eq!(Some(logs[1].clone()), log_tree.get(&4)?); - assert_eq!(None, log_tree.get(&5)?); - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_range_remove() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - Entry { - log_id: new_log_id(1, 0, 9), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 10), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(1, 0, 256), - payload: EntryPayload::Blank, - }, - ]; - - log_tree.append(logs.clone()).await?; - log_tree.range_remove(0.., false).await?; - assert_eq!(logs[5..], log_tree.range_values(0..)?); - - log_tree.append(logs.clone()).await?; - log_tree.range_remove(1.., false).await?; - assert_eq!(logs[5..], log_tree.range_values(0..)?); - - log_tree.append(logs.clone()).await?; - log_tree.range_remove(3.., true).await?; - assert_eq!(logs[0..1], log_tree.range_values(0..)?); - - log_tree.append(logs.clone()).await?; - log_tree.range_remove(3..10, true).await?; - assert_eq!(logs[0..1], log_tree.range_values(0..5)?); - assert_eq!(logs[3..], log_tree.range_values(5..)?); - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_key_space_multi_types() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - let sm_meta = tree.key_space::(); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - log_tree.append(logs.clone()).await?; - - let metas = vec![ - ( - LastApplied, - StateMachineMetaValue::LogId(new_log_id(1, 0, 2)), - ), - (Initialized, StateMachineMetaValue::Bool(true)), - ]; - sm_meta.append(metas.clone()).await?; - - // range get/keys are limited to its own namespace. - { - let got = log_tree.range_values(..)?; - assert_eq!(logs, got); - - let got = sm_meta.range_values(..=LastApplied)?; - assert_eq!(vec![StateMachineMetaValue::LogId(new_log_id(1, 0, 2))], got); - - let got = sm_meta.range_values(Initialized..)?; - assert_eq!(vec![StateMachineMetaValue::Bool(true)], got); - - let got = sm_meta - .range(Initialized..)? - .map(|x| x.unwrap().key().unwrap()) - .collect::>(); - assert_eq!(vec![Initialized], got); - } - - // range remove are limited to its own namespace. - { - sm_meta.range_remove(.., false).await?; - - let got = log_tree.range_values(..)?; - assert_eq!(logs, got); - } - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_export() -> anyhow::Result<()> { - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - let log_tree = tree.key_space::(); - - let logs: Vec = vec![ - Entry { - log_id: new_log_id(1, 0, 2), - payload: EntryPayload::Blank, - }, - Entry { - log_id: new_log_id(3, 0, 4), - payload: EntryPayload::Normal(LogEntry { - txid: None, - time_ms: None, - cmd: Cmd::UpsertKV(UpsertKV::insert("foo", b"foo")), - }), - }, - ]; - - log_tree.append(logs.clone()).await?; - - let data = tree.export()?; - - for kv in data.iter() { - println!("{:?}", kv); - } - - Ok(()) -} diff --git a/src/meta/sled-store/tests/it/sled_txn_tree.rs b/src/meta/sled-store/tests/it/sled_txn_tree.rs deleted file mode 100644 index 4dfba71f911b..000000000000 --- a/src/meta/sled-store/tests/it/sled_txn_tree.rs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_sled_store::SledTree; -use databend_common_meta_sled_store::Store; -use databend_common_meta_types::Endpoint; -use databend_common_meta_types::Node; -use test_harness::test; - -use crate::testing::fake_key_spaces::Nodes; -use crate::testing::new_sled_test_context; -use crate::testing::sled_test_harness; - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_sled_txn_tree_key_space_insert_get_remove() -> anyhow::Result<()> { - // Test transactional API insert, get, remove on a sub key space of TransactionSledTree - - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - - let k = 100; - - tree.txn(false, |txn_tree| { - // sub tree key space - let nodes_ks = txn_tree.key_space::(); - - let got = nodes_ks.insert(&101, &Node::new("foo", Endpoint::new("", 100)))?; - - assert!(got.is_none()); - - let got = nodes_ks.insert(&k, &Node::new("n", Endpoint::new("", 100)))?; - - assert!(got.is_none()); - - let got = nodes_ks.get(&k)?; - - assert_eq!(Some(Node::new("n", Endpoint::new("", 100))), got); - - let got = nodes_ks.insert(&k, &Node::new("m", Endpoint::new("", 101)))?; - - assert_eq!(Some(Node::new("n", Endpoint::new("", 100))), got); - - Ok(()) - })?; - - let got = tree.key_space::().get(&k)?; - assert_eq!(Some(Node::new("m", Endpoint::new("", 101))), got); - - let got = tree.key_space::().get(&101)?; - assert_eq!(Some(Node::new("foo", Endpoint::new("", 100))), got); - - Ok(()) -} - -#[test(harness = sled_test_harness)] -#[fastrace::trace] -async fn test_sled_txn_tree_key_space_remove() -> anyhow::Result<()> { - // Test transactional API insert, get, remove on a sub key space of TransactionSledTree - - let tc = new_sled_test_context(); - let db = &tc.db; - let tree = SledTree::open(db, tc.tree_name, true)?; - - let k = 100; - - tree.txn(false, |txn_tree| { - // sub tree key space - let nodes_ks = txn_tree.key_space::(); - - let _got = nodes_ks.insert(&k, &Node::new("n", Endpoint::new("", 100)))?; - - let got = nodes_ks.get(&k)?; - assert_eq!(Some(Node::new("n", Endpoint::new("", 100))), got); - - let got = nodes_ks.remove(&k)?; - assert_eq!(Some(Node::new("n", Endpoint::new("", 100))), got); - - let got = nodes_ks.get(&k)?; - assert!(got.is_none()); - - Ok(()) - })?; - - let got = tree.key_space::().get(&k)?; - assert!(got.is_none()); - - Ok(()) -} diff --git a/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs b/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs deleted file mode 100644 index f9c80e47c2e6..000000000000 --- a/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_sled_store::SledKeySpace; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::LogIndex; -use databend_common_meta_types::raft_types::NodeId; -use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::Node; - -use crate::testing::fake_state_machine_meta::StateMachineMetaKey; -use crate::testing::fake_state_machine_meta::StateMachineMetaValue; - -/// Types for raft log in SledTree -pub struct Logs {} - -impl SledKeySpace for Logs { - const PREFIX: u8 = 1; - const NAME: &'static str = "log"; - type K = LogIndex; - type V = Entry; -} - -/// Types for Node in SledTree -pub struct Nodes {} -impl SledKeySpace for Nodes { - const PREFIX: u8 = 2; - const NAME: &'static str = "node"; - type K = NodeId; - type V = Node; -} - -/// Key-Value Types for storing meta data of a raft state machine in sled::Tree, e.g. the last applied log id. -pub struct StateMachineMeta {} -impl SledKeySpace for StateMachineMeta { - const PREFIX: u8 = 3; - const NAME: &'static str = "sm-meta"; - type K = StateMachineMetaKey; - type V = StateMachineMetaValue; -} - -/// Key-Value Types for storing DFS files in sled::Tree: -pub struct Files {} -impl SledKeySpace for Files { - const PREFIX: u8 = 5; - const NAME: &'static str = "files"; - type K = String; - type V = String; -} - -/// Key-Value Types for storing general purpose kv in sled::Tree: -pub struct GenericKV {} -impl SledKeySpace for GenericKV { - const PREFIX: u8 = 6; - const NAME: &'static str = "generic-kv"; - type K = String; - type V = SeqV>; -} diff --git a/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs b/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs deleted file mode 100644 index 0d3f2e0f03c1..000000000000 --- a/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt; - -use databend_common_meta_sled_store::SledBytesError; -use databend_common_meta_sled_store::SledOrderedSerde; -use databend_common_meta_sled_store::SledSerde; -use databend_common_meta_types::anyerror::AnyError; -use databend_common_meta_types::raft_types::LogId; -use databend_common_meta_types::raft_types::Membership; -use serde::Deserialize; -use serde::Serialize; -use sled::IVec; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum StateMachineMetaKey { - /// The last applied log id in the state machine. - LastApplied, - - /// Whether the state machine is initialized. - Initialized, - - /// The last membership config - LastMembership, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum StateMachineMetaValue { - LogId(LogId), - Bool(bool), - Membership(Membership), -} - -impl fmt::Display for StateMachineMetaKey { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - StateMachineMetaKey::LastApplied => { - write!(f, "last-applied") - } - StateMachineMetaKey::Initialized => { - write!(f, "initialized") - } - StateMachineMetaKey::LastMembership => { - write!(f, "last-membership") - } - } - } -} - -impl SledOrderedSerde for StateMachineMetaKey { - fn ser(&self) -> Result { - let i = match self { - StateMachineMetaKey::LastApplied => 1, - StateMachineMetaKey::Initialized => 2, - StateMachineMetaKey::LastMembership => 3, - }; - - Ok(IVec::from(&[i])) - } - - fn de>(v: V) -> Result - where Self: Sized { - let slice = v.as_ref(); - if slice[0] == 1 { - return Ok(StateMachineMetaKey::LastApplied); - } else if slice[0] == 2 { - return Ok(StateMachineMetaKey::Initialized); - } else if slice[0] == 3 { - return Ok(StateMachineMetaKey::LastMembership); - } - - Err(SledBytesError::new(&AnyError::error("invalid key IVec"))) - } -} - -impl From for LogId { - fn from(v: StateMachineMetaValue) -> Self { - match v { - StateMachineMetaValue::LogId(x) => x, - _ => panic!("expect LogId"), - } - } -} - -impl From for bool { - fn from(v: StateMachineMetaValue) -> Self { - match v { - StateMachineMetaValue::Bool(x) => x, - _ => panic!("expect LogId"), - } - } -} -impl From for Membership { - fn from(v: StateMachineMetaValue) -> Self { - match v { - StateMachineMetaValue::Membership(x) => x, - _ => panic!("expect Membership"), - } - } -} - -impl SledSerde for StateMachineMetaValue { - fn de>(v: T) -> Result - where Self: Sized { - let s = serde_json::from_slice(v.as_ref())?; - Ok(s) - } -} diff --git a/src/meta/sled-store/tests/it/testing/mod.rs b/src/meta/sled-store/tests/it/testing/mod.rs deleted file mode 100644 index 62d3f84932a1..000000000000 --- a/src/meta/sled-store/tests/it/testing/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod fake_key_spaces; -pub mod fake_state_machine_meta; - -use std::collections::BTreeMap; -use std::sync::Once; - -use databend_common_base::base::GlobalSequence; -use databend_common_meta_sled_store::get_sled_db; -use databend_common_tracing::closure_name; -use databend_common_tracing::init_logging; -use databend_common_tracing::Config; -use fastrace::prelude::*; - -pub struct SledTestContext { - pub tree_name: String, - pub db: sled::Db, -} - -/// Create a new context for testing sled -pub fn new_sled_test_context() -> SledTestContext { - SledTestContext { - tree_name: format!("test-{}-", next_seq()), - db: get_sled_db(), - } -} - -fn next_seq() -> u32 { - 29000u32 + (GlobalSequence::next() as u32) -} - -pub fn sled_test_harness(test: F) -where - F: FnOnce() -> Fut + 'static, - Fut: std::future::Future> + Send + 'static, -{ - setup_test(); - - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .enable_all() - .build() - .unwrap(); - let root = Span::root(closure_name::(), SpanContext::random()); - let test = test().in_span(root); - rt.block_on(test).unwrap(); - - shutdown_test(); -} - -fn setup_test() { - static INIT: Once = Once::new(); - INIT.call_once(|| { - let t = tempfile::tempdir().expect("create temp dir to sled db"); - databend_common_meta_sled_store::init_temp_sled_db(t); - - let guards = init_logging("meta_unittests", &Config::new_testing(), BTreeMap::new()); - Box::leak(Box::new(guards)); - }); -} - -fn shutdown_test() { - fastrace::flush(); -} diff --git a/src/meta/store/Cargo.toml b/src/meta/store/Cargo.toml index d117182cc146..29d667f095b4 100644 --- a/src/meta/store/Cargo.toml +++ b/src/meta/store/Cargo.toml @@ -12,7 +12,7 @@ doctest = false test = true [features] -io-uring = ["databend-common-meta-embedded/io-uring"] +io-uring = [] [dependencies] async-trait = { workspace = true } diff --git a/src/meta/store/src/lib.rs b/src/meta/store/src/lib.rs index e6aad2a70acc..2daa684e9cb0 100644 --- a/src/meta/store/src/lib.rs +++ b/src/meta/store/src/lib.rs @@ -20,7 +20,7 @@ use std::task::Poll; use databend_common_grpc::RpcClientConf; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; -use databend_common_meta_embedded::MetaEmbedded; +use databend_common_meta_embedded::MemMeta; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::UpsertKVReply; @@ -44,7 +44,7 @@ pub struct MetaStoreProvider { /// MetaStore is impl with either a local embedded meta store, or a grpc-client of metasrv #[derive(Clone)] pub enum MetaStore { - L(Arc), + L(Arc), R(Arc), } @@ -127,8 +127,8 @@ impl MetaStoreProvider { ); // NOTE: This can only be used for test: data will be removed when program quit. - let meta_store = MetaEmbedded::get_meta().await?; - Ok(MetaStore::L(meta_store)) + let meta_store = MemMeta::default(); + Ok(MetaStore::L(Arc::new(meta_store))) } else { info!(conf :? =(&self.rpc_conf); "use remote meta"); let client = MetaGrpcClient::try_new(&self.rpc_conf)?; diff --git a/src/meta/types/src/errors/meta_errors.rs b/src/meta/types/src/errors/meta_errors.rs index ad4d47c400f9..696c2ad83c07 100644 --- a/src/meta/types/src/errors/meta_errors.rs +++ b/src/meta/types/src/errors/meta_errors.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::io; + use databend_common_exception::ErrorCode; use databend_common_meta_stoerr::MetaStorageError; use thiserror::Error; @@ -51,6 +53,13 @@ impl MetaError { } } +impl From for MetaError { + fn from(e: io::Error) -> Self { + let net_err = MetaStorageError::from(e); + MetaError::StorageError(net_err) + } +} + impl From for MetaError { fn from(status: tonic::Status) -> Self { let net_err = MetaNetworkError::from(status); diff --git a/src/query/management/tests/it/cluster.rs b/src/query/management/tests/it/cluster.rs index 8166bc6696bb..567627849ba9 100644 --- a/src/query/management/tests/it/cluster.rs +++ b/src/query/management/tests/it/cluster.rs @@ -18,7 +18,7 @@ use std::time::Duration; use databend_common_base::base::tokio; use databend_common_exception::Result; use databend_common_management::*; -use databend_common_meta_embedded::MetaEmbedded; +use databend_common_meta_embedded::MemMeta; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_store::MetaStore; use databend_common_meta_types::seq_value::SeqV; @@ -154,7 +154,7 @@ fn create_test_node_info() -> NodeInfo { } async fn new_cluster_api() -> Result<(MetaStore, ClusterMgr)> { - let test_api = MetaStore::L(Arc::new(MetaEmbedded::new_temp().await?)); + let test_api = MetaStore::L(Arc::new(MemMeta::default())); let cluster_manager = ClusterMgr::create( test_api.clone(), "test-tenant-id", diff --git a/src/query/management/tests/it/quota.rs b/src/query/management/tests/it/quota.rs index 5e72b53ae4b4..c61b1bf6fd32 100644 --- a/src/query/management/tests/it/quota.rs +++ b/src/query/management/tests/it/quota.rs @@ -20,7 +20,7 @@ use databend_common_management::*; use databend_common_meta_api::deserialize_struct; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant::TenantQuota; -use databend_common_meta_embedded::MetaEmbedded; +use databend_common_meta_embedded::MemMeta; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_types::MatchSeq; @@ -84,8 +84,8 @@ async fn test_update_quota_from_json_to_pb() -> Result<()> { Ok(()) } -async fn new_quota_api() -> Result<(Arc, QuotaMgr, QuotaMgr)> { - let test_api = Arc::new(MetaEmbedded::new_temp().await?); +async fn new_quota_api() -> Result<(Arc, QuotaMgr, QuotaMgr)> { + let test_api = Arc::new(MemMeta::default()); let mgr_json = QuotaMgr::::create(test_api.clone(), &Tenant::new_literal("admin")); let mgr_pb = QuotaMgr::::create(test_api.clone(), &Tenant::new_literal("admin")); Ok((test_api, mgr_json, mgr_pb)) diff --git a/src/query/management/tests/it/role.rs b/src/query/management/tests/it/role.rs index 59ec69abcfe6..e662269c9f7c 100644 --- a/src/query/management/tests/it/role.rs +++ b/src/query/management/tests/it/role.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use databend_common_base::base::tokio; use databend_common_management::*; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_embedded::MetaEmbedded; +use databend_common_meta_embedded::MemMeta; use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MatchSeq; use mockall::predicate::*; @@ -89,8 +89,8 @@ mod add { async fn new_role_api( enable_meta_data_upgrade_json_to_pb_from_v307: bool, -) -> databend_common_exception::Result<(Arc, RoleMgr)> { - let test_api = Arc::new(MetaEmbedded::new_temp().await?); +) -> databend_common_exception::Result<(Arc, RoleMgr)> { + let test_api = Arc::new(MemMeta::default()); let tenant = Tenant::new_literal("admin"); let mgr = RoleMgr::create( test_api.clone(), diff --git a/src/query/management/tests/it/setting.rs b/src/query/management/tests/it/setting.rs index 2d42269c87f8..8c08a2c9870d 100644 --- a/src/query/management/tests/it/setting.rs +++ b/src/query/management/tests/it/setting.rs @@ -20,7 +20,7 @@ use databend_common_management::*; use databend_common_meta_app::principal::UserSetting; use databend_common_meta_app::principal::UserSettingValue; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_embedded::MetaEmbedded; +use databend_common_meta_embedded::MemMeta; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MatchSeq; @@ -114,8 +114,8 @@ async fn test_set_setting() -> Result<()> { Ok(()) } -async fn new_setting_api() -> Result<(Arc, SettingMgr)> { - let test_api = Arc::new(MetaEmbedded::new_temp().await?); +async fn new_setting_api() -> Result<(Arc, SettingMgr)> { + let test_api = Arc::new(MemMeta::default()); let mgr = SettingMgr::create( test_api.clone(), &Tenant::new_or_err("databend_query", func_name!()).unwrap(), diff --git a/src/query/management/tests/it/stage.rs b/src/query/management/tests/it/stage.rs index 1f39f98f616b..e104b96e7b28 100644 --- a/src/query/management/tests/it/stage.rs +++ b/src/query/management/tests/it/stage.rs @@ -25,7 +25,7 @@ use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::storage::StorageParams; use databend_common_meta_app::storage::StorageS3Config; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_embedded::MetaEmbedded; +use databend_common_meta_embedded::MemMeta; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_types::seq_value::SeqV; use fastrace::func_name; @@ -138,8 +138,8 @@ fn create_test_stage_info() -> StageInfo { } } -async fn new_stage_api() -> Result<(Arc, StageMgr)> { - let test_api = Arc::new(MetaEmbedded::new_temp().await?); +async fn new_stage_api() -> Result<(Arc, StageMgr)> { + let test_api = Arc::new(MemMeta::default()); let mgr = StageMgr::create( test_api.clone(), &Tenant::new_or_err("admin", func_name!()).unwrap(), diff --git a/src/query/management/tests/it/udf.rs b/src/query/management/tests/it/udf.rs index 253204f80c3c..adb09f25f229 100644 --- a/src/query/management/tests/it/udf.rs +++ b/src/query/management/tests/it/udf.rs @@ -24,7 +24,7 @@ use databend_common_management::*; use databend_common_meta_app::principal::UserDefinedFunction; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_embedded::MetaEmbedded; +use databend_common_meta_embedded::MemMeta; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MatchSeq; @@ -232,8 +232,8 @@ fn create_test_udf_script() -> UserDefinedFunction { ) } -async fn new_udf_api() -> Result<(Arc, UdfMgr)> { - let test_api = Arc::new(MetaEmbedded::new_temp().await?); +async fn new_udf_api() -> Result<(Arc, UdfMgr)> { + let test_api = Arc::new(MemMeta::default()); let mgr = UdfMgr::create(test_api.clone(), &Tenant::new_literal("admin")); Ok((test_api, mgr)) } diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index 5825ca3ebd7d..b712b9ccbf4d 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -20,10 +20,7 @@ jemalloc = ["databend-common-storages-system/jemalloc"] memory-profiling = ["databend-common-base/memory-profiling", "databend-common-http/memory-profiling"] storage-hdfs = ["opendal/services-hdfs", "databend-common-storage/storage-hdfs"] io-uring = [ - # "databend-common-meta-embedded/io-uring", "databend-common-meta-store/io-uring", - # "databend-common-meta-sled-store/io-uring", - # "databend-common-meta-raft-store/io-uring", ] enable_queries_executor = [] @@ -73,7 +70,6 @@ databend-common-license = { workspace = true } databend-common-management = { workspace = true } databend-common-meta-api = { workspace = true } databend-common-meta-app = { workspace = true } -databend-common-meta-embedded = { workspace = true } databend-common-meta-kvapi = { workspace = true } databend-common-meta-store = { workspace = true } databend-common-meta-types = { workspace = true } diff --git a/src/query/service/src/local/mod.rs b/src/query/service/src/local/mod.rs index 728e93c05b59..33f5a9452f03 100644 --- a/src/query/service/src/local/mod.rs +++ b/src/query/service/src/local/mod.rs @@ -29,7 +29,6 @@ use databend_common_license::license_manager::LicenseManager; use databend_common_license::license_manager::OssLicenseManager; use databend_common_meta_app::storage::StorageFsConfig; use databend_common_meta_app::storage::StorageParams; -use databend_common_meta_embedded::MetaEmbedded; use crate::clusters::ClusterDiscovery; use crate::GlobalServices; @@ -49,11 +48,6 @@ pub async fn query_local(query_sql: &str, output_format: &str) -> Result<()> { root: path.join("_data").to_str().unwrap().to_owned(), }); - let meta_dir = path.join("_meta"); - MetaEmbedded::init_global_meta_store(meta_dir.to_string_lossy().to_string()) - .await - .unwrap(); - GlobalServices::init(&conf).await?; // init oss license manager OssLicenseManager::init(conf.query.tenant_id.tenant_name().to_string()).unwrap(); From 3dbc785a0f84724a662bd1e37a5a0fc73392b59d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Sat, 16 Nov 2024 12:30:00 +0800 Subject: [PATCH 51/92] refactor: remove unused sled related codes (#16857) * refactor: remove unused sled related codes * M src/meta/raft-store/tests/it/types.rs * M src/meta/sled-store/Cargo.toml * M src/meta/raft-store/src/ondisk/upgrade_to_v003.rs * M src/meta/service/src/store/mod.rs --- .../raft-store/src/ondisk/upgrade_to_v003.rs | 2 +- src/meta/raft-store/src/state_machine/mod.rs | 4 - src/meta/raft-store/src/state_machine/sm.rs | 1064 ----------------- .../src/state_machine/sm_kv_api_impl.rs | 106 -- .../raft-store/src/state_machine/testing.rs | 26 - src/meta/raft-store/tests/it/types.rs | 16 - src/meta/service/src/store/mod.rs | 2 - src/meta/service/src/store/store_inner.rs | 4 +- .../service/src/store/to_storage_error.rs | 36 - src/meta/sled-store/Cargo.toml | 1 - src/meta/sled-store/src/bytes_error.rs | 2 +- src/meta/sled-store/src/lib.rs | 6 - src/meta/sled-store/src/sled_serde.rs | 60 - src/meta/sled-store/src/sled_tree.rs | 412 ------- src/meta/sled-store/src/store.rs | 26 - src/meta/stoerr/src/meta_storage_errors.rs | 40 +- 16 files changed, 13 insertions(+), 1794 deletions(-) delete mode 100644 src/meta/raft-store/src/state_machine/sm_kv_api_impl.rs delete mode 100644 src/meta/service/src/store/to_storage_error.rs delete mode 100644 src/meta/sled-store/src/store.rs diff --git a/src/meta/raft-store/src/ondisk/upgrade_to_v003.rs b/src/meta/raft-store/src/ondisk/upgrade_to_v003.rs index 028aa4081f73..9a6f1c87f585 100644 --- a/src/meta/raft-store/src/ondisk/upgrade_to_v003.rs +++ b/src/meta/raft-store/src/ondisk/upgrade_to_v003.rs @@ -76,7 +76,7 @@ impl OnDisk { let last_snapshot = loader.load_last_snapshot().await.map_err(|e| { let ae = AnyError::new(&e).add_context(|| "load last snapshot"); - MetaStorageError::Damaged(ae) + MetaStorageError(ae) })?; if last_snapshot.is_some() { diff --git a/src/meta/raft-store/src/state_machine/mod.rs b/src/meta/raft-store/src/state_machine/mod.rs index 6c500c5de091..e7a8b535ba2c 100644 --- a/src/meta/raft-store/src/state_machine/mod.rs +++ b/src/meta/raft-store/src/state_machine/mod.rs @@ -17,9 +17,6 @@ pub use expire::ExpireKey; pub use expire::ExpireValue; pub use log_meta::LogMetaKey; pub use log_meta::LogMetaValue; -pub use sm::SerializableSnapshot; -pub use sm::SnapshotKeyValue; -pub use sm::StateMachine; pub use sm::StateMachineSubscriber; pub use snapshot_id::MetaSnapshotId; pub use state_machine_meta::StateMachineMetaKey; @@ -29,7 +26,6 @@ pub mod client_last_resp; mod expire; pub mod log_meta; pub mod sm; -mod sm_kv_api_impl; mod snapshot_id; pub mod state_machine_meta; diff --git a/src/meta/raft-store/src/state_machine/sm.rs b/src/meta/raft-store/src/state_machine/sm.rs index 33d69f5b09e1..828cf9f00a3e 100644 --- a/src/meta/raft-store/src/state_machine/sm.rs +++ b/src/meta/raft-store/src/state_machine/sm.rs @@ -12,1075 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; -use std::convert::TryInto; use std::fmt::Debug; -use std::time::Duration; -use std::time::Instant; -use databend_common_base::display::display_unix_epoch::DisplayUnixTimeStampExt; -use databend_common_meta_sled_store::init_get_sled_db; -use databend_common_meta_sled_store::openraft::MessageSummary; -use databend_common_meta_sled_store::AsKeySpace; -use databend_common_meta_sled_store::SledKeySpace; -use databend_common_meta_sled_store::SledTree; -use databend_common_meta_sled_store::Store; -use databend_common_meta_sled_store::TransactionSledTree; -use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::protobuf as pb; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::EntryPayload; -use databend_common_meta_types::raft_types::LogId; -use databend_common_meta_types::raft_types::NodeId; -use databend_common_meta_types::raft_types::StoredMembership; -use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::seq_value::SeqValue; -use databend_common_meta_types::txn_condition; -use databend_common_meta_types::txn_op; -use databend_common_meta_types::txn_op_response; -use databend_common_meta_types::AppliedState; use databend_common_meta_types::Change; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::CmdContext; -use databend_common_meta_types::ConditionResult; -use databend_common_meta_types::Interval; -use databend_common_meta_types::MatchSeq; -use databend_common_meta_types::MatchSeqExt; -use databend_common_meta_types::MetaSpec; -use databend_common_meta_types::Node; -use databend_common_meta_types::Operation; -use databend_common_meta_types::TxnCondition; -use databend_common_meta_types::TxnDeleteByPrefixRequest; -use databend_common_meta_types::TxnDeleteByPrefixResponse; -use databend_common_meta_types::TxnDeleteRequest; -use databend_common_meta_types::TxnDeleteResponse; -use databend_common_meta_types::TxnGetRequest; -use databend_common_meta_types::TxnGetResponse; -use databend_common_meta_types::TxnOp; -use databend_common_meta_types::TxnOpResponse; -use databend_common_meta_types::TxnPutRequest; -use databend_common_meta_types::TxnPutResponse; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; -use databend_common_meta_types::UpsertKV; -use databend_common_meta_types::With; -use log::debug; -use log::error; -use log::info; -use num::FromPrimitive; -use serde::Deserialize; -use serde::Serialize; - -use crate::config::RaftConfig; -use crate::key_spaces::ClientLastResps; -use crate::key_spaces::Expire; -use crate::key_spaces::GenericKV; -use crate::key_spaces::Nodes; -use crate::key_spaces::Sequences; -use crate::key_spaces::StateMachineMeta; -use crate::state_machine::ClientLastRespValue; -use crate::state_machine::ExpireKey; -use crate::state_machine::ExpireValue; -use crate::state_machine::StateMachineMetaKey; -use crate::state_machine::StateMachineMetaKey::Initialized; -use crate::state_machine::StateMachineMetaKey::LastApplied; -use crate::state_machine::StateMachineMetaKey::LastMembership; -use crate::state_machine::StateMachineMetaValue; - -/// sled db tree name for nodes -// const TREE_NODES: &str = "nodes"; -// const TREE_META: &str = "meta"; -const TREE_STATE_MACHINE: &str = "state_machine"; /// StateMachine subscriber trait pub trait StateMachineSubscriber: Debug + Sync + Send { fn kv_changed(&self, change: Change, String>); } - -/// The state machine of the `MemStore`. -/// It includes user data and two raft-related information: -/// `last_applied_logs` and `client_serial_responses` to achieve idempotence. -#[derive(Debug)] -pub struct StateMachine { - /// The internal sled::Tree to store everything about a state machine: - /// - Store initialization state and last applied in keyspace `StateMachineMeta`. - /// - Every other state is store in its own keyspace such as `Nodes`. - pub sm_tree: SledTree, - - /// subscriber of state machine data - pub subscriber: Option>, -} - -/// A key-value pair in a snapshot is a vec of two `Vec`. -pub type SnapshotKeyValue = Vec>; -pub(crate) type DeleteByPrefixKeyMap = BTreeMap>; - -/// Snapshot data for serialization and for transport. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] -pub struct SerializableSnapshot { - /// A list of kv pairs. - pub kvs: Vec, -} - -impl SerializableSnapshot { - /// Convert the snapshot to a `Vec<(type, name, iter)>` format for sled to import. - pub fn sled_importable(self) -> Vec<(Vec, Vec, impl Iterator>>)> { - vec![( - "tree".as_bytes().to_vec(), - TREE_STATE_MACHINE.as_bytes().to_vec(), - self.kvs.into_iter(), - )] - } -} - -impl StateMachine { - pub fn tree_name(config: &RaftConfig, sm_id: u64) -> String { - config.tree_name(format!("{}/{}", TREE_STATE_MACHINE, sm_id)) - } - - #[fastrace::trace] - pub async fn open(config: &RaftConfig, sm_id: u64) -> Result { - let db = init_get_sled_db(config.raft_dir.clone(), config.sled_cache_size()); - - let tree_name = StateMachine::tree_name(config, sm_id); - debug!("opening tree: {}", &tree_name); - - let sm_tree = SledTree::open(&db, &tree_name, config.is_sync())?; - - let sm = StateMachine { - sm_tree, - subscriber: None, - }; - - let inited = { - let sm_meta = sm.sm_meta(); - sm_meta.get(&Initialized)? - }; - - if inited.is_some() { - Ok(sm) - } else { - let sm_meta = sm.sm_meta(); - sm_meta - .insert(&Initialized, &StateMachineMetaValue::Bool(true)) - .await?; - Ok(sm) - } - } - - pub fn set_subscriber(&mut self, subscriber: Box) { - self.subscriber = Some(subscriber); - } - - fn scan_prefix_if_needed( - &self, - entry: &Entry, - ) -> Result, MetaStorageError> { - match entry.payload { - EntryPayload::Normal(ref data) => match &data.cmd { - Cmd::Transaction(txn) => { - let kvs = self.kvs(); - let mut if_map = BTreeMap::new(); - let mut else_map = BTreeMap::new(); - for op in txn.if_then.iter() { - if let Some(txn_op::Request::DeleteByPrefix(delete_by_prefix)) = &op.request - { - if_map.insert( - delete_by_prefix.clone(), - kvs.scan_prefix(&delete_by_prefix.prefix)?, - ); - } - } - for op in txn.else_then.iter() { - if let Some(txn_op::Request::DeleteByPrefix(delete_by_prefix)) = &op.request - { - else_map.insert( - delete_by_prefix.clone(), - kvs.scan_prefix(&delete_by_prefix.prefix)?, - ); - } - } - Ok(Some((if_map, else_map))) - } - _ => Ok(None), - }, - _ => Ok(None), - } - } - - /// Apply an log entry to state machine. - /// - /// If a duplicated log entry is detected by checking data.txid, no update - /// will be made and the previous resp is returned. In this way a client is able to re-send a - /// command safely in case of network failure etc. - #[fastrace::trace] - pub async fn apply(&self, entry: &Entry) -> Result { - info!("apply: summary: {}", entry.summary(),); - debug!(log_id :% =(&entry.log_id); "sled tx start: {:?}", entry); - - let log_id = &entry.log_id; - let log_time_ms = Self::get_log_time(entry); - - let expired = self.list_expired_kvs(log_time_ms)?; - debug!("expired keys: {:?}", expired); - - let kv_pairs = self.scan_prefix_if_needed(entry)?; - - let result = self.sm_tree.txn(true, move |mut txn_tree| { - self.clean_expired_kvs(&mut txn_tree, &expired)?; - - let txn_sm_meta = txn_tree.key_space::(); - txn_sm_meta.insert(&LastApplied, &StateMachineMetaValue::LogId(*log_id))?; - - match entry.payload { - EntryPayload::Blank => { - info!("apply: blank"); - } - EntryPayload::Normal(ref data) => { - info!("apply: {}", data); - if let Some(ref txid) = data.txid { - let (serial, resp) = - self.txn_get_client_last_resp(&txid.client, &txn_tree)?; - if serial == txid.serial { - return Ok((Some(resp), txn_tree.changes)); - } - } - - let res = - self.apply_cmd(&data.cmd, &mut txn_tree, kv_pairs.as_ref(), log_time_ms); - if let Ok(ok) = &res { - info!("apply_result: summary: {}; res ok: {}", entry.summary(), ok); - } - if let Err(err) = &res { - info!( - "apply_result: summary: {}; res err: {:?}", - entry.summary(), - err - ); - } - - let applied_state = res?; - - if let Some(ref txid) = data.txid { - self.txn_client_last_resp_update( - &txid.client, - (txid.serial, applied_state.clone()), - &txn_tree, - )?; - } - return Ok((Some(applied_state), txn_tree.changes)); - } - EntryPayload::Membership(ref mem) => { - info!("apply: membership: {:?}", mem); - txn_sm_meta.insert( - &LastMembership, - &StateMachineMetaValue::Membership(StoredMembership::new( - Some(*log_id), - mem.clone(), - )), - )?; - return Ok((Some(AppliedState::None), txn_tree.changes)); - } - }; - - Ok((None, txn_tree.changes)) - }); - - let (opt_applied_state, changes) = result?; - - debug!("sled tx done: {:?}", entry); - - let applied_state = opt_applied_state.unwrap_or(AppliedState::None); - - // Send queued change events to subscriber - if let Some(subscriber) = &self.subscriber { - for event in changes { - subscriber.kv_changed(event); - } - } - - Ok(applied_state) - } - - /// Retrieve the proposing time from a raft-log. - /// - /// Only `Normal` log has a time embedded. - #[fastrace::trace] - fn get_log_time(entry: &Entry) -> u64 { - match &entry.payload { - EntryPayload::Normal(data) => match data.time_ms { - None => { - error!( - "log has no time: {}, treat every record with non-none `expire` as timed out", - entry.summary() - ); - 0 - } - Some(x) => { - info!( - "apply: raft-log time: {}", - Duration::from_millis(x).display_unix_timestamp() - ); - x - } - }, - _ => 0, - } - } - - #[fastrace::trace] - fn apply_add_node_cmd( - &self, - node_id: &u64, - node: &Node, - overriding: bool, - txn_tree: &TransactionSledTree, - ) -> Result { - let sm_nodes = txn_tree.key_space::(); - - let prev = sm_nodes.get(node_id)?; - - if prev.is_none() { - sm_nodes.insert(node_id, node)?; - info!("applied AddNode(non-overriding): {}={:?}", node_id, node); - return Ok((prev, Some(node.clone())).into()); - } - - if overriding { - sm_nodes.insert(node_id, node)?; - info!("applied AddNode(overriding): {}={:?}", node_id, node); - Ok((prev, Some(node.clone())).into()) - } else { - Ok((prev.clone(), prev).into()) - } - } - - #[fastrace::trace] - fn apply_remove_node_cmd( - &self, - node_id: &u64, - txn_tree: &TransactionSledTree, - ) -> Result { - let sm_nodes = txn_tree.key_space::(); - - let prev = sm_nodes.get(node_id)?; - - if prev.is_some() { - info!("applied RemoveNode: {}={:?}", node_id, prev); - sm_nodes.remove(node_id)?; - } - Ok((prev, None).into()) - } - - #[fastrace::trace] - fn apply_update_kv_cmd( - &self, - upsert_kv: &UpsertKV, - txn_tree: &mut TransactionSledTree, - log_time_ms: u64, - ) -> Result { - debug!(upsert_kv :? =(upsert_kv); "apply_update_kv_cmd"); - - let (expired, prev, result) = Self::txn_upsert_kv(txn_tree, upsert_kv, log_time_ms)?; - - debug!("applied UpsertKV: {:?} {:?}", upsert_kv, result); - - if expired.is_some() { - txn_tree.push_change(&upsert_kv.key, expired, None); - } - txn_tree.push_change(&upsert_kv.key, prev.clone(), result.clone()); - - Ok(Change::new(prev, result).into()) - } - - fn return_value_condition_result( - &self, - expected: i32, - target_value: &Vec, - value: &SeqV, - ) -> bool { - match FromPrimitive::from_i32(expected) { - Some(ConditionResult::Eq) => value.data == *target_value, - Some(ConditionResult::Gt) => value.data > *target_value, - Some(ConditionResult::Lt) => value.data < *target_value, - Some(ConditionResult::Ne) => value.data != *target_value, - Some(ConditionResult::Ge) => value.data >= *target_value, - Some(ConditionResult::Le) => value.data <= *target_value, - _ => false, - } - } - - pub fn return_seq_condition_result( - &self, - expected: i32, - target_seq: &u64, - value: &SeqV, - ) -> bool { - match FromPrimitive::from_i32(expected) { - Some(ConditionResult::Eq) => value.seq == *target_seq, - Some(ConditionResult::Gt) => value.seq > *target_seq, - Some(ConditionResult::Lt) => value.seq < *target_seq, - Some(ConditionResult::Ne) => value.seq != *target_seq, - Some(ConditionResult::Ge) => value.seq >= *target_seq, - Some(ConditionResult::Le) => value.seq <= *target_seq, - _ => false, - } - } - - #[fastrace::trace] - fn txn_execute_one_condition( - &self, - txn_tree: &TransactionSledTree, - cond: &TxnCondition, - ) -> Result { - debug!(cond :% =(cond); "txn_execute_one_condition"); - - let key = cond.key.clone(); - - let sub_tree = txn_tree.key_space::(); - let sv = sub_tree.get(&key)?; - - debug!("txn_execute_one_condition: {:?} {:?}", key, sv); - - if let Some(target) = &cond.target { - match target { - txn_condition::Target::Seq(target_seq) => { - return Ok(self.return_seq_condition_result( - cond.expected, - target_seq, - // seq is 0 if the record does not exist. - &sv.unwrap_or_default(), - )); - } - txn_condition::Target::Value(target_value) => { - if let Some(sv) = sv { - return Ok(self.return_value_condition_result( - cond.expected, - target_value, - &sv, - )); - } else { - return Ok(false); - } - } - } - }; - - Ok(false) - } - - #[fastrace::trace] - fn txn_execute_condition( - &self, - txn_tree: &TransactionSledTree, - condition: &Vec, - ) -> Result { - for cond in condition { - let res = self.txn_execute_one_condition(txn_tree, cond)?; - debug!(condition :% =(cond), res=res; "txn_execute_condition"); - - if !res { - return Ok(false); - } - } - - Ok(true) - } - - fn txn_execute_get_operation( - &self, - txn_tree: &TransactionSledTree, - get: &TxnGetRequest, - resp: &mut TxnReply, - ) -> Result<(), MetaStorageError> { - let sub_tree = txn_tree.key_space::(); - let sv = sub_tree.get(&get.key)?; - let value = sv.map(pb::SeqV::from); - let get_resp = TxnGetResponse { - key: get.key.clone(), - value, - }; - - resp.responses.push(TxnOpResponse { - response: Some(txn_op_response::Response::Get(get_resp)), - }); - - Ok(()) - } - - fn txn_execute_put_operation( - &self, - txn_tree: &mut TransactionSledTree, - put: &TxnPutRequest, - resp: &mut TxnReply, - log_time_ms: u64, - ) -> Result<(), MetaStorageError> { - let (expired, prev, result) = Self::txn_upsert_kv( - txn_tree, - &UpsertKV::update(&put.key, &put.value).with(MetaSpec::new( - put.expire_at, - put.ttl_ms.map(Interval::from_millis), - )), - log_time_ms, - )?; - - if expired.is_some() { - txn_tree.push_change(&put.key, expired, None); - } - txn_tree.push_change(&put.key, prev.clone(), result); - - let put_resp = TxnPutResponse { - key: put.key.clone(), - prev_value: prev.map(pb::SeqV::from), - }; - - resp.responses.push(TxnOpResponse { - response: Some(txn_op_response::Response::Put(put_resp)), - }); - - Ok(()) - } - - fn txn_execute_delete_operation( - &self, - txn_tree: &mut TransactionSledTree, - delete: &TxnDeleteRequest, - resp: &mut TxnReply, - log_time_ms: u64, - ) -> Result<(), MetaStorageError> { - let upsert = UpsertKV::delete(&delete.key); - - // If `delete.match_seq` is `Some`, only delete the record with the exact `seq`. - let upsert = if let Some(seq) = delete.match_seq { - upsert.with(MatchSeq::Exact(seq)) - } else { - upsert - }; - - let (expired, prev, result) = Self::txn_upsert_kv(txn_tree, &upsert, log_time_ms)?; - let is_deleted = prev.is_some() && result.is_none(); - - if expired.is_some() { - txn_tree.push_change(&delete.key, expired, None); - } - txn_tree.push_change(&delete.key, prev.clone(), result); - - let del_resp = TxnDeleteResponse { - key: delete.key.clone(), - success: is_deleted, - prev_value: prev.map(pb::SeqV::from), - }; - - resp.responses.push(TxnOpResponse { - response: Some(txn_op_response::Response::Delete(del_resp)), - }); - - Ok(()) - } - - fn txn_execute_delete_by_prefix_operation( - &self, - txn_tree: &mut TransactionSledTree, - delete_by_prefix: &TxnDeleteByPrefixRequest, - kv_pairs: Option<&DeleteByPrefixKeyMap>, - resp: &mut TxnReply, - log_time_ms: u64, - ) -> Result<(), MetaStorageError> { - let mut count: u32 = 0; - if let Some(kv_pairs) = kv_pairs { - if let Some(kv_pairs) = kv_pairs.get(delete_by_prefix) { - for (key, _seq) in kv_pairs.iter() { - let (expired, prev, res) = - Self::txn_upsert_kv(txn_tree, &UpsertKV::delete(key), log_time_ms)?; - - count += 1; - - if expired.is_some() { - txn_tree.push_change(key, expired, None); - } - txn_tree.push_change(key, prev, res); - } - } - } - - let del_resp = TxnDeleteByPrefixResponse { - prefix: delete_by_prefix.prefix.clone(), - count, - }; - - resp.responses.push(TxnOpResponse { - response: Some(txn_op_response::Response::DeleteByPrefix(del_resp)), - }); - - Ok(()) - } - - #[fastrace::trace] - fn txn_execute_operation( - &self, - txn_tree: &mut TransactionSledTree, - op: &TxnOp, - kv_pairs: Option<&DeleteByPrefixKeyMap>, - resp: &mut TxnReply, - log_time_ms: u64, - ) -> Result<(), MetaStorageError> { - debug!(op :% =(op); "txn execute TxnOp"); - match &op.request { - Some(txn_op::Request::Get(get)) => { - self.txn_execute_get_operation(txn_tree, get, resp)?; - } - Some(txn_op::Request::Put(put)) => { - self.txn_execute_put_operation(txn_tree, put, resp, log_time_ms)?; - } - Some(txn_op::Request::Delete(delete)) => { - self.txn_execute_delete_operation(txn_tree, delete, resp, log_time_ms)?; - } - Some(txn_op::Request::DeleteByPrefix(delete_by_prefix)) => { - self.txn_execute_delete_by_prefix_operation( - txn_tree, - delete_by_prefix, - kv_pairs, - resp, - log_time_ms, - )?; - } - None => {} - } - - Ok(()) - } - - #[fastrace::trace] - fn apply_txn_cmd( - &self, - req: &TxnRequest, - txn_tree: &mut TransactionSledTree, - kv_pairs: Option<&(DeleteByPrefixKeyMap, DeleteByPrefixKeyMap)>, - log_time_ms: u64, - ) -> Result { - debug!(txn :% =(req); "apply txn cmd"); - - let condition = &req.condition; - - let ops: &Vec; - let kv_op_pairs: Option<&DeleteByPrefixKeyMap>; - let success = if self.txn_execute_condition(txn_tree, condition)? { - ops = &req.if_then; - kv_op_pairs = if let Some(kv_pairs) = kv_pairs { - Some(&kv_pairs.0) - } else { - None - }; - true - } else { - ops = &req.else_then; - kv_op_pairs = if let Some(kv_pairs) = kv_pairs { - Some(&kv_pairs.1) - } else { - None - }; - false - }; - - let mut resp: TxnReply = TxnReply { - success, - error: "".to_string(), - responses: vec![], - }; - - for op in ops { - self.txn_execute_operation(txn_tree, op, kv_op_pairs, &mut resp, log_time_ms)?; - } - - Ok(AppliedState::TxnReply(resp)) - } - - /// Apply a `Cmd` to state machine. - /// - /// Already applied log should be filtered out before passing into this function. - /// This is the only entry to modify state machine. - /// The `cmd` is always committed by raft before applying. - #[fastrace::trace] - pub fn apply_cmd( - &self, - cmd: &Cmd, - txn_tree: &mut TransactionSledTree, - kv_pairs: Option<&(DeleteByPrefixKeyMap, DeleteByPrefixKeyMap)>, - log_time_ms: u64, - ) -> Result { - info!("apply_cmd: {}", cmd); - - let now = Instant::now(); - - let res = match cmd { - Cmd::AddNode { - ref node_id, - ref node, - overriding, - } => self.apply_add_node_cmd(node_id, node, *overriding, txn_tree), - - Cmd::RemoveNode { ref node_id } => self.apply_remove_node_cmd(node_id, txn_tree), - - Cmd::UpsertKV(ref upsert_kv) => { - self.apply_update_kv_cmd(upsert_kv, txn_tree, log_time_ms) - } - - Cmd::Transaction(txn) => self.apply_txn_cmd(txn, txn_tree, kv_pairs, log_time_ms), - }; - - let elapsed = now.elapsed().as_micros(); - debug!("apply_cmd: elapsed: {}", elapsed); - - res - } - - /// Before applying, list expired keys to clean. - /// - /// Apply is done in a sled-txn tree, which does not provide listing function. - #[fastrace::trace] - pub fn list_expired_kvs( - &self, - log_time_ms: u64, - ) -> Result, MetaStorageError> { - if log_time_ms == 0 { - return Ok(vec![]); - } - - let at_most = 32; - let mut to_clean = Vec::with_capacity(at_most); - - info!("list_expired_kv, log_time_ts: {}", log_time_ms); - - let expires = self.sm_tree.key_space::(); - - let it = expires.range(..)?.take(at_most); - for item_res in it { - let item = item_res?; - let k: ExpireKey = item.key()?; - if log_time_ms > k.time_ms { - let v: ExpireValue = item.value()?; - to_clean.push((v.key, k)) - } - } - - Ok(to_clean) - } - - /// Remove expired key-values, and corresponding secondary expiration index record. - /// - /// This should be done inside a sled-transaction. - #[fastrace::trace] - fn clean_expired_kvs( - &self, - txn_tree: &mut TransactionSledTree, - expired: &[(String, ExpireKey)], - ) -> Result<(), MetaStorageError> { - for (key, expire_key) in expired.iter() { - let sv = txn_tree.key_space::().get(key)?; - - if let Some(seq_v) = &sv { - if expire_key.seq == seq_v.seq { - info!("clean expired: {}, {}", key, expire_key); - - txn_tree.key_space::().remove(key)?; - txn_tree.key_space::().remove(expire_key)?; - - txn_tree.push_change(key, sv, None); - continue; - } - } - - unreachable!( - "trying to remove un-cleanable: {}, {}, kv-record: {:?}", - key, expire_key, sv - ); - } - Ok(()) - } - - fn txn_incr_seq(key: &str, txn_tree: &TransactionSledTree) -> Result { - let seqs = txn_tree.key_space::(); - - let key = key.to_string(); - - let curr = seqs.get(&key)?; - let new_value = curr.unwrap_or_default() + 1; - seqs.insert(&key, &new_value)?; - - debug!("txn_incr_seq: {}={}", key, new_value); - - Ok(new_value.0) - } - - /// Execute an upsert-kv operation on a transactional sled tree. - /// - /// KV has two indexes: - /// - The primary index: `key -> (seq, meta(expire_time), value)`, - /// - and a secondary expiration index: `(expire_time, seq) -> key`. - /// - /// Thus upsert a kv record is done in two steps: - /// update the primary index and optionally update the secondary index. - /// - /// It returns 3 SeqV: - /// - `(None, None, x)`: upsert nonexistent key; - /// - `(None, Some, x)`: upsert existent and non-expired key; - /// - `(Some, None, x)`: upsert existent but expired key; - #[allow(clippy::type_complexity)] - fn txn_upsert_kv( - txn_tree: &TransactionSledTree, - upsert_kv: &UpsertKV, - log_time_ms: u64, - ) -> Result<(Option, Option, Option), MetaStorageError> { - let (expired, prev, res) = - Self::txn_upsert_kv_primary_index(txn_tree, upsert_kv, log_time_ms)?; - - let expires = txn_tree.key_space::(); - - if let Some(sv) = &expired { - if let Some(m) = &sv.meta { - if let Some(exp_ms) = m.get_expire_at_ms() { - expires.remove(&ExpireKey::new(exp_ms, sv.seq))?; - } - } - } - - // No change, no need to update expiration index - if prev == res { - return Ok((expired, prev, res)); - } - - // Remove previous expiration index, add a new one. - - if let Some(sv) = &prev { - if let Some(m) = &sv.meta { - if let Some(exp_ms) = m.get_expire_at_ms() { - expires.remove(&ExpireKey::new(exp_ms, sv.seq))?; - } - } - } - - if let Some(sv) = &res { - if let Some(m) = &sv.meta { - if let Some(exp_ms) = m.get_expire_at_ms() { - let k = ExpireKey::new(exp_ms, sv.seq); - let v = ExpireValue::new(&upsert_kv.key, 0); - expires.insert(&k, &v)?; - } - } - } - - Ok((expired, prev, res)) - } - - /// It returns 3 SeqV: - /// - The first one is `Some` if an existent record expired. - /// - The second and the third represent the change that is made by the upsert operation. - /// - /// Only one of the first and second can be `Some`. - #[allow(clippy::type_complexity)] - fn txn_upsert_kv_primary_index( - txn_tree: &TransactionSledTree, - upsert_kv: &UpsertKV, - log_time_ms: u64, - ) -> Result<(Option, Option, Option), MetaStorageError> { - let cmd_ctx = CmdContext::from_millis(log_time_ms); - - let kvs = txn_tree.key_space::(); - - let prev = kvs.get(&upsert_kv.key)?; - - // If prev is timed out, treat it as a None. But still keep the original value for cleaning up it. - let (expired, prev) = Self::expire_seq_v(prev, log_time_ms); - - if upsert_kv.seq.match_seq(&prev).is_err() { - return Ok((expired, prev.clone(), prev)); - } - - let mut new_seq_v = match &upsert_kv.value { - Operation::Update(v) => SeqV::with_meta( - 0, - upsert_kv - .value_meta - .as_ref() - .map(|x| x.to_kv_meta(&cmd_ctx)), - v.clone(), - ), - Operation::Delete => { - kvs.remove(&upsert_kv.key)?; - return Ok((expired, prev, None)); - } - Operation::AsIs => match prev { - None => return Ok((expired, prev, None)), - Some(ref prev_kv_value) => prev_kv_value.clone().set_meta( - upsert_kv - .value_meta - .as_ref() - .map(|m| m.to_kv_meta(&cmd_ctx)), - ), - }, - }; - - new_seq_v.seq = Self::txn_incr_seq(GenericKV::NAME, txn_tree)?; - kvs.insert(&upsert_kv.key, &new_seq_v)?; - - debug!("applied upsert: {:?} res: {:?}", upsert_kv, new_seq_v); - Ok((expired, prev, Some(new_seq_v))) - } - - fn txn_client_last_resp_update( - &self, - key: &str, - value: (u64, AppliedState), - txn_tree: &TransactionSledTree, - ) -> Result { - let v = ClientLastRespValue { - req_serial_num: value.0, - res: value.1.clone(), - }; - let txn_ks = txn_tree.key_space::(); - txn_ks.insert(&key.to_string(), &v)?; - - Ok(value.1) - } - - pub fn get_membership(&self) -> Result, MetaStorageError> { - let sm_meta = self.sm_meta(); - let mem = sm_meta - .get(&StateMachineMetaKey::LastMembership)? - .map(|x| x.try_into().expect("Membership")); - - Ok(mem) - } - - pub fn get_last_applied(&self) -> Result, MetaStorageError> { - let sm_meta = self.sm_meta(); - let last_applied = sm_meta - .get(&LastApplied)? - .map(|x| x.try_into().expect("LogId")); - - Ok(last_applied) - } - - pub async fn add_node(&self, node_id: u64, node: &Node) -> Result<(), MetaStorageError> { - let sm_nodes = self.nodes(); - sm_nodes.insert(&node_id, node).await?; - Ok(()) - } - - pub fn get_client_last_resp( - &self, - key: &str, - ) -> Result, MetaStorageError> { - let client_last_resps = self.client_last_resps(); - let v: Option = client_last_resps.get(&key.to_string())?; - - if let Some(resp) = v { - return Ok(Some((resp.req_serial_num, resp.res))); - } - - Ok(Some((0, AppliedState::None))) - } - - pub fn txn_get_client_last_resp( - &self, - key: &str, - txn_tree: &TransactionSledTree, - ) -> Result<(u64, AppliedState), MetaStorageError> { - let client_last_resps = txn_tree.key_space::(); - let v = client_last_resps.get(&key.to_string())?; - - if let Some(resp) = v { - return Ok((resp.req_serial_num, resp.res)); - } - Ok((0, AppliedState::None)) - } - - pub fn get_node(&self, node_id: &NodeId) -> Result, MetaStorageError> { - let sm_nodes = self.nodes(); - sm_nodes.get(node_id) - } - - pub fn get_nodes(&self) -> Result, MetaStorageError> { - let sm_nodes = self.nodes(); - sm_nodes.range_values(..) - } - - /// Expire an `SeqV` and returns the value discarded by expiration and the unexpired value: - /// - `(Some, None)` if it expires. - /// - `(None, Some)` if it does not. - /// - `(None, None)` if the input is None. - pub fn expire_seq_v( - seq_value: Option>, - log_time_ms: u64, - ) -> (Option>, Option>) { - if let Some(s) = &seq_value { - if s.eval_expire_at_ms() < log_time_ms { - (seq_value, None) - } else { - (None, seq_value) - } - } else { - (None, None) - } - } -} - -/// Key space support -impl StateMachine { - pub fn sm_meta(&self) -> AsKeySpace { - self.sm_tree.key_space() - } - - pub fn nodes(&self) -> AsKeySpace { - self.sm_tree.key_space() - } - - /// A kv store of all other general purpose information. - /// The value is tuple of a monotonic sequence number and userdata value in string. - /// The sequence number is guaranteed to increment(by some value greater than 0) everytime the record changes. - pub fn kvs(&self) -> AsKeySpace { - self.sm_tree.key_space() - } - - /// storage of auto-incremental number. - pub fn sequences(&self) -> AsKeySpace { - self.sm_tree.key_space() - } - - /// storage of client last resp to keep idempotent. - pub fn client_last_resps(&self) -> AsKeySpace { - self.sm_tree.key_space() - } -} - -#[cfg(test)] -mod tests { - use databend_common_meta_types::seq_value::KVMeta; - use databend_common_meta_types::seq_value::SeqV; - - use crate::state_machine::StateMachine; - - #[test] - fn test_expire_seq_v() -> anyhow::Result<()> { - let sv = || SeqV::new(1, ()); - let expire_seq_v = StateMachine::expire_seq_v; - - assert_eq!((None, None), expire_seq_v(None, 10_000)); - assert_eq!((None, Some(sv())), expire_seq_v(Some(sv()), 10_000)); - - assert_eq!( - (None, Some(sv().set_meta(Some(KVMeta::new(None))))), - expire_seq_v(Some(sv().set_meta(Some(KVMeta::new(None)))), 10_000) - ); - assert_eq!( - (None, Some(sv().set_meta(Some(KVMeta::new_expire(20))))), - expire_seq_v(Some(sv().set_meta(Some(KVMeta::new_expire(20)))), 10_000) - ); - assert_eq!( - (Some(sv().set_meta(Some(KVMeta::new_expire(5)))), None), - expire_seq_v(Some(sv().set_meta(Some(KVMeta::new_expire(5)))), 10_000) - ); - - Ok(()) - } -} diff --git a/src/meta/raft-store/src/state_machine/sm_kv_api_impl.rs b/src/meta/raft-store/src/state_machine/sm_kv_api_impl.rs deleted file mode 100644 index 61466bcb3e7e..000000000000 --- a/src/meta/raft-store/src/state_machine/sm_kv_api_impl.rs +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_kvapi::kvapi; -use databend_common_meta_kvapi::kvapi::KVStream; -use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; -use databend_common_meta_types::protobuf::StreamItem; -use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::AppliedState; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::MetaError; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; -use databend_common_meta_types::UpsertKV; -use futures_util::StreamExt; - -use crate::state_machine::StateMachine; - -#[async_trait::async_trait] -impl kvapi::KVApi for StateMachine { - type Error = MetaError; - - async fn upsert_kv(&self, act: UpsertKVReq) -> Result { - let cmd = Cmd::UpsertKV(UpsertKV { - key: act.key, - seq: act.seq, - value: act.value, - value_meta: act.value_meta, - }); - - let res = self.sm_tree.txn(true, |mut txn_sled_tree| { - let r = self - .apply_cmd(&cmd, &mut txn_sled_tree, None, SeqV::<()>::now_ms()) - .unwrap(); - Ok(r) - })?; - - match res { - AppliedState::KV(x) => Ok(x), - _ => { - panic!("expect AppliedState::KV"); - } - } - } - - async fn transaction(&self, txn: TxnRequest) -> Result { - let cmd = Cmd::Transaction(txn); - - let res = self.sm_tree.txn(true, |mut txn_sled_tree| { - let r = self.apply_cmd(&cmd, &mut txn_sled_tree, None, SeqV::<()>::now_ms())?; - Ok(r) - })?; - - match res { - AppliedState::TxnReply(x) => Ok(x), - _ => { - unreachable!("expect AppliedState::TxnReply"); - } - } - } - - async fn get_kv_stream(&self, keys: &[String]) -> Result, Self::Error> { - let kvs = self.kvs(); - let mut items = vec![]; - - let local_now_ms = SeqV::<()>::now_ms(); - - for k in keys.iter() { - let v = kvs.get(k)?; - let (_, v) = Self::expire_seq_v(v, local_now_ms); - items.push(Ok(StreamItem::from((k.clone(), v)))) - } - - Ok(futures::stream::iter(items).boxed()) - } - - async fn list_kv(&self, prefix: &str) -> Result, Self::Error> { - let kvs = self.kvs(); - let kv_pairs = kvs.scan_prefix(&prefix.to_string())?; - - let x = kv_pairs.into_iter(); - - let local_now_ms = SeqV::<()>::now_ms(); - - // Convert expired to None - let x = x.map(move |(k, v)| (k, Self::expire_seq_v(Some(v), local_now_ms).1)); - // Remove None - let x = x.filter(|(_k, v)| v.is_some()); - - let x = x.map(|kv: (String, Option)| Ok(StreamItem::from(kv))); - - Ok(futures::stream::iter(x).boxed()) - } -} diff --git a/src/meta/raft-store/src/state_machine/testing.rs b/src/meta/raft-store/src/state_machine/testing.rs index c141766047dd..cc4a8dd31fa7 100644 --- a/src/meta/raft-store/src/state_machine/testing.rs +++ b/src/meta/raft-store/src/state_machine/testing.rs @@ -24,9 +24,6 @@ use maplit::btreeset; use openraft::entry::RaftEntry; use openraft::Membership; -use crate::key_spaces::RaftStoreEntry; -use crate::state_machine::SnapshotKeyValue; - /// Logs and the expected snapshot for testing snapshot. pub fn snapshot_logs() -> (Vec, Vec) { let logs = vec![ @@ -83,29 +80,6 @@ pub fn snapshot_logs() -> (Vec, Vec) { (logs, want) } -pub fn pretty_snapshot(snap: &[SnapshotKeyValue]) -> Vec { - let mut res = vec![]; - for kv in snap.iter() { - let k = kv[0].clone(); - let v = kv[1].clone(); - let line = format!("{:?}:{}", k, String::from_utf8(v.to_vec()).unwrap()); - res.push(line); - } - res -} - -pub fn pretty_snapshot_entries<'a>( - snap: impl IntoIterator, -) -> Vec { - let mut res = vec![]; - - for kv in snap.into_iter() { - let line = serde_json::to_string(kv).unwrap(); - res.push(line); - } - res -} - // test cases fro Cmd::IncrSeq: // case_name, txid, key, want pub fn cases_incr_seq() -> Vec<(&'static str, Option, &'static str, u64)> { diff --git a/src/meta/raft-store/tests/it/types.rs b/src/meta/raft-store/tests/it/types.rs index c24a8b6b5469..22a1ce82f2a6 100644 --- a/src/meta/raft-store/tests/it/types.rs +++ b/src/meta/raft-store/tests/it/types.rs @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::ops::Bound; - use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::SledOrderedSerde; -use databend_common_meta_sled_store::SledRangeSerde; use databend_common_meta_types::raft_types::NodeId; #[test] @@ -49,16 +46,3 @@ fn test_node_id_serde_de() -> anyhow::Result<()> { Ok(()) } - -#[test] -fn test_node_id_range_serde() -> anyhow::Result<()> { - let a: NodeId = 8; - let b: NodeId = 11; - let got = (a..b).ser()?; - let want = ( - Bound::Included(sled::IVec::from(vec![0, 0, 0, 0, 0, 0, 0, 8])), - Bound::Excluded(sled::IVec::from(vec![0, 0, 0, 0, 0, 0, 0, 11])), - ); - assert_eq!(want, got); - Ok(()) -} diff --git a/src/meta/service/src/store/mod.rs b/src/meta/service/src/store/mod.rs index 19fcbb03c335..257c6d07f0b6 100644 --- a/src/meta/service/src/store/mod.rs +++ b/src/meta/service/src/store/mod.rs @@ -17,8 +17,6 @@ mod raft_state_machine_impl; #[allow(clippy::module_inception)] mod store; mod store_inner; -mod to_storage_error; pub use store::RaftStore; pub use store_inner::StoreInner; -pub use to_storage_error::ToStorageError; diff --git a/src/meta/service/src/store/store_inner.rs b/src/meta/service/src/store/store_inner.rs index 2d39defc66fc..80701815ba22 100644 --- a/src/meta/service/src/store/store_inner.rs +++ b/src/meta/service/src/store/store_inner.rs @@ -104,7 +104,7 @@ impl StoreInner { fn to_startup_err(e: impl std::error::Error + 'static) -> MetaStartupError { let ae = AnyError::new(&e); - let store_err = MetaStorageError::Damaged(ae); + let store_err = MetaStorageError(ae); MetaStartupError::StoreOpenError(store_err) } @@ -303,7 +303,7 @@ impl StoreInner { pub async fn do_install_snapshot(&self, db: DB) -> Result<(), MetaStorageError> { let mut sm = self.state_machine.write().await; sm.install_snapshot_v003(db).await.map_err(|e| { - MetaStorageError::Damaged( + MetaStorageError( AnyError::new(&e).add_context(|| "replacing state-machine with snapshot"), ) })?; diff --git a/src/meta/service/src/store/to_storage_error.rs b/src/meta/service/src/store/to_storage_error.rs deleted file mode 100644 index ea72405bab16..000000000000 --- a/src/meta/service/src/store/to_storage_error.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_meta_sled_store::openraft; -use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::raft_types::ErrorSubject; -use databend_common_meta_types::raft_types::StorageError; -use openraft::ErrorVerb; - -/// Convert MetaStorageError to openraft::StorageError; -pub trait ToStorageError { - fn map_to_sto_err(self, subject: ErrorSubject, verb: ErrorVerb) -> Result; -} - -impl ToStorageError for Result { - fn map_to_sto_err(self, subject: ErrorSubject, verb: ErrorVerb) -> Result { - match self { - Ok(x) => Ok(x), - Err(e) => { - let io_err = StorageError::new(subject, verb, &e); - Err(io_err) - } - } - } -} diff --git a/src/meta/sled-store/Cargo.toml b/src/meta/sled-store/Cargo.toml index 8eb0a9c794d0..7e0f149868c0 100644 --- a/src/meta/sled-store/Cargo.toml +++ b/src/meta/sled-store/Cargo.toml @@ -27,7 +27,6 @@ serde_json = { workspace = true } sled = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true } [dev-dependencies] anyhow = { workspace = true } diff --git a/src/meta/sled-store/src/bytes_error.rs b/src/meta/sled-store/src/bytes_error.rs index 47a13c7bf3e9..b9eef48c5253 100644 --- a/src/meta/sled-store/src/bytes_error.rs +++ b/src/meta/sled-store/src/bytes_error.rs @@ -46,6 +46,6 @@ impl From for SledBytesError { // TODO: remove this: after refactoring, sled should not use MetaStorageError directly. impl From for MetaStorageError { fn from(e: SledBytesError) -> Self { - MetaStorageError::Damaged(AnyError::new(&e)) + MetaStorageError(AnyError::new(&e)) } } diff --git a/src/meta/sled-store/src/lib.rs b/src/meta/sled-store/src/lib.rs index fe7bce06d972..6405319b00bb 100644 --- a/src/meta/sled-store/src/lib.rs +++ b/src/meta/sled-store/src/lib.rs @@ -26,15 +26,10 @@ pub use openraft; pub use sled; pub use sled_key_space::SledKeySpace; pub use sled_serde::SledOrderedSerde; -pub use sled_serde::SledRangeSerde; pub use sled_serde::SledSerde; pub use sled_tree::AsKeySpace; -pub use sled_tree::SledAsRef; pub use sled_tree::SledItem; pub use sled_tree::SledTree; -pub use sled_tree::TransactionSledTree; -pub use sled_tree::TxnKeySpace; -pub use store::Store; mod bytes_error; mod db; @@ -42,4 +37,3 @@ mod sled_key_space; mod sled_serde; mod sled_serde_impl; mod sled_tree; -mod store; diff --git a/src/meta/sled-store/src/sled_serde.rs b/src/meta/sled-store/src/sled_serde.rs index d12141373069..afe5c3314110 100644 --- a/src/meta/sled-store/src/sled_serde.rs +++ b/src/meta/sled-store/src/sled_serde.rs @@ -13,17 +13,13 @@ // limitations under the License. use std::mem::size_of_val; -use std::ops::Bound; -use std::ops::RangeBounds; use byteorder::BigEndian; use byteorder::ByteOrder; -use databend_common_meta_types::raft_types::Entry; use serde::de::DeserializeOwned; use serde::Serialize; use sled::IVec; -use crate::SledAsRef; use crate::SledBytesError; /// Serialize/deserialize(ser/de) to/from sled values. @@ -55,60 +51,6 @@ pub trait SledOrderedSerde: Serialize + DeserializeOwned { where Self: Sized; } -/// Serialize/deserialize(ser/de) to/from range to sled IVec range. -/// The type must impl SledOrderedSerde so that after serialization the order is preserved. -pub trait SledRangeSerde -where - SD: SledOrderedSerde, - V: RangeBounds, - R: RangeBounds, -{ - /// (ser)ialize a range to range of `sled::IVec`. - fn ser(&self) -> Result; - - // TODO(xp): do we need this? - // /// (de)serialize a value from `sled::IVec`. - // fn de>(v: T) -> Result - // where Self: Sized; -} - -/// Impl ser/de for range of value that can be ser/de to `sled::IVec` -impl SledRangeSerde, Bound)> for V -where - SD: SledOrderedSerde, - V: RangeBounds, -{ - fn ser(&self) -> Result<(Bound, Bound), SledBytesError> { - let s = self.start_bound(); - let e = self.end_bound(); - - let s = bound_ser(s)?; - let e = bound_ser(e)?; - - Ok((s, e)) - } -} - -fn bound_ser(v: Bound<&SD>) -> Result, SledBytesError> { - let res = match v { - Bound::Included(v) => Bound::Included(v.ser()?), - Bound::Excluded(v) => Bound::Excluded(v.ser()?), - Bound::Unbounded => Bound::Unbounded, - }; - Ok(res) -} - -/// Extract log index from log entry -impl SledAsRef for Entry { - fn as_key(&self) -> &u64 { - &self.log_id.index - } - - fn as_value(&self) -> &Entry { - self - } -} - /// NodeId, LogIndex and Term need to be serialized with order preserved, for listing items. impl SledOrderedSerde for u64 { fn ser(&self) -> Result { @@ -138,5 +80,3 @@ impl SledOrderedSerde for String { Ok(String::from_utf8(v.as_ref().to_vec())?) } } - -// impl SledSerde for T where T: Serialize + DeserializeOwned {} diff --git a/src/meta/sled-store/src/sled_tree.rs b/src/meta/sled-store/src/sled_tree.rs index c5e7266d0abf..dbcb76399e7e 100644 --- a/src/meta/sled-store/src/sled_tree.rs +++ b/src/meta/sled-store/src/sled_tree.rs @@ -14,48 +14,15 @@ use std::fmt::Display; use std::marker::PhantomData; -use std::ops::Bound; -use std::ops::Deref; -use std::ops::RangeBounds; -use std::time::Duration; use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::anyerror::AnyError; -use databend_common_meta_types::seq_value::SeqV; -use databend_common_meta_types::Change; use fastrace::func_name; use log::debug; -use log::warn; -use sled::transaction::ConflictableTransactionError; -use sled::transaction::TransactionResult; -use sled::transaction::TransactionalTree; use sled::IVec; -use crate::sled::transaction::TransactionError; -use crate::store::Store; use crate::SledBytesError; use crate::SledKeySpace; -const DEFAULT_CHUNK_SIZE: usize = 256; - -/// Get a ref to the key or to the value. -/// -/// It is used as an abstract representation of key/value used in the sled store. -pub trait SledAsRef { - fn as_key(&self) -> &K; - fn as_value(&self) -> &V; -} - -impl SledAsRef for (K, V) { - fn as_key(&self) -> &K { - &self.0 - } - - fn as_value(&self) -> &V { - &self.1 - } -} - /// SledTree is a wrapper of sled::Tree that provides access of more than one key-value /// types. /// A `SledKVType` defines a key-value type to be stored. @@ -109,17 +76,6 @@ impl SledItem { #[allow(clippy::type_complexity)] impl SledTree { - /// Return true if the tree exists. - pub fn has_tree + Display>(db: &sled::Db, tree_name: N) -> bool { - // During testing, every tree name must be unique. - if cfg!(test) { - let x = tree_name.as_ref(); - let x = &x[0..5]; - assert_eq!(x, b"test-"); - } - db.contains_tree(&tree_name) - } - /// Open SledTree pub fn open + Display>( db: &sled::Db, @@ -167,49 +123,6 @@ impl SledTree { Ok(kvs) } - pub fn txn( - &self, - sync: bool, - f: impl Fn(TransactionSledTree<'_>) -> Result, - ) -> Result { - let sync = sync && self.sync; - - let result: TransactionResult = self.tree.transaction(move |tree| { - let txn_sled_tree = TransactionSledTree::new(tree); - let r = f(txn_sled_tree); - match r { - Ok(r) => { - if sync { - tree.flush(); - } - Ok(r) - } - Err(meta_sto_err) => { - warn!("txn error: {:?}", meta_sto_err); - - match &meta_sto_err { - MetaStorageError::TransactionConflict => { - Err(ConflictableTransactionError::Conflict) - } - MetaStorageError::Damaged(_e) => { - Err(ConflictableTransactionError::Abort(meta_sto_err)) - } - } - } - } - }); - - match result { - Ok(x) => Ok(x), - Err(txn_err) => match txn_err { - TransactionError::Abort(meta_sto_err) => Err(meta_sto_err), - TransactionError::Storage(sto_err) => { - Err(MetaStorageError::Damaged(AnyError::new(&sto_err))) - } - }, - } - } - /// Retrieve the value of key. pub(crate) fn get( &self, @@ -225,155 +138,6 @@ impl SledTree { Ok(v) } - /// Delete kvs that are in `range`. - #[fastrace::trace] - pub(crate) async fn range_remove( - &self, - range: R, - flush: bool, - ) -> Result<(), MetaStorageError> - where - KV: SledKeySpace, - R: RangeBounds, - { - loop { - // Convert K range into sled::IVec range - let sled_range = KV::serialize_range(&range)?; - - // Removing should not leave a hole. Only when removing from left, it is allowed to delete in chunks. - let chunk_size = if let Bound::Unbounded = sled_range.0 { - // Do chunked delete - DEFAULT_CHUNK_SIZE - } else { - // Do one shot delete - usize::MAX - }; - - let mut found = false; - let mut batch = sled::Batch::default(); - - for item in self.tree.range(sled_range).take(chunk_size) { - let (k, _) = item?; - batch.remove(k); - found = true; - } - - if !found { - break; - } - - self.tree.apply_batch(batch)?; - - self.flush_async(flush).await?; - - // Do not block for too long if there are many keys to delete. - tokio::time::sleep(Duration::from_millis(5)).await; - } - - Ok(()) - } - - /// Get key-values in `range` - pub(crate) fn range( - &self, - range: R, - ) -> Result< - impl DoubleEndedIterator, MetaStorageError>>, - MetaStorageError, - > - where - KV: SledKeySpace, - R: RangeBounds, - { - // Convert K range into sled::IVec range - let range = KV::serialize_range(&range)?; - - let it = self.tree.range(range); - let it = it.map(move |item| { - let (k, v) = item?; - - let item = SledItem::new(k, v); - Ok(item) - }); - - Ok(it) - } - - /// Get key-values in with the same prefix - pub(crate) fn scan_prefix( - &self, - prefix: &KV::K, - ) -> Result, MetaStorageError> - where - KV: SledKeySpace, - { - let mut res = vec![]; - - let pref = KV::serialize_key(prefix)?; - for item in self.tree.scan_prefix(pref) { - let (k, v) = item?; - - let key = KV::deserialize_key(k)?; - let value = KV::deserialize_value(v)?; - res.push((key, value)); - } - - Ok(res) - } - - /// Append many key-values into SledTree. - pub(crate) async fn append(&self, kvs: I) -> Result<(), MetaStorageError> - where - KV: SledKeySpace, - T: SledAsRef, - I: IntoIterator, - { - let mut batch = sled::Batch::default(); - - for t in kvs.into_iter() { - debug!("{}: append kvs", func_name!()); - let key = t.as_key(); - let value = t.as_value(); - - let k = KV::serialize_key(key)?; - let v = KV::serialize_value(value)?; - - batch.insert(k, v); - } - - debug!("{}: applying the batch", func_name!()); - self.tree.apply_batch(batch)?; - - self.flush_async(true).await?; - - Ok(()) - } - - /// Insert a single kv. - /// Returns the last value if it is set. - pub(crate) async fn insert( - &self, - key: &KV::K, - value: &KV::V, - ) -> Result, MetaStorageError> - where - KV: SledKeySpace, - { - let k = KV::serialize_key(key)?; - let v = KV::serialize_value(value)?; - - let prev = self.tree.insert(k, v)?; - - let prev = match prev { - None => None, - Some(x) => Some(KV::deserialize_value(x)?), - }; - - self.flush_async(true).await?; - - Ok(prev) - } - /// Remove a key without returning the previous value. /// /// Just return the size of the removed value if the key is removed. @@ -396,22 +160,6 @@ impl SledTree { Ok(removed) } - /// Build a string describing the range for a range operation. - #[allow(dead_code)] - fn range_message(&self, range: &R) -> String - where - KV: SledKeySpace, - R: RangeBounds, - { - format!( - "{}:{}/[{:?}, {:?}]", - self.name, - KV::NAME, - range.start_bound(), - range.end_bound() - ) - } - #[fastrace::trace] async fn flush_async(&self, flush: bool) -> Result<(), MetaStorageError> { debug!("{}: flush: {}", func_name!(), flush); @@ -424,127 +172,18 @@ impl SledTree { } } -#[derive(Clone)] -pub struct TransactionSledTree<'a> { - pub txn_tree: &'a TransactionalTree, - - /// The changes that are collected during transaction execution. - pub changes: Vec, String>>, -} - -impl<'a> TransactionSledTree<'a> { - pub fn new(txn_tree: &'a TransactionalTree) -> Self { - Self { - txn_tree, - changes: vec![], - } - } - - pub fn key_space(&self) -> TxnKeySpace { - TxnKeySpace:: { - inner: self, - phantom: PhantomData, - } - } - - /// Push a **change** that is applied to `key`. - /// - /// It does nothing if `prev == result` - pub fn push_change(&mut self, key: impl ToString, prev: Option, result: Option) { - if prev == result { - return; - } - - self.changes - .push(Change::new(prev, result).with_id(key.to_string())) - } -} - /// It borrows the internal SledTree with access limited to a specified namespace `KV`. pub struct AsKeySpace<'a, KV: SledKeySpace> { inner: &'a SledTree, phantom: PhantomData, } -pub struct TxnKeySpace<'a, KV: SledKeySpace> { - inner: &'a TransactionSledTree<'a>, - phantom: PhantomData, -} - -impl<'a, KV: SledKeySpace> Store for TxnKeySpace<'a, KV> { - type Error = MetaStorageError; - - fn insert(&self, key: &KV::K, value: &KV::V) -> Result, Self::Error> { - let k = KV::serialize_key(key)?; - let v = KV::serialize_value(value)?; - - let prev = self.txn_tree.insert(k, v)?; - match prev { - Some(v) => Ok(Some(KV::deserialize_value(v)?)), - None => Ok(None), - } - } - - fn get(&self, key: &KV::K) -> Result, Self::Error> { - let k = KV::serialize_key(key)?; - let got = self.txn_tree.get(k)?; - - match got { - Some(v) => Ok(Some(KV::deserialize_value(v)?)), - None => Ok(None), - } - } - - fn remove(&self, key: &KV::K) -> Result, Self::Error> { - let k = KV::serialize_key(key)?; - let removed = self.txn_tree.remove(k)?; - - match removed { - Some(v) => Ok(Some(KV::deserialize_value(v)?)), - None => Ok(None), - } - } -} - -/// Some methods that take `&TransactionSledTree` as parameter need to be called -/// in subTree method, since subTree(aka: AsTxnKeySpace) already ref to `TransactionSledTree` -/// we impl deref here to fetch inner `&TransactionSledTree`. -/// # Example: -/// -/// ``` -/// fn txn_incr_seq(&self, key: &str, txn_tree: &TransactionSledTree) {} -/// -/// fn sub_txn_tree_do_update<'s, KS>(&'s self, sub_tree: &AsTxnKeySpace<'s, KS>) { -/// seq_kv_value.seq = self.txn_incr_seq(KS::NAME, &*sub_tree); -/// sub_tree.insert(key, &seq_kv_value); -/// } -/// ``` -impl<'a, KV: SledKeySpace> Deref for TxnKeySpace<'a, KV> { - type Target = &'a TransactionSledTree<'a>; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - #[allow(clippy::type_complexity)] impl<'a, KV: SledKeySpace> AsKeySpace<'a, KV> { pub fn get(&self, key: &KV::K) -> Result, MetaStorageError> { self.inner.get::(key) } - pub fn last(&self) -> Result, MetaStorageError> { - let mut it = self.inner.range::(..)?.rev(); - let last = it.next(); - let last = match last { - None => return Ok(None), - Some(x) => x, - }; - - let kv = last?.kv()?; - Ok(Some(kv)) - } - pub async fn remove_no_return( &self, key: &KV::K, @@ -552,55 +191,4 @@ impl<'a, KV: SledKeySpace> AsKeySpace<'a, KV> { ) -> Result, MetaStorageError> { self.inner.remove_no_return::(key, flush).await } - - pub async fn range_remove(&self, range: R, flush: bool) -> Result<(), MetaStorageError> - where R: RangeBounds { - self.inner.range_remove::(range, flush).await - } - - pub fn range( - &self, - range: R, - ) -> Result< - impl DoubleEndedIterator, MetaStorageError>>, - MetaStorageError, - > - where - R: RangeBounds, - { - self.inner.range::(range) - } - - pub fn scan_prefix(&self, prefix: &KV::K) -> Result, MetaStorageError> { - self.inner.scan_prefix::(prefix) - } - - pub fn range_values(&self, range: R) -> Result, MetaStorageError> - where R: RangeBounds { - let it = self.inner.range::(range)?; - let mut res = vec![]; - for r in it { - let item = r?; - let v = item.value()?; - res.push(v); - } - - Ok(res) - } - - pub async fn append(&self, kvs: I) -> Result<(), MetaStorageError> - where - T: SledAsRef, - I: IntoIterator, - { - self.inner.append::(kvs).await - } - - pub async fn insert( - &self, - key: &KV::K, - value: &KV::V, - ) -> Result, MetaStorageError> { - self.inner.insert::(key, value).await - } } diff --git a/src/meta/sled-store/src/store.rs b/src/meta/sled-store/src/store.rs deleted file mode 100644 index 6677c02440de..000000000000 --- a/src/meta/sled-store/src/store.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::SledKeySpace; - -/// Defines low level storage API -pub trait Store { - type Error: std::error::Error; - - fn insert(&self, key: &KV::K, value: &KV::V) -> Result, Self::Error>; - - fn get(&self, key: &KV::K) -> Result, Self::Error>; - - fn remove(&self, key: &KV::K) -> Result, Self::Error>; -} diff --git a/src/meta/stoerr/src/meta_storage_errors.rs b/src/meta/stoerr/src/meta_storage_errors.rs index a80c953fb93e..e0bad5e1b426 100644 --- a/src/meta/stoerr/src/meta_storage_errors.rs +++ b/src/meta/stoerr/src/meta_storage_errors.rs @@ -17,68 +17,46 @@ use std::io; use anyerror::AnyError; use databend_common_exception::ErrorCode; -use sled::transaction::UnabortableTransactionError; /// Storage level error that is raised by meta service. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum MetaStorageError { - #[error("Data damaged: {0}")] - Damaged(AnyError), - - // TODO(1): remove this error - /// An internal error that inform txn to retry. - #[error("Conflict when execute transaction, just retry")] - TransactionConflict, -} +#[error("{0}")] +pub struct MetaStorageError(pub AnyError); impl MetaStorageError { pub fn damaged D>( error: &(impl std::error::Error + 'static), context: F, ) -> Self { - MetaStorageError::Damaged(AnyError::new(error).add_context(context)) + MetaStorageError(AnyError::new(error).add_context(context)) } pub fn name(&self) -> &'static str { - match self { - MetaStorageError::Damaged(_) => "Damaged", - MetaStorageError::TransactionConflict => "TransactionConflict", - } + "MetaStorageError" } } impl From for MetaStorageError { fn from(error: std::string::FromUtf8Error) -> Self { - MetaStorageError::Damaged(AnyError::new(&error)) + MetaStorageError(AnyError::new(&error)) } } impl From for MetaStorageError { fn from(error: serde_json::Error) -> MetaStorageError { - MetaStorageError::Damaged(AnyError::new(&error)) + MetaStorageError(AnyError::new(&error)) } } impl From for MetaStorageError { fn from(error: sled::Error) -> MetaStorageError { - MetaStorageError::Damaged(AnyError::new(&error)) - } -} - -impl From for MetaStorageError { - fn from(error: UnabortableTransactionError) -> Self { - match error { - UnabortableTransactionError::Storage(error) => { - MetaStorageError::Damaged(AnyError::new(&error)) - } - UnabortableTransactionError::Conflict => MetaStorageError::TransactionConflict, - } + MetaStorageError(AnyError::new(&error)) } } impl From for MetaStorageError { fn from(error: io::Error) -> Self { - MetaStorageError::Damaged(AnyError::new(&error)) + MetaStorageError(AnyError::new(&error)) } } @@ -90,6 +68,6 @@ impl From for io::Error { impl From for ErrorCode { fn from(e: MetaStorageError) -> Self { - ErrorCode::MetaServiceError(e.to_string()) + ErrorCode::MetaStorageError(e.to_string()) } } From 93eb1e3a674744f6a840e3e1da45caebb10d0bc4 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Sun, 17 Nov 2024 12:03:16 +0800 Subject: [PATCH 52/92] chore(query): remove useless new line for query log (#16860) --- src/common/tracing/src/loggers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/tracing/src/loggers.rs b/src/common/tracing/src/loggers.rs index d5dd662edabe..e3b2d099a19c 100644 --- a/src/common/tracing/src/loggers.rs +++ b/src/common/tracing/src/loggers.rs @@ -55,7 +55,7 @@ pub fn get_layout(format: &str) -> Layout { } fn identical_layout() -> Layout { - CustomLayout::new(|record: &Record| Ok(format!("{}\n", record.args()).into_bytes())).into() + CustomLayout::new(|record: &Record| Ok(format!("{}", record.args()).into_bytes())).into() } fn text_layout() -> Layout { From 55cef11019851c858f22e6d9531824bbd4038607 Mon Sep 17 00:00:00 2001 From: Sky Fan <3374614481@qq.com> Date: Sun, 17 Nov 2024 22:02:38 +0800 Subject: [PATCH 53/92] chore: add some log (#16863) * chore: add some log * remove flood log * add log --- src/query/storages/common/cache/src/read/loader.rs | 1 + src/query/storages/fuse/src/io/snapshots.rs | 1 + .../fuse/src/table_functions/fuse_time_travel_size.rs | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/src/query/storages/common/cache/src/read/loader.rs b/src/query/storages/common/cache/src/read/loader.rs index cbc4b67e18ff..947ee8334bea 100644 --- a/src/query/storages/common/cache/src/read/loader.rs +++ b/src/query/storages/common/cache/src/read/loader.rs @@ -14,6 +14,7 @@ use databend_common_exception::Result; +#[derive(Debug)] pub struct LoadParams { pub location: String, pub len_hint: Option, diff --git a/src/query/storages/fuse/src/io/snapshots.rs b/src/query/storages/fuse/src/io/snapshots.rs index c09998fed3c1..2b720723a1f0 100644 --- a/src/query/storages/fuse/src/io/snapshots.rs +++ b/src/query/storages/fuse/src/io/snapshots.rs @@ -85,6 +85,7 @@ impl SnapshotsIO { ver, put_cache: true, }; + info!("read_snapshot will read: {:?}", load_params); let snapshot = reader.read(&load_params).await?; Ok((snapshot, ver)) } diff --git a/src/query/storages/fuse/src/table_functions/fuse_time_travel_size.rs b/src/query/storages/fuse/src/table_functions/fuse_time_travel_size.rs index a7d98c193fe7..ad4575480a2a 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_time_travel_size.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_time_travel_size.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use databend_common_catalog::plan::DataSourcePlan; +use databend_common_catalog::table::Table; use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; @@ -173,6 +174,10 @@ async fn get_time_travel_size(storage_prefix: &str, op: &Operator) -> Result Result<(u64, u64)> { + info!( + "fuse_time_travel_size start calc_tbl_size:{}", + tbl.get_table_info().desc + ); let operator = tbl.get_operator(); let storage_prefix = tbl.get_storage_prefix(); let start = std::time::Instant::now(); @@ -182,6 +187,7 @@ async fn calc_tbl_size(tbl: &FuseTable) -> Result<(u64, u64)> { let latest_snapshot_size = match snapshot_location { Some(snapshot_location) => { let start = std::time::Instant::now(); + info!("fuse_time_travel_size will read: {}", snapshot_location); let (snapshot, _) = SnapshotsIO::read_snapshot(snapshot_location, operator).await?; info!("read_snapshot cost: {:?}", start.elapsed()); snapshot.summary.compressed_byte_size + snapshot.summary.index_size From 1d55d5764c576686633e9fefcde2a690aee3dcaf Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Mon, 18 Nov 2024 12:53:14 +0800 Subject: [PATCH 54/92] feat(query): add distributed pruning settings (#16864) chore(query): add distributed pruning settings --- src/query/settings/src/settings_default.rs | 11 +++-- .../settings/src/settings_getter_setter.rs | 4 ++ .../fuse/src/operations/read_partitions.rs | 3 +- .../09_0102_distributed_pruning.sql | 41 +++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 tests/sqllogictests/suites/base/09_fuse_engine/09_0102_distributed_pruning.sql diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index 35e74afbb312..90e50c14782f 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -275,7 +275,7 @@ impl DefaultSettings { mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=1)), }), - ("enable_dio", DefaultSettingValue{ + ("enable_dio", DefaultSettingValue { value: UserSettingValue::UInt64(1), desc: "Enables Direct IO.", mode: SettingMode::Both, @@ -880,7 +880,6 @@ impl DefaultSettings { mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=u64::MAX)), }), - ("enable_auto_fix_missing_bloom_index", DefaultSettingValue { value: UserSettingValue::UInt64(0), desc: "Enables auto fix missing bloom index", @@ -939,7 +938,13 @@ impl DefaultSettings { value: UserSettingValue::UInt64(128), desc: "Sets the maximum length for truncating SQL queries in short_sql function.", mode: SettingMode::Both, - range: Some(SettingRange::Numeric(1..=1024*1024)), + range: Some(SettingRange::Numeric(1..=1024 * 1024)), + }), + ("enable_distributed_pruning", DefaultSettingValue { + value: UserSettingValue::UInt64(1), + desc: "Enable distributed index pruning, as it is very necessary and should remain enabled in the vast majority of cases.", + mode: SettingMode::Both, + range: Some(SettingRange::Numeric(0..=1)), }), ]); diff --git a/src/query/settings/src/settings_getter_setter.rs b/src/query/settings/src/settings_getter_setter.rs index 8938fcd616dd..fcaead2fe2c6 100644 --- a/src/query/settings/src/settings_getter_setter.rs +++ b/src/query/settings/src/settings_getter_setter.rs @@ -809,4 +809,8 @@ impl Settings { pub fn set_short_sql_max_length(&self, val: u64) -> Result<()> { self.try_set_u64("short_sql_max_length", val) } + + pub fn get_enable_distributed_pruning(&self) -> Result { + Ok(self.try_get_u64("enable_distributed_pruning")? == 1) + } } diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index e39e54a4237a..058ea80e573f 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -61,6 +61,7 @@ impl FuseTable { push_downs: Option, dry_run: bool, ) -> Result<(PartStatistics, Partitions)> { + let distributed_pruning = ctx.get_settings().get_enable_distributed_pruning()?; debug!("fuse table do read partitions, push downs:{:?}", push_downs); if let Some(changes_desc) = &self.changes_desc { // For "ANALYZE TABLE" statement, we need set the default change type to "Insert". @@ -86,7 +87,7 @@ impl FuseTable { nodes_num = cluster.nodes.len(); } - if !dry_run && snapshot.segments.len() > nodes_num { + if !dry_run && snapshot.segments.len() > nodes_num && distributed_pruning { let mut segments = Vec::with_capacity(snapshot.segments.len()); for (idx, segment_location) in snapshot.segments.iter().enumerate() { segments.push(FuseLazyPartInfo::create(idx, segment_location.clone())) diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0102_distributed_pruning.sql b/tests/sqllogictests/suites/base/09_fuse_engine/09_0102_distributed_pruning.sql new file mode 100644 index 000000000000..16495c3b74a4 --- /dev/null +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0102_distributed_pruning.sql @@ -0,0 +1,41 @@ + +statement ok +create or replace database distributed_pruning; + +statement ok +use distributed_pruning; + +statement ok +CREATE or replace TABLE test_table(TEXT String); + + +statement ok +INSERT INTO test_table VALUES('test_text1'); + +statement ok +INSERT INTO test_table VALUES('test_text2'); + +statement ok +INSERT INTO test_table VALUES('test_text3'); + +statement ok +INSERT INTO test_table VALUES('test_text4'); + +statement ok +SET enable_distributed_pruning = 1; + +query TTTTT +SELECT COUNT() FROM test_table; +---- +4 + +statement ok +SET enable_distributed_pruning = 0; + +query TTTTT +SELECT COUNT() FROM test_table; +---- +4 + +statement ok +SET enable_distributed_pruning = 1; From ed6a0c7682eede432e39c1d2d43bdcb0a9ff7cf9 Mon Sep 17 00:00:00 2001 From: Jk Xu <54522439+Dousir9@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:24:18 +0800 Subject: [PATCH 55/92] chore(query): improve distributed runtime filter (#16862) * chore(query): improve distributed runtime filter * chore(code): refine code * chore(code): refine code * chore(code); fix make lint * chore(query): add scan id to fix same table index * chore(code): make lint * chore(pipeline): fix deserializer runtime filter key --- .../src/plan/datasource/datasource_plan.rs | 1 + src/query/catalog/src/runtime_filter_info.rs | 23 +++++++++ src/query/catalog/src/table_context.rs | 9 ++++ .../interpreter_copy_into_table.rs | 1 + .../pipelines/builders/builder_aggregate.rs | 1 + .../src/pipelines/builders/builder_join.rs | 1 + .../pipelines/builders/builder_recluster.rs | 1 + .../src/pipelines/builders/builder_scan.rs | 2 + .../service/src/pipelines/pipeline_builder.rs | 3 ++ .../transforms/hash_join/build_state.rs | 5 ++ .../hash_join/hash_join_build_state.rs | 51 +++++++++++++++++++ .../src/schedulers/fragments/plan_fragment.rs | 1 + src/query/service/src/sessions/query_ctx.rs | 34 +++++++++++++ .../service/src/sessions/query_ctx_shared.rs | 8 +++ .../tests/it/sql/exec/get_table_bind_test.rs | 17 +++++++ .../it/storages/fuse/operations/commit.rs | 17 +++++++ .../physical_plans/physical_hash_join.rs | 9 +++- .../physical_plans/physical_table_scan.rs | 4 ++ src/query/sql/src/executor/table_read_plan.rs | 1 + src/query/sql/src/planner/binder/table.rs | 8 +++ src/query/sql/src/planner/metadata.rs | 26 +++++++++- .../optimizer/decorrelate/flatten_plan.rs | 1 + src/query/sql/src/planner/plans/scan.rs | 2 + .../read/parquet_data_source_deserializer.rs | 41 +++++++++++++++ 24 files changed, 265 insertions(+), 2 deletions(-) diff --git a/src/query/catalog/src/plan/datasource/datasource_plan.rs b/src/query/catalog/src/plan/datasource/datasource_plan.rs index 649c87da63ba..a8d56c2bacae 100644 --- a/src/query/catalog/src/plan/datasource/datasource_plan.rs +++ b/src/query/catalog/src/plan/datasource/datasource_plan.rs @@ -47,6 +47,7 @@ pub struct DataSourcePlan { pub data_mask_policy: Option>, pub table_index: usize, + pub scan_id: usize, } impl DataSourcePlan { diff --git a/src/query/catalog/src/runtime_filter_info.rs b/src/query/catalog/src/runtime_filter_info.rs index 7c29f193879f..5caeae409078 100644 --- a/src/query/catalog/src/runtime_filter_info.rs +++ b/src/query/catalog/src/runtime_filter_info.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_base::base::tokio::sync::watch; +use databend_common_base::base::tokio::sync::watch::Receiver; +use databend_common_base::base::tokio::sync::watch::Sender; use databend_common_expression::Expr; use xorf::BinaryFuse16; @@ -62,4 +65,24 @@ impl RuntimeFilterInfo { pub fn is_empty(&self) -> bool { self.inlist.is_empty() && self.bloom.is_empty() && self.min_max.is_empty() } + + pub fn is_blooms_empty(&self) -> bool { + self.bloom.is_empty() + } +} + +pub struct RuntimeFilterReady { + pub runtime_filter_watcher: Sender>, + /// A dummy receiver to make runtime_filter_watcher channel open. + pub _runtime_filter_dummy_receiver: Receiver>, +} + +impl Default for RuntimeFilterReady { + fn default() -> Self { + let (watcher, dummy_receiver) = watch::channel(None); + Self { + runtime_filter_watcher: watcher, + _runtime_filter_dummy_receiver: dummy_receiver, + } + } } diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index 6683f176875a..9ff28e69347e 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -74,6 +74,7 @@ use crate::plan::PartInfoPtr; use crate::plan::Partitions; use crate::query_kind::QueryKind; use crate::runtime_filter_info::RuntimeFilterInfo; +use crate::runtime_filter_info::RuntimeFilterReady; use crate::statistics::data_cache_statistics::DataCacheMetrics; use crate::table::Table; @@ -317,6 +318,14 @@ pub trait TableContext: Send + Sync { fn set_runtime_filter(&self, filters: (usize, RuntimeFilterInfo)); + fn set_runtime_filter_ready(&self, table_index: usize, ready: Arc); + + fn get_runtime_filter_ready(&self, table_index: usize) -> Vec>; + + fn set_wait_runtime_filter(&self, table_index: usize, need_to_wait: bool); + + fn get_wait_runtime_filter(&self, table_index: usize) -> bool; + fn clear_runtime_filter(&self); fn set_merge_into_join(&self, join: MergeIntoJoin); diff --git a/src/query/service/src/interpreters/interpreter_copy_into_table.rs b/src/query/service/src/interpreters/interpreter_copy_into_table.rs index e03d5500c87c..ff1894c9db06 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_table.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_table.rs @@ -137,6 +137,7 @@ impl CopyIntoTableInterpreter { ( CopyIntoTableSource::Stage(Box::new(PhysicalPlan::TableScan(TableScan { plan_id: 0, + scan_id: 0, name_mapping, stat_info: None, table_index: None, diff --git a/src/query/service/src/pipelines/builders/builder_aggregate.rs b/src/query/service/src/pipelines/builders/builder_aggregate.rs index bdd562923c8c..9c16033ae25d 100644 --- a/src/query/service/src/pipelines/builders/builder_aggregate.rs +++ b/src/query/service/src/pipelines/builders/builder_aggregate.rs @@ -97,6 +97,7 @@ impl PipelineBuilder { } pub(crate) fn build_aggregate_partial(&mut self, aggregate: &AggregatePartial) -> Result<()> { + self.contain_sink_processor = true; self.build_pipeline(&aggregate.input)?; let max_block_size = self.settings.get_max_block_size()?; diff --git a/src/query/service/src/pipelines/builders/builder_join.rs b/src/query/service/src/pipelines/builders/builder_join.rs index fd5ef8c2f906..07289a4ae51d 100644 --- a/src/query/service/src/pipelines/builders/builder_join.rs +++ b/src/query/service/src/pipelines/builders/builder_join.rs @@ -163,6 +163,7 @@ impl PipelineBuilder { join_state.clone(), output_len, )?; + build_state.add_runtime_filter_ready(); let create_sink_processor = |input| { Ok(ProcessorPtr::create(TransformHashJoinBuild::try_create( diff --git a/src/query/service/src/pipelines/builders/builder_recluster.rs b/src/query/service/src/pipelines/builders/builder_recluster.rs index 0690db51ebf4..dfc414490f9b 100644 --- a/src/query/service/src/pipelines/builders/builder_recluster.rs +++ b/src/query/service/src/pipelines/builders/builder_recluster.rs @@ -68,6 +68,7 @@ impl PipelineBuilder { update_stream_columns: table.change_tracking_enabled(), data_mask_policy: None, table_index: usize::MAX, + scan_id: usize::MAX, }; { diff --git a/src/query/service/src/pipelines/builders/builder_scan.rs b/src/query/service/src/pipelines/builders/builder_scan.rs index 2ba6f9ce135e..4e85b4e6542e 100644 --- a/src/query/service/src/pipelines/builders/builder_scan.rs +++ b/src/query/service/src/pipelines/builders/builder_scan.rs @@ -41,6 +41,8 @@ impl PipelineBuilder { pub(crate) fn build_table_scan(&mut self, scan: &TableScan) -> Result<()> { let table = self.ctx.build_table_from_source_plan(&scan.source)?; self.ctx.set_partitions(scan.source.parts.clone())?; + self.ctx + .set_wait_runtime_filter(scan.scan_id, self.contain_sink_processor); table.read_data( self.ctx.clone(), &scan.source, diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index 653324ffbdab..12e8463ee028 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -60,6 +60,8 @@ pub struct PipelineBuilder { pub r_cte_scan_interpreters: Vec, pub(crate) is_exchange_neighbor: bool, + + pub contain_sink_processor: bool, } impl PipelineBuilder { @@ -83,6 +85,7 @@ impl PipelineBuilder { hash_join_states: HashMap::new(), r_cte_scan_interpreters: vec![], is_exchange_neighbor: false, + contain_sink_processor: false, } } diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs index bddb87c0cf2a..59dbf495053f 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + +use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_expression::types::DataType; use databend_common_expression::ColumnVec; use databend_common_expression::DataBlock; @@ -23,6 +26,7 @@ pub struct BuildState { pub(crate) outer_scan_map: Vec>, /// LeftMarkScan map, initialized at `HashJoinBuildState`, used in `HashJoinProbeState` pub(crate) mark_scan_map: Vec>, + pub(crate) runtime_filter_ready: Vec>, } impl BuildState { @@ -31,6 +35,7 @@ impl BuildState { generation_state: BuildBlockGenerationState::new(), outer_scan_map: Vec::new(), mark_scan_map: Vec::new(), + runtime_filter_ready: Vec::new(), } } } diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs index 3af426c9401a..54e6e73e5236 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs @@ -23,6 +23,7 @@ use std::sync::Arc; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_base::base::tokio::sync::Barrier; use databend_common_catalog::runtime_filter_info::RuntimeFilterInfo; +use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -331,6 +332,7 @@ impl HashJoinBuildState { .build_watcher .send(HashTableType::Empty) .map_err(|_| ErrorCode::TokioError("build_watcher channel is closed"))?; + self.set_bloom_filter_ready(false)?; return Ok(()); } @@ -348,6 +350,8 @@ impl HashJoinBuildState { // If spilling happened, skip adding runtime filter, because probe data is ready and spilled. if self.hash_join_state.spilled_partitions.read().is_empty() { self.add_runtime_filter(&build_chunks, build_num_rows)?; + } else { + self.set_bloom_filter_ready(false)?; } // Divide the finalize phase into multiple tasks. @@ -848,7 +852,52 @@ impl HashJoinBuildState { Ok(()) } + pub fn add_runtime_filter_ready(&self) { + if self.ctx.get_cluster().is_empty() { + return; + } + + let mut wait_runtime_filter_table_indexes = HashSet::new(); + for (build_key, probe_key, table_index) in self + .hash_join_state + .hash_join_desc + .build_keys + .iter() + .zip(self.hash_join_state.hash_join_desc.probe_keys_rt.iter()) + .filter_map(|(b, p)| p.as_ref().map(|(p, index)| (b, p, index))) + { + if !build_key.data_type().remove_nullable().is_numeric() + && !build_key.data_type().remove_nullable().is_string() + { + continue; + } + if let Expr::ColumnRef { .. } = probe_key { + wait_runtime_filter_table_indexes.insert(*table_index); + } + } + + let build_state = unsafe { &mut *self.hash_join_state.build_state.get() }; + let runtime_filter_ready = &mut build_state.runtime_filter_ready; + for table_index in wait_runtime_filter_table_indexes.into_iter() { + let ready = Arc::new(RuntimeFilterReady::default()); + runtime_filter_ready.push(ready.clone()); + self.ctx.set_runtime_filter_ready(table_index, ready); + } + } + + pub fn set_bloom_filter_ready(&self, ready: bool) -> Result<()> { + let build_state = unsafe { &mut *self.hash_join_state.build_state.get() }; + for runtime_filter_ready in build_state.runtime_filter_ready.iter() { + runtime_filter_ready + .runtime_filter_watcher + .send(Some(ready)) + .map_err(|_| ErrorCode::TokioError("watcher channel is closed"))?; + } + Ok(()) + } + fn add_runtime_filter(&self, build_chunks: &[DataBlock], build_num_rows: usize) -> Result<()> { + let mut bloom_filter_ready = false; for (build_key, probe_key, table_index) in self .hash_join_state .hash_join_desc @@ -879,9 +928,11 @@ impl HashJoinBuildState { )?; } if !runtime_filter.is_empty() { + bloom_filter_ready |= !runtime_filter.is_blooms_empty(); self.ctx.set_runtime_filter((*table_index, runtime_filter)); } } + self.set_bloom_filter_ready(bloom_filter_ready)?; Ok(()) } diff --git a/src/query/service/src/schedulers/fragments/plan_fragment.rs b/src/query/service/src/schedulers/fragments/plan_fragment.rs index db415a5bc29c..da76a11358df 100644 --- a/src/query/service/src/schedulers/fragments/plan_fragment.rs +++ b/src/query/service/src/schedulers/fragments/plan_fragment.rs @@ -509,6 +509,7 @@ impl PhysicalPlanReplacer for ReplaceReadSource { Ok(PhysicalPlan::TableScan(TableScan { plan_id: plan.plan_id, + scan_id: plan.scan_id, source: Box::new(source), name_mapping: plan.name_mapping.clone(), table_index: plan.table_index, diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 30f64ea68377..c5822b6da492 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -50,6 +50,7 @@ use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::StageTableInfo; use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::runtime_filter_info::RuntimeFilterInfo; +use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::statistics::data_cache_statistics::DataCacheMetrics; use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::ContextError; @@ -1177,6 +1178,39 @@ impl TableContext for QueryContext { } } + fn set_runtime_filter_ready(&self, table_index: usize, ready: Arc) { + let mut runtime_filter_ready = self.shared.runtime_filter_ready.write(); + match runtime_filter_ready.entry(table_index) { + Entry::Vacant(v) => { + v.insert(vec![ready]); + } + Entry::Occupied(mut v) => { + v.get_mut().push(ready); + } + } + } + + fn get_runtime_filter_ready(&self, table_index: usize) -> Vec> { + let runtime_filter_ready = self.shared.runtime_filter_ready.read(); + match runtime_filter_ready.get(&table_index) { + Some(v) => v.to_vec(), + None => vec![], + } + } + + fn set_wait_runtime_filter(&self, table_index: usize, need_to_wait: bool) { + let mut wait_runtime_filter = self.shared.wait_runtime_filter.write(); + wait_runtime_filter.insert(table_index, need_to_wait); + } + + fn get_wait_runtime_filter(&self, table_index: usize) -> bool { + let wait_runtime_filter = self.shared.wait_runtime_filter.read(); + match wait_runtime_filter.get(&table_index) { + Some(v) => *v, + None => false, + } + } + fn get_merge_into_join(&self) -> MergeIntoJoin { let merge_into_join = self.shared.merge_into_join.read(); MergeIntoJoin { diff --git a/src/query/service/src/sessions/query_ctx_shared.rs b/src/query/service/src/sessions/query_ctx_shared.rs index 547babc3b606..145057beb63e 100644 --- a/src/query/service/src/sessions/query_ctx_shared.rs +++ b/src/query/service/src/sessions/query_ctx_shared.rs @@ -31,6 +31,7 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::merge_into_join::MergeIntoJoin; use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::runtime_filter_info::RuntimeFilterInfo; +use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::statistics::data_cache_statistics::DataCacheMetrics; use databend_common_catalog::table_context::ContextError; use databend_common_catalog::table_context::MaterializedCtesBlocks; @@ -132,6 +133,11 @@ pub struct QueryContextShared { pub(in crate::sessions) runtime_filters: Arc>>, + pub(in crate::sessions) runtime_filter_ready: + Arc>>>>, + + pub(in crate::sessions) wait_runtime_filter: Arc>>, + pub(in crate::sessions) merge_into_join: Arc>, // Records query level data cache metrics @@ -189,6 +195,8 @@ impl QueryContextShared { query_cache_metrics: DataCacheMetrics::new(), query_profiles: Arc::new(RwLock::new(HashMap::new())), runtime_filters: Default::default(), + runtime_filter_ready: Default::default(), + wait_runtime_filter: Default::default(), merge_into_join: Default::default(), multi_table_insert_status: Default::default(), query_queued_duration: Arc::new(RwLock::new(Duration::from_secs(0))), diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 9bd031cf5598..16a8824240a8 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -33,6 +33,7 @@ use databend_common_catalog::plan::PartInfoPtr; use databend_common_catalog::plan::Partitions; use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::runtime_filter_info::RuntimeFilterInfo; +use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::statistics::data_cache_statistics::DataCacheMetrics; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::ContextError; @@ -928,6 +929,22 @@ impl TableContext for CtxDelegation { todo!() } + fn set_runtime_filter_ready(&self, _table_index: usize, _ready: Arc) { + todo!() + } + + fn get_runtime_filter_ready(&self, _table_index: usize) -> Vec> { + todo!() + } + + fn set_wait_runtime_filter(&self, _table_index: usize, _need_to_wait: bool) { + todo!() + } + + fn get_wait_runtime_filter(&self, _table_index: usize) -> bool { + todo!() + } + fn clear_runtime_filter(&self) { todo!() } diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 3453e4d96647..94b3f39457c9 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -32,6 +32,7 @@ use databend_common_catalog::plan::PartInfoPtr; use databend_common_catalog::plan::Partitions; use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::runtime_filter_info::RuntimeFilterInfo; +use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::statistics::data_cache_statistics::DataCacheMetrics; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::ContextError; @@ -808,6 +809,22 @@ impl TableContext for CtxDelegation { todo!() } + fn set_runtime_filter_ready(&self, _table_index: usize, _ready: Arc) { + todo!() + } + + fn get_runtime_filter_ready(&self, _table_index: usize) -> Vec> { + todo!() + } + + fn set_wait_runtime_filter(&self, _table_index: usize, _need_to_wait: bool) { + todo!() + } + + fn get_wait_runtime_filter(&self, _table_index: usize) -> bool { + todo!() + } + fn clear_runtime_filter(&self) { todo!() } diff --git a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs index 0d947551bd4b..641648cce95e 100644 --- a/src/query/sql/src/executor/physical_plans/physical_hash_join.rs +++ b/src/query/sql/src/executor/physical_plans/physical_hash_join.rs @@ -215,6 +215,7 @@ impl PhysicalPlanBuilder { let mut left_join_conditions_rt = Vec::new(); let mut probe_to_build_index = Vec::new(); let mut table_index = None; + let mut scan_id = None; for condition in join.equi_conditions.iter() { let left_condition = &condition.left; let right_condition = &condition.right; @@ -243,13 +244,19 @@ impl PhysicalPlanBuilder { .table_index() .unwrap(), ); + scan_id = Some( + self.metadata + .read() + .base_column_scan_id(*column_idx) + .unwrap(), + ); } Some(( left_condition .as_raw_expr() .type_check(&*self.metadata.read())? .project_column_ref(|col| col.column_name.clone()), - table_index.unwrap(), + scan_id.unwrap(), )) } else { None diff --git a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs index 09d3bc58b6a4..891c892f7531 100644 --- a/src/query/sql/src/executor/physical_plans/physical_table_scan.rs +++ b/src/query/sql/src/executor/physical_plans/physical_table_scan.rs @@ -71,6 +71,7 @@ use crate::DUMMY_TABLE_INDEX; pub struct TableScan { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, + pub scan_id: usize, pub name_mapping: BTreeMap, pub source: Box, pub internal_column: Option>, @@ -263,6 +264,7 @@ impl PhysicalPlanBuilder { } } source.table_index = scan.table_index; + source.scan_id = scan.scan_id; if let Some(agg_index) = &scan.agg_index { let source_schema = source.schema(); let push_down = source.push_downs.as_mut().unwrap(); @@ -283,6 +285,7 @@ impl PhysicalPlanBuilder { let mut plan = PhysicalPlan::TableScan(TableScan { plan_id: 0, + scan_id: scan.scan_id, name_mapping, source: Box::new(source), table_index: Some(scan.table_index), @@ -319,6 +322,7 @@ impl PhysicalPlanBuilder { .await?; Ok(PhysicalPlan::TableScan(TableScan { plan_id: 0, + scan_id: DUMMY_TABLE_INDEX, name_mapping: BTreeMap::from([("dummy".to_string(), DUMMY_COLUMN_INDEX)]), source: Box::new(source), table_index: Some(DUMMY_TABLE_INDEX), diff --git a/src/query/sql/src/executor/table_read_plan.rs b/src/query/sql/src/executor/table_read_plan.rs index 83a39b4e286c..2d401e806503 100644 --- a/src/query/sql/src/executor/table_read_plan.rs +++ b/src/query/sql/src/executor/table_read_plan.rs @@ -281,6 +281,7 @@ impl ToReadDataSourcePlan for dyn Table { data_mask_policy, // Set a dummy id, will be set real id later table_index: usize::MAX, + scan_id: usize::MAX, }) } } diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index c8136d07d089..45c646fbc2c8 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::BTreeMap; +use std::collections::HashMap; use std::default::Default; use std::sync::Arc; @@ -455,6 +456,8 @@ impl Binder { let table = self.metadata.read().table(table_index).clone(); let table_name = table.name(); let columns = self.metadata.read().columns_by_table_index(table_index); + let scan_id = self.metadata.write().next_scan_id(); + let mut base_column_scan_id = HashMap::new(); for column in columns.iter() { match column { ColumnEntry::BaseTableColumn(BaseTableColumn { @@ -484,6 +487,7 @@ impl Binder { .virtual_computed_expr(virtual_computed_expr.clone()) .build(); bind_context.add_column_binding(column_binding); + base_column_scan_id.insert(*column_index, scan_id); } other => { return Err(ErrorCode::Internal(format!( @@ -494,6 +498,9 @@ impl Binder { } } } + self.metadata + .write() + .add_base_column_scan_id(base_column_scan_id); Ok(( SExpr::create_leaf(Arc::new( @@ -503,6 +510,7 @@ impl Binder { statistics: Arc::new(Statistics::default()), change_type, sample: sample.clone(), + scan_id, ..Default::default() } .into(), diff --git a/src/query/sql/src/planner/metadata.rs b/src/query/sql/src/planner/metadata.rs index 7ddce096658d..042766c848d5 100644 --- a/src/query/sql/src/planner/metadata.rs +++ b/src/query/sql/src/planner/metadata.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; -use ahash::HashMap; use databend_common_ast::ast::Expr; use databend_common_ast::ast::Literal; use databend_common_catalog::plan::DataSourcePlan; @@ -74,6 +74,11 @@ pub struct Metadata { table_row_id_index: HashMap, agg_indexes: HashMap>, max_column_position: usize, // for CSV + + /// Scan id of each scan operator. + next_scan_id: usize, + /// Mappings from base column index to scan id. + base_column_scan_id: HashMap, } impl Metadata { @@ -462,9 +467,24 @@ impl Metadata { pub fn set_max_column_position(&mut self, max_pos: usize) { self.max_column_position = max_pos } + pub fn get_max_column_position(&self) -> usize { self.max_column_position } + + pub fn next_scan_id(&mut self) -> usize { + let next_scan_id = self.next_scan_id; + self.next_scan_id += 1; + next_scan_id + } + + pub fn add_base_column_scan_id(&mut self, base_column_scan_id: HashMap) { + self.base_column_scan_id.extend(base_column_scan_id); + } + + pub fn base_column_scan_id(&self, column_index: usize) -> Option { + self.base_column_scan_id.get(&column_index).cloned() + } } #[derive(Clone)] @@ -569,6 +589,10 @@ impl TableEntry { pub fn is_consume(&self) -> bool { self.consume } + + pub fn update_table_index(&mut self, table_index: IndexType) { + self.index = table_index; + } } #[derive(Clone, Debug)] diff --git a/src/query/sql/src/planner/optimizer/decorrelate/flatten_plan.rs b/src/query/sql/src/planner/optimizer/decorrelate/flatten_plan.rs index 3d18059714ed..c208878c78fd 100644 --- a/src/query/sql/src/planner/optimizer/decorrelate/flatten_plan.rs +++ b/src/query/sql/src/planner/optimizer/decorrelate/flatten_plan.rs @@ -108,6 +108,7 @@ impl SubqueryRewriter { Scan { table_index, columns: scan_columns, + scan_id: metadata.next_scan_id(), ..Default::default() } .into(), diff --git a/src/query/sql/src/planner/plans/scan.rs b/src/query/sql/src/planner/plans/scan.rs index 72124bf43945..7a7c8f2d18e3 100644 --- a/src/query/sql/src/planner/plans/scan.rs +++ b/src/query/sql/src/planner/plans/scan.rs @@ -107,6 +107,7 @@ pub struct Scan { // Lazy row fetch. pub is_lazy_table: bool, pub sample: Option, + pub scan_id: usize, pub statistics: Arc, } @@ -147,6 +148,7 @@ impl Scan { inverted_index: self.inverted_index.clone(), is_lazy_table: self.is_lazy_table, sample: self.sample.clone(), + scan_id: self.scan_id, } } diff --git a/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs b/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs index c4c6d9d916f6..d58dffde4a7a 100644 --- a/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs +++ b/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs @@ -25,7 +25,9 @@ use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::PartInfoPtr; +use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_expression::BlockMetaInfoDowncast; @@ -56,6 +58,7 @@ use crate::operations::read::runtime_filter_prunner::update_bitmap_with_bloom_fi pub struct DeserializeDataTransform { ctx: Arc, table_index: IndexType, + scan_id: usize, scan_progress: Arc, block_reader: Arc, @@ -74,6 +77,8 @@ pub struct DeserializeDataTransform { cached_runtime_filter: Option>, // for merge_into target build. need_reserve_block_info: bool, + need_wait_runtime_filter: bool, + runtime_filter_ready: Option>, } unsafe impl Send for DeserializeDataTransform {} @@ -89,6 +94,8 @@ impl DeserializeDataTransform { virtual_reader: Arc>, ) -> Result { let scan_progress = ctx.get_scan_progress(); + let need_wait_runtime_filter = + !ctx.get_cluster().is_empty() && ctx.get_wait_runtime_filter(plan.scan_id); let mut src_schema: DataSchema = (block_reader.schema().as_ref()).into(); if let Some(virtual_reader) = virtual_reader.as_ref() { @@ -110,6 +117,7 @@ impl DeserializeDataTransform { Ok(ProcessorPtr::create(Box::new(DeserializeDataTransform { ctx, table_index: plan.table_index, + scan_id: plan.scan_id, scan_progress, block_reader, input, @@ -124,6 +132,8 @@ impl DeserializeDataTransform { base_block_ids: plan.base_block_ids.clone(), cached_runtime_filter: None, need_reserve_block_info, + need_wait_runtime_filter, + runtime_filter_ready: None, }))) } @@ -169,6 +179,20 @@ impl DeserializeDataTransform { Ok(None) } } + + fn need_wait_runtime_filter(&mut self) -> bool { + if !self.need_wait_runtime_filter { + return false; + } + self.need_wait_runtime_filter = false; + let runtime_filter_ready = self.ctx.get_runtime_filter_ready(self.scan_id); + if runtime_filter_ready.len() == 1 { + self.runtime_filter_ready = Some(runtime_filter_ready[0].clone()); + true + } else { + false + } + } } #[async_trait::async_trait] @@ -182,6 +206,10 @@ impl Processor for DeserializeDataTransform { } fn event(&mut self) -> Result { + if self.need_wait_runtime_filter() { + return Ok(Event::Async); + } + if self.output.is_finished() { self.input.finish(); return Ok(Event::Finished); @@ -326,4 +354,17 @@ impl Processor for DeserializeDataTransform { Ok(()) } + + #[async_backtrace::framed] + async fn async_process(&mut self) -> Result<()> { + let runtime_filter_ready = self.runtime_filter_ready.as_mut().unwrap(); + let mut rx = runtime_filter_ready.runtime_filter_watcher.subscribe(); + if (*rx.borrow()).is_some() { + return Ok(()); + } + rx.changed() + .await + .map_err(|_| ErrorCode::TokioError("watcher's sender is dropped"))?; + Ok(()) + } } From 357404c19e9c22db0c7a819c4331c664fcb3584e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Tue, 19 Nov 2024 11:58:34 +0800 Subject: [PATCH 56/92] refactor: make ListKV non-blocking by returning stream directly (#16868) perf: make ListKV non-blocking by returning stream directly Return Stream to client instead of collecting results into Vec first. This change eliminates unnecessary buffering of ListKV results in databend-meta server, reducing memory usage and improving response times. The direct streaming approach better matches current usage patterns and removes legacy collection behavior. Although a large ListKV response body won't block the databend-meta server, but it will still block on the client side. --- Cargo.lock | 1 + src/meta/raft-store/src/state_machine_api_ext.rs | 5 ----- src/meta/service/Cargo.toml | 1 + src/meta/service/src/meta_service/meta_leader.rs | 16 ++++++++++------ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e91d4d9ca32f..61c501affe8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4979,6 +4979,7 @@ dependencies = [ "temp-env", "tempfile", "test-harness", + "tokio", "tokio-stream", "tonic", "tonic-reflection", diff --git a/src/meta/raft-store/src/state_machine_api_ext.rs b/src/meta/raft-store/src/state_machine_api_ext.rs index 359af9a21681..5da204110602 100644 --- a/src/meta/raft-store/src/state_machine_api_ext.rs +++ b/src/meta/raft-store/src/state_machine_api_ext.rs @@ -107,11 +107,6 @@ pub trait StateMachineApiExt: StateMachineApi { future::ready(Ok(res)) }); - // Make it static - - let vs = strm.collect::>().await; - let strm = futures::stream::iter(vs); - Ok(strm.boxed()) } diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index 2ea2d38a0f46..f10704003d81 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -59,6 +59,7 @@ semver = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serfig = { workspace = true } +tokio = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true } tonic-reflection = { workspace = true } diff --git a/src/meta/service/src/meta_service/meta_leader.rs b/src/meta/service/src/meta_service/meta_leader.rs index 9dfce821c98c..719401559a43 100644 --- a/src/meta/service/src/meta_service/meta_leader.rs +++ b/src/meta/service/src/meta_service/meta_leader.rs @@ -36,11 +36,13 @@ use databend_common_meta_types::MetaOperationError; use databend_common_meta_types::Node; use databend_common_metrics::count::Count; use futures::StreamExt; +use futures::TryStreamExt; use log::debug; use log::info; use maplit::btreemap; use maplit::btreeset; use tonic::codegen::BoxStream; +use tonic::Status; use crate::message::ForwardRequest; use crate::message::ForwardRequestBody; @@ -147,12 +149,14 @@ impl<'a> Handler for MetaLeader<'a> { } MetaGrpcReadReq::ListKV(req) => { - // safe unwrap(): Infallible - let kvs = kv_api.prefix_list_kv(&req.prefix).await.unwrap(); - - let kv_iter = kvs.into_iter().map(|kv| Ok(StreamItem::from(kv))); - - let strm = futures::stream::iter(kv_iter); + let strm = + kv_api.list_kv(&req.prefix).await.map_err(|e| { + MetaOperationError::DataError(MetaDataError::ReadError( + MetaDataReadError::new("list_kv", &req.prefix, &e), + )) + })?; + + let strm = strm.map_err(|e| Status::internal(e.to_string())); Ok(strm.boxed()) } From aac6e097ce8600f9482866edc2c259325d0dfb7a Mon Sep 17 00:00:00 2001 From: dantengsky Date: Tue, 19 Nov 2024 12:33:46 +0800 Subject: [PATCH 57/92] chore: enable stderr logging for `table_meta_inspector` (#16865) --- src/binaries/tool/table_meta_inspector.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/binaries/tool/table_meta_inspector.rs b/src/binaries/tool/table_meta_inspector.rs index 2aef10d22e63..4bf93b379953 100644 --- a/src/binaries/tool/table_meta_inspector.rs +++ b/src/binaries/tool/table_meta_inspector.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; use std::env; use std::fs::File; use std::io; @@ -27,6 +28,8 @@ use databend_common_config::DATABEND_COMMIT_VERSION; use databend_common_exception::Result; use databend_common_storage::init_operator; use databend_common_storage::StorageConfig; +use databend_common_tracing::init_logging; +use databend_common_tracing::Config as LogConfig; use databend_query::GlobalServices; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; @@ -121,6 +124,10 @@ async fn run(config: &InspectorConfig) -> Result<()> { #[tokio::main] async fn main() -> Result<()> { + let mut log_config = LogConfig::default(); + log_config.stderr.on = true; + let _guards = init_logging("table_meta_inspector", &log_config, BTreeMap::new()); + let config = InspectorConfig::parse(); if let Err(err) = run(&config).await { From b40336b3d92fd849904ad5fe5e5f76d4acc138a7 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Tue, 19 Nov 2024 20:31:54 +0800 Subject: [PATCH 58/92] chore(query): remove common/arrow crate (#16846) * add column crate * add column crate * update * update * update * update * finish todo * finish todos * remove common/arrow * remove tests * refactor functions * native-part * update * update * update * add column tests * update * update * update * update * update * update * update * update * update * update * fix * update * update * update --- Cargo.lock | 151 ++-- Cargo.toml | 7 +- src/common/arrow/Cargo.toml | 79 -- src/common/arrow/src/arrow/array/README.md | 73 -- .../arrow/src/arrow/array/binary/data.rs | 61 -- .../arrow/src/arrow/array/binary/from.rs | 27 - .../arrow/src/arrow/array/binary/iterator.rs | 60 -- .../arrow/src/arrow/array/binary/mod.rs | 455 ---------- .../arrow/src/arrow/array/binary/mutable.rs | 494 ---------- .../src/arrow/array/binary/mutable_values.rs | 402 --------- .../arrow/src/arrow/array/binview/from.rs | 100 --- .../arrow/src/arrow/array/boolean/data.rs | 54 -- .../arrow/src/arrow/array/boolean/fmt.rs | 35 - .../arrow/src/arrow/array/boolean/from.rs | 31 - .../arrow/src/arrow/array/boolean/iterator.rs | 72 -- .../arrow/src/arrow/array/boolean/mod.rs | 411 --------- .../arrow/src/arrow/array/boolean/mutable.rs | 580 ------------ .../arrow/src/arrow/array/dictionary/data.rs | 69 -- .../arrow/src/arrow/array/dictionary/fmt.rs | 51 -- .../src/arrow/array/dictionary/iterator.rs | 84 -- .../arrow/src/arrow/array/dictionary/mod.rs | 443 --------- .../src/arrow/array/dictionary/mutable.rs | 260 ------ .../arrow/array/dictionary/typed_iterator.rs | 123 --- .../src/arrow/array/dictionary/value_map.rs | 192 ---- .../arrow/src/arrow/array/equal/binary.rs | 21 - .../src/arrow/array/equal/binary_view.rs | 25 - .../arrow/src/arrow/array/equal/boolean.rs | 20 - .../arrow/src/arrow/array/equal/dictionary.rs | 30 - .../arrow/array/equal/fixed_size_binary.rs | 21 - .../src/arrow/array/equal/fixed_size_list.rs | 21 - .../arrow/src/arrow/array/equal/list.rs | 22 - src/common/arrow/src/arrow/array/equal/map.rs | 21 - src/common/arrow/src/arrow/array/equal/mod.rs | 313 ------- .../arrow/src/arrow/array/equal/null.rs | 22 - .../arrow/src/arrow/array/equal/primitive.rs | 21 - .../arrow/src/arrow/array/equal/struct_.rs | 70 -- .../arrow/src/arrow/array/equal/union.rs | 21 - .../arrow/src/arrow/array/equal/utf8.rs | 21 - .../src/arrow/array/fixed_size_binary/data.rs | 54 -- .../src/arrow/array/fixed_size_binary/fmt.rs | 38 - .../arrow/array/fixed_size_binary/iterator.rs | 66 -- .../src/arrow/array/fixed_size_binary/mod.rs | 307 ------- .../arrow/array/fixed_size_binary/mutable.rs | 345 ------- .../src/arrow/array/fixed_size_list/data.rs | 55 -- .../src/arrow/array/fixed_size_list/fmt.rs | 43 - .../arrow/array/fixed_size_list/iterator.rs | 61 -- .../src/arrow/array/fixed_size_list/mod.rs | 244 ----- .../arrow/array/fixed_size_list/mutable.rs | 283 ------ src/common/arrow/src/arrow/array/fmt.rs | 213 ----- .../arrow/src/arrow/array/growable/binary.rs | 121 --- .../arrow/src/arrow/array/growable/binview.rs | 218 ----- .../arrow/src/arrow/array/growable/boolean.rs | 108 --- .../src/arrow/array/growable/dictionary.rs | 178 ---- .../src/arrow/array/growable/fixed_binary.rs | 115 --- .../arrow/array/growable/fixed_size_list.rs | 125 --- .../arrow/src/arrow/array/growable/list.rs | 131 --- .../arrow/src/arrow/array/growable/map.rs | 125 --- .../arrow/src/arrow/array/growable/mod.rs | 186 ---- .../arrow/src/arrow/array/growable/null.rs | 72 -- .../src/arrow/array/growable/primitive.rs | 118 --- .../src/arrow/array/growable/structure.rs | 150 ---- .../arrow/src/arrow/array/growable/union.rs | 137 --- .../arrow/src/arrow/array/growable/utf8.rs | 123 --- .../arrow/src/arrow/array/growable/utils.rs | 84 -- src/common/arrow/src/arrow/array/indexable.rs | 212 ----- src/common/arrow/src/arrow/array/list/data.rs | 58 -- src/common/arrow/src/arrow/array/list/fmt.rs | 49 - .../arrow/src/arrow/array/list/iterator.rs | 86 -- src/common/arrow/src/arrow/array/list/mod.rs | 266 ------ .../arrow/src/arrow/array/list/mutable.rs | 343 ------- src/common/arrow/src/arrow/array/map/data.rs | 57 -- src/common/arrow/src/arrow/array/map/fmt.rs | 43 - .../arrow/src/arrow/array/map/iterator.rs | 97 -- src/common/arrow/src/arrow/array/map/mod.rs | 227 ----- src/common/arrow/src/arrow/array/mod.rs | 842 ------------------ src/common/arrow/src/arrow/array/null.rs | 195 ---- src/common/arrow/src/arrow/array/ord.rs | 258 ------ .../arrow/src/arrow/array/physical_binary.rs | 246 ----- .../arrow/src/arrow/array/primitive/data.rs | 50 -- .../arrow/src/arrow/array/primitive/fmt.rs | 171 ---- .../src/arrow/array/primitive/from_natural.rs | 32 - .../src/arrow/array/primitive/iterator.rs | 64 -- .../arrow/src/arrow/array/primitive/mod.rs | 541 ----------- .../src/arrow/array/primitive/mutable.rs | 682 -------------- .../arrow/src/arrow/array/specification.rs | 196 ---- .../arrow/src/arrow/array/struct_/data.rs | 47 - .../arrow/src/arrow/array/struct_/fmt.rs | 54 -- .../arrow/src/arrow/array/struct_/iterator.rs | 113 --- .../arrow/src/arrow/array/struct_/mod.rs | 272 ------ .../arrow/src/arrow/array/struct_/mutable.rs | 261 ------ .../arrow/src/arrow/array/union/data.rs | 89 -- src/common/arrow/src/arrow/array/union/fmt.rs | 43 - .../arrow/src/arrow/array/union/iterator.rs | 74 -- src/common/arrow/src/arrow/array/union/mod.rs | 402 --------- src/common/arrow/src/arrow/array/utf8/data.rs | 60 -- src/common/arrow/src/arrow/array/utf8/fmt.rs | 41 - src/common/arrow/src/arrow/array/utf8/from.rs | 27 - .../arrow/src/arrow/array/utf8/iterator.rs | 98 -- src/common/arrow/src/arrow/array/utf8/mod.rs | 581 ------------ .../arrow/src/arrow/array/utf8/mutable.rs | 581 ------------ .../src/arrow/array/utf8/mutable_values.rs | 440 --------- src/common/arrow/src/arrow/chunk.rs | 100 --- src/common/arrow/src/arrow/compute/README.md | 32 - .../arrow/src/arrow/compute/concatenate.rs | 64 -- src/common/arrow/src/arrow/compute/mod.rs | 32 - src/common/arrow/src/arrow/compute/utils.rs | 39 - src/common/arrow/src/arrow/datatypes/field.rs | 135 --- src/common/arrow/src/arrow/datatypes/mod.rs | 539 ----------- .../src/arrow/datatypes/physical_type.rs | 97 -- .../arrow/src/arrow/datatypes/schema.rs | 78 -- src/common/arrow/src/arrow/mod.rs | 29 - src/common/arrow/src/arrow/scalar/README.md | 28 - src/common/arrow/src/arrow/scalar/binary.rs | 70 -- src/common/arrow/src/arrow/scalar/binview.rs | 88 -- src/common/arrow/src/arrow/scalar/boolean.rs | 61 -- .../arrow/src/arrow/scalar/dictionary.rs | 70 -- src/common/arrow/src/arrow/scalar/equal.rs | 77 -- .../src/arrow/scalar/fixed_size_binary.rs | 73 -- .../arrow/src/arrow/scalar/fixed_size_list.rs | 75 -- src/common/arrow/src/arrow/scalar/list.rs | 84 -- src/common/arrow/src/arrow/scalar/map.rs | 82 -- src/common/arrow/src/arrow/scalar/mod.rs | 223 ----- src/common/arrow/src/arrow/scalar/null.rs | 52 -- .../arrow/src/arrow/scalar/primitive.rs | 82 -- src/common/arrow/src/arrow/scalar/struct_.rs | 69 -- src/common/arrow/src/arrow/scalar/union.rs | 67 -- src/common/arrow/src/arrow/scalar/utf8.rs | 70 -- .../arrow/src/arrow/temporal_conversions.rs | 578 ------------ src/common/arrow/src/arrow/trusted_len.rs | 72 -- src/common/arrow/src/arrow/util/bench_util.rs | 115 --- src/common/arrow/src/arrow/util/mod.rs | 18 - src/common/arrow/src/native/nested.rs | 403 --------- .../arrow/src/native/read/array/binary.rs | 176 ---- .../arrow/src/native/read/batch_read.rs | 166 ---- .../arrow/src/native/read/deserialize.rs | 204 ----- src/common/arrow/src/native/util/mod.rs | 101 --- .../arrow/src/native/write/primitive.rs | 93 -- .../arrow/src/native/write/serialize.rs | 147 --- src/common/arrow/src/schema_projection.rs | 60 -- .../arrow/tests/it/arrow/array/binary/mod.rs | 231 ----- .../tests/it/arrow/array/binary/mutable.rs | 233 ----- .../it/arrow/array/binary/mutable_values.rs | 113 --- .../tests/it/arrow/array/binary/to_mutable.rs | 85 -- .../arrow/tests/it/arrow/array/binview/mod.rs | 211 ----- .../tests/it/arrow/array/binview/mutable.rs | 50 -- .../it/arrow/array/binview/mutable_values.rs | 45 - .../it/arrow/array/binview/to_mutable.rs | 46 - .../arrow/tests/it/arrow/array/boolean/mod.rs | 157 ---- .../tests/it/arrow/array/boolean/mutable.rs | 194 ---- .../tests/it/arrow/array/dictionary/mod.rs | 229 ----- .../it/arrow/array/dictionary/mutable.rs | 185 ---- .../tests/it/arrow/array/equal/boolean.rs | 68 -- .../tests/it/arrow/array/equal/dictionary.rs | 135 --- .../it/arrow/array/equal/fixed_size_list.rs | 98 -- .../arrow/tests/it/arrow/array/equal/list.rs | 109 --- .../arrow/tests/it/arrow/array/equal/mod.rs | 65 -- .../tests/it/arrow/array/equal/primitive.rs | 105 --- .../it/arrow/array/fixed_size_binary/mod.rs | 114 --- .../arrow/array/fixed_size_binary/mutable.rs | 186 ---- .../it/arrow/array/fixed_size_list/mod.rs | 124 --- .../it/arrow/array/fixed_size_list/mutable.rs | 104 --- .../tests/it/arrow/array/growable/binary.rs | 101 --- .../tests/it/arrow/array/growable/boolean.rs | 33 - .../it/arrow/array/growable/dictionary.rs | 82 -- .../it/arrow/array/growable/fixed_binary.rs | 144 --- .../arrow/array/growable/fixed_size_list.rs | 104 --- .../tests/it/arrow/array/growable/list.rs | 156 ---- .../tests/it/arrow/array/growable/map.rs | 132 --- .../tests/it/arrow/array/growable/mod.rs | 89 -- .../tests/it/arrow/array/growable/null.rs | 33 - .../it/arrow/array/growable/primitive.rs | 84 -- .../tests/it/arrow/array/growable/struct_.rs | 149 ---- .../tests/it/arrow/array/growable/union.rs | 165 ---- .../tests/it/arrow/array/growable/utf8.rs | 101 --- .../arrow/tests/it/arrow/array/list/mod.rs | 85 -- .../tests/it/arrow/array/list/mutable.rs | 91 -- .../arrow/tests/it/arrow/array/map/mod.rs | 68 -- src/common/arrow/tests/it/arrow/array/mod.rs | 165 ---- src/common/arrow/tests/it/arrow/array/ord.rs | 132 --- .../tests/it/arrow/array/primitive/fmt.rs | 237 ----- .../tests/it/arrow/array/primitive/mod.rs | 157 ---- .../tests/it/arrow/array/primitive/mutable.rs | 346 ------- .../it/arrow/array/primitive/to_mutable.rs | 68 -- .../tests/it/arrow/array/struct_/iterator.rs | 43 - .../arrow/tests/it/arrow/array/struct_/mod.rs | 42 - .../tests/it/arrow/array/struct_/mutable.rs | 47 - .../arrow/tests/it/arrow/array/union.rs | 390 -------- .../arrow/tests/it/arrow/array/utf8/mod.rs | 253 ------ .../tests/it/arrow/array/utf8/mutable.rs | 257 ------ .../it/arrow/array/utf8/mutable_values.rs | 117 --- .../tests/it/arrow/array/utf8/to_mutable.rs | 86 -- src/common/arrow/tests/it/arrow/arrow_data.rs | 385 -------- src/common/arrow/tests/it/arrow/buffer/mod.rs | 16 - .../tests/it/arrow/compute/concatenate.rs | 132 --- .../arrow/tests/it/arrow/compute/mod.rs | 17 - .../arrow/tests/it/arrow/scalar/binary.rs | 47 - .../arrow/tests/it/arrow/scalar/boolean.rs | 42 - .../it/arrow/scalar/fixed_size_binary.rs | 42 - .../tests/it/arrow/scalar/fixed_size_list.rs | 58 -- .../arrow/tests/it/arrow/scalar/list.rs | 52 -- src/common/arrow/tests/it/arrow/scalar/map.rs | 85 -- src/common/arrow/tests/it/arrow/scalar/mod.rs | 34 - .../arrow/tests/it/arrow/scalar/null.rs | 35 - .../arrow/tests/it/arrow/scalar/primitive.rs | 52 -- .../arrow/tests/it/arrow/scalar/struct_.rs | 59 -- .../arrow/tests/it/arrow/scalar/utf8.rs | 47 - .../tests/it/arrow/temporal_conversions.rs | 307 ------- src/common/arrow/tests/it/arrow/types.rs | 59 -- src/common/arrow/tests/it/native/io.rs | 557 ------------ src/common/base/Cargo.toml | 1 + src/common/base/src/base/ordered_float.rs | 7 + src/common/column/Cargo.toml | 47 + src/common/column/src/binary/builder.rs | 247 +++++ .../arrow/array => column/src}/binary/fmt.rs | 37 +- src/common/column/src/binary/iterator.rs | 61 ++ src/common/column/src/binary/mod.rs | 170 ++++ .../src/binview/builder.rs} | 227 ++--- .../arrow/array => column/src}/binview/fmt.rs | 25 +- .../array => column/src}/binview/iterator.rs | 28 +- .../arrow/array => column/src}/binview/mod.rs | 535 ++++++----- .../array => column/src}/binview/view.rs | 93 +- .../arrow => column/src}/bitmap/assign_ops.rs | 12 +- .../arrow => column/src}/bitmap/bitmap_ops.rs | 12 +- .../arrow => column/src}/bitmap/bitmask.rs | 2 +- .../arrow => column/src}/bitmap/immutable.rs | 72 +- .../arrow => column/src}/bitmap/iterator.rs | 7 +- .../src/arrow => column/src}/bitmap/mod.rs | 0 .../arrow => column/src}/bitmap/mutable.rs | 24 +- .../utils/chunk_iterator/chunks_exact.rs | 3 +- .../src}/bitmap/utils/chunk_iterator/merge.rs | 0 .../src}/bitmap/utils/chunk_iterator/mod.rs | 6 +- .../src}/bitmap/utils/chunks_exact_mut.rs | 0 .../arrow => column/src}/bitmap/utils/fmt.rs | 0 .../src}/bitmap/utils/iterator.rs | 3 +- .../arrow => column/src}/bitmap/utils/mod.rs | 0 .../src}/bitmap/utils/slice_iterator.rs | 4 +- .../src}/bitmap/utils/zip_validity.rs | 9 +- .../arrow => column/src}/buffer/immutable.rs | 61 +- .../arrow => column/src}/buffer/iterator.rs | 3 +- .../src/arrow => column/src}/buffer/mod.rs | 15 +- .../{arrow/src/arrow => column/src}/error.rs | 11 + src/common/column/src/fmt.rs | 85 ++ .../arrow/array => column/src}/iterator.rs | 37 +- src/common/{arrow => column}/src/lib.rs | 24 +- .../{arrow/src/arrow => column/src}/offset.rs | 6 +- .../arrow => column/src}/types/bit_chunk.rs | 4 +- .../src/arrow => column/src}/types/index.rs | 2 +- .../src/arrow => column/src}/types/mod.rs | 9 +- .../src/arrow => column/src}/types/native.rs | 47 + .../src/arrow => column/src}/types/offset.rs | 0 .../arrow => column/src}/types/simd/mod.rs | 0 .../arrow => column/src}/types/simd/native.rs | 2 +- .../arrow => column/src}/types/simd/packed.rs | 0 src/common/column/src/utils.rs | 56 ++ src/common/column/tests/it/binview/builder.rs | 65 ++ src/common/column/tests/it/binview/mod.rs | 156 ++++ .../tests/it}/bitmap/assign_ops.rs | 10 +- .../tests/it}/bitmap/bitmap_ops.rs | 10 +- .../tests/it}/bitmap/immutable.rs | 11 +- .../arrow => column/tests/it}/bitmap/mod.rs | 10 +- .../tests/it}/bitmap/mutable.rs | 8 +- .../it}/bitmap/utils/bit_chunks_exact.rs | 2 +- .../tests/it}/bitmap/utils/chunk_iter.rs | 4 +- .../tests/it}/bitmap/utils/fmt.rs | 2 +- .../tests/it}/bitmap/utils/iterator.rs | 2 +- .../tests/it}/bitmap/utils/mod.rs | 6 +- .../tests/it}/bitmap/utils/slice_iterator.rs | 12 +- .../tests/it}/bitmap/utils/zip_validity.rs | 6 +- .../tests/it}/buffer/immutable.rs | 7 +- .../utf8.rs => column/tests/it/buffer/mod.rs} | 39 +- .../arrow/mod.rs => column/tests/it/main.rs} | 10 +- src/common/exception/Cargo.toml | 1 - src/common/exception/src/exception_into.rs | 10 - src/common/hashtable/Cargo.toml | 2 +- .../hashtable/src/hashjoin_hashtable.rs | 14 +- .../src/hashjoin_string_hashtable.rs | 14 +- src/common/hashtable/src/traits.rs | 2 +- src/common/native/Cargo.toml | 46 + .../src}/compression/basic.rs | 14 +- .../src}/compression/binary/dict.rs | 52 +- .../src}/compression/binary/freq.rs | 31 +- .../src}/compression/binary/mod.rs | 108 +-- .../src}/compression/binary/one_value.rs | 33 +- .../src}/compression/boolean/mod.rs | 94 +- .../src}/compression/boolean/one_value.rs | 26 +- .../src}/compression/boolean/rle.rs | 31 +- .../src}/compression/double/dict.rs | 34 +- .../src}/compression/double/freq.rs | 32 +- .../src}/compression/double/mod.rs | 105 ++- .../src}/compression/double/one_value.rs | 19 +- .../src}/compression/double/patas.rs | 20 +- .../src}/compression/double/rle.rs | 42 +- .../src}/compression/double/traits.rs | 47 +- .../src}/compression/integer/bp.rs | 16 +- .../src}/compression/integer/delta_bp.rs | 16 +- .../src}/compression/integer/dict.rs | 30 +- .../src}/compression/integer/freq.rs | 20 +- .../src}/compression/integer/mod.rs | 112 +-- .../src}/compression/integer/one_value.rs | 19 +- .../src}/compression/integer/rle.rs | 26 +- .../src}/compression/integer/traits.rs | 6 +- .../native => native/src}/compression/mod.rs | 8 +- .../native/errors.rs => native/src/error.rs} | 3 + .../src/native/mod.rs => native/src/lib.rs} | 4 +- src/common/native/src/nested.rs | 282 ++++++ src/common/native/src/read/array/binary.rs | 156 ++++ .../src}/read/array/boolean.rs | 66 +- .../src/read/array/decimal.rs} | 88 +- .../src}/read/array/double.rs | 72 +- src/common/native/src/read/array/integer.rs | 152 ++++ .../native => native/src}/read/array/list.rs | 35 +- .../native => native/src}/read/array/map.rs | 33 +- .../native => native/src}/read/array/mod.rs | 3 +- .../native => native/src}/read/array/null.rs | 54 +- .../src}/read/array/struct_.rs | 37 +- .../native => native/src}/read/array/view.rs | 88 +- src/common/native/src/read/batch_read.rs | 197 ++++ src/common/native/src/read/deserialize.rs | 237 +++++ .../src/native => native/src}/read/mod.rs | 38 +- .../native => native/src}/read/read_basic.rs | 20 +- .../src/native => native/src}/read/reader.rs | 33 +- .../{arrow/src/native => native/src}/stat.rs | 154 ++-- .../native => native/src}/util/bit_util.rs | 33 +- .../native => native/src}/util/byte_writer.rs | 0 .../src/native => native/src}/util/env.rs | 0 .../src/native => native/src}/util/memory.rs | 0 src/common/native/src/util/mod.rs | 71 ++ .../src/native => native/src}/write/binary.rs | 16 +- .../native => native/src}/write/boolean.rs | 12 +- .../src/native => native/src}/write/common.rs | 56 +- .../src/native => native/src}/write/mod.rs | 0 src/common/native/src/write/primitive.rs | 95 ++ src/common/native/src/write/serialize.rs | 130 +++ .../src/native => native/src}/write/view.rs | 9 +- .../src/native => native/src}/write/writer.rs | 26 +- src/common/{arrow => native}/tests/it/main.rs | 1 - src/common/native/tests/it/native/io.rs | 295 ++++++ .../{arrow => native}/tests/it/native/mod.rs | 0 .../tests/it/native/read_meta.rs | 38 +- src/common/storage/Cargo.toml | 2 +- src/common/storage/src/column_node.rs | 28 +- src/common/storage/tests/it/column_node.rs | 4 +- src/meta/binaries/Cargo.toml | 3 +- src/meta/service/Cargo.toml | 4 +- src/meta/service/src/version.rs | 6 +- .../it/meta_node/meta_node_replication.rs | 3 +- src/query/catalog/Cargo.toml | 2 +- src/query/catalog/src/plan/internal_column.rs | 2 +- src/query/codegen/src/writes/register.rs | 6 +- src/query/expression/Cargo.toml | 6 +- .../src/aggregate/aggregate_function.rs | 2 +- .../expression/src/aggregate/group_hash.rs | 12 +- src/query/expression/src/aggregate/payload.rs | 4 +- .../expression/src/aggregate/payload_row.rs | 12 +- src/query/expression/src/block.rs | 76 +- .../expression/src/converts/arrow/from.rs | 285 +++++- .../expression/src/converts/arrow/mod.rs | 8 +- src/query/expression/src/converts/arrow/to.rs | 270 +++++- .../expression/src/converts/arrow2/from.rs | 781 ---------------- .../expression/src/converts/arrow2/mod.rs | 25 - .../expression/src/converts/arrow2/to.rs | 418 --------- .../expression/src/converts/meta/bincode.rs | 15 +- src/query/expression/src/converts/mod.rs | 1 - src/query/expression/src/evaluator.rs | 18 +- .../expression/src/filter/filter_executor.rs | 2 +- .../expression/src/filter/select_value/mod.rs | 2 +- .../src/filter/select_value/select_column.rs | 2 +- .../select_value/select_column_scalar.rs | 2 +- src/query/expression/src/function.rs | 10 +- src/query/expression/src/kernels/concat.rs | 47 +- src/query/expression/src/kernels/filter.rs | 33 +- .../src/kernels/group_by_hash/method.rs | 2 +- .../group_by_hash/method_fixed_keys.rs | 4 +- .../group_by_hash/method_serializer.rs | 4 +- .../group_by_hash/method_single_string.rs | 10 +- src/query/expression/src/kernels/scatter.rs | 4 +- .../expression/src/kernels/sort_compare.rs | 6 +- src/query/expression/src/kernels/take.rs | 43 +- .../expression/src/kernels/take_chunks.rs | 6 +- .../expression/src/kernels/take_compact.rs | 19 +- .../expression/src/kernels/take_ranges.rs | 25 +- src/query/expression/src/kernels/topk.rs | 2 +- src/query/expression/src/lib.rs | 1 + src/query/expression/src/register.rs | 442 +++++---- src/query/expression/src/row/fixed.rs | 2 +- src/query/expression/src/row/variable.rs | 2 +- src/query/expression/src/schema.rs | 18 +- src/query/expression/src/types.rs | 15 +- src/query/expression/src/types/array.rs | 4 +- src/query/expression/src/types/binary.rs | 355 +------- src/query/expression/src/types/bitmap.rs | 4 +- src/query/expression/src/types/boolean.rs | 5 +- src/query/expression/src/types/date.rs | 2 +- src/query/expression/src/types/decimal.rs | 53 +- src/query/expression/src/types/geography.rs | 9 +- src/query/expression/src/types/geometry.rs | 4 +- src/query/expression/src/types/map.rs | 3 +- src/query/expression/src/types/nullable.rs | 8 +- src/query/expression/src/types/number.rs | 63 +- src/query/expression/src/types/string.rs | 343 +------ src/query/expression/src/types/timestamp.rs | 2 +- src/query/expression/src/types/variant.rs | 4 +- src/query/expression/src/utils/arrow.rs | 23 +- src/query/expression/src/utils/column_from.rs | 2 +- src/query/expression/src/utils/display.rs | 22 - .../expression/src/utils/filter_helper.rs | 4 +- src/query/expression/src/utils/mod.rs | 2 +- src/query/expression/src/utils/visitor.rs | 4 +- src/query/expression/src/values.rs | 39 +- src/query/expression/tests/it/column.rs | 32 - .../tests/it/fill_field_default_value.rs | 39 - src/query/expression/tests/it/kernel.rs | 4 +- src/query/expression/tests/it/main.rs | 1 - src/query/expression/tests/it/row.rs | 24 +- src/query/expression/tests/it/schema.rs | 17 +- .../it/testdata/fill_field_default_value.txt | 19 - .../tests/it/testdata/kernel-pass.txt | 36 +- src/query/expression/tests/it/types.rs | 7 +- src/query/formats/Cargo.toml | 4 +- .../formats/src/field_decoder/fast_values.rs | 2 +- .../formats/src/field_decoder/json_ast.rs | 2 +- src/query/formats/src/field_decoder/nested.rs | 2 +- .../src/field_decoder/separated_text.rs | 2 +- src/query/formats/src/field_encoder/values.rs | 6 +- .../formats/tests/it/output_format_utils.rs | 2 +- src/query/functions/Cargo.toml | 2 +- .../adaptors/aggregate_null_unary_adaptor.rs | 12 +- .../aggregate_null_variadic_adaptor.rs | 12 +- .../adaptors/aggregate_ornull_adaptor.rs | 8 +- .../src/aggregates/aggregate_arg_min_max.rs | 4 +- .../src/aggregates/aggregate_array_agg.rs | 2 +- .../src/aggregates/aggregate_array_moving.rs | 4 +- .../src/aggregates/aggregate_bitmap.rs | 12 +- .../aggregate_combinator_distinct.rs | 2 +- .../src/aggregates/aggregate_combinator_if.rs | 6 +- .../aggregates/aggregate_combinator_state.rs | 2 +- .../src/aggregates/aggregate_count.rs | 10 +- .../src/aggregates/aggregate_covariance.rs | 2 +- .../aggregates/aggregate_distinct_state.rs | 4 +- .../aggregates/aggregate_json_array_agg.rs | 2 +- .../aggregates/aggregate_json_object_agg.rs | 5 +- .../src/aggregates/aggregate_min_max_any.rs | 6 +- .../src/aggregates/aggregate_null_result.rs | 2 +- .../aggregates/aggregate_quantile_tdigest.rs | 2 +- .../aggregate_quantile_tdigest_weighted.rs | 2 +- .../src/aggregates/aggregate_retention.rs | 2 +- .../src/aggregates/aggregate_scalar_state.rs | 4 +- .../src/aggregates/aggregate_string_agg.rs | 2 +- .../functions/src/aggregates/aggregate_sum.rs | 6 +- .../src/aggregates/aggregate_unary.rs | 2 +- .../src/aggregates/aggregate_window_funnel.rs | 2 +- src/query/functions/src/scalars/arithmetic.rs | 2 +- src/query/functions/src/scalars/binary.rs | 12 +- src/query/functions/src/scalars/boolean.rs | 5 +- src/query/functions/src/scalars/comparison.rs | 6 +- src/query/functions/src/scalars/datetime.rs | 4 +- .../src/scalars/string_multi_args.rs | 2 +- src/query/functions/src/scalars/variant.rs | 6 +- src/query/functions/src/scalars/vector.rs | 2 +- src/query/functions/src/srfs/variant.rs | 2 +- .../tests/it/aggregates/testdata/agg.txt | 114 +-- .../it/aggregates/testdata/agg_group_by.txt | 66 +- src/query/functions/tests/it/scalars/mod.rs | 4 +- .../tests/it/scalars/testdata/arithmetic.txt | 96 +- .../tests/it/scalars/testdata/array.txt | 36 +- .../tests/it/scalars/testdata/binary.txt | 72 +- .../tests/it/scalars/testdata/cast.txt | 266 +++--- .../tests/it/scalars/testdata/comparison.txt | 294 +++--- .../tests/it/scalars/testdata/geo_h3.txt | 24 +- .../tests/it/scalars/testdata/geometry.txt | 28 +- .../tests/it/scalars/testdata/hash.txt | 88 +- .../tests/it/scalars/testdata/map.txt | 322 +++---- .../tests/it/scalars/testdata/regexp.txt | 450 +++++----- .../tests/it/scalars/testdata/string.txt | 842 +++++++++--------- .../tests/it/scalars/testdata/tuple.txt | 36 +- .../tests/it/scalars/testdata/variant.txt | 656 +++++++------- .../processors/transforms/sort/rows/common.rs | 2 +- src/query/service/Cargo.toml | 5 +- .../group_by/aggregator_keys_builder.rs | 2 +- .../group_by/aggregator_keys_iter.rs | 6 +- .../group_by/aggregator_polymorphic_keys.rs | 1 - .../processors/transforms/hash_join/common.rs | 6 +- .../hash_join/hash_join_build_state.rs | 14 +- .../hash_join/hash_join_probe_state.rs | 6 +- .../hash_join/probe_join/left_mark_join.rs | 4 +- .../transforms/hash_join/probe_state.rs | 2 +- .../transforms/hash_join/result_blocks.rs | 2 +- .../transforms/hash_join/spill_common.rs | 2 +- .../transforms/range_join/ie_join_state.rs | 4 +- .../processors/transforms/transform_srf.rs | 12 +- .../transforms/transform_udf_script.rs | 2 +- .../flight/v1/scatter/flight_scatter_hash.rs | 6 +- src/query/service/src/spillers/serialize.rs | 4 +- .../tests/it/servers/http/json_block.rs | 2 +- .../it/storages/fuse/operations/read_plan.rs | 21 +- .../src/planner/plans/constant_table_scan.rs | 2 +- .../storages/common/blocks/src/parquet_rs.rs | 3 +- src/query/storages/common/cache/Cargo.toml | 3 +- src/query/storages/common/cache/src/caches.rs | 6 +- src/query/storages/common/index/Cargo.toml | 4 +- .../storages/common/index/src/bloom_index.rs | 8 +- .../index/tests/it/filters/bloom_filter.rs | 2 +- .../storages/common/table_meta/Cargo.toml | 3 +- .../common/table_meta/src/meta/v2/segment.rs | 2 +- .../src/meta/v3/frozen/block_meta.rs | 4 +- .../table_meta/src/table/table_compression.rs | 12 +- src/query/storages/fuse/Cargo.toml | 3 +- .../read/agg_index/agg_index_reader_native.rs | 14 +- .../fuse/src/io/read/block/block_reader.rs | 15 +- .../io/read/block/block_reader_deserialize.rs | 6 +- .../src/io/read/block/block_reader_native.rs | 19 +- .../block/block_reader_native_deserialize.rs | 113 ++- .../src/io/read/block/parquet/deserialize.rs | 4 +- .../fuse/src/io/read/block/parquet/mod.rs | 8 +- .../virtual_column_reader_parquet.rs | 4 +- .../fuse/src/io/write/block_writer.rs | 16 +- .../src/io/write/inverted_index_writer.rs | 4 +- .../merge_into/mutator/matched_mutator.rs | 2 +- .../mutator/merge_into_split_mutator.rs | 2 +- .../mutation/processors/mutation_source.rs | 2 +- .../fuse/src/operations/mutation_source.rs | 2 +- .../src/operations/read/fuse_rows_fetcher.rs | 4 +- .../read/native_data_source_deserializer.rs | 162 ++-- .../operations/read/native_rows_fetcher.rs | 8 +- .../read/parquet_data_source_deserializer.rs | 4 +- .../operations/read/runtime_filter_prunner.rs | 2 +- .../mutator/merge_into_mutator.rs | 4 +- .../mutator/mutator_replace_into.rs | 4 +- .../fuse/src/statistics/column_statistic.rs | 2 +- .../fuse/src/table_functions/fuse_encoding.rs | 12 +- src/query/storages/parquet/Cargo.toml | 2 +- .../parquet_rs/parquet_reader/predicate.rs | 2 +- .../read_policy/predicate_and_topk.rs | 2 +- .../parquet_reader/read_policy/utils.rs | 2 +- .../src/parquet_rs/parquet_reader/topk.rs | 2 +- .../src/parquet_rs/parquet_reader/utils.rs | 13 +- .../append/parquet_file/writer_processor.rs | 9 +- .../mode/cluster/memo/aggregate_property.test | 4 +- .../mode/cluster/memo/join_property.test | 10 +- .../mode/cluster/memo/mix_property.test | 2 +- 540 files changed, 7558 insertions(+), 36360 deletions(-) delete mode 100644 src/common/arrow/Cargo.toml delete mode 100644 src/common/arrow/src/arrow/array/README.md delete mode 100644 src/common/arrow/src/arrow/array/binary/data.rs delete mode 100644 src/common/arrow/src/arrow/array/binary/from.rs delete mode 100644 src/common/arrow/src/arrow/array/binary/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/binary/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/binary/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/binary/mutable_values.rs delete mode 100644 src/common/arrow/src/arrow/array/binview/from.rs delete mode 100644 src/common/arrow/src/arrow/array/boolean/data.rs delete mode 100644 src/common/arrow/src/arrow/array/boolean/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/boolean/from.rs delete mode 100644 src/common/arrow/src/arrow/array/boolean/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/boolean/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/boolean/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/dictionary/data.rs delete mode 100644 src/common/arrow/src/arrow/array/dictionary/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/dictionary/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/dictionary/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/dictionary/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/dictionary/typed_iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/dictionary/value_map.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/binary.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/binary_view.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/boolean.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/fixed_size_binary.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/fixed_size_list.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/list.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/map.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/null.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/primitive.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/struct_.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/union.rs delete mode 100644 src/common/arrow/src/arrow/array/equal/utf8.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_binary/data.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_binary/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_binary/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_binary/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_binary/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_list/data.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_list/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_list/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_list/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/fixed_size_list/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/binary.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/binview.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/boolean.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/fixed_binary.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/fixed_size_list.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/list.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/map.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/null.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/primitive.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/structure.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/union.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/utf8.rs delete mode 100644 src/common/arrow/src/arrow/array/growable/utils.rs delete mode 100644 src/common/arrow/src/arrow/array/indexable.rs delete mode 100644 src/common/arrow/src/arrow/array/list/data.rs delete mode 100644 src/common/arrow/src/arrow/array/list/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/list/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/list/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/list/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/map/data.rs delete mode 100644 src/common/arrow/src/arrow/array/map/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/map/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/map/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/null.rs delete mode 100644 src/common/arrow/src/arrow/array/ord.rs delete mode 100644 src/common/arrow/src/arrow/array/physical_binary.rs delete mode 100644 src/common/arrow/src/arrow/array/primitive/data.rs delete mode 100644 src/common/arrow/src/arrow/array/primitive/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/primitive/from_natural.rs delete mode 100644 src/common/arrow/src/arrow/array/primitive/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/primitive/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/primitive/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/specification.rs delete mode 100644 src/common/arrow/src/arrow/array/struct_/data.rs delete mode 100644 src/common/arrow/src/arrow/array/struct_/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/struct_/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/struct_/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/struct_/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/union/data.rs delete mode 100644 src/common/arrow/src/arrow/array/union/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/union/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/union/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/utf8/data.rs delete mode 100644 src/common/arrow/src/arrow/array/utf8/fmt.rs delete mode 100644 src/common/arrow/src/arrow/array/utf8/from.rs delete mode 100644 src/common/arrow/src/arrow/array/utf8/iterator.rs delete mode 100644 src/common/arrow/src/arrow/array/utf8/mod.rs delete mode 100644 src/common/arrow/src/arrow/array/utf8/mutable.rs delete mode 100644 src/common/arrow/src/arrow/array/utf8/mutable_values.rs delete mode 100644 src/common/arrow/src/arrow/chunk.rs delete mode 100644 src/common/arrow/src/arrow/compute/README.md delete mode 100644 src/common/arrow/src/arrow/compute/concatenate.rs delete mode 100644 src/common/arrow/src/arrow/compute/mod.rs delete mode 100644 src/common/arrow/src/arrow/compute/utils.rs delete mode 100644 src/common/arrow/src/arrow/datatypes/field.rs delete mode 100644 src/common/arrow/src/arrow/datatypes/mod.rs delete mode 100644 src/common/arrow/src/arrow/datatypes/physical_type.rs delete mode 100644 src/common/arrow/src/arrow/datatypes/schema.rs delete mode 100644 src/common/arrow/src/arrow/mod.rs delete mode 100644 src/common/arrow/src/arrow/scalar/README.md delete mode 100644 src/common/arrow/src/arrow/scalar/binary.rs delete mode 100644 src/common/arrow/src/arrow/scalar/binview.rs delete mode 100644 src/common/arrow/src/arrow/scalar/boolean.rs delete mode 100644 src/common/arrow/src/arrow/scalar/dictionary.rs delete mode 100644 src/common/arrow/src/arrow/scalar/equal.rs delete mode 100644 src/common/arrow/src/arrow/scalar/fixed_size_binary.rs delete mode 100644 src/common/arrow/src/arrow/scalar/fixed_size_list.rs delete mode 100644 src/common/arrow/src/arrow/scalar/list.rs delete mode 100644 src/common/arrow/src/arrow/scalar/map.rs delete mode 100644 src/common/arrow/src/arrow/scalar/mod.rs delete mode 100644 src/common/arrow/src/arrow/scalar/null.rs delete mode 100644 src/common/arrow/src/arrow/scalar/primitive.rs delete mode 100644 src/common/arrow/src/arrow/scalar/struct_.rs delete mode 100644 src/common/arrow/src/arrow/scalar/union.rs delete mode 100644 src/common/arrow/src/arrow/scalar/utf8.rs delete mode 100644 src/common/arrow/src/arrow/temporal_conversions.rs delete mode 100644 src/common/arrow/src/arrow/trusted_len.rs delete mode 100644 src/common/arrow/src/arrow/util/bench_util.rs delete mode 100644 src/common/arrow/src/arrow/util/mod.rs delete mode 100644 src/common/arrow/src/native/nested.rs delete mode 100644 src/common/arrow/src/native/read/array/binary.rs delete mode 100644 src/common/arrow/src/native/read/batch_read.rs delete mode 100644 src/common/arrow/src/native/read/deserialize.rs delete mode 100644 src/common/arrow/src/native/util/mod.rs delete mode 100644 src/common/arrow/src/native/write/primitive.rs delete mode 100644 src/common/arrow/src/native/write/serialize.rs delete mode 100644 src/common/arrow/src/schema_projection.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/binary/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/binary/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/binary/mutable_values.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/binary/to_mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/binview/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/binview/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/binview/mutable_values.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/binview/to_mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/boolean/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/boolean/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/dictionary/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/dictionary/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/equal/boolean.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/equal/dictionary.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/equal/fixed_size_list.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/equal/list.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/equal/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/equal/primitive.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/fixed_size_binary/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/fixed_size_binary/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/fixed_size_list/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/fixed_size_list/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/binary.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/boolean.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/dictionary.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/fixed_binary.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/fixed_size_list.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/list.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/map.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/null.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/primitive.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/struct_.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/union.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/growable/utf8.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/list/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/list/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/map/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/ord.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/primitive/fmt.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/primitive/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/primitive/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/primitive/to_mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/struct_/iterator.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/struct_/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/struct_/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/union.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/utf8/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/utf8/mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/utf8/mutable_values.rs delete mode 100644 src/common/arrow/tests/it/arrow/array/utf8/to_mutable.rs delete mode 100644 src/common/arrow/tests/it/arrow/arrow_data.rs delete mode 100644 src/common/arrow/tests/it/arrow/buffer/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/concatenate.rs delete mode 100644 src/common/arrow/tests/it/arrow/compute/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/binary.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/boolean.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/fixed_size_binary.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/fixed_size_list.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/list.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/map.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/mod.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/null.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/primitive.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/struct_.rs delete mode 100644 src/common/arrow/tests/it/arrow/scalar/utf8.rs delete mode 100644 src/common/arrow/tests/it/arrow/temporal_conversions.rs delete mode 100644 src/common/arrow/tests/it/arrow/types.rs delete mode 100644 src/common/arrow/tests/it/native/io.rs create mode 100644 src/common/column/Cargo.toml create mode 100644 src/common/column/src/binary/builder.rs rename src/common/{arrow/src/arrow/array => column/src}/binary/fmt.rs (52%) create mode 100644 src/common/column/src/binary/iterator.rs create mode 100644 src/common/column/src/binary/mod.rs rename src/common/{arrow/src/arrow/array/binview/mutable.rs => column/src/binview/builder.rs} (62%) rename src/common/{arrow/src/arrow/array => column/src}/binview/fmt.rs (68%) rename src/common/{arrow/src/arrow/array => column/src}/binview/iterator.rs (55%) rename src/common/{arrow/src/arrow/array => column/src}/binview/mod.rs (51%) rename src/common/{arrow/src/arrow/array => column/src}/binview/view.rs (76%) rename src/common/{arrow/src/arrow => column/src}/bitmap/assign_ops.rs (96%) rename src/common/{arrow/src/arrow => column/src}/bitmap/bitmap_ops.rs (96%) rename src/common/{arrow/src/arrow => column/src}/bitmap/bitmask.rs (99%) rename src/common/{arrow/src/arrow => column/src}/bitmap/immutable.rs (90%) rename src/common/{arrow/src/arrow => column/src}/bitmap/iterator.rs (96%) rename src/common/{arrow/src/arrow => column/src}/bitmap/mod.rs (100%) rename src/common/{arrow/src/arrow => column/src}/bitmap/mutable.rs (97%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/chunk_iterator/chunks_exact.rs (98%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/chunk_iterator/merge.rs (100%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/chunk_iterator/mod.rs (98%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/chunks_exact_mut.rs (100%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/fmt.rs (100%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/iterator.rs (98%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/mod.rs (100%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/slice_iterator.rs (98%) rename src/common/{arrow/src/arrow => column/src}/bitmap/utils/zip_validity.rs (97%) rename src/common/{arrow/src/arrow => column/src}/buffer/immutable.rs (88%) rename src/common/{arrow/src/arrow => column/src}/buffer/iterator.rs (98%) rename src/common/{arrow/src/arrow => column/src}/buffer/mod.rs (91%) rename src/common/{arrow/src/arrow => column/src}/error.rs (93%) create mode 100644 src/common/column/src/fmt.rs rename src/common/{arrow/src/arrow/array => column/src}/iterator.rs (71%) rename src/common/{arrow => column}/src/lib.rs (71%) rename src/common/{arrow/src/arrow => column/src}/offset.rs (99%) rename src/common/{arrow/src/arrow => column/src}/types/bit_chunk.rs (96%) rename src/common/{arrow/src/arrow => column/src}/types/index.rs (98%) rename src/common/{arrow/src/arrow => column/src}/types/mod.rs (94%) rename src/common/{arrow/src/arrow => column/src}/types/native.rs (93%) rename src/common/{arrow/src/arrow => column/src}/types/offset.rs (100%) rename src/common/{arrow/src/arrow => column/src}/types/simd/mod.rs (100%) rename src/common/{arrow/src/arrow => column/src}/types/simd/native.rs (96%) rename src/common/{arrow/src/arrow => column/src}/types/simd/packed.rs (100%) create mode 100644 src/common/column/src/utils.rs create mode 100644 src/common/column/tests/it/binview/builder.rs create mode 100644 src/common/column/tests/it/binview/mod.rs rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/assign_ops.rs (91%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/bitmap_ops.rs (89%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/immutable.rs (91%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/mod.rs (95%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/mutable.rs (98%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/utils/bit_chunks_exact.rs (95%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/utils/chunk_iter.rs (97%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/utils/fmt.rs (97%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/utils/iterator.rs (96%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/utils/mod.rs (95%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/utils/slice_iterator.rs (92%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/bitmap/utils/zip_validity.rs (95%) rename src/common/{arrow/tests/it/arrow => column/tests/it}/buffer/immutable.rs (96%) rename src/common/{arrow/tests/it/arrow/array/equal/utf8.rs => column/tests/it/buffer/mod.rs} (50%) rename src/common/{arrow/tests/it/arrow/mod.rs => column/tests/it/main.rs} (86%) create mode 100644 src/common/native/Cargo.toml rename src/common/{arrow/src/native => native/src}/compression/basic.rs (92%) rename src/common/{arrow/src/native => native/src}/compression/binary/dict.rs (74%) rename src/common/{arrow/src/native => native/src}/compression/binary/freq.rs (86%) rename src/common/{arrow/src/native => native/src}/compression/binary/mod.rs (78%) rename src/common/{arrow/src/native => native/src}/compression/binary/one_value.rs (71%) rename src/common/{arrow/src/native => native/src}/compression/boolean/mod.rs (75%) rename src/common/{arrow/src/native => native/src}/compression/boolean/one_value.rs (73%) rename src/common/{arrow/src/native => native/src}/compression/boolean/rle.rs (68%) rename src/common/{arrow/src/native => native/src}/compression/double/dict.rs (80%) rename src/common/{arrow/src/native => native/src}/compression/double/freq.rs (82%) rename src/common/{arrow/src/native => native/src}/compression/double/mod.rs (78%) rename src/common/{arrow/src/native => native/src}/compression/double/one_value.rs (85%) rename src/common/{arrow/src/native => native/src}/compression/double/patas.rs (93%) rename src/common/{arrow/src/native => native/src}/compression/double/rle.rs (77%) rename src/common/{arrow/src/native => native/src}/compression/double/traits.rs (65%) rename src/common/{arrow/src/native => native/src}/compression/integer/bp.rs (89%) rename src/common/{arrow/src/native => native/src}/compression/integer/delta_bp.rs (90%) rename src/common/{arrow/src/native => native/src}/compression/integer/dict.rs (91%) rename src/common/{arrow/src/native => native/src}/compression/integer/freq.rs (90%) rename src/common/{arrow/src/native => native/src}/compression/integer/mod.rs (78%) rename src/common/{arrow/src/native => native/src}/compression/integer/one_value.rs (85%) rename src/common/{arrow/src/native => native/src}/compression/integer/rle.rs (86%) rename src/common/{arrow/src/native => native/src}/compression/integer/traits.rs (90%) rename src/common/{arrow/src/native => native/src}/compression/mod.rs (92%) rename src/common/{arrow/src/native/errors.rs => native/src/error.rs} (92%) rename src/common/{arrow/src/native/mod.rs => native/src/lib.rs} (97%) create mode 100644 src/common/native/src/nested.rs create mode 100644 src/common/native/src/read/array/binary.rs rename src/common/{arrow/src/native => native/src}/read/array/boolean.rs (66%) rename src/common/{arrow/src/native/read/array/integer.rs => native/src/read/array/decimal.rs} (59%) rename src/common/{arrow/src/native => native/src}/read/array/double.rs (65%) create mode 100644 src/common/native/src/read/array/integer.rs rename src/common/{arrow/src/native => native/src}/read/array/list.rs (63%) rename src/common/{arrow/src/native => native/src}/read/array/map.rs (66%) rename src/common/{arrow/src/native => native/src}/read/array/mod.rs (96%) rename src/common/{arrow/src/native => native/src}/read/array/null.rs (57%) rename src/common/{arrow/src/native => native/src}/read/array/struct_.rs (73%) rename src/common/{arrow/src/native => native/src}/read/array/view.rs (65%) create mode 100644 src/common/native/src/read/batch_read.rs create mode 100644 src/common/native/src/read/deserialize.rs rename src/common/{arrow/src/native => native/src}/read/mod.rs (75%) rename src/common/{arrow/src/native => native/src}/read/read_basic.rs (87%) rename src/common/{arrow/src/native => native/src}/read/reader.rs (90%) rename src/common/{arrow/src/native => native/src}/stat.rs (62%) rename src/common/{arrow/src/native => native/src}/util/bit_util.rs (92%) rename src/common/{arrow/src/native => native/src}/util/byte_writer.rs (100%) rename src/common/{arrow/src/native => native/src}/util/env.rs (100%) rename src/common/{arrow/src/native => native/src}/util/memory.rs (100%) create mode 100644 src/common/native/src/util/mod.rs rename src/common/{arrow/src/native => native/src}/write/binary.rs (71%) rename src/common/{arrow/src/native => native/src}/write/boolean.rs (77%) rename src/common/{arrow/src/native => native/src}/write/common.rs (67%) rename src/common/{arrow/src/native => native/src}/write/mod.rs (100%) create mode 100644 src/common/native/src/write/primitive.rs create mode 100644 src/common/native/src/write/serialize.rs rename src/common/{arrow/src/native => native/src}/write/view.rs (92%) rename src/common/{arrow/src/native => native/src}/write/writer.rs (90%) rename src/common/{arrow => native}/tests/it/main.rs (98%) create mode 100644 src/common/native/tests/it/native/io.rs rename src/common/{arrow => native}/tests/it/native/mod.rs (100%) rename src/common/{arrow => native}/tests/it/native/read_meta.rs (65%) delete mode 100644 src/query/expression/src/converts/arrow2/from.rs delete mode 100644 src/query/expression/src/converts/arrow2/mod.rs delete mode 100644 src/query/expression/src/converts/arrow2/to.rs delete mode 100644 src/query/expression/tests/it/column.rs diff --git a/Cargo.lock b/Cargo.lock index 61c501affe8a..b1deb8a980af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3026,50 +3026,6 @@ dependencies = [ "itertools 0.13.0", ] -[[package]] -name = "databend-common-arrow" -version = "0.1.0" -dependencies = [ - "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "bitpacking 0.8.4", - "bytemuck", - "byteorder", - "bytes", - "chrono", - "chrono-tz 0.8.6", - "dyn-clone", - "either", - "env_logger 0.11.5", - "ethnum", - "flate2", - "foreign_vec", - "hashbrown 0.14.5", - "indexmap 2.6.0", - "log", - "lz4", - "num", - "num-traits", - "opendal", - "ordered-float 4.2.2", - "proptest", - "quanta 0.11.1", - "rand", - "ringbuffer", - "roaring", - "serde", - "serde_derive", - "serde_json", - "simdutf8", - "snap", - "tokio", - "tokio-util", - "zstd 0.12.4", -] - [[package]] name = "databend-common-ast" version = "0.0.4" @@ -3122,6 +3078,7 @@ dependencies = [ "async-backtrace", "async-trait", "borsh", + "bytemuck", "bytes", "bytesize", "chrono", @@ -3193,7 +3150,6 @@ dependencies = [ "async-trait", "chrono", "dashmap 6.1.0", - "databend-common-arrow", "databend-common-ast", "databend-common-base", "databend-common-config", @@ -3244,6 +3200,29 @@ dependencies = [ "tower 0.5.1", ] +[[package]] +name = "databend-common-column" +version = "0.1.0" +dependencies = [ + "arrow-buffer", + "arrow-data", + "arrow-schema", + "bytemuck", + "databend-common-base", + "databend-common-exception", + "either", + "ethnum", + "foreign_vec", + "hex", + "match-template", + "num-traits", + "proptest", + "serde", + "serde_derive", + "serde_json", + "simdutf8", +] + [[package]] name = "databend-common-compress" version = "0.1.0" @@ -3301,7 +3280,6 @@ dependencies = [ "arrow-schema", "backtrace", "bincode 2.0.0-rc.3", - "databend-common-arrow", "databend-common-ast", "geos", "geozero 0.14.0", @@ -3329,6 +3307,9 @@ name = "databend-common-expression" version = "0.1.0" dependencies = [ "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", "arrow-flight", "arrow-ipc", "arrow-ord", @@ -3343,9 +3324,9 @@ dependencies = [ "comfy-table", "criterion", "dashmap 6.1.0", - "databend-common-arrow", "databend-common-ast", "databend-common-base", + "databend-common-column", "databend-common-datavalues", "databend-common-exception", "databend-common-grpc", @@ -3379,7 +3360,6 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "simdutf8", "strength_reduce", "terminal_size 0.2.6", "tonic", @@ -3396,7 +3376,6 @@ dependencies = [ "base64 0.22.1", "bstr", "chrono-tz 0.8.6", - "databend-common-arrow", "databend-common-base", "databend-common-exception", "databend-common-expression", @@ -3435,7 +3414,6 @@ dependencies = [ "crc32fast", "criterion", "ctor 0.2.8", - "databend-common-arrow", "databend-common-ast", "databend-common-base", "databend-common-exception", @@ -3508,8 +3486,8 @@ dependencies = [ "ahash 0.8.11", "bumpalo", "cfg-if", - "databend-common-arrow", "databend-common-base", + "databend-common-column", "ethnum", "rand", ] @@ -3882,6 +3860,34 @@ dependencies = [ "tokio", ] +[[package]] +name = "databend-common-native" +version = "0.1.0" +dependencies = [ + "ahash 0.8.11", + "bitpacking 0.8.4", + "bytemuck", + "byteorder", + "bytes", + "databend-common-column", + "databend-common-expression", + "env_logger 0.11.5", + "ethnum", + "hashbrown 0.14.5", + "log", + "lz4", + "match-template", + "num", + "opendal", + "rand", + "ringbuffer", + "roaring", + "serde", + "serde_json", + "snap", + "zstd 0.12.4", +] + [[package]] name = "databend-common-openai" version = "0.1.0" @@ -4106,13 +4112,13 @@ dependencies = [ "async-backtrace", "chrono", "dashmap 6.1.0", - "databend-common-arrow", "databend-common-auth", "databend-common-base", "databend-common-exception", "databend-common-expression", "databend-common-meta-app", "databend-common-metrics", + "databend-common-native", "databend-enterprise-storage-encryption", "flagset", "futures", @@ -4197,7 +4203,6 @@ dependencies = [ "bytes", "chrono", "criterion", - "databend-common-arrow", "databend-common-base", "databend-common-catalog", "databend-common-exception", @@ -4209,6 +4214,7 @@ dependencies = [ "databend-common-meta-app", "databend-common-meta-types", "databend-common-metrics", + "databend-common-native", "databend-common-pipeline-core", "databend-common-pipeline-sinks", "databend-common-pipeline-sources", @@ -4412,7 +4418,6 @@ dependencies = [ "async-trait", "bytes", "chrono", - "databend-common-arrow", "databend-common-base", "databend-common-catalog", "databend-common-exception", @@ -4939,7 +4944,6 @@ dependencies = [ "async-trait", "backon", "clap", - "databend-common-arrow", "databend-common-base", "databend-common-building", "databend-common-grpc", @@ -5050,13 +5054,13 @@ dependencies = [ "criterion", "ctor 0.2.8", "dashmap 6.1.0", - "databend-common-arrow", "databend-common-ast", "databend-common-base", "databend-common-building", "databend-common-cache", "databend-common-catalog", "databend-common-cloud-control", + "databend-common-column", "databend-common-config", "databend-common-exception", "databend-common-expression", @@ -5254,12 +5258,12 @@ dependencies = [ name = "databend-storages-common-cache" version = "0.1.0" dependencies = [ + "arrow", "async-backtrace", "async-trait", "bytes", "crc32fast", "crossbeam-channel", - "databend-common-arrow", "databend-common-base", "databend-common-cache", "databend-common-catalog", @@ -5285,7 +5289,6 @@ dependencies = [ "anyerror", "cbordata", "criterion", - "databend-common-arrow", "databend-common-ast", "databend-common-exception", "databend-common-expression", @@ -5372,12 +5375,12 @@ version = "0.1.0" dependencies = [ "bincode 1.3.3", "chrono", - "databend-common-arrow", "databend-common-base", "databend-common-datavalues", "databend-common-exception", "databend-common-expression", "databend-common-io", + "databend-common-native", "databend-common-storage", "enum-as-inner 0.5.1", "log", @@ -9857,7 +9860,7 @@ dependencies = [ "futures-util", "once_cell", "parking_lot 0.12.3", - "quanta 0.12.3", + "quanta", "rustc_version", "smallvec", "tagptr", @@ -11878,22 +11881,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" -[[package]] -name = "quanta" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" -dependencies = [ - "crossbeam-utils", - "libc", - "mach2", - "once_cell", - "raw-cpuid 10.7.0", - "wasi", - "web-sys", - "winapi", -] - [[package]] name = "quanta" version = "0.12.3" @@ -11903,7 +11890,7 @@ dependencies = [ "crossbeam-utils", "libc", "once_cell", - "raw-cpuid 11.1.0", + "raw-cpuid", "wasi", "web-sys", "winapi", @@ -12133,15 +12120,6 @@ dependencies = [ "random-number", ] -[[package]] -name = "raw-cpuid" -version = "10.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "raw-cpuid" version = "11.1.0" @@ -14736,7 +14714,6 @@ checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", - "futures-io", "futures-sink", "pin-project-lite", "tokio", diff --git a/Cargo.toml b/Cargo.toml index d7618d0f5c19..80de5122a3e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,12 +9,13 @@ edition = "2021" resolver = "2" members = [ "src/binaries", - "src/common/arrow", "src/common/auth", "src/common/base", "src/common/building", "src/common/cache", "src/common/cloud_control", + "src/common/column", + "src/common/native", "src/common/compress", "src/common/exception", "src/common/grpc", @@ -102,7 +103,6 @@ members = [ # Workspace dependencies [workspace.dependencies] -databend-common-arrow = { path = "src/common/arrow" } databend-common-ast = { path = "src/query/ast" } databend-common-async-functions = { path = "src/query/async_functions" } databend-common-auth = { path = "src/common/auth" } @@ -113,6 +113,7 @@ databend-common-cache = { path = "src/common/cache" } databend-common-catalog = { path = "src/query/catalog" } databend-common-cloud-control = { path = "src/common/cloud_control" } databend-common-codegen = { path = "src/query/codegen" } +databend-common-column = { path = "src/common/column" } databend-common-compress = { path = "src/common/compress" } databend-common-config = { path = "src/query/config" } databend-common-datavalues = { path = "src/query/datavalues" } @@ -138,6 +139,7 @@ databend-common-meta-stoerr = { path = "src/meta/stoerr" } databend-common-meta-store = { path = "src/meta/store" } databend-common-meta-types = { path = "src/meta/types" } databend-common-metrics = { path = "src/common/metrics" } +databend-common-native = { path = "src/common/native" } databend-common-openai = { path = "src/common/openai" } databend-common-pipeline-core = { path = "src/query/pipeline/core" } databend-common-pipeline-sinks = { path = "src/query/pipeline/sinks" } @@ -577,7 +579,6 @@ incremental = false debug-assertions = true # [profile.release.package] -# databend-common-arrow = { codegen-units = 16 } # databend-query = { codegen-units = 4 } # databend-binaries = { codegen-units = 4 } diff --git a/src/common/arrow/Cargo.toml b/src/common/arrow/Cargo.toml deleted file mode 100644 index 0380e70f9f9f..000000000000 --- a/src/common/arrow/Cargo.toml +++ /dev/null @@ -1,79 +0,0 @@ -[package] -description = "Arrow implementation forked from arrow2 and native format implementation forked from strawboat." -edition = "2021" -license = "Apache-2.0" -name = "databend-common-arrow" -publish = false -version = "0.1.0" - -[lib] -doctest = false -test = true - -[features] -default = ["arrow-default"] - -arrow = ["arrow-buffer", "arrow-schema", "arrow-data", "arrow-array"] - -# sample testing of generated arrow data - -compute = [ - "compute_concatenate", -] -compute_concatenate = [] - -serde_types = ["serde", "serde_derive"] -simd = [] - -arrow-default = [ - "arrow", - "compute", - "serde_types", - "simd", -] - -[dependencies] -ahash = { workspace = true } -arrow-array = { workspace = true, optional = true } -arrow-buffer = { workspace = true, optional = true } -arrow-data = { workspace = true, optional = true } -arrow-schema = { workspace = true, optional = true } -bitpacking = { workspace = true } -bytemuck = { workspace = true } -byteorder = { workspace = true } -bytes = { workspace = true } -chrono = { workspace = true } -chrono-tz = { workspace = true, optional = true } -dyn-clone = { workspace = true } -either = { workspace = true } -ethnum = { workspace = true } -foreign_vec = { workspace = true } -hashbrown_v0_14 = { workspace = true } -indexmap = { workspace = true } -log = { workspace = true } -lz4 = { workspace = true } -num = { workspace = true, features = ["std"] } -num-traits = { workspace = true } -opendal = { workspace = true } -ordered-float = { workspace = true } -rand = { workspace = true } -ringbuffer = { workspace = true } -roaring = { workspace = true } -serde = { workspace = true, features = ["rc"], optional = true } -serde_derive = { workspace = true, optional = true } -serde_json = { workspace = true } -simdutf8 = { workspace = true } -snap = { workspace = true } -zstd = { workspace = true } - -[dev-dependencies] -# used to test async readers -env_logger = { workspace = true } -flate2 = { workspace = true } -proptest = { workspace = true, default-features = false, features = ["std"] } -quanta = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt", "fs", "io-util"] } -tokio-util = { workspace = true, features = ["compat"] } - -[lints] -workspace = true diff --git a/src/common/arrow/src/arrow/array/README.md b/src/common/arrow/src/arrow/array/README.md deleted file mode 100644 index 497348610aad..000000000000 --- a/src/common/arrow/src/arrow/array/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Array module - -This document describes the overall design of this module. - -## Notation: - -* "array" in this module denotes any struct that implements the trait `Array`. -* "mutable array" in this module denotes any struct that implements the trait `MutableArray`. -* words in `code` denote existing terms on this implementation. - -## Arrays: - -* Every arrow array with a different physical representation MUST be implemented as a struct or generic struct. - -* An array MAY have its own module. E.g. `primitive/mod.rs` - -* An array with a null bitmap MUST implement it as `Option` - -* An array MUST be `#[derive(Clone)]` - -* The trait `Array` MUST only be implemented by structs in this module. - -* Every child array on the struct MUST be `Box`. - -* An array MUST implement `try_new(...) -> Self`. This method MUST error iff - the data does not follow the arrow specification, including any sentinel types such as utf8. - -* An array MAY implement `unsafe try_new_unchecked` that skips validation steps that are `O(N)`. - -* An array MUST implement either `new_empty()` or `new_empty(DataType)` that returns a zero-len of `Self`. - -* An array MUST implement either `new_null(length: usize)` or `new_null(DataType, length: usize)` that returns a valid array of length `length` whose all elements are null. - -* An array MAY implement `value(i: usize)` that returns the value at slot `i` ignoring the validity bitmap. - -* functions to create new arrays from native Rust SHOULD be named as follows: - * `from`: from a slice of optional values (e.g. `AsRef<[Option]` for `BooleanArray`) - * `from_slice`: from a slice of values (e.g. `AsRef<[bool]>` for `BooleanArray`) - * `from_trusted_len_iter` from an iterator of trusted len of optional values - * `from_trusted_len_values_iter` from an iterator of trusted len of values - * `try_from_trusted_len_iter` from an fallible iterator of trusted len of optional values - -### Slot offsets - -* An array MUST have a `offset: usize` measuring the number of slots that the array is currently offsetted by if the specification requires. - -* An array MUST implement `fn slice(&self, offset: usize, length: usize) -> Self` that returns an offsetted and/or truncated clone of the array. This function MUST increase the array's offset if it exists. - -* Conversely, `offset` MUST only be changed by `slice`. - -The rational of the above is that it enable us to be fully interoperable with the offset logic supported by the C data interface, while at the same time easily perform array slices -within Rust's type safety mechanism. - -### Mutable Arrays - -* An array MAY have a mutable counterpart. E.g. `MutablePrimitiveArray` is the mutable counterpart of `PrimitiveArray`. - -* Arrays with mutable counterparts MUST have its own module, and have the mutable counterpart declared in `{module}/mutable.rs`. - -* The trait `MutableArray` MUST only be implemented by mutable arrays in this module. - -* A mutable array MUST be `#[derive(Debug)]` - -* A mutable array with a null bitmap MUST implement it as `Option` - -* Converting a `MutableArray` to its immutable counterpart MUST be `O(1)`. Specifically: - * it must not allocate - * it must not cause `O(N)` data transformations - - This is achieved by converting mutable versions to immutable counterparts (e.g. `MutableBitmap -> Bitmap`). - - The rational is that `MutableArray`s can be used to perform in-place operations under - the arrow spec. diff --git a/src/common/arrow/src/arrow/array/binary/data.rs b/src/common/arrow/src/arrow/array/binary/data.rs deleted file mode 100644 index ec3837e18a87..000000000000 --- a/src/common/arrow/src/arrow/array/binary/data.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::BinaryArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::offset::Offset; -use crate::arrow::offset::OffsetsBuffer; - -impl Arrow2Arrow for BinaryArray { - fn to_data(&self) -> ArrayData { - let data_type = self.data_type.clone().into(); - let builder = ArrayDataBuilder::new(data_type) - .len(self.offsets().len_proxy()) - .buffers(vec![ - self.offsets.clone().into_inner().into(), - self.values.clone().into(), - ]) - .nulls(self.validity.as_ref().map(|b| b.clone().into())); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type = data.data_type().clone().into(); - - if data.is_empty() { - // Handle empty offsets - return Self::new_empty(data_type); - } - - let buffers = data.buffers(); - - // Safety: ArrayData is valid - let mut offsets = unsafe { OffsetsBuffer::new_unchecked(buffers[0].clone().into()) }; - offsets.slice(data.offset(), data.len() + 1); - - Self { - data_type, - offsets, - values: buffers[1].clone().into(), - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/binary/from.rs b/src/common/arrow/src/arrow/array/binary/from.rs deleted file mode 100644 index bcb2e3358328..000000000000 --- a/src/common/arrow/src/arrow/array/binary/from.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; - -use super::BinaryArray; -use super::MutableBinaryArray; -use crate::arrow::offset::Offset; - -impl> FromIterator> for BinaryArray { - #[inline] - fn from_iter>>(iter: I) -> Self { - MutableBinaryArray::::from_iter(iter).into() - } -} diff --git a/src/common/arrow/src/arrow/array/binary/iterator.rs b/src/common/arrow/src/arrow/array/binary/iterator.rs deleted file mode 100644 index f84206413cdc..000000000000 --- a/src/common/arrow/src/arrow/array/binary/iterator.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::BinaryArray; -use super::MutableBinaryValuesArray; -use crate::arrow::array::ArrayAccessor; -use crate::arrow::array::ArrayValuesIter; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::offset::Offset; - -unsafe impl<'a, O: Offset> ArrayAccessor<'a> for BinaryArray { - type Item = &'a [u8]; - - #[inline] - unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) - } - - #[inline] - fn len(&self) -> usize { - self.len() - } -} - -/// Iterator of values of an [`BinaryArray`]. -pub type BinaryValueIter<'a, O> = ArrayValuesIter<'a, BinaryArray>; - -impl<'a, O: Offset> IntoIterator for &'a BinaryArray { - type Item = Option<&'a [u8]>; - type IntoIter = ZipValidity<&'a [u8], BinaryValueIter<'a, O>, BitmapIter<'a>>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -/// Iterator of values of an [`MutableBinaryValuesArray`]. -pub type MutableBinaryValuesIter<'a, O> = ArrayValuesIter<'a, MutableBinaryValuesArray>; - -impl<'a, O: Offset> IntoIterator for &'a MutableBinaryValuesArray { - type Item = &'a [u8]; - type IntoIter = MutableBinaryValuesIter<'a, O>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} diff --git a/src/common/arrow/src/arrow/array/binary/mod.rs b/src/common/arrow/src/arrow/array/binary/mod.rs deleted file mode 100644 index c19f4fc217ee..000000000000 --- a/src/common/arrow/src/arrow/array/binary/mod.rs +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use either::Either; - -use super::specification::try_check_offsets_bounds; -use super::Array; -use super::GenericBinaryArray; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::offset::OffsetsBuffer; -use crate::arrow::trusted_len::TrustedLen; - -pub(super) mod fmt; -mod iterator; -pub use iterator::*; -mod from; -mod mutable_values; -pub use mutable_values::*; -mod mutable; -pub use mutable::*; - -#[cfg(feature = "arrow")] -mod data; - -/// A [`BinaryArray`] is Arrow's semantically equivalent of an immutable `Vec>>`. -/// It implements [`Array`]. -/// -/// The size of this struct is `O(1)`, as all data is stored behind an [`std::sync::Arc`]. -/// # Example -/// ``` -/// use arrow2::array::BinaryArray; -/// use arrow2::bitmap::Bitmap; -/// use arrow2::buffer::Buffer; -/// -/// let array = BinaryArray::::from([Some([1, 2].as_ref()), None, Some([3].as_ref())]); -/// assert_eq!(array.value(0), &[1, 2]); -/// assert_eq!(array.iter().collect::>(), vec![ -/// Some([1, 2].as_ref()), -/// None, -/// Some([3].as_ref()) -/// ]); -/// assert_eq!(array.values_iter().collect::>(), vec![ -/// [1, 2].as_ref(), -/// &[], -/// &[3] -/// ]); -/// // the underlying representation: -/// assert_eq!(array.values(), &Buffer::from(vec![1, 2, 3])); -/// assert_eq!(array.offsets().buffer(), &Buffer::from(vec![0, 2, 2, 3])); -/// assert_eq!(array.validity(), Some(&Bitmap::from([true, false, true]))); -/// ``` -/// -/// # Generic parameter -/// The generic parameter [`Offset`] can only be `i32` or `i64` and tradeoffs maximum array length with -/// memory usage: -/// * the sum of lengths of all elements cannot exceed `Offset::MAX` -/// * the total size of the underlying data is `array.len() * size_of::() + sum of lengths of all elements` -/// -/// # Safety -/// The following invariants hold: -/// * Two consecutives `offsets` casted (`as`) to `usize` are valid slices of `values`. -/// * `len` is equal to `validity.len()`, when defined. -#[derive(Clone)] -pub struct BinaryArray { - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, -} - -impl BinaryArray { - /// Returns a [`BinaryArray`] created from its internal representation. - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Binary` or `LargeBinary`. - /// # Implementation - /// This function is `O(1)` - pub fn try_new( - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, - ) -> Result { - try_check_offsets_bounds(&offsets, values.len())?; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != offsets.len_proxy()) - { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - - if data_type.to_physical_type() != Self::default_data_type().to_physical_type() { - return Err(Error::oos( - "BinaryArray can only be initialized with DataType::Binary or DataType::LargeBinary", - )); - } - - Ok(Self { - data_type, - offsets, - values, - validity, - }) - } - - /// Creates a new [`BinaryArray`] from slices of `&[u8]`. - pub fn from_slice, P: AsRef<[T]>>(slice: P) -> Self { - Self::from_trusted_len_values_iter(slice.as_ref().iter()) - } - - /// Creates a new [`BinaryArray`] from a slice of optional `&[u8]`. - // Note: this can't be `impl From` because Rust does not allow double `AsRef` on it. - pub fn from, P: AsRef<[Option]>>(slice: P) -> Self { - MutableBinaryArray::::from(slice).into() - } - - /// Returns an iterator of `Option<&[u8]>` over every element of this array. - pub fn iter(&self) -> ZipValidity<&[u8], BinaryValueIter, BitmapIter> { - ZipValidity::new_with_validity(self.values_iter(), self.validity.as_ref()) - } - - /// Returns an iterator of `&[u8]` over every element of this array, ignoring the validity - pub fn values_iter(&self) -> BinaryValueIter { - BinaryValueIter::new(self) - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.offsets.len_proxy() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the element at index `i` - /// # Panics - /// iff `i >= self.len()` - #[inline] - pub fn value(&self, i: usize) -> &[u8] { - assert!(i < self.len()); - unsafe { self.value_unchecked(i) } - } - - /// Returns the element at index `i` - /// # Safety - /// Assumes that the `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] { - // soundness: the invariant of the function - let (start, end) = self.offsets.start_end_unchecked(i); - - // soundness: the invariant of the struct - self.values.get_unchecked(start..end) - } - - /// Returns the element at index `i` or `None` if it is null - /// # Panics - /// iff `i >= self.len()` - #[inline] - pub fn get(&self, i: usize) -> Option<&[u8]> { - if !self.is_null(i) { - // soundness: Array::is_null panics if i >= self.len - unsafe { Some(self.value_unchecked(i)) } - } else { - None - } - } - - /// Returns the [`DataType`] of this array. - #[inline] - pub fn data_type(&self) -> &DataType { - &self.data_type - } - - /// Returns the values of this [`BinaryArray`]. - #[inline] - pub fn values(&self) -> &Buffer { - &self.values - } - - /// Returns the offsets of this [`BinaryArray`]. - #[inline] - pub fn offsets(&self) -> &OffsetsBuffer { - &self.offsets - } - - /// The optional validity. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - /// Slices this [`BinaryArray`]. - /// # Implementation - /// This function is `O(1)`. - /// # Panics - /// iff `offset + length > self.len()`. - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new Buffer cannot exceed the existing length" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Slices this [`BinaryArray`]. - /// # Implementation - /// This function is `O(1)`. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.offsets.slice_unchecked(offset, length + 1); - } - - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); - - /// Returns its internal representation - #[must_use] - pub fn into_inner(self) -> (DataType, OffsetsBuffer, Buffer, Option) { - let Self { - data_type, - offsets, - values, - validity, - } = self; - (data_type, offsets, values, validity) - } - - /// Try to convert this `BinaryArray` to a `MutableBinaryArray` - #[must_use] - pub fn into_mut(self) -> Either> { - use Either::*; - if let Some(bitmap) = self.validity { - match bitmap.into_mut() { - // Safety: invariants are preserved - Left(bitmap) => Left(BinaryArray::new( - self.data_type, - self.offsets, - self.values, - Some(bitmap), - )), - Right(mutable_bitmap) => match (self.values.into_mut(), self.offsets.into_mut()) { - (Left(values), Left(offsets)) => Left(BinaryArray::new( - self.data_type, - offsets, - values, - Some(mutable_bitmap.into()), - )), - (Left(values), Right(offsets)) => Left(BinaryArray::new( - self.data_type, - offsets.into(), - values, - Some(mutable_bitmap.into()), - )), - (Right(values), Left(offsets)) => Left(BinaryArray::new( - self.data_type, - offsets, - values.into(), - Some(mutable_bitmap.into()), - )), - (Right(values), Right(offsets)) => Right( - MutableBinaryArray::try_new( - self.data_type, - offsets, - values, - Some(mutable_bitmap), - ) - .unwrap(), - ), - }, - } - } else { - match (self.values.into_mut(), self.offsets.into_mut()) { - (Left(values), Left(offsets)) => { - Left(BinaryArray::new(self.data_type, offsets, values, None)) - } - (Left(values), Right(offsets)) => Left(BinaryArray::new( - self.data_type, - offsets.into(), - values, - None, - )), - (Right(values), Left(offsets)) => Left(BinaryArray::new( - self.data_type, - offsets, - values.into(), - None, - )), - (Right(values), Right(offsets)) => Right( - MutableBinaryArray::try_new(self.data_type, offsets, values, None).unwrap(), - ), - } - } - } - - /// Creates an empty [`BinaryArray`], i.e. whose `.len` is zero. - pub fn new_empty(data_type: DataType) -> Self { - Self::new(data_type, OffsetsBuffer::new(), Buffer::new(), None) - } - - /// Creates an null [`BinaryArray`], i.e. whose `.null_count() == .len()`. - #[inline] - pub fn new_null(data_type: DataType, length: usize) -> Self { - Self::new( - data_type, - Offsets::new_zeroed(length).into(), - Buffer::new(), - Some(Bitmap::new_zeroed(length)), - ) - } - - /// Returns the default [`DataType`], `DataType::Binary` or `DataType::LargeBinary` - pub fn default_data_type() -> DataType { - if O::IS_LARGE { - DataType::LargeBinary - } else { - DataType::Binary - } - } - - /// Alias for unwrapping [`Self::try_new`] - pub fn new( - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, - ) -> Self { - Self::try_new(data_type, offsets, values, validity).unwrap() - } - - /// Returns a [`BinaryArray`] from an iterator of trusted length. - /// - /// The [`BinaryArray`] is guaranteed to not have a validity - #[inline] - pub fn from_trusted_len_values_iter, I: TrustedLen>( - iterator: I, - ) -> Self { - MutableBinaryArray::::from_trusted_len_values_iter(iterator).into() - } - - /// Returns a new [`BinaryArray`] from a [`Iterator`] of `&[u8]`. - /// - /// The [`BinaryArray`] is guaranteed to not have a validity - pub fn from_iter_values, I: Iterator>(iterator: I) -> Self { - MutableBinaryArray::::from_iter_values(iterator).into() - } - - /// Creates a [`BinaryArray`] from an iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: AsRef<[u8]>, - I: Iterator>, - { - MutableBinaryArray::::from_trusted_len_iter_unchecked(iterator).into() - } - - /// Creates a [`BinaryArray`] from a [`TrustedLen`] - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: AsRef<[u8]>, - I: TrustedLen>, - { - // soundness: I is `TrustedLen` - unsafe { Self::from_trusted_len_iter_unchecked(iterator) } - } - - /// Creates a [`BinaryArray`] from an falible iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn try_from_trusted_len_iter_unchecked(iterator: I) -> Result - where - P: AsRef<[u8]>, - I: IntoIterator, E>>, - { - MutableBinaryArray::::try_from_trusted_len_iter_unchecked(iterator).map(|x| x.into()) - } - - /// Creates a [`BinaryArray`] from an fallible iterator of trusted length. - #[inline] - pub fn try_from_trusted_len_iter(iter: I) -> Result - where - P: AsRef<[u8]>, - I: TrustedLen, E>>, - { - // soundness: I: TrustedLen - unsafe { Self::try_from_trusted_len_iter_unchecked(iter) } - } -} - -impl Array for BinaryArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} - -unsafe impl GenericBinaryArray for BinaryArray { - #[inline] - fn values(&self) -> &[u8] { - self.values() - } - - #[inline] - fn offsets(&self) -> &[O] { - self.offsets().buffer() - } -} diff --git a/src/common/arrow/src/arrow/array/binary/mutable.rs b/src/common/arrow/src/arrow/array/binary/mutable.rs deleted file mode 100644 index 4efda527e5e2..000000000000 --- a/src/common/arrow/src/arrow/array/binary/mutable.rs +++ /dev/null @@ -1,494 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; -use std::sync::Arc; - -use super::BinaryArray; -use super::MutableBinaryValuesArray; -use super::MutableBinaryValuesIter; -use crate::arrow::array::physical_binary::*; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::trusted_len::TrustedLen; - -/// The Arrow's equivalent to `Vec>>`. -/// Converting a [`MutableBinaryArray`] into a [`BinaryArray`] is `O(1)`. -/// # Implementation -/// This struct does not allocate a validity until one is required (i.e. push a null to it). -#[derive(Debug, Clone)] -pub struct MutableBinaryArray { - values: MutableBinaryValuesArray, - validity: Option, -} - -impl From> for BinaryArray { - fn from(other: MutableBinaryArray) -> Self { - let validity = other.validity.and_then(|x| { - let validity: Option = x.into(); - validity - }); - let array: BinaryArray = other.values.into(); - array.with_validity(validity) - } -} - -impl Default for MutableBinaryArray { - fn default() -> Self { - Self::new() - } -} - -impl MutableBinaryArray { - /// Creates a new empty [`MutableBinaryArray`]. - /// # Implementation - /// This allocates a [`Vec`] of one element - pub fn new() -> Self { - Self::with_capacity(0) - } - - /// Returns a [`MutableBinaryArray`] created from its internal representation. - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Binary` or `LargeBinary`. - /// # Implementation - /// This function is `O(1)` - pub fn try_new( - data_type: DataType, - offsets: Offsets, - values: Vec, - validity: Option, - ) -> Result { - let values = MutableBinaryValuesArray::try_new(data_type, offsets, values)?; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != values.len()) - { - return Err(Error::oos( - "validity's length must be equal to the number of values", - )); - } - - Ok(Self { values, validity }) - } - - /// Creates a new [`MutableBinaryArray`] from a slice of optional `&[u8]`. - // Note: this can't be `impl From` because Rust does not allow double `AsRef` on it. - pub fn from, P: AsRef<[Option]>>(slice: P) -> Self { - Self::from_trusted_len_iter(slice.as_ref().iter().map(|x| x.as_ref())) - } - - fn default_data_type() -> DataType { - BinaryArray::::default_data_type() - } - - /// Initializes a new [`MutableBinaryArray`] with a pre-allocated capacity of slots. - pub fn with_capacity(capacity: usize) -> Self { - Self::with_capacities(capacity, 0) - } - - /// Initializes a new [`MutableBinaryArray`] with a pre-allocated capacity of slots and values. - /// # Implementation - /// This does not allocate the validity. - pub fn with_capacities(capacity: usize, values: usize) -> Self { - Self { - values: MutableBinaryValuesArray::with_capacities(capacity, values), - validity: None, - } - } - - /// Reserves `additional` elements and `additional_values` on the values buffer. - pub fn reserve(&mut self, additional: usize, additional_values: usize) { - self.values.reserve(additional, additional_values); - if let Some(x) = self.validity.as_mut() { - x.reserve(additional) - } - } - - /// Pushes a new element to the array. - /// # Panic - /// This operation panics iff the length of all values (in bytes) exceeds `O` maximum value. - pub fn push>(&mut self, value: Option) { - self.try_push(value).unwrap() - } - - /// Pop the last entry from [`MutableBinaryArray`]. - /// This function returns `None` iff this array is empty - pub fn pop(&mut self) -> Option> { - let value = self.values.pop()?; - self.validity - .as_mut() - .map(|x| x.pop()?.then_some(())) - .unwrap_or_else(|| Some(())) - .map(|_| value) - } - - fn try_from_iter, I: IntoIterator>>(iter: I) -> Result { - let iterator = iter.into_iter(); - let (lower, _) = iterator.size_hint(); - let mut primitive = Self::with_capacity(lower); - for item in iterator { - primitive.try_push(item.as_ref())? - } - Ok(primitive) - } - - fn init_validity(&mut self) { - let mut validity = MutableBitmap::with_capacity(self.values.capacity()); - validity.extend_constant(self.len(), true); - validity.set(self.len() - 1, false); - self.validity = Some(validity); - } - - /// Converts itself into an [`Array`]. - pub fn into_arc(self) -> Arc { - let a: BinaryArray = self.into(); - Arc::new(a) - } - - /// Shrinks the capacity of the [`MutableBinaryArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - if let Some(validity) = &mut self.validity { - validity.shrink_to_fit() - } - } - - impl_mutable_array_mut_validity!(); -} - -impl MutableBinaryArray { - /// returns its values. - pub fn values(&self) -> &Vec { - self.values.values() - } - - /// returns its offsets. - pub fn offsets(&self) -> &Offsets { - self.values.offsets() - } - - /// Returns an iterator of `Option<&[u8]>` - pub fn iter(&self) -> ZipValidity<&[u8], MutableBinaryValuesIter, BitmapIter> { - ZipValidity::new(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) - } - - /// Returns an iterator over the values of this array - pub fn values_iter(&self) -> MutableBinaryValuesIter { - self.values.iter() - } -} - -impl MutableArray for MutableBinaryArray { - fn len(&self) -> usize { - self.values.len() - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - let array: BinaryArray = std::mem::take(self).into(); - array.boxed() - } - - fn as_arc(&mut self) -> Arc { - let array: BinaryArray = std::mem::take(self).into(); - array.arced() - } - - fn data_type(&self) -> &DataType { - self.values.data_type() - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - #[inline] - fn push_null(&mut self) { - self.push::<&[u8]>(None) - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional, 0) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl> FromIterator> for MutableBinaryArray { - fn from_iter>>(iter: I) -> Self { - Self::try_from_iter(iter).unwrap() - } -} - -impl MutableBinaryArray { - /// Creates a [`MutableBinaryArray`] from an iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: AsRef<[u8]>, - I: Iterator>, - { - let (validity, offsets, values) = trusted_len_unzip(iterator); - - Self::try_new(Self::default_data_type(), offsets, values, validity).unwrap() - } - - /// Creates a [`MutableBinaryArray`] from an iterator of trusted length. - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: AsRef<[u8]>, - I: TrustedLen>, - { - // soundness: I is `TrustedLen` - unsafe { Self::from_trusted_len_iter_unchecked(iterator) } - } - - /// Creates a new [`BinaryArray`] from a [`TrustedLen`] of `&[u8]`. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_values_iter_unchecked, I: Iterator>( - iterator: I, - ) -> Self { - let (offsets, values) = trusted_len_values_iter(iterator); - Self::try_new(Self::default_data_type(), offsets, values, None).unwrap() - } - - /// Creates a new [`BinaryArray`] from a [`TrustedLen`] of `&[u8]`. - #[inline] - pub fn from_trusted_len_values_iter, I: TrustedLen>( - iterator: I, - ) -> Self { - // soundness: I is `TrustedLen` - unsafe { Self::from_trusted_len_values_iter_unchecked(iterator) } - } - - /// Creates a [`MutableBinaryArray`] from an falible iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn try_from_trusted_len_iter_unchecked( - iterator: I, - ) -> std::result::Result - where - P: AsRef<[u8]>, - I: IntoIterator, E>>, - { - let iterator = iterator.into_iter(); - - // soundness: assumed trusted len - let (mut validity, offsets, values) = try_trusted_len_unzip(iterator)?; - - if validity.as_mut().unwrap().unset_bits() == 0 { - validity = None; - } - - Ok(Self::try_new(Self::default_data_type(), offsets, values, validity).unwrap()) - } - - /// Creates a [`MutableBinaryArray`] from an falible iterator of trusted length. - #[inline] - pub fn try_from_trusted_len_iter(iterator: I) -> std::result::Result - where - P: AsRef<[u8]>, - I: TrustedLen, E>>, - { - // soundness: I: TrustedLen - unsafe { Self::try_from_trusted_len_iter_unchecked(iterator) } - } - - /// Extends the [`MutableBinaryArray`] from an iterator of trusted length. - /// This differs from `extend_trusted_len` which accepts iterator of optional values. - #[inline] - pub fn extend_trusted_len_values(&mut self, iterator: I) - where - P: AsRef<[u8]>, - I: TrustedLen, - { - // Safety: The iterator is `TrustedLen` - unsafe { self.extend_trusted_len_values_unchecked(iterator) } - } - - /// Extends the [`MutableBinaryArray`] from an iterator of values. - /// This differs from `extended_trusted_len` which accepts iterator of optional values. - #[inline] - pub fn extend_values(&mut self, iterator: I) - where - P: AsRef<[u8]>, - I: Iterator, - { - let length = self.values.len(); - self.values.extend(iterator); - let additional = self.values.len() - length; - - if let Some(validity) = self.validity.as_mut() { - validity.extend_constant(additional, true); - } - } - - /// Extends the [`MutableBinaryArray`] from an `iterator` of values of trusted length. - /// This differs from `extend_trusted_len_unchecked` which accepts iterator of optional - /// values. - /// # Safety - /// The `iterator` must be [`TrustedLen`] - #[inline] - pub unsafe fn extend_trusted_len_values_unchecked(&mut self, iterator: I) - where - P: AsRef<[u8]>, - I: Iterator, - { - let length = self.values.len(); - self.values.extend_trusted_len_unchecked(iterator); - let additional = self.values.len() - length; - - if let Some(validity) = self.validity.as_mut() { - validity.extend_constant(additional, true); - } - } - - /// Extends the [`MutableBinaryArray`] from an iterator of [`TrustedLen`] - #[inline] - pub fn extend_trusted_len(&mut self, iterator: I) - where - P: AsRef<[u8]>, - I: TrustedLen>, - { - // Safety: The iterator is `TrustedLen` - unsafe { self.extend_trusted_len_unchecked(iterator) } - } - - /// Extends the [`MutableBinaryArray`] from an iterator of [`TrustedLen`] - /// # Safety - /// The `iterator` must be [`TrustedLen`] - #[inline] - pub unsafe fn extend_trusted_len_unchecked(&mut self, iterator: I) - where - P: AsRef<[u8]>, - I: Iterator>, - { - if self.validity.is_none() { - let mut validity = MutableBitmap::new(); - validity.extend_constant(self.len(), true); - self.validity = Some(validity); - } - - self.values - .extend_from_trusted_len_iter(self.validity.as_mut().unwrap(), iterator); - } - - /// Creates a new [`MutableBinaryArray`] from a [`Iterator`] of `&[u8]`. - pub fn from_iter_values, I: Iterator>(iterator: I) -> Self { - let (offsets, values) = values_iter(iterator); - Self::try_new(Self::default_data_type(), offsets, values, None).unwrap() - } - - /// Extend with a fallible iterator - pub fn extend_fallible(&mut self, iter: I) -> std::result::Result<(), E> - where - E: std::error::Error, - I: IntoIterator, E>>, - T: AsRef<[u8]>, - { - let mut iter = iter.into_iter(); - self.reserve(iter.size_hint().0, 0); - iter.try_for_each(|x| { - self.push(x?); - Ok(()) - }) - } -} - -impl> Extend> for MutableBinaryArray { - fn extend>>(&mut self, iter: I) { - self.try_extend(iter).unwrap(); - } -} - -impl> TryExtend> for MutableBinaryArray { - fn try_extend>>(&mut self, iter: I) -> Result<()> { - let mut iter = iter.into_iter(); - self.reserve(iter.size_hint().0, 0); - iter.try_for_each(|x| self.try_push(x)) - } -} - -impl> TryPush> for MutableBinaryArray { - fn try_push(&mut self, value: Option) -> Result<()> { - match value { - Some(value) => { - self.values.try_push(value.as_ref())?; - - match &mut self.validity { - Some(validity) => validity.push(true), - None => {} - } - } - None => { - self.values.push(""); - match &mut self.validity { - Some(validity) => validity.push(false), - None => self.init_validity(), - } - } - } - Ok(()) - } -} - -impl PartialEq for MutableBinaryArray { - fn eq(&self, other: &Self) -> bool { - self.iter().eq(other.iter()) - } -} - -impl TryExtendFromSelf for MutableBinaryArray { - fn try_extend_from_self(&mut self, other: &Self) -> Result<()> { - extend_validity(self.len(), &mut self.validity, &other.validity); - - self.values.try_extend_from_self(&other.values) - } -} diff --git a/src/common/arrow/src/arrow/array/binary/mutable_values.rs b/src/common/arrow/src/arrow/array/binary/mutable_values.rs deleted file mode 100644 index 80f9bf41c181..000000000000 --- a/src/common/arrow/src/arrow/array/binary/mutable_values.rs +++ /dev/null @@ -1,402 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; -use std::sync::Arc; - -use super::BinaryArray; -use super::MutableBinaryArray; -use crate::arrow::array::physical_binary::*; -use crate::arrow::array::specification::try_check_offsets_bounds; -use crate::arrow::array::Array; -use crate::arrow::array::ArrayAccessor; -use crate::arrow::array::ArrayValuesIter; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::trusted_len::TrustedLen; - -/// A [`MutableArray`] that builds a [`BinaryArray`]. It differs -/// from [`MutableBinaryArray`] in that it builds non-null [`BinaryArray`]. -#[derive(Debug, Clone)] -pub struct MutableBinaryValuesArray { - data_type: DataType, - offsets: Offsets, - values: Vec, -} - -impl From> for BinaryArray { - fn from(other: MutableBinaryValuesArray) -> Self { - BinaryArray::::new( - other.data_type, - other.offsets.into(), - other.values.into(), - None, - ) - } -} - -impl From> for MutableBinaryArray { - fn from(other: MutableBinaryValuesArray) -> Self { - MutableBinaryArray::::try_new(other.data_type, other.offsets, other.values, None) - .expect("MutableBinaryValuesArray is consistent with MutableBinaryArray") - } -} - -impl Default for MutableBinaryValuesArray { - fn default() -> Self { - Self::new() - } -} - -impl MutableBinaryValuesArray { - /// Returns an empty [`MutableBinaryValuesArray`]. - pub fn new() -> Self { - Self { - data_type: Self::default_data_type(), - offsets: Offsets::new(), - values: Vec::::new(), - } - } - - /// Returns a [`MutableBinaryValuesArray`] created from its internal representation. - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Binary` or `LargeBinary`. - /// # Implementation - /// This function is `O(1)` - pub fn try_new(data_type: DataType, offsets: Offsets, values: Vec) -> Result { - try_check_offsets_bounds(&offsets, values.len())?; - - if data_type.to_physical_type() != Self::default_data_type().to_physical_type() { - return Err(Error::oos( - "MutableBinaryValuesArray can only be initialized with DataType::Binary or DataType::LargeBinary", - )); - } - - Ok(Self { - data_type, - offsets, - values, - }) - } - - /// Returns the default [`DataType`] of this container: [`DataType::Utf8`] or [`DataType::LargeUtf8`] - /// depending on the generic [`Offset`]. - pub fn default_data_type() -> DataType { - BinaryArray::::default_data_type() - } - - /// Initializes a new [`MutableBinaryValuesArray`] with a pre-allocated capacity of items. - pub fn with_capacity(capacity: usize) -> Self { - Self::with_capacities(capacity, 0) - } - - /// Initializes a new [`MutableBinaryValuesArray`] with a pre-allocated capacity of items and values. - pub fn with_capacities(capacity: usize, values: usize) -> Self { - Self { - data_type: Self::default_data_type(), - offsets: Offsets::::with_capacity(capacity), - values: Vec::::with_capacity(values), - } - } - - /// returns its values. - #[inline] - pub fn values(&self) -> &Vec { - &self.values - } - - /// returns its offsets. - #[inline] - pub fn offsets(&self) -> &Offsets { - &self.offsets - } - - /// Reserves `additional` elements and `additional_values` on the values. - #[inline] - pub fn reserve(&mut self, additional: usize, additional_values: usize) { - self.offsets.reserve(additional); - self.values.reserve(additional_values); - } - - /// Returns the capacity in number of items - pub fn capacity(&self) -> usize { - self.offsets.capacity() - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.offsets.len_proxy() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Pushes a new item to the array. - /// # Panic - /// This operation panics iff the length of all values (in bytes) exceeds `O` maximum value. - #[inline] - pub fn push>(&mut self, value: T) { - self.try_push(value).unwrap() - } - - /// Pop the last entry from [`MutableBinaryValuesArray`]. - /// This function returns `None` iff this array is empty. - pub fn pop(&mut self) -> Option> { - if self.is_empty() { - return None; - } - self.offsets.pop()?; - let start = self.offsets.last().to_usize(); - let value = self.values.split_off(start); - Some(value.to_vec()) - } - - /// Returns the value of the element at index `i`. - /// # Panic - /// This function panics iff `i >= self.len`. - #[inline] - pub fn value(&self, i: usize) -> &[u8] { - assert!(i < self.len()); - unsafe { self.value_unchecked(i) } - } - - /// Returns the value of the element at index `i`. - /// # Safety - /// This function is safe iff `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] { - // soundness: the invariant of the function - let (start, end) = self.offsets.start_end(i); - - // soundness: the invariant of the struct - self.values.get_unchecked(start..end) - } - - /// Returns an iterator of `&[u8]` - pub fn iter(&self) -> ArrayValuesIter { - ArrayValuesIter::new(self) - } - - /// Shrinks the capacity of the [`MutableBinaryValuesArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - self.offsets.shrink_to_fit(); - } - - /// Extract the low-end APIs from the [`MutableBinaryValuesArray`]. - pub fn into_inner(self) -> (DataType, Offsets, Vec) { - (self.data_type, self.offsets, self.values) - } -} - -impl MutableArray for MutableBinaryValuesArray { - fn len(&self) -> usize { - self.len() - } - - fn validity(&self) -> Option<&MutableBitmap> { - None - } - - fn as_box(&mut self) -> Box { - let (data_type, offsets, values) = std::mem::take(self).into_inner(); - BinaryArray::new(data_type, offsets.into(), values.into(), None).boxed() - } - - fn as_arc(&mut self) -> Arc { - let (data_type, offsets, values) = std::mem::take(self).into_inner(); - BinaryArray::new(data_type, offsets.into(), values.into(), None).arced() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - #[inline] - fn push_null(&mut self) { - self.push::<&[u8]>(b"") - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional, 0) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl> FromIterator

for MutableBinaryValuesArray { - fn from_iter>(iter: I) -> Self { - let (offsets, values) = values_iter(iter.into_iter()); - Self::try_new(Self::default_data_type(), offsets, values).unwrap() - } -} - -impl MutableBinaryValuesArray { - pub(crate) unsafe fn extend_from_trusted_len_iter( - &mut self, - validity: &mut MutableBitmap, - iterator: I, - ) where - P: AsRef<[u8]>, - I: Iterator>, - { - extend_from_trusted_len_iter(&mut self.offsets, &mut self.values, validity, iterator); - } - - /// Extends the [`MutableBinaryValuesArray`] from a [`TrustedLen`] - #[inline] - pub fn extend_trusted_len(&mut self, iterator: I) - where - P: AsRef<[u8]>, - I: TrustedLen, - { - unsafe { self.extend_trusted_len_unchecked(iterator) } - } - - /// Extends [`MutableBinaryValuesArray`] from an iterator of trusted len. - /// # Safety - /// The iterator must be trusted len. - #[inline] - pub unsafe fn extend_trusted_len_unchecked(&mut self, iterator: I) - where - P: AsRef<[u8]>, - I: Iterator, - { - extend_from_trusted_len_values_iter(&mut self.offsets, &mut self.values, iterator); - } - - /// Creates a [`MutableBinaryValuesArray`] from a [`TrustedLen`] - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: AsRef<[u8]>, - I: TrustedLen, - { - // soundness: I is `TrustedLen` - unsafe { Self::from_trusted_len_iter_unchecked(iterator) } - } - - /// Returns a new [`MutableBinaryValuesArray`] from an iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: AsRef<[u8]>, - I: Iterator, - { - let (offsets, values) = trusted_len_values_iter(iterator); - Self::try_new(Self::default_data_type(), offsets, values).unwrap() - } - - /// Returns a new [`MutableBinaryValuesArray`] from an iterator. - /// # Error - /// This operation errors iff the total length in bytes on the iterator exceeds `O`'s maximum value. - /// (`i32::MAX` or `i64::MAX` respectively). - pub fn try_from_iter, I: IntoIterator>(iter: I) -> Result { - let iterator = iter.into_iter(); - let (lower, _) = iterator.size_hint(); - let mut array = Self::with_capacity(lower); - for item in iterator { - array.try_push(item)?; - } - Ok(array) - } - - /// Extend with a fallible iterator - pub fn extend_fallible(&mut self, iter: I) -> std::result::Result<(), E> - where - E: std::error::Error, - I: IntoIterator>, - T: AsRef<[u8]>, - { - let mut iter = iter.into_iter(); - self.reserve(iter.size_hint().0, 0); - iter.try_for_each(|x| { - self.push(x?); - Ok(()) - }) - } -} - -impl> Extend for MutableBinaryValuesArray { - fn extend>(&mut self, iter: I) { - extend_from_values_iter(&mut self.offsets, &mut self.values, iter.into_iter()); - } -} - -impl> TryExtend for MutableBinaryValuesArray { - fn try_extend>(&mut self, iter: I) -> Result<()> { - let mut iter = iter.into_iter(); - self.reserve(iter.size_hint().0, 0); - iter.try_for_each(|x| self.try_push(x)) - } -} - -impl> TryPush for MutableBinaryValuesArray { - #[inline] - fn try_push(&mut self, value: T) -> Result<()> { - let bytes = value.as_ref(); - self.values.extend_from_slice(bytes); - self.offsets.try_push_usize(bytes.len()) - } -} - -unsafe impl<'a, O: Offset> ArrayAccessor<'a> for MutableBinaryValuesArray { - type Item = &'a [u8]; - - #[inline] - unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) - } - - #[inline] - fn len(&self) -> usize { - self.len() - } -} - -impl TryExtendFromSelf for MutableBinaryValuesArray { - fn try_extend_from_self(&mut self, other: &Self) -> Result<()> { - self.values.extend_from_slice(&other.values); - self.offsets.try_extend_from_self(&other.offsets) - } -} diff --git a/src/common/arrow/src/arrow/array/binview/from.rs b/src/common/arrow/src/arrow/array/binview/from.rs deleted file mode 100644 index 40304c7c0279..000000000000 --- a/src/common/arrow/src/arrow/array/binview/from.rs +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; -use arrow_schema::DataType; - -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::BinaryViewArray; -use crate::arrow::array::BinaryViewArrayGeneric; -use crate::arrow::array::MutableBinaryViewArray; -use crate::arrow::array::Utf8ViewArray; -use crate::arrow::array::ViewType; -use crate::arrow::bitmap::Bitmap; - -impl> FromIterator> for BinaryViewArrayGeneric { - #[inline] - fn from_iter>>(iter: I) -> Self { - MutableBinaryViewArray::::from_iter(iter).into() - } -} - -impl Arrow2Arrow for BinaryViewArray { - fn to_data(&self) -> ArrayData { - let builder = ArrayDataBuilder::new(DataType::BinaryView) - .len(self.len()) - .add_buffer(self.views.clone().into()) - .add_buffers( - self.buffers - .iter() - .map(|x| x.clone().into()) - .collect::>(), - ) - .nulls(self.validity.clone().map(Into::into)); - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let views = crate::arrow::buffer::Buffer::from(data.buffers()[0].clone()); - let buffers = data.buffers()[1..] - .iter() - .map(|x| crate::arrow::buffer::Buffer::from(x.clone())) - .collect(); - let validity = data.nulls().map(|x| Bitmap::from_null_buffer(x.clone())); - unsafe { - Self::new_unchecked_unknown_md( - crate::arrow::datatypes::DataType::BinaryView, - views, - buffers, - validity, - None, - ) - } - } -} - -impl Arrow2Arrow for Utf8ViewArray { - fn to_data(&self) -> ArrayData { - let builder = ArrayDataBuilder::new(DataType::Utf8View) - .len(self.len()) - .add_buffer(self.views.clone().into()) - .add_buffers( - self.buffers - .iter() - .map(|x| x.clone().into()) - .collect::>(), - ) - .nulls(self.validity.clone().map(Into::into)); - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let views = crate::arrow::buffer::Buffer::from(data.buffers()[0].clone()); - let buffers = data.buffers()[1..] - .iter() - .map(|x| crate::arrow::buffer::Buffer::from(x.clone())) - .collect(); - let validity = data.nulls().map(|x| Bitmap::from_null_buffer(x.clone())); - unsafe { - Self::new_unchecked_unknown_md( - crate::arrow::datatypes::DataType::Utf8View, - views, - buffers, - validity, - None, - ) - } - } -} diff --git a/src/common/arrow/src/arrow/array/boolean/data.rs b/src/common/arrow/src/arrow/array/boolean/data.rs deleted file mode 100644 index 066881e8cede..000000000000 --- a/src/common/arrow/src/arrow/array/boolean/data.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_buffer::BooleanBuffer; -use arrow_buffer::NullBuffer; -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::BooleanArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::DataType; - -impl Arrow2Arrow for BooleanArray { - fn to_data(&self) -> ArrayData { - let buffer = NullBuffer::from(self.values.clone()); - - let builder = ArrayDataBuilder::new(arrow_schema::DataType::Boolean) - .len(buffer.len()) - .offset(buffer.offset()) - .buffers(vec![buffer.into_inner().into_inner()]) - .nulls(self.validity.as_ref().map(|b| b.clone().into())); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - assert_eq!(data.data_type(), &arrow_schema::DataType::Boolean); - - let buffers = data.buffers(); - let buffer = BooleanBuffer::new(buffers[0].clone(), data.offset(), data.len()); - // Use NullBuffer to compute set count - let values = Bitmap::from_null_buffer(NullBuffer::new(buffer)); - - Self { - data_type: DataType::Boolean, - values, - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/boolean/fmt.rs b/src/common/arrow/src/arrow/array/boolean/fmt.rs deleted file mode 100644 index a53acb3a8887..000000000000 --- a/src/common/arrow/src/arrow/array/boolean/fmt.rs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::write_vec; -use super::BooleanArray; - -pub fn write_value(array: &BooleanArray, index: usize, f: &mut W) -> Result { - write!(f, "{}", array.value(index)) -} - -impl Debug for BooleanArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, f); - - write!(f, "BooleanArray")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/boolean/from.rs b/src/common/arrow/src/arrow/array/boolean/from.rs deleted file mode 100644 index 5fe474c93227..000000000000 --- a/src/common/arrow/src/arrow/array/boolean/from.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; - -use super::BooleanArray; -use super::MutableBooleanArray; - -impl]>> From

for BooleanArray { - fn from(slice: P) -> Self { - MutableBooleanArray::from(slice).into() - } -} - -impl>> FromIterator for BooleanArray { - fn from_iter>(iter: I) -> Self { - MutableBooleanArray::from_iter(iter).into() - } -} diff --git a/src/common/arrow/src/arrow/array/boolean/iterator.rs b/src/common/arrow/src/arrow/array/boolean/iterator.rs deleted file mode 100644 index db6cd20ccf00..000000000000 --- a/src/common/arrow/src/arrow/array/boolean/iterator.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::super::MutableArray; -use super::BooleanArray; -use super::MutableBooleanArray; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::IntoIter; - -impl<'a> IntoIterator for &'a BooleanArray { - type Item = Option; - type IntoIter = ZipValidity, BitmapIter<'a>>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl IntoIterator for BooleanArray { - type Item = Option; - type IntoIter = ZipValidity; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - let (_, values, validity) = self.into_inner(); - let values = values.into_iter(); - let validity = - validity.and_then(|validity| (validity.unset_bits() > 0).then(|| validity.into_iter())); - ZipValidity::new(values, validity) - } -} - -impl<'a> IntoIterator for &'a MutableBooleanArray { - type Item = Option; - type IntoIter = ZipValidity, BitmapIter<'a>>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> MutableBooleanArray { - /// Returns an iterator over the optional values of this [`MutableBooleanArray`]. - #[inline] - pub fn iter(&'a self) -> ZipValidity, BitmapIter<'a>> { - ZipValidity::new( - self.values().iter(), - self.validity().as_ref().map(|x| x.iter()), - ) - } - - /// Returns an iterator over the values of this [`MutableBooleanArray`] - #[inline] - pub fn values_iter(&'a self) -> BitmapIter<'a> { - self.values().iter() - } -} diff --git a/src/common/arrow/src/arrow/array/boolean/mod.rs b/src/common/arrow/src/arrow/array/boolean/mod.rs deleted file mode 100644 index 82cbd63bab32..000000000000 --- a/src/common/arrow/src/arrow/array/boolean/mod.rs +++ /dev/null @@ -1,411 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use either::Either; - -use super::Array; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; -use crate::arrow::trusted_len::TrustedLen; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod from; -mod iterator; -mod mutable; - -pub use mutable::*; - -/// A [`BooleanArray`] is Arrow's semantically equivalent of an immutable `Vec>`. -/// It implements [`Array`]. -/// -/// One way to think about a [`BooleanArray`] is `(DataType, Arc>, Option>>)` -/// where: -/// * the first item is the array's logical type -/// * the second is the immutable values -/// * the third is the immutable validity (whether a value is null or not as a bitmap). -/// -/// The size of this struct is `O(1)`, as all data is stored behind an [`std::sync::Arc`]. -/// # Example -/// ``` -/// use arrow2::array::BooleanArray; -/// use arrow2::bitmap::Bitmap; -/// use arrow2::buffer::Buffer; -/// -/// let array = BooleanArray::from([Some(true), None, Some(false)]); -/// assert_eq!(array.value(0), true); -/// assert_eq!(array.iter().collect::>(), vec![ -/// Some(true), -/// None, -/// Some(false) -/// ]); -/// assert_eq!(array.values_iter().collect::>(), vec![ -/// true, false, false -/// ]); -/// // the underlying representation -/// assert_eq!(array.values(), &Bitmap::from([true, false, false])); -/// assert_eq!(array.validity(), Some(&Bitmap::from([true, false, true]))); -/// ``` -#[derive(Clone)] -pub struct BooleanArray { - data_type: DataType, - values: Bitmap, - validity: Option, -} - -impl BooleanArray { - /// The canonical method to create a [`BooleanArray`] out of low-end APIs. - /// # Errors - /// This function errors iff: - /// * The validity is not `None` and its length is different from `values`'s length - /// * The `data_type`'s [`PhysicalType`] is not equal to [`PhysicalType::Boolean`]. - pub fn try_new( - data_type: DataType, - values: Bitmap, - validity: Option, - ) -> Result { - if validity - .as_ref() - .map_or(false, |validity| validity.len() != values.len()) - { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - - if data_type.to_physical_type() != PhysicalType::Boolean { - return Err(Error::oos( - "BooleanArray can only be initialized with a DataType whose physical type is Boolean", - )); - } - - Ok(Self { - data_type, - values, - validity, - }) - } - - /// Alias to `Self::try_new().unwrap()` - pub fn new(data_type: DataType, values: Bitmap, validity: Option) -> Self { - Self::try_new(data_type, values, validity).unwrap() - } - - /// Returns an iterator over the optional values of this [`BooleanArray`]. - #[inline] - pub fn iter(&self) -> ZipValidity { - ZipValidity::new_with_validity(self.values().iter(), self.validity()) - } - - /// Returns an iterator over the values of this [`BooleanArray`]. - #[inline] - pub fn values_iter(&self) -> BitmapIter { - self.values().iter() - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.values.len() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The values [`Bitmap`]. - /// Values on null slots are undetermined (they can be anything). - #[inline] - pub fn values(&self) -> &Bitmap { - &self.values - } - - /// Returns the optional validity. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - /// Returns the arrays' [`DataType`]. - #[inline] - pub fn data_type(&self) -> &DataType { - &self.data_type - } - - /// Returns the value at index `i` - /// # Panic - /// This function panics iff `i >= self.len()`. - #[inline] - pub fn value(&self, i: usize) -> bool { - self.values.get_bit(i) - } - - /// Returns the element at index `i` as bool - /// # Safety - /// Caller must be sure that `i < self.len()` - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> bool { - self.values.get_bit_unchecked(i) - } - - /// Returns the element at index `i` or `None` if it is null - /// # Panics - /// iff `i >= self.len()` - #[inline] - pub fn get(&self, i: usize) -> Option { - if !self.is_null(i) { - // soundness: Array::is_null panics if i >= self.len - unsafe { Some(self.value_unchecked(i)) } - } else { - None - } - } - - /// Slices this [`BooleanArray`]. - /// # Implementation - /// This operation is `O(1)` as it amounts to increase up to two ref counts. - /// # Panic - /// This function panics iff `offset + length > self.len()`. - #[inline] - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new Buffer cannot exceed the existing length" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Slices this [`BooleanArray`]. - /// # Implementation - /// This operation is `O(1)` as it amounts to increase two ref counts. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - #[inline] - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.values.slice_unchecked(offset, length); - } - - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); - - /// Returns a clone of this [`BooleanArray`] with new values. - /// # Panics - /// This function panics iff `values.len() != self.len()`. - #[must_use] - pub fn with_values(&self, values: Bitmap) -> Self { - let mut out = self.clone(); - out.set_values(values); - out - } - - /// Sets the values of this [`BooleanArray`]. - /// # Panics - /// This function panics iff `values.len() != self.len()`. - pub fn set_values(&mut self, values: Bitmap) { - assert_eq!( - values.len(), - self.len(), - "values length must be equal to this arrays length" - ); - self.values = values; - } - - /// Applies a function `f` to the values of this array, cloning the values - /// iff they are being shared with others - /// - /// This is an API to use clone-on-write - /// # Implementation - /// This function is `O(f)` if the data is not being shared, and `O(N) + O(f)` - /// if it is being shared (since it results in a `O(N)` memcopy). - /// # Panics - /// This function panics if the function modifies the length of the [`MutableBitmap`]. - pub fn apply_values_mut(&mut self, f: F) { - let values = std::mem::take(&mut self.values); - let mut values = values.make_mut(); - f(&mut values); - if let Some(validity) = &self.validity { - assert_eq!(validity.len(), values.len()); - } - self.values = values.into(); - } - - /// Try to convert this [`BooleanArray`] to a [`MutableBooleanArray`] - pub fn into_mut(self) -> Either { - use Either::*; - - if let Some(bitmap) = self.validity { - match bitmap.into_mut() { - Left(bitmap) => Left(BooleanArray::new(self.data_type, self.values, Some(bitmap))), - Right(mutable_bitmap) => match self.values.into_mut() { - Left(immutable) => Left(BooleanArray::new( - self.data_type, - immutable, - Some(mutable_bitmap.into()), - )), - Right(mutable) => Right( - MutableBooleanArray::try_new(self.data_type, mutable, Some(mutable_bitmap)) - .unwrap(), - ), - }, - } - } else { - match self.values.into_mut() { - Left(immutable) => Left(BooleanArray::new(self.data_type, immutable, None)), - Right(mutable) => { - Right(MutableBooleanArray::try_new(self.data_type, mutable, None).unwrap()) - } - } - } - } - - /// Returns a new empty [`BooleanArray`]. - pub fn new_empty(data_type: DataType) -> Self { - Self::new(data_type, Bitmap::new(), None) - } - - /// Returns a new [`BooleanArray`] whose all slots are null / `None`. - pub fn new_null(data_type: DataType, length: usize) -> Self { - let bitmap = Bitmap::new_zeroed(length); - Self::new(data_type, bitmap.clone(), Some(bitmap)) - } - - /// Creates a new [`BooleanArray`] from an [`TrustedLen`] of `bool`. - #[inline] - pub fn from_trusted_len_values_iter>(iterator: I) -> Self { - MutableBooleanArray::from_trusted_len_values_iter(iterator).into() - } - - /// Creates a new [`BooleanArray`] from an [`TrustedLen`] of `bool`. - /// Use this over [`BooleanArray::from_trusted_len_iter`] when the iterator is trusted len - /// but this crate does not mark it as such. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_values_iter_unchecked>( - iterator: I, - ) -> Self { - MutableBooleanArray::from_trusted_len_values_iter_unchecked(iterator).into() - } - - /// Creates a new [`BooleanArray`] from a slice of `bool`. - #[inline] - pub fn from_slice>(slice: P) -> Self { - MutableBooleanArray::from_slice(slice).into() - } - - /// Creates a [`BooleanArray`] from an iterator of trusted length. - /// Use this over [`BooleanArray::from_trusted_len_iter`] when the iterator is trusted len - /// but this crate does not mark it as such. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: std::borrow::Borrow, - I: Iterator>, - { - MutableBooleanArray::from_trusted_len_iter_unchecked(iterator).into() - } - - /// Creates a [`BooleanArray`] from a [`TrustedLen`]. - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: std::borrow::Borrow, - I: TrustedLen>, - { - MutableBooleanArray::from_trusted_len_iter(iterator).into() - } - - /// Creates a [`BooleanArray`] from an falible iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn try_from_trusted_len_iter_unchecked(iterator: I) -> Result - where - P: std::borrow::Borrow, - I: Iterator, E>>, - { - Ok(MutableBooleanArray::try_from_trusted_len_iter_unchecked(iterator)?.into()) - } - - /// Creates a [`BooleanArray`] from a [`TrustedLen`]. - #[inline] - pub fn try_from_trusted_len_iter(iterator: I) -> Result - where - P: std::borrow::Borrow, - I: TrustedLen, E>>, - { - Ok(MutableBooleanArray::try_from_trusted_len_iter(iterator)?.into()) - } - - /// Returns its internal representation - #[must_use] - pub fn into_inner(self) -> (DataType, Bitmap, Option) { - let Self { - data_type, - values, - validity, - } = self; - (data_type, values, validity) - } - - /// Creates a `[BooleanArray]` from its internal representation. - /// This is the inverted from `[BooleanArray::into_inner]` - /// - /// # Safety - /// Callers must ensure all invariants of this struct are upheld. - pub unsafe fn from_inner_unchecked( - data_type: DataType, - values: Bitmap, - validity: Option, - ) -> Self { - Self { - data_type, - values, - validity, - } - } -} - -impl Array for BooleanArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/boolean/mutable.rs b/src/common/arrow/src/arrow/array/boolean/mutable.rs deleted file mode 100644 index 12d9114ef0c5..000000000000 --- a/src/common/arrow/src/arrow/array/boolean/mutable.rs +++ /dev/null @@ -1,580 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; -use std::sync::Arc; - -use super::BooleanArray; -use crate::arrow::array::physical_binary::extend_validity; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; -use crate::arrow::trusted_len::TrustedLen; - -/// The Arrow's equivalent to `Vec>`, but with `1/16` of its size. -/// Converting a [`MutableBooleanArray`] into a [`BooleanArray`] is `O(1)`. -/// # Implementation -/// This struct does not allocate a validity until one is required (i.e. push a null to it). -#[derive(Debug, Clone)] -pub struct MutableBooleanArray { - data_type: DataType, - values: MutableBitmap, - validity: Option, -} - -impl From for BooleanArray { - fn from(other: MutableBooleanArray) -> Self { - BooleanArray::new( - other.data_type, - other.values.into(), - other.validity.map(|x| x.into()), - ) - } -} - -impl]>> From

for MutableBooleanArray { - /// Creates a new [`MutableBooleanArray`] out of a slice of Optional `bool`. - fn from(slice: P) -> Self { - Self::from_trusted_len_iter(slice.as_ref().iter().map(|x| x.as_ref())) - } -} - -impl Default for MutableBooleanArray { - fn default() -> Self { - Self::new() - } -} - -impl MutableBooleanArray { - /// Creates an new empty [`MutableBooleanArray`]. - pub fn new() -> Self { - Self::with_capacity(0) - } - - /// The canonical method to create a [`MutableBooleanArray`] out of low-end APIs. - /// # Errors - /// This function errors iff: - /// * The validity is not `None` and its length is different from `values`'s length - /// * The `data_type`'s [`PhysicalType`] is not equal to [`PhysicalType::Boolean`]. - pub fn try_new( - data_type: DataType, - values: MutableBitmap, - validity: Option, - ) -> Result { - if validity - .as_ref() - .map_or(false, |validity| validity.len() != values.len()) - { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - - if data_type.to_physical_type() != PhysicalType::Boolean { - return Err(Error::oos( - "MutableBooleanArray can only be initialized with a DataType whose physical type is Boolean", - )); - } - - Ok(Self { - data_type, - values, - validity, - }) - } - - /// Creates an new [`MutableBooleanArray`] with a capacity of values. - pub fn with_capacity(capacity: usize) -> Self { - Self { - data_type: DataType::Boolean, - values: MutableBitmap::with_capacity(capacity), - validity: None, - } - } - - /// Reserves `additional` slots. - pub fn reserve(&mut self, additional: usize) { - self.values.reserve(additional); - if let Some(x) = self.validity.as_mut() { - x.reserve(additional) - } - } - - /// Pushes a new entry to [`MutableBooleanArray`]. - pub fn push(&mut self, value: Option) { - match value { - Some(value) => { - self.values.push(value); - match &mut self.validity { - Some(validity) => validity.push(true), - None => {} - } - } - None => { - self.values.push(false); - match &mut self.validity { - Some(validity) => validity.push(false), - None => self.init_validity(), - } - } - } - } - - /// Pop an entry from [`MutableBooleanArray`]. - /// Note If the values is empty, this method will return None. - pub fn pop(&mut self) -> Option { - let value = self.values.pop()?; - self.validity - .as_mut() - .map(|x| x.pop()?.then_some(value)) - .unwrap_or_else(|| Some(value)) - } - - /// Extends the [`MutableBooleanArray`] from an iterator of values of trusted len. - /// This differs from `extend_trusted_len` which accepts in iterator of optional values. - #[inline] - pub fn extend_trusted_len_values(&mut self, iterator: I) - where I: TrustedLen { - // Safety: `I` is `TrustedLen` - unsafe { self.extend_trusted_len_values_unchecked(iterator) } - } - - /// Extends the [`MutableBooleanArray`] from an iterator of values of trusted len. - /// This differs from `extend_trusted_len_unchecked`, which accepts in iterator of optional values. - /// # Safety - /// The iterator must be trusted len. - #[inline] - pub unsafe fn extend_trusted_len_values_unchecked(&mut self, iterator: I) - where I: Iterator { - let (_, upper) = iterator.size_hint(); - let additional = - upper.expect("extend_trusted_len_values_unchecked requires an upper limit"); - - if let Some(validity) = self.validity.as_mut() { - validity.extend_constant(additional, true); - } - - self.values.extend_from_trusted_len_iter_unchecked(iterator) - } - - /// Extends the [`MutableBooleanArray`] from an iterator of trusted len. - #[inline] - pub fn extend_trusted_len(&mut self, iterator: I) - where - P: std::borrow::Borrow, - I: TrustedLen>, - { - // Safety: `I` is `TrustedLen` - unsafe { self.extend_trusted_len_unchecked(iterator) } - } - - /// Extends the [`MutableBooleanArray`] from an iterator of trusted len. - /// # Safety - /// The iterator must be trusted len. - #[inline] - pub unsafe fn extend_trusted_len_unchecked(&mut self, iterator: I) - where - P: std::borrow::Borrow, - I: Iterator>, - { - if let Some(validity) = self.validity.as_mut() { - extend_trusted_len_unzip(iterator, validity, &mut self.values); - } else { - let mut validity = MutableBitmap::new(); - validity.extend_constant(self.len(), true); - - extend_trusted_len_unzip(iterator, &mut validity, &mut self.values); - - if validity.unset_bits() > 0 { - self.validity = Some(validity); - } - } - } - - fn init_validity(&mut self) { - let mut validity = MutableBitmap::with_capacity(self.values.capacity()); - validity.extend_constant(self.len(), true); - validity.set(self.len() - 1, false); - self.validity = Some(validity) - } - - /// Converts itself into an [`Array`]. - pub fn into_arc(self) -> Arc { - let a: BooleanArray = self.into(); - Arc::new(a) - } -} - -/// Getters -impl MutableBooleanArray { - /// Returns its values. - pub fn values(&self) -> &MutableBitmap { - &self.values - } -} - -/// Setters -impl MutableBooleanArray { - /// Sets position `index` to `value`. - /// Note that if it is the first time a null appears in this array, - /// this initializes the validity bitmap (`O(N)`). - /// # Panic - /// Panics iff index is larger than `self.len()`. - pub fn set(&mut self, index: usize, value: Option) { - self.values.set(index, value.unwrap_or_default()); - - if value.is_none() && self.validity.is_none() { - // When the validity is None, all elements so far are valid. When one of the elements is set to null, - // the validity must be initialized. - self.validity = Some(MutableBitmap::from_trusted_len_iter( - std::iter::repeat(true).take(self.len()), - )); - } - if let Some(x) = self.validity.as_mut() { - x.set(index, value.is_some()) - } - } -} - -/// From implementations -impl MutableBooleanArray { - /// Creates a new [`MutableBooleanArray`] from an [`TrustedLen`] of `bool`. - #[inline] - pub fn from_trusted_len_values_iter>(iterator: I) -> Self { - Self::try_new( - DataType::Boolean, - MutableBitmap::from_trusted_len_iter(iterator), - None, - ) - .unwrap() - } - - /// Creates a new [`MutableBooleanArray`] from an [`TrustedLen`] of `bool`. - /// Use this over [`BooleanArray::from_trusted_len_iter`] when the iterator is trusted len - /// but this crate does not mark it as such. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_values_iter_unchecked>( - iterator: I, - ) -> Self { - let mut mutable = MutableBitmap::new(); - mutable.extend_from_trusted_len_iter_unchecked(iterator); - MutableBooleanArray::try_new(DataType::Boolean, mutable, None).unwrap() - } - - /// Creates a new [`MutableBooleanArray`] from a slice of `bool`. - #[inline] - pub fn from_slice>(slice: P) -> Self { - Self::from_trusted_len_values_iter(slice.as_ref().iter().copied()) - } - - /// Creates a [`BooleanArray`] from an iterator of trusted length. - /// Use this over [`BooleanArray::from_trusted_len_iter`] when the iterator is trusted len - /// but this crate does not mark it as such. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: std::borrow::Borrow, - I: Iterator>, - { - let (validity, values) = trusted_len_unzip(iterator); - - Self::try_new(DataType::Boolean, values, validity).unwrap() - } - - /// Creates a [`BooleanArray`] from a [`TrustedLen`]. - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: std::borrow::Borrow, - I: TrustedLen>, - { - // Safety: `I` is `TrustedLen` - unsafe { Self::from_trusted_len_iter_unchecked(iterator) } - } - - /// Creates a [`BooleanArray`] from an falible iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn try_from_trusted_len_iter_unchecked( - iterator: I, - ) -> std::result::Result - where - P: std::borrow::Borrow, - I: Iterator, E>>, - { - let (validity, values) = try_trusted_len_unzip(iterator)?; - - let validity = if validity.unset_bits() > 0 { - Some(validity) - } else { - None - }; - - Ok(Self::try_new(DataType::Boolean, values, validity).unwrap()) - } - - /// Creates a [`BooleanArray`] from a [`TrustedLen`]. - #[inline] - pub fn try_from_trusted_len_iter(iterator: I) -> std::result::Result - where - P: std::borrow::Borrow, - I: TrustedLen, E>>, - { - // Safety: `I` is `TrustedLen` - unsafe { Self::try_from_trusted_len_iter_unchecked(iterator) } - } - - /// Shrinks the capacity of the [`MutableBooleanArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - if let Some(validity) = &mut self.validity { - validity.shrink_to_fit() - } - } -} - -/// Creates a Bitmap and an optional [`MutableBitmap`] from an iterator of `Option`. -/// The first buffer corresponds to a bitmap buffer, the second one -/// corresponds to a values buffer. -/// # Safety -/// The caller must ensure that `iterator` is `TrustedLen`. -#[inline] -pub(crate) unsafe fn trusted_len_unzip(iterator: I) -> (Option, MutableBitmap) -where - P: std::borrow::Borrow, - I: Iterator>, -{ - let mut validity = MutableBitmap::new(); - let mut values = MutableBitmap::new(); - - extend_trusted_len_unzip(iterator, &mut validity, &mut values); - - let validity = if validity.unset_bits() > 0 { - Some(validity) - } else { - None - }; - - (validity, values) -} - -/// Extends validity [`MutableBitmap`] and values [`MutableBitmap`] from an iterator of `Option`. -/// # Safety -/// The caller must ensure that `iterator` is `TrustedLen`. -#[inline] -pub(crate) unsafe fn extend_trusted_len_unzip( - iterator: I, - validity: &mut MutableBitmap, - values: &mut MutableBitmap, -) where - P: std::borrow::Borrow, - I: Iterator>, -{ - let (_, upper) = iterator.size_hint(); - let additional = upper.expect("extend_trusted_len_unzip requires an upper limit"); - - // Length of the array before new values are pushed, - // variable created for assertion post operation - let pre_length = values.len(); - - validity.reserve(additional); - values.reserve(additional); - - for item in iterator { - let item = if let Some(item) = item { - validity.push_unchecked(true); - *item.borrow() - } else { - validity.push_unchecked(false); - bool::default() - }; - values.push_unchecked(item); - } - - debug_assert_eq!( - values.len(), - pre_length + additional, - "Trusted iterator length was not accurately reported" - ); -} - -/// # Safety -/// The caller must ensure that `iterator` is `TrustedLen`. -#[inline] -pub(crate) unsafe fn try_trusted_len_unzip( - iterator: I, -) -> std::result::Result<(MutableBitmap, MutableBitmap), E> -where - P: std::borrow::Borrow, - I: Iterator, E>>, -{ - let (_, upper) = iterator.size_hint(); - let len = upper.expect("trusted_len_unzip requires an upper limit"); - - let mut null = MutableBitmap::with_capacity(len); - let mut values = MutableBitmap::with_capacity(len); - - for item in iterator { - let item = if let Some(item) = item? { - null.push(true); - *item.borrow() - } else { - null.push(false); - false - }; - values.push(item); - } - assert_eq!( - values.len(), - len, - "Trusted iterator length was not accurately reported" - ); - values.set_len(len); - null.set_len(len); - - Ok((null, values)) -} - -impl>> FromIterator for MutableBooleanArray { - fn from_iter>(iter: I) -> Self { - let iter = iter.into_iter(); - let (lower, _) = iter.size_hint(); - - let mut validity = MutableBitmap::with_capacity(lower); - - let values: MutableBitmap = iter - .map(|item| { - if let Some(a) = item.borrow() { - validity.push(true); - *a - } else { - validity.push(false); - false - } - }) - .collect(); - - let validity = if validity.unset_bits() > 0 { - Some(validity) - } else { - None - }; - - MutableBooleanArray::try_new(DataType::Boolean, values, validity).unwrap() - } -} - -impl MutableArray for MutableBooleanArray { - fn len(&self) -> usize { - self.values.len() - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - let array: BooleanArray = std::mem::take(self).into(); - array.boxed() - } - - fn as_arc(&mut self) -> Arc { - let array: BooleanArray = std::mem::take(self).into(); - array.arced() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - #[inline] - fn push_null(&mut self) { - self.push(None) - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl Extend> for MutableBooleanArray { - fn extend>>(&mut self, iter: I) { - let iter = iter.into_iter(); - self.reserve(iter.size_hint().0); - iter.for_each(|x| self.push(x)) - } -} - -impl TryExtend> for MutableBooleanArray { - /// This is infalible and is implemented for consistency with all other types - fn try_extend>>(&mut self, iter: I) -> Result<(), Error> { - self.extend(iter); - Ok(()) - } -} - -impl TryPush> for MutableBooleanArray { - /// This is infalible and is implemented for consistency with all other types - fn try_push(&mut self, item: Option) -> Result<(), Error> { - self.push(item); - Ok(()) - } -} - -impl PartialEq for MutableBooleanArray { - fn eq(&self, other: &Self) -> bool { - self.iter().eq(other.iter()) - } -} - -impl TryExtendFromSelf for MutableBooleanArray { - fn try_extend_from_self(&mut self, other: &Self) -> Result<(), Error> { - extend_validity(self.len(), &mut self.validity, &other.validity); - - let slice = other.values.as_slice(); - // safety: invariant offset + length <= slice.len() - unsafe { - self.values - .extend_from_slice_unchecked(slice, 0, other.values.len()); - } - Ok(()) - } -} diff --git a/src/common/arrow/src/arrow/array/dictionary/data.rs b/src/common/arrow/src/arrow/array/dictionary/data.rs deleted file mode 100644 index 6f43e876843a..000000000000 --- a/src/common/arrow/src/arrow/array/dictionary/data.rs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::from_data; -use crate::arrow::array::to_data; -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; - -impl Arrow2Arrow for DictionaryArray { - fn to_data(&self) -> ArrayData { - let keys = self.keys.to_data(); - let builder = keys - .into_builder() - .data_type(self.data_type.clone().into()) - .child_data(vec![to_data(self.values.as_ref())]); - - // Safety: Dictionary is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let key = match data.data_type() { - arrow_schema::DataType::Dictionary(k, _) => k.as_ref(), - d => panic!("unsupported dictionary type {d}"), - }; - - let data_type = DataType::from(data.data_type().clone()); - assert_eq!( - data_type.to_physical_type(), - PhysicalType::Dictionary(K::KEY_TYPE) - ); - - let key_builder = ArrayDataBuilder::new(key.clone()) - .buffers(vec![data.buffers()[0].clone()]) - .offset(data.offset()) - .len(data.len()) - .nulls(data.nulls().cloned()); - - // Safety: Dictionary is valid - let key_data = unsafe { key_builder.build_unchecked() }; - let keys = PrimitiveArray::from_data(&key_data); - let values = from_data(&data.child_data()[0]); - - Self { - data_type, - keys, - values, - } - } -} diff --git a/src/common/arrow/src/arrow/array/dictionary/fmt.rs b/src/common/arrow/src/arrow/array/dictionary/fmt.rs deleted file mode 100644 index 7274d6c72f52..000000000000 --- a/src/common/arrow/src/arrow/array/dictionary/fmt.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::get_display; -use super::super::fmt::write_vec; -use super::DictionaryArray; -use super::DictionaryKey; -use crate::arrow::array::Array; - -pub fn write_value( - array: &DictionaryArray, - index: usize, - null: &'static str, - f: &mut W, -) -> Result { - let keys = array.keys(); - let values = array.values(); - - if keys.is_valid(index) { - let key = array.key_value(index); - get_display(values.as_ref(), null)(f, key) - } else { - write!(f, "{null}") - } -} - -impl Debug for DictionaryArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, "None", f); - - write!(f, "DictionaryArray")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/dictionary/iterator.rs b/src/common/arrow/src/arrow/array/dictionary/iterator.rs deleted file mode 100644 index b48fb8007e6e..000000000000 --- a/src/common/arrow/src/arrow/array/dictionary/iterator.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::DictionaryArray; -use super::DictionaryKey; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::scalar::Scalar; -use crate::arrow::trusted_len::TrustedLen; - -/// Iterator of values of an `ListArray`. -pub struct DictionaryValuesIter<'a, K: DictionaryKey> { - array: &'a DictionaryArray, - index: usize, - end: usize, -} - -impl<'a, K: DictionaryKey> DictionaryValuesIter<'a, K> { - #[inline] - pub fn new(array: &'a DictionaryArray) -> Self { - Self { - array, - index: 0, - end: array.len(), - } - } -} - -impl<'a, K: DictionaryKey> Iterator for DictionaryValuesIter<'a, K> { - type Item = Box; - - #[inline] - fn next(&mut self) -> Option { - if self.index == self.end { - return None; - } - let old = self.index; - self.index += 1; - Some(self.array.value(old)) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.end - self.index, Some(self.end - self.index)) - } -} - -unsafe impl<'a, K: DictionaryKey> TrustedLen for DictionaryValuesIter<'a, K> {} - -impl<'a, K: DictionaryKey> DoubleEndedIterator for DictionaryValuesIter<'a, K> { - #[inline] - fn next_back(&mut self) -> Option { - if self.index == self.end { - None - } else { - self.end -= 1; - Some(self.array.value(self.end)) - } - } -} - -type ValuesIter<'a, K> = DictionaryValuesIter<'a, K>; -type ZipIter<'a, K> = ZipValidity, ValuesIter<'a, K>, BitmapIter<'a>>; - -impl<'a, K: DictionaryKey> IntoIterator for &'a DictionaryArray { - type Item = Option>; - type IntoIter = ZipIter<'a, K>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} diff --git a/src/common/arrow/src/arrow/array/dictionary/mod.rs b/src/common/arrow/src/arrow/array/dictionary/mod.rs deleted file mode 100644 index 4ccc27135824..000000000000 --- a/src/common/arrow/src/arrow/array/dictionary/mod.rs +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::hash::Hash; -use std::hint::unreachable_unchecked; - -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::IntegerType; -use crate::arrow::error::Error; -use crate::arrow::scalar::new_scalar; -use crate::arrow::scalar::Scalar; -use crate::arrow::trusted_len::TrustedLen; -use crate::arrow::types::NativeType; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod iterator; -mod mutable; -use crate::arrow::array::specification::check_indexes_unchecked; -mod typed_iterator; -mod value_map; - -pub use iterator::*; -pub use mutable::*; - -use super::new_empty_array; -use super::new_null_array; -use super::primitive::PrimitiveArray; -use super::specification::check_indexes; -use super::Array; -use crate::arrow::array::dictionary::typed_iterator::DictValue; -use crate::arrow::array::dictionary::typed_iterator::DictionaryValuesIterTyped; - -/// Trait denoting [`NativeType`]s that can be used as keys of a dictionary. -/// # Safety -/// -/// Any implementation of this trait must ensure that `always_fits_usize` only -/// returns `true` if all values succeeds on `value::try_into::().unwrap()`. -pub unsafe trait DictionaryKey: NativeType + TryInto + TryFrom + Hash { - /// The corresponding [`IntegerType`] of this key - const KEY_TYPE: IntegerType; - - /// Represents this key as a `usize`. - /// # Safety - /// The caller _must_ have checked that the value can be casted to `usize`. - #[inline] - unsafe fn as_usize(self) -> usize { - match self.try_into() { - Ok(v) => v, - Err(_) => unreachable_unchecked(), - } - } - - /// If the key type always can be converted to `usize`. - fn always_fits_usize() -> bool { - false - } -} - -unsafe impl DictionaryKey for i8 { - const KEY_TYPE: IntegerType = IntegerType::Int8; -} -unsafe impl DictionaryKey for i16 { - const KEY_TYPE: IntegerType = IntegerType::Int16; -} -unsafe impl DictionaryKey for i32 { - const KEY_TYPE: IntegerType = IntegerType::Int32; -} -unsafe impl DictionaryKey for i64 { - const KEY_TYPE: IntegerType = IntegerType::Int64; -} -unsafe impl DictionaryKey for u8 { - const KEY_TYPE: IntegerType = IntegerType::UInt8; - - fn always_fits_usize() -> bool { - true - } -} -unsafe impl DictionaryKey for u16 { - const KEY_TYPE: IntegerType = IntegerType::UInt16; - - fn always_fits_usize() -> bool { - true - } -} -unsafe impl DictionaryKey for u32 { - const KEY_TYPE: IntegerType = IntegerType::UInt32; - - fn always_fits_usize() -> bool { - true - } -} -unsafe impl DictionaryKey for u64 { - const KEY_TYPE: IntegerType = IntegerType::UInt64; - - #[cfg(target_pointer_width = "64")] - fn always_fits_usize() -> bool { - true - } -} - -/// An [`Array`] whose values are stored as indices. This [`Array`] is useful when the cardinality of -/// values is low compared to the length of the [`Array`]. -/// -/// # Safety -/// This struct guarantees that each item of [`DictionaryArray::keys`] is castable to `usize` and -/// its value is smaller than [`DictionaryArray::values`]`.len()`. In other words, you can safely -/// use `unchecked` calls to retrieve the values -#[derive(Clone)] -pub struct DictionaryArray { - data_type: DataType, - keys: PrimitiveArray, - values: Box, -} - -fn check_data_type( - key_type: IntegerType, - data_type: &DataType, - values_data_type: &DataType, -) -> Result<(), Error> { - if let DataType::Dictionary(key, value, _) = data_type.to_logical_type() { - if *key != key_type { - return Err(Error::oos( - "DictionaryArray must be initialized with a DataType::Dictionary whose integer is compatible to its keys", - )); - } - if value.as_ref().to_logical_type() != values_data_type.to_logical_type() { - return Err(Error::oos( - "DictionaryArray must be initialized with a DataType::Dictionary whose value is equal to its values", - )); - } - } else { - return Err(Error::oos( - "DictionaryArray must be initialized with logical DataType::Dictionary", - )); - } - Ok(()) -} - -impl DictionaryArray { - /// Returns a new [`DictionaryArray`]. - /// # Implementation - /// This function is `O(N)` where `N` is the length of keys - /// # Errors - /// This function errors iff - /// * the `data_type`'s logical type is not a `DictionaryArray` - /// * the `data_type`'s keys is not compatible with `keys` - /// * the `data_type`'s values's data_type is not equal with `values.data_type()` - /// * any of the keys's values is not represented in `usize` or is `>= values.len()` - pub fn try_new( - data_type: DataType, - keys: PrimitiveArray, - values: Box, - ) -> Result { - check_data_type(K::KEY_TYPE, &data_type, values.data_type())?; - - if keys.null_count() != keys.len() { - if K::always_fits_usize() { - // safety: we just checked that conversion to `usize` always - // succeeds - unsafe { check_indexes_unchecked(keys.values(), values.len()) }?; - } else { - check_indexes(keys.values(), values.len())?; - } - } - - Ok(Self { - data_type, - keys, - values, - }) - } - - /// Returns a new [`DictionaryArray`]. - /// # Implementation - /// This function is `O(N)` where `N` is the length of keys - /// # Errors - /// This function errors iff - /// * any of the keys's values is not represented in `usize` or is `>= values.len()` - pub fn try_from_keys(keys: PrimitiveArray, values: Box) -> Result { - let data_type = Self::default_data_type(values.data_type().clone()); - Self::try_new(data_type, keys, values) - } - - /// Returns a new [`DictionaryArray`]. - /// # Errors - /// This function errors iff - /// * the `data_type`'s logical type is not a `DictionaryArray` - /// * the `data_type`'s keys is not compatible with `keys` - /// * the `data_type`'s values's data_type is not equal with `values.data_type()` - /// # Safety - /// The caller must ensure that every keys's values is represented in `usize` and is `< values.len()` - pub unsafe fn try_new_unchecked( - data_type: DataType, - keys: PrimitiveArray, - values: Box, - ) -> Result { - check_data_type(K::KEY_TYPE, &data_type, values.data_type())?; - - Ok(Self { - data_type, - keys, - values, - }) - } - - /// Returns a new empty [`DictionaryArray`]. - pub fn new_empty(data_type: DataType) -> Self { - let values = Self::try_get_child(&data_type).unwrap(); - let values = new_empty_array(values.clone()); - Self::try_new( - data_type, - PrimitiveArray::::new_empty(K::PRIMITIVE.into()), - values, - ) - .unwrap() - } - - /// Returns an [`DictionaryArray`] whose all elements are null - #[inline] - pub fn new_null(data_type: DataType, length: usize) -> Self { - let values = Self::try_get_child(&data_type).unwrap(); - let values = new_null_array(values.clone(), 1); - Self::try_new( - data_type, - PrimitiveArray::::new_null(K::PRIMITIVE.into(), length), - values, - ) - .unwrap() - } - - /// Returns an iterator of [`Option>`]. - /// # Implementation - /// This function will allocate a new [`Scalar`] per item and is usually not performant. - /// Consider calling `keys_iter` and `values`, downcasting `values`, and iterating over that. - pub fn iter(&self) -> ZipValidity, DictionaryValuesIter, BitmapIter> { - ZipValidity::new_with_validity(DictionaryValuesIter::new(self), self.keys.validity()) - } - - /// Returns an iterator of [`Box`] - /// # Implementation - /// This function will allocate a new [`Scalar`] per item and is usually not performant. - /// Consider calling `keys_iter` and `values`, downcasting `values`, and iterating over that. - pub fn values_iter(&self) -> DictionaryValuesIter { - DictionaryValuesIter::new(self) - } - - /// Returns an iterator over the the values [`V::IterValue`]. - /// - /// # Panics - /// - /// Panics if the keys of this [`DictionaryArray`] have any null types. - /// If they do [`DictionaryArray::iter_typed`] should be called - pub fn values_iter_typed( - &self, - ) -> Result, Error> { - let keys = &self.keys; - assert_eq!(keys.null_count(), 0); - let values = self.values.as_ref(); - let values = V::downcast_values(values)?; - Ok(unsafe { DictionaryValuesIterTyped::new(keys, values) }) - } - - /// Returns an iterator over the the optional values of [`Option`]. - /// - /// # Panics - /// - /// This function panics if the `values` array - #[allow(clippy::type_complexity)] - pub fn iter_typed( - &self, - ) -> Result, DictionaryValuesIterTyped, BitmapIter>, Error> - { - let keys = &self.keys; - let values = self.values.as_ref(); - let values = V::downcast_values(values)?; - let values_iter = unsafe { DictionaryValuesIterTyped::new(keys, values) }; - Ok(ZipValidity::new_with_validity(values_iter, self.validity())) - } - - /// Returns the [`DataType`] of this [`DictionaryArray`] - #[inline] - pub fn data_type(&self) -> &DataType { - &self.data_type - } - - /// Returns whether the values of this [`DictionaryArray`] are ordered - #[inline] - pub fn is_ordered(&self) -> bool { - match self.data_type.to_logical_type() { - DataType::Dictionary(_, _, is_ordered) => *is_ordered, - _ => unreachable!(), - } - } - - pub(crate) fn default_data_type(values_datatype: DataType) -> DataType { - DataType::Dictionary(K::KEY_TYPE, Box::new(values_datatype), false) - } - - /// Slices this [`DictionaryArray`]. - /// # Panics - /// iff `offset + length > self.len()`. - pub fn slice(&mut self, offset: usize, length: usize) { - self.keys.slice(offset, length); - } - - /// Slices this [`DictionaryArray`]. - /// # Safety - /// Safe iff `offset + length <= self.len()`. - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.keys.slice_unchecked(offset, length); - } - - impl_sliced!(); - - /// Returns this [`DictionaryArray`] with a new validity. - /// # Panic - /// This function panics iff `validity.len() != self.len()`. - #[must_use] - pub fn with_validity(mut self, validity: Option) -> Self { - self.set_validity(validity); - self - } - - /// Sets the validity of the keys of this [`DictionaryArray`]. - /// # Panics - /// This function panics iff `validity.len() != self.len()`. - pub fn set_validity(&mut self, validity: Option) { - self.keys.set_validity(validity); - } - - impl_into_array!(); - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.keys.len() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The optional validity. Equivalent to `self.keys().validity()`. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.keys.validity() - } - - /// Returns the keys of the [`DictionaryArray`]. These keys can be used to fetch values - /// from `values`. - #[inline] - pub fn keys(&self) -> &PrimitiveArray { - &self.keys - } - - /// Returns an iterator of the keys' values of the [`DictionaryArray`] as `usize` - #[inline] - pub fn keys_values_iter(&self) -> impl TrustedLen + Clone + '_ { - // safety - invariant of the struct - self.keys.values_iter().map(|x| unsafe { x.as_usize() }) - } - - /// Returns an iterator of the keys' of the [`DictionaryArray`] as `usize` - #[inline] - pub fn keys_iter(&self) -> impl TrustedLen> + Clone + '_ { - // safety - invariant of the struct - self.keys.iter().map(|x| x.map(|x| unsafe { x.as_usize() })) - } - - /// Returns the keys' value of the [`DictionaryArray`] as `usize` - /// # Panics - /// This function panics iff `index >= self.len()` - #[inline] - pub fn key_value(&self, index: usize) -> usize { - assert!(index < self.keys.values().len(), "index is out of bounds"); - // safety - invariant of the struct - unsafe { self.keys.values()[index].as_usize() } - } - - /// Returns the values of the [`DictionaryArray`]. - #[inline] - #[allow(clippy::borrowed_box)] - pub fn values(&self) -> &Box { - &self.values - } - - /// Returns the value of the [`DictionaryArray`] at position `i`. - /// # Implementation - /// This function will allocate a new [`Scalar`] and is usually not performant. - /// Consider calling `keys` and `values`, downcasting `values`, and iterating over that. - /// # Panic - /// This function panics iff `index >= self.len()` - #[inline] - pub fn value(&self, index: usize) -> Box { - // safety - invariant of this struct - let index = unsafe { self.keys.value(index).as_usize() }; - new_scalar(self.values.as_ref(), index) - } - - pub(crate) fn try_get_child(data_type: &DataType) -> Result<&DataType, Error> { - Ok(match data_type.to_logical_type() { - DataType::Dictionary(_, values, _) => values.as_ref(), - _ => { - return Err(Error::oos( - "Dictionaries must be initialized with DataType::Dictionary", - )); - } - }) - } -} - -impl Array for DictionaryArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.keys.validity() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/dictionary/mutable.rs b/src/common/arrow/src/arrow/array/dictionary/mutable.rs deleted file mode 100644 index 1fa8cfac7490..000000000000 --- a/src/common/arrow/src/arrow/array/dictionary/mutable.rs +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::hash::Hash; -use std::sync::Arc; - -use super::value_map::ValueMap; -use super::DictionaryArray; -use super::DictionaryKey; -use crate::arrow::array::indexable::AsIndexed; -use crate::arrow::array::indexable::Indexable; -use crate::arrow::array::primitive::MutablePrimitiveArray; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; - -/// A mutable, strong-typed version of [`DictionaryArray`]. -/// -/// # Example -/// Building a UTF8 dictionary with `i32` keys. -/// ``` -/// # use arrow2::array::{MutableDictionaryArray, MutableUtf8Array, TryPush}; -/// # fn main() -> Result<(), Box> { -/// let mut array: MutableDictionaryArray> = -/// MutableDictionaryArray::new(); -/// array.try_push(Some("A"))?; -/// array.try_push(Some("B"))?; -/// array.push_null(); -/// array.try_push(Some("C"))?; -/// # Ok(()) -/// # } -/// ``` -#[derive(Debug)] -pub struct MutableDictionaryArray { - data_type: DataType, - map: ValueMap, - // invariant: `max(keys) < map.values().len()` - keys: MutablePrimitiveArray, -} - -impl From> for DictionaryArray { - fn from(other: MutableDictionaryArray) -> Self { - // Safety - the invariant of this struct ensures that this is up-held - unsafe { - DictionaryArray::::try_new_unchecked( - other.data_type, - other.keys.into(), - other.map.into_values().as_box(), - ) - .unwrap() - } - } -} - -impl MutableDictionaryArray { - /// Creates an empty [`MutableDictionaryArray`]. - pub fn new() -> Self { - Self::try_empty(M::default()).unwrap() - } -} - -impl Default for MutableDictionaryArray { - fn default() -> Self { - Self::new() - } -} - -impl MutableDictionaryArray { - /// Creates an empty [`MutableDictionaryArray`] from a given empty values array. - /// # Errors - /// Errors if the array is non-empty. - pub fn try_empty(values: M) -> Result { - Ok(Self::from_value_map(ValueMap::::try_empty(values)?)) - } - - /// Creates an empty [`MutableDictionaryArray`] preloaded with a given dictionary of values. - /// Indices associated with those values are automatically assigned based on the order of - /// the values. - /// # Errors - /// Errors if there's more values than the maximum value of `K` or if values are not unique. - pub fn from_values(values: M) -> Result - where - M: Indexable, - M::Type: Eq + Hash, - { - Ok(Self::from_value_map(ValueMap::::from_values(values)?)) - } - - fn from_value_map(value_map: ValueMap) -> Self { - let keys = MutablePrimitiveArray::::new(); - let data_type = - DataType::Dictionary(K::KEY_TYPE, Box::new(value_map.data_type().clone()), false); - Self { - data_type, - map: value_map, - keys, - } - } - - /// Creates an empty [`MutableDictionaryArray`] retaining the same dictionary as the current - /// mutable dictionary array, but with no data. This may come useful when serializing the - /// array into multiple chunks, where there's a requirement that the dictionary is the same. - /// No copying is performed, the value map is moved over to the new array. - pub fn into_empty(self) -> Self { - Self::from_value_map(self.map) - } - - /// Same as `into_empty` but clones the inner value map instead of taking full ownership. - pub fn to_empty(&self) -> Self - where M: Clone { - Self::from_value_map(self.map.clone()) - } - - /// pushes a null value - pub fn push_null(&mut self) { - self.keys.push(None) - } - - /// returns a reference to the inner values. - pub fn values(&self) -> &M { - self.map.values() - } - - /// converts itself into [`Arc`] - pub fn into_arc(self) -> Arc { - let a: DictionaryArray = self.into(); - Arc::new(a) - } - - /// converts itself into [`Box`] - pub fn into_box(self) -> Box { - let a: DictionaryArray = self.into(); - Box::new(a) - } - - /// Reserves `additional` slots. - pub fn reserve(&mut self, additional: usize) { - self.keys.reserve(additional); - } - - /// Shrinks the capacity of the [`MutableDictionaryArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.map.shrink_to_fit(); - self.keys.shrink_to_fit(); - } - - /// Returns the dictionary keys - pub fn keys(&self) -> &MutablePrimitiveArray { - &self.keys - } - - fn take_into(&mut self) -> DictionaryArray { - DictionaryArray::::try_new( - self.data_type.clone(), - std::mem::take(&mut self.keys).into(), - self.map.take_into(), - ) - .unwrap() - } -} - -impl MutableArray for MutableDictionaryArray { - fn len(&self) -> usize { - self.keys.len() - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.keys.validity() - } - - fn as_box(&mut self) -> Box { - Box::new(self.take_into()) - } - - fn as_arc(&mut self) -> Arc { - Arc::new(self.take_into()) - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn push_null(&mut self) { - self.keys.push(None) - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl TryExtend> for MutableDictionaryArray -where - K: DictionaryKey, - M: MutableArray + Indexable + TryExtend>, - T: AsIndexed, - M::Type: Eq + Hash, -{ - fn try_extend>>(&mut self, iter: II) -> Result<()> { - for value in iter { - if let Some(value) = value { - let key = self - .map - .try_push_valid(value, |arr, v| arr.try_extend(std::iter::once(Some(v))))?; - self.keys.try_push(Some(key))?; - } else { - self.push_null(); - } - } - Ok(()) - } -} - -impl TryPush> for MutableDictionaryArray -where - K: DictionaryKey, - M: MutableArray + Indexable + TryPush>, - T: AsIndexed, - M::Type: Eq + Hash, -{ - fn try_push(&mut self, item: Option) -> Result<()> { - if let Some(value) = item { - let key = self - .map - .try_push_valid(value, |arr, v| arr.try_push(Some(v)))?; - self.keys.try_push(Some(key))?; - } else { - self.push_null(); - } - Ok(()) - } -} diff --git a/src/common/arrow/src/arrow/array/dictionary/typed_iterator.rs b/src/common/arrow/src/arrow/array/dictionary/typed_iterator.rs deleted file mode 100644 index 4bb4e56a0597..000000000000 --- a/src/common/arrow/src/arrow/array/dictionary/typed_iterator.rs +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::DictionaryKey; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::trusted_len::TrustedLen; -use crate::arrow::types::Offset; - -pub trait DictValue { - type IterValue<'this> - where Self: 'this; - - /// # Safety - /// Will not do any bound checks but must check validity. - unsafe fn get_unchecked(&self, item: usize) -> Self::IterValue<'_>; - - /// Take a [`dyn Array`] an try to downcast it to the type of `DictValue`. - fn downcast_values(array: &dyn Array) -> Result<&Self> - where Self: Sized; -} - -impl DictValue for Utf8Array { - type IterValue<'a> = &'a str; - - unsafe fn get_unchecked(&self, item: usize) -> Self::IterValue<'_> { - self.value_unchecked(item) - } - - fn downcast_values(array: &dyn Array) -> Result<&Self> - where Self: Sized { - array - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::InvalidArgumentError("could not convert array to dictionary value".into()) - }) - .inspect(|arr| { - assert_eq!( - arr.null_count(), - 0, - "null values in values not supported in iteration" - ); - }) - } -} - -/// Iterator of values of an `ListArray`. -pub struct DictionaryValuesIterTyped<'a, K: DictionaryKey, V: DictValue> { - keys: &'a PrimitiveArray, - values: &'a V, - index: usize, - end: usize, -} - -impl<'a, K: DictionaryKey, V: DictValue> DictionaryValuesIterTyped<'a, K, V> { - pub(super) unsafe fn new(keys: &'a PrimitiveArray, values: &'a V) -> Self { - Self { - keys, - values, - index: 0, - end: keys.len(), - } - } -} - -impl<'a, K: DictionaryKey, V: DictValue> Iterator for DictionaryValuesIterTyped<'a, K, V> { - type Item = V::IterValue<'a>; - - #[inline] - fn next(&mut self) -> Option { - if self.index == self.end { - return None; - } - let old = self.index; - self.index += 1; - unsafe { - let key = self.keys.value_unchecked(old); - let idx = key.as_usize(); - Some(self.values.get_unchecked(idx)) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.end - self.index, Some(self.end - self.index)) - } -} - -unsafe impl<'a, K: DictionaryKey, V: DictValue> TrustedLen for DictionaryValuesIterTyped<'a, K, V> {} - -impl<'a, K: DictionaryKey, V: DictValue> DoubleEndedIterator - for DictionaryValuesIterTyped<'a, K, V> -{ - #[inline] - fn next_back(&mut self) -> Option { - if self.index == self.end { - None - } else { - self.end -= 1; - unsafe { - let key = self.keys.value_unchecked(self.end); - let idx = key.as_usize(); - Some(self.values.get_unchecked(idx)) - } - } - } -} diff --git a/src/common/arrow/src/arrow/array/dictionary/value_map.rs b/src/common/arrow/src/arrow/array/dictionary/value_map.rs deleted file mode 100644 index 3b0d17221e54..000000000000 --- a/src/common/arrow/src/arrow/array/dictionary/value_map.rs +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::borrow::Borrow; -use std::fmt::Debug; -use std::fmt::{self}; -use std::hash::BuildHasher; -use std::hash::BuildHasherDefault; -use std::hash::Hash; -use std::hash::Hasher; - -use hashbrown_v0_14::hash_map::RawEntryMut; -use hashbrown_v0_14::HashMap; - -use super::DictionaryKey; -use crate::arrow::array::indexable::AsIndexed; -use crate::arrow::array::indexable::Indexable; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// Hasher for pre-hashed values; similar to `hash_hasher` but with native endianness. -/// -/// We know that we'll only use it for `u64` values, so we can avoid endian conversion. -/// -/// Invariant: hash of a u64 value is always equal to itself. -#[derive(Copy, Clone, Default)] -pub struct PassthroughHasher(u64); - -impl Hasher for PassthroughHasher { - #[inline] - fn write_u64(&mut self, value: u64) { - self.0 = value; - } - - fn write(&mut self, _: &[u8]) { - unreachable!(); - } - - #[inline] - fn finish(&self) -> u64 { - self.0 - } -} - -#[derive(Clone)] -pub struct Hashed { - hash: u64, - key: K, -} - -#[inline] -fn ahash_hash(value: &T) -> u64 { - BuildHasherDefault::::default().hash_one(value) -} - -impl Hash for Hashed { - #[inline] - fn hash(&self, state: &mut H) { - self.hash.hash(state) - } -} - -#[derive(Clone)] -pub struct ValueMap { - pub values: M, - pub map: HashMap, (), BuildHasherDefault>, /* NB: *only* use insert_hashed_nocheck() and no other hashmap API */ -} - -impl ValueMap { - pub fn try_empty(values: M) -> Result { - if !values.is_empty() { - return Err(Error::InvalidArgumentError( - "initializing value map with non-empty values array".into(), - )); - } - Ok(Self { - values, - map: HashMap::default(), - }) - } - - pub fn from_values(values: M) -> Result - where - M: Indexable, - M::Type: Eq + Hash, - { - let mut map = HashMap::, _, _>::with_capacity_and_hasher( - values.len(), - BuildHasherDefault::::default(), - ); - for index in 0..values.len() { - let key = K::try_from(index).map_err(|_| Error::Overflow)?; - // safety: we only iterate within bounds - let value = unsafe { values.value_unchecked_at(index) }; - let hash = ahash_hash(value.borrow()); - let entry = map.raw_entry_mut().from_hash(hash, |item| { - // safety: invariant of the struct, it's always in bounds since we maintain it - let stored_value = unsafe { values.value_unchecked_at(item.key.as_usize()) }; - stored_value.borrow() == value.borrow() - }); - match entry { - RawEntryMut::Occupied(_) => { - return Err(Error::InvalidArgumentError( - "duplicate value in dictionary values array".into(), - )); - } - RawEntryMut::Vacant(entry) => { - // NB: don't use .insert() here! - entry.insert_hashed_nocheck(hash, Hashed { hash, key }, ()); - } - } - } - Ok(Self { values, map }) - } - - pub fn data_type(&self) -> &DataType { - self.values.data_type() - } - - pub fn into_values(self) -> M { - self.values - } - - pub fn take_into(&mut self) -> Box { - let arr = self.values.as_box(); - self.map.clear(); - arr - } - - #[inline] - pub fn values(&self) -> &M { - &self.values - } - - /// Try to insert a value and return its index (it may or may not get inserted). - pub fn try_push_valid( - &mut self, - value: V, - mut push: impl FnMut(&mut M, V) -> Result<()>, - ) -> Result - where - M: Indexable, - V: AsIndexed, - M::Type: Eq + Hash, - { - let hash = ahash_hash(value.as_indexed()); - let entry = self.map.raw_entry_mut().from_hash(hash, |item| { - // safety: we've already checked (the inverse) when we pushed it, so it should be ok? - let index = unsafe { item.key.as_usize() }; - // safety: invariant of the struct, it's always in bounds since we maintain it - let stored_value = unsafe { self.values.value_unchecked_at(index) }; - stored_value.borrow() == value.as_indexed() - }); - let result = match entry { - RawEntryMut::Occupied(entry) => entry.key().key, - RawEntryMut::Vacant(entry) => { - let index = self.values.len(); - let key = K::try_from(index).map_err(|_| Error::Overflow)?; - entry.insert_hashed_nocheck(hash, Hashed { hash, key }, ()); // NB: don't use .insert() here! - push(&mut self.values, value)?; - debug_assert_eq!(self.values.len(), index + 1); - key - } - }; - Ok(result) - } - - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - } -} - -impl Debug for ValueMap { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.values.fmt(f) - } -} diff --git a/src/common/arrow/src/arrow/array/equal/binary.rs b/src/common/arrow/src/arrow/array/equal/binary.rs deleted file mode 100644 index 78f59ecf2c06..000000000000 --- a/src/common/arrow/src/arrow/array/equal/binary.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::BinaryArray; -use crate::arrow::offset::Offset; - -pub(super) fn equal(lhs: &BinaryArray, rhs: &BinaryArray) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/binary_view.rs b/src/common/arrow/src/arrow/array/equal/binary_view.rs deleted file mode 100644 index 1fe1111b68ca..000000000000 --- a/src/common/arrow/src/arrow/array/equal/binary_view.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::BinaryViewArrayGeneric; -use crate::arrow::array::ViewType; - -pub(super) fn equal( - lhs: &BinaryViewArrayGeneric, - rhs: &BinaryViewArrayGeneric, -) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/boolean.rs b/src/common/arrow/src/arrow/array/equal/boolean.rs deleted file mode 100644 index de2c0b4122c4..000000000000 --- a/src/common/arrow/src/arrow/array/equal/boolean.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::BooleanArray; - -pub(super) fn equal(lhs: &BooleanArray, rhs: &BooleanArray) -> bool { - lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/dictionary.rs b/src/common/arrow/src/arrow/array/equal/dictionary.rs deleted file mode 100644 index 4fbcf05f1224..000000000000 --- a/src/common/arrow/src/arrow/array/equal/dictionary.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; - -pub(super) fn equal(lhs: &DictionaryArray, rhs: &DictionaryArray) -> bool { - if !(lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len()) { - return false; - }; - - // if x is not valid and y is but its child is not, the slots are equal. - lhs.iter().zip(rhs.iter()).all(|(x, y)| match (&x, &y) { - (None, Some(y)) => !y.is_valid(), - (Some(x), None) => !x.is_valid(), - _ => x == y, - }) -} diff --git a/src/common/arrow/src/arrow/array/equal/fixed_size_binary.rs b/src/common/arrow/src/arrow/array/equal/fixed_size_binary.rs deleted file mode 100644 index 69c3db887acf..000000000000 --- a/src/common/arrow/src/arrow/array/equal/fixed_size_binary.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::FixedSizeBinaryArray; - -pub(super) fn equal(lhs: &FixedSizeBinaryArray, rhs: &FixedSizeBinaryArray) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/fixed_size_list.rs b/src/common/arrow/src/arrow/array/equal/fixed_size_list.rs deleted file mode 100644 index eba3e276f167..000000000000 --- a/src/common/arrow/src/arrow/array/equal/fixed_size_list.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::FixedSizeListArray; - -pub(super) fn equal(lhs: &FixedSizeListArray, rhs: &FixedSizeListArray) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/list.rs b/src/common/arrow/src/arrow/array/equal/list.rs deleted file mode 100644 index fc5a2417b962..000000000000 --- a/src/common/arrow/src/arrow/array/equal/list.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::ListArray; -use crate::arrow::offset::Offset; - -pub(super) fn equal(lhs: &ListArray, rhs: &ListArray) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/map.rs b/src/common/arrow/src/arrow/array/equal/map.rs deleted file mode 100644 index 61ca0fccdb9d..000000000000 --- a/src/common/arrow/src/arrow/array/equal/map.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::MapArray; - -pub(super) fn equal(lhs: &MapArray, rhs: &MapArray) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/mod.rs b/src/common/arrow/src/arrow/array/equal/mod.rs deleted file mode 100644 index c6c2f409808c..000000000000 --- a/src/common/arrow/src/arrow/array/equal/mod.rs +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::*; -use crate::arrow::offset::Offset; -use crate::arrow::types::NativeType; - -mod binary; -mod binary_view; -mod boolean; -mod dictionary; -mod fixed_size_binary; -mod fixed_size_list; -mod list; -mod map; -mod null; -mod primitive; -mod struct_; -mod union; -mod utf8; - -impl PartialEq for dyn Array + '_ { - fn eq(&self, that: &dyn Array) -> bool { - equal(self, that) - } -} - -impl PartialEq for std::sync::Arc { - fn eq(&self, that: &dyn Array) -> bool { - equal(&**self, that) - } -} - -impl PartialEq for Box { - fn eq(&self, that: &dyn Array) -> bool { - equal(&**self, that) - } -} - -impl PartialEq for NullArray { - fn eq(&self, other: &Self) -> bool { - null::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for NullArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq<&dyn Array> for PrimitiveArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq> for &dyn Array { - fn eq(&self, other: &PrimitiveArray) -> bool { - equal(*self, other) - } -} - -impl PartialEq> for PrimitiveArray { - fn eq(&self, other: &Self) -> bool { - primitive::equal::(self, other) - } -} - -impl PartialEq for BooleanArray { - fn eq(&self, other: &Self) -> bool { - equal(self, other) - } -} - -impl PartialEq<&dyn Array> for BooleanArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq> for Utf8Array { - fn eq(&self, other: &Self) -> bool { - utf8::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for Utf8Array { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq> for &dyn Array { - fn eq(&self, other: &Utf8Array) -> bool { - equal(*self, other) - } -} - -impl PartialEq> for BinaryArray { - fn eq(&self, other: &Self) -> bool { - binary::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for BinaryArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq> for &dyn Array { - fn eq(&self, other: &BinaryArray) -> bool { - equal(*self, other) - } -} - -impl PartialEq for FixedSizeBinaryArray { - fn eq(&self, other: &Self) -> bool { - fixed_size_binary::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for FixedSizeBinaryArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq> for ListArray { - fn eq(&self, other: &Self) -> bool { - list::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for ListArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq for FixedSizeListArray { - fn eq(&self, other: &Self) -> bool { - fixed_size_list::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for FixedSizeListArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq for StructArray { - fn eq(&self, other: &Self) -> bool { - struct_::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for StructArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq> for DictionaryArray { - fn eq(&self, other: &Self) -> bool { - dictionary::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for DictionaryArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq for UnionArray { - fn eq(&self, other: &Self) -> bool { - union::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for UnionArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -impl PartialEq for MapArray { - fn eq(&self, other: &Self) -> bool { - map::equal(self, other) - } -} - -impl PartialEq<&dyn Array> for MapArray { - fn eq(&self, other: &&dyn Array) -> bool { - equal(self, *other) - } -} - -/// Logically compares two [`Array`]s. -/// Two arrays are logically equal if and only if: -/// * their data types are equal -/// * each of their items are equal -pub fn equal(lhs: &dyn Array, rhs: &dyn Array) -> bool { - if lhs.data_type() != rhs.data_type() { - return false; - } - - use crate::arrow::datatypes::PhysicalType::*; - match lhs.data_type().to_physical_type() { - Null => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - null::equal(lhs, rhs) - } - Boolean => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - boolean::equal(lhs, rhs) - } - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - primitive::equal::<$T>(lhs, rhs) - }), - Utf8 => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - utf8::equal::(lhs, rhs) - } - LargeUtf8 => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - utf8::equal::(lhs, rhs) - } - Binary => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - binary::equal::(lhs, rhs) - } - LargeBinary => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - binary::equal::(lhs, rhs) - } - List => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - list::equal::(lhs, rhs) - } - LargeList => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - list::equal::(lhs, rhs) - } - Struct => { - let lhs = lhs.as_any().downcast_ref::().unwrap(); - let rhs = rhs.as_any().downcast_ref::().unwrap(); - struct_::equal(lhs, rhs) - } - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - dictionary::equal::<$T>(lhs, rhs) - }) - } - FixedSizeBinary => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - fixed_size_binary::equal(lhs, rhs) - } - FixedSizeList => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - fixed_size_list::equal(lhs, rhs) - } - Union => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - union::equal(lhs, rhs) - } - Map => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - map::equal(lhs, rhs) - } - BinaryView => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - binary_view::equal::<[u8]>(lhs, rhs) - } - Utf8View => { - let lhs = lhs.as_any().downcast_ref().unwrap(); - let rhs = rhs.as_any().downcast_ref().unwrap(); - binary_view::equal::(lhs, rhs) - } - } -} diff --git a/src/common/arrow/src/arrow/array/equal/null.rs b/src/common/arrow/src/arrow/array/equal/null.rs deleted file mode 100644 index 71fca317b953..000000000000 --- a/src/common/arrow/src/arrow/array/equal/null.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::NullArray; - -#[inline] -pub(super) fn equal(lhs: &NullArray, rhs: &NullArray) -> bool { - lhs.len() == rhs.len() -} diff --git a/src/common/arrow/src/arrow/array/equal/primitive.rs b/src/common/arrow/src/arrow/array/equal/primitive.rs deleted file mode 100644 index f5b4c2f58e9e..000000000000 --- a/src/common/arrow/src/arrow/array/equal/primitive.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::PrimitiveArray; -use crate::arrow::types::NativeType; - -pub(super) fn equal(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/struct_.rs b/src/common/arrow/src/arrow/array/equal/struct_.rs deleted file mode 100644 index 59f765de22bc..000000000000 --- a/src/common/arrow/src/arrow/array/equal/struct_.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::StructArray; - -pub(super) fn equal(lhs: &StructArray, rhs: &StructArray) -> bool { - lhs.data_type() == rhs.data_type() - && lhs.len() == rhs.len() - && match (lhs.validity(), rhs.validity()) { - (None, None) => lhs.values().iter().eq(rhs.values().iter()), - (Some(l_validity), Some(r_validity)) => lhs - .values() - .iter() - .zip(rhs.values().iter()) - .all(|(lhs, rhs)| { - l_validity.iter().zip(r_validity.iter()).enumerate().all( - |(i, (lhs_is_valid, rhs_is_valid))| { - if lhs_is_valid && rhs_is_valid { - lhs.sliced(i, 1) == rhs.sliced(i, 1) - } else { - lhs_is_valid == rhs_is_valid - } - }, - ) - }), - (Some(l_validity), None) => { - lhs.values() - .iter() - .zip(rhs.values().iter()) - .all(|(lhs, rhs)| { - l_validity.iter().enumerate().all(|(i, lhs_is_valid)| { - if lhs_is_valid { - lhs.sliced(i, 1) == rhs.sliced(i, 1) - } else { - // rhs is always valid => different - false - } - }) - }) - } - (None, Some(r_validity)) => { - lhs.values() - .iter() - .zip(rhs.values().iter()) - .all(|(lhs, rhs)| { - r_validity.iter().enumerate().all(|(i, rhs_is_valid)| { - if rhs_is_valid { - lhs.sliced(i, 1) == rhs.sliced(i, 1) - } else { - // lhs is always valid => different - false - } - }) - }) - } - } -} diff --git a/src/common/arrow/src/arrow/array/equal/union.rs b/src/common/arrow/src/arrow/array/equal/union.rs deleted file mode 100644 index 990592deaae9..000000000000 --- a/src/common/arrow/src/arrow/array/equal/union.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::UnionArray; - -pub(super) fn equal(lhs: &UnionArray, rhs: &UnionArray) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/equal/utf8.rs b/src/common/arrow/src/arrow/array/equal/utf8.rs deleted file mode 100644 index 7a0628dea066..000000000000 --- a/src/common/arrow/src/arrow/array/equal/utf8.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Utf8Array; -use crate::arrow::offset::Offset; - -pub(super) fn equal(lhs: &Utf8Array, rhs: &Utf8Array) -> bool { - lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_binary/data.rs b/src/common/arrow/src/arrow/array/fixed_size_binary/data.rs deleted file mode 100644 index 8081aafa90d0..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_binary/data.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::FixedSizeBinaryArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; - -impl Arrow2Arrow for FixedSizeBinaryArray { - fn to_data(&self) -> ArrayData { - let data_type = self.data_type.clone().into(); - let builder = ArrayDataBuilder::new(data_type) - .len(self.len()) - .buffers(vec![self.values.clone().into()]) - .nulls(self.validity.as_ref().map(|b| b.clone().into())); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type: DataType = data.data_type().clone().into(); - let size = match data_type { - DataType::FixedSizeBinary(size) => size, - _ => unreachable!("must be FixedSizeBinary"), - }; - - let mut values: Buffer = data.buffers()[0].clone().into(); - values.slice(data.offset() * size, data.len() * size); - - Self { - size, - data_type, - values, - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_binary/fmt.rs b/src/common/arrow/src/arrow/array/fixed_size_binary/fmt.rs deleted file mode 100644 index 22c24cacb2fe..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_binary/fmt.rs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::write_vec; -use super::FixedSizeBinaryArray; - -pub fn write_value(array: &FixedSizeBinaryArray, index: usize, f: &mut W) -> Result { - let values = array.value(index); - let writer = |f: &mut W, index| write!(f, "{}", values[index]); - - write_vec(f, writer, None, values.len(), "None", false) -} - -impl Debug for FixedSizeBinaryArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, f); - - write!(f, "{:?}", self.data_type)?; - write_vec(f, writer, self.validity(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_binary/iterator.rs b/src/common/arrow/src/arrow/array/fixed_size_binary/iterator.rs deleted file mode 100644 index 855e488384a5..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_binary/iterator.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::FixedSizeBinaryArray; -use super::MutableFixedSizeBinaryArray; -use crate::arrow::array::MutableArray; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; - -impl<'a> IntoIterator for &'a FixedSizeBinaryArray { - type Item = Option<&'a [u8]>; - type IntoIter = ZipValidity<&'a [u8], std::slice::ChunksExact<'a, u8>, BitmapIter<'a>>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> FixedSizeBinaryArray { - /// constructs a new iterator - pub fn iter( - &'a self, - ) -> ZipValidity<&'a [u8], std::slice::ChunksExact<'a, u8>, BitmapIter<'a>> { - ZipValidity::new_with_validity(self.values_iter(), self.validity()) - } - - /// Returns iterator over the values of [`FixedSizeBinaryArray`] - pub fn values_iter(&'a self) -> std::slice::ChunksExact<'a, u8> { - self.values().chunks_exact(self.size) - } -} - -impl<'a> IntoIterator for &'a MutableFixedSizeBinaryArray { - type Item = Option<&'a [u8]>; - type IntoIter = ZipValidity<&'a [u8], std::slice::ChunksExact<'a, u8>, BitmapIter<'a>>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> MutableFixedSizeBinaryArray { - /// constructs a new iterator - pub fn iter( - &'a self, - ) -> ZipValidity<&'a [u8], std::slice::ChunksExact<'a, u8>, BitmapIter<'a>> { - ZipValidity::new(self.iter_values(), self.validity().map(|x| x.iter())) - } - - /// Returns iterator over the values of [`MutableFixedSizeBinaryArray`] - pub fn iter_values(&'a self) -> std::slice::ChunksExact<'a, u8> { - self.values().chunks_exact(self.size()) - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_binary/mod.rs b/src/common/arrow/src/arrow/array/fixed_size_binary/mod.rs deleted file mode 100644 index 7787790e4ffd..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_binary/mod.rs +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod iterator; -mod mutable; -pub use mutable::*; - -/// The Arrow's equivalent to an immutable `Vec>`. -/// Cloning and slicing this struct is `O(1)`. -#[derive(Clone)] -pub struct FixedSizeBinaryArray { - size: usize, /* this is redundant with `data_type`, but useful to not have to deconstruct the data_type. */ - data_type: DataType, - values: Buffer, - validity: Option, -} - -impl FixedSizeBinaryArray { - /// Creates a new [`FixedSizeBinaryArray`]. - /// - /// # Errors - /// This function returns an error iff: - /// * The `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::FixedSizeBinary`] - /// * The length of `values` is not a multiple of `size` in `data_type` - /// * the validity's length is not equal to `values.len() / size`. - pub fn try_new( - data_type: DataType, - values: Buffer, - validity: Option, - ) -> Result { - let size = Self::maybe_get_size(&data_type)?; - - if values.len() % size != 0 { - return Err(Error::oos(format!( - "values (of len {}) must be a multiple of size ({}) in FixedSizeBinaryArray.", - values.len(), - size - ))); - } - let len = values.len() / size; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != len) - { - return Err(Error::oos( - "validity mask length must be equal to the number of values divided by size", - )); - } - - Ok(Self { - size, - data_type, - values, - validity, - }) - } - - /// Creates a new [`FixedSizeBinaryArray`]. - /// # Panics - /// This function panics iff: - /// * The `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::FixedSizeBinary`] - /// * The length of `values` is not a multiple of `size` in `data_type` - /// * the validity's length is not equal to `values.len() / size`. - pub fn new(data_type: DataType, values: Buffer, validity: Option) -> Self { - Self::try_new(data_type, values, validity).unwrap() - } - - /// Returns a new empty [`FixedSizeBinaryArray`]. - pub fn new_empty(data_type: DataType) -> Self { - Self::new(data_type, Buffer::new(), None) - } - - /// Returns a new null [`FixedSizeBinaryArray`]. - pub fn new_null(data_type: DataType, length: usize) -> Self { - let size = Self::maybe_get_size(&data_type).unwrap(); - Self::new( - data_type, - vec![0u8; length * size].into(), - Some(Bitmap::new_zeroed(length)), - ) - } -} - -// must use -impl FixedSizeBinaryArray { - /// Slices this [`FixedSizeBinaryArray`]. - /// # Implementation - /// This operation is `O(1)`. - /// # Panics - /// panics iff `offset + length > self.len()` - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new Buffer cannot exceed the existing length" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Slices this [`FixedSizeBinaryArray`]. - /// # Implementation - /// This operation is `O(1)`. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.values - .slice_unchecked(offset * self.size, length * self.size); - } - - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); -} - -// accessors -impl FixedSizeBinaryArray { - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.values.len() / self.size - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The optional validity. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - /// Returns the values allocated on this [`FixedSizeBinaryArray`]. - pub fn values(&self) -> &Buffer { - &self.values - } - - /// Returns value at position `i`. - /// # Panic - /// Panics iff `i >= self.len()`. - #[inline] - pub fn value(&self, i: usize) -> &[u8] { - assert!(i < self.len()); - unsafe { self.value_unchecked(i) } - } - - /// Returns the element at index `i` as &str - /// # Safety - /// Assumes that the `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] { - // soundness: invariant of the function. - self.values - .get_unchecked(i * self.size..(i + 1) * self.size) - } - - /// Returns the element at index `i` or `None` if it is null - /// # Panics - /// iff `i >= self.len()` - #[inline] - pub fn get(&self, i: usize) -> Option<&[u8]> { - if !self.is_null(i) { - // soundness: Array::is_null panics if i >= self.len - unsafe { Some(self.value_unchecked(i)) } - } else { - None - } - } - - /// Returns a new [`FixedSizeBinaryArray`] with a different logical type. - /// This is `O(1)`. - /// # Panics - /// Panics iff the data_type is not supported for the physical type. - #[inline] - pub fn to(self, data_type: DataType) -> Self { - match ( - data_type.to_logical_type(), - self.data_type().to_logical_type(), - ) { - (DataType::FixedSizeBinary(size_a), DataType::FixedSizeBinary(size_b)) - if size_a == size_b => {} - _ => panic!("Wrong DataType"), - } - - Self { - size: self.size, - data_type, - values: self.values, - validity: self.validity, - } - } - - /// Returns the size - pub fn size(&self) -> usize { - self.size - } -} - -impl FixedSizeBinaryArray { - pub(crate) fn maybe_get_size(data_type: &DataType) -> Result { - match data_type.to_logical_type() { - DataType::FixedSizeBinary(size) => { - if *size == 0 { - return Err(Error::oos("FixedSizeBinaryArray expects a positive size")); - } - Ok(*size) - } - _ => Err(Error::oos( - "FixedSizeBinaryArray expects DataType::FixedSizeBinary", - )), - } - } - - pub(crate) fn get_size(data_type: &DataType) -> usize { - Self::maybe_get_size(data_type).unwrap() - } -} - -impl Array for FixedSizeBinaryArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} - -impl FixedSizeBinaryArray { - /// Creates a [`FixedSizeBinaryArray`] from an fallible iterator of optional `[u8]`. - pub fn try_from_iter, I: IntoIterator>>( - iter: I, - size: usize, - ) -> Result { - MutableFixedSizeBinaryArray::try_from_iter(iter, size).map(|x| x.into()) - } - - /// Creates a [`FixedSizeBinaryArray`] from an iterator of optional `[u8]`. - pub fn from_iter, I: IntoIterator>>( - iter: I, - size: usize, - ) -> Self { - MutableFixedSizeBinaryArray::try_from_iter(iter, size) - .unwrap() - .into() - } - - /// Creates a [`FixedSizeBinaryArray`] from a slice of arrays of bytes - pub fn from_slice>(a: P) -> Self { - let values = a.as_ref().iter().flatten().copied().collect::>(); - Self::new(DataType::FixedSizeBinary(N), values.into(), None) - } - - /// Creates a new [`FixedSizeBinaryArray`] from a slice of optional `[u8]`. - // Note: this can't be `impl From` because Rust does not allow double `AsRef` on it. - pub fn from]>>(slice: P) -> Self { - MutableFixedSizeBinaryArray::from(slice).into() - } -} - -pub trait FixedSizeBinaryValues { - fn values(&self) -> &[u8]; - fn size(&self) -> usize; -} - -impl FixedSizeBinaryValues for FixedSizeBinaryArray { - #[inline] - fn values(&self) -> &[u8] { - &self.values - } - - #[inline] - fn size(&self) -> usize { - self.size - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_binary/mutable.rs b/src/common/arrow/src/arrow/array/fixed_size_binary/mutable.rs deleted file mode 100644 index 8436952cd616..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_binary/mutable.rs +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::FixedSizeBinaryArray; -use super::FixedSizeBinaryValues; -use crate::arrow::array::physical_binary::extend_validity; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; - -/// The Arrow's equivalent to a mutable `Vec>`. -/// Converting a [`MutableFixedSizeBinaryArray`] into a [`FixedSizeBinaryArray`] is `O(1)`. -/// # Implementation -/// This struct does not allocate a validity until one is required (i.e. push a null to it). -#[derive(Debug, Clone)] -pub struct MutableFixedSizeBinaryArray { - data_type: DataType, - size: usize, - values: Vec, - validity: Option, -} - -impl From for FixedSizeBinaryArray { - fn from(other: MutableFixedSizeBinaryArray) -> Self { - FixedSizeBinaryArray::new( - other.data_type, - other.values.into(), - other.validity.map(|x| x.into()), - ) - } -} - -impl MutableFixedSizeBinaryArray { - /// Creates a new [`MutableFixedSizeBinaryArray`]. - /// - /// # Errors - /// This function returns an error iff: - /// * The `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::FixedSizeBinary`] - /// * The length of `values` is not a multiple of `size` in `data_type` - /// * the validity's length is not equal to `values.len() / size`. - pub fn try_new( - data_type: DataType, - values: Vec, - validity: Option, - ) -> Result { - let size = FixedSizeBinaryArray::maybe_get_size(&data_type)?; - - if values.len() % size != 0 { - return Err(Error::oos(format!( - "values (of len {}) must be a multiple of size ({}) in FixedSizeBinaryArray.", - values.len(), - size - ))); - } - let len = values.len() / size; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != len) - { - return Err(Error::oos( - "validity mask length must be equal to the number of values divided by size", - )); - } - - Ok(Self { - size, - data_type, - values, - validity, - }) - } - - /// Creates a new empty [`MutableFixedSizeBinaryArray`]. - pub fn new(size: usize) -> Self { - Self::with_capacity(size, 0) - } - - /// Creates a new [`MutableFixedSizeBinaryArray`] with capacity for `capacity` entries. - pub fn with_capacity(size: usize, capacity: usize) -> Self { - Self::try_new( - DataType::FixedSizeBinary(size), - Vec::::with_capacity(capacity * size), - None, - ) - .unwrap() - } - - /// Creates a new [`MutableFixedSizeBinaryArray`] from a slice of optional `[u8]`. - // Note: this can't be `impl From` because Rust does not allow double `AsRef` on it. - pub fn from]>>(slice: P) -> Self { - let values = slice - .as_ref() - .iter() - .copied() - .flat_map(|x| x.unwrap_or([0; N])) - .collect::>(); - let validity = slice - .as_ref() - .iter() - .map(|x| x.is_some()) - .collect::(); - Self::try_new(DataType::FixedSizeBinary(N), values, validity.into()).unwrap() - } - - /// tries to push a new entry to [`MutableFixedSizeBinaryArray`]. - /// # Error - /// Errors iff the size of `value` is not equal to its own size. - #[inline] - pub fn try_push>(&mut self, value: Option

) -> Result<(), Error> { - match value { - Some(bytes) => { - let bytes = bytes.as_ref(); - if self.size != bytes.len() { - return Err(Error::InvalidArgumentError( - "FixedSizeBinaryArray requires every item to be of its length".to_string(), - )); - } - self.values.extend_from_slice(bytes); - - match &mut self.validity { - Some(validity) => validity.push(true), - None => {} - } - } - None => { - self.values.resize(self.values.len() + self.size, 0); - match &mut self.validity { - Some(validity) => validity.push(false), - None => self.init_validity(), - } - } - } - Ok(()) - } - - /// pushes a new entry to [`MutableFixedSizeBinaryArray`]. - /// # Panics - /// Panics iff the size of `value` is not equal to its own size. - #[inline] - pub fn push>(&mut self, value: Option

) { - self.try_push(value).unwrap() - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.values.len() / self.size - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Pop the last entry from [`MutableFixedSizeBinaryArray`]. - /// This function returns `None` iff this array is empty - pub fn pop(&mut self) -> Option> { - if self.values.len() < self.size { - return None; - } - let value_start = self.values.len() - self.size; - let value = self.values.split_off(value_start); - self.validity - .as_mut() - .map(|x| x.pop()?.then_some(())) - .unwrap_or_else(|| Some(())) - .map(|_| value) - } - - /// Creates a new [`MutableFixedSizeBinaryArray`] from an iterator of values. - /// # Errors - /// Errors iff the size of any of the `value` is not equal to its own size. - pub fn try_from_iter, I: IntoIterator>>( - iter: I, - size: usize, - ) -> Result { - let iterator = iter.into_iter(); - let (lower, _) = iterator.size_hint(); - let mut primitive = Self::with_capacity(size, lower); - for item in iterator { - primitive.try_push(item)? - } - Ok(primitive) - } - - /// returns the (fixed) size of the [`MutableFixedSizeBinaryArray`]. - #[inline] - pub fn size(&self) -> usize { - self.size - } - - /// Returns the capacity of this array - pub fn capacity(&self) -> usize { - self.values.capacity() / self.size - } - - fn init_validity(&mut self) { - let mut validity = MutableBitmap::new(); - validity.extend_constant(self.len(), true); - validity.set(self.len() - 1, false); - self.validity = Some(validity) - } - - /// Returns the element at index `i` as `&[u8]` - #[inline] - pub fn value(&self, i: usize) -> &[u8] { - &self.values[i * self.size..(i + 1) * self.size] - } - - /// Returns the element at index `i` as `&[u8]` - /// # Safety - /// Assumes that the `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] { - std::slice::from_raw_parts(self.values.as_ptr().add(i * self.size), self.size) - } - - /// Reserves `additional` slots. - pub fn reserve(&mut self, additional: usize) { - self.values.reserve(additional * self.size); - if let Some(x) = self.validity.as_mut() { - x.reserve(additional) - } - } - - /// Shrinks the capacity of the [`MutableFixedSizeBinaryArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - if let Some(validity) = &mut self.validity { - validity.shrink_to_fit() - } - } -} - -/// Accessors -impl MutableFixedSizeBinaryArray { - /// Returns its values. - pub fn values(&self) -> &Vec { - &self.values - } - - /// Returns a mutable slice of values. - pub fn values_mut_slice(&mut self) -> &mut [u8] { - self.values.as_mut_slice() - } -} - -impl MutableArray for MutableFixedSizeBinaryArray { - fn len(&self) -> usize { - self.values.len() / self.size - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(self.size), - std::mem::take(&mut self.values).into(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .boxed() - } - - fn as_arc(&mut self) -> Arc { - FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(self.size), - std::mem::take(&mut self.values).into(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .arced() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn push_null(&mut self) { - self.push::<&[u8]>(None); - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl FixedSizeBinaryValues for MutableFixedSizeBinaryArray { - #[inline] - fn values(&self) -> &[u8] { - &self.values - } - - #[inline] - fn size(&self) -> usize { - self.size - } -} - -impl PartialEq for MutableFixedSizeBinaryArray { - fn eq(&self, other: &Self) -> bool { - self.iter().eq(other.iter()) - } -} - -impl TryExtendFromSelf for MutableFixedSizeBinaryArray { - fn try_extend_from_self(&mut self, other: &Self) -> Result<(), Error> { - extend_validity(self.len(), &mut self.validity, &other.validity); - - let slice = other.values.as_slice(); - self.values.extend_from_slice(slice); - Ok(()) - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_list/data.rs b/src/common/arrow/src/arrow/array/fixed_size_list/data.rs deleted file mode 100644 index fad809298fc7..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_list/data.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::from_data; -use crate::arrow::array::to_data; -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::FixedSizeListArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::DataType; - -impl Arrow2Arrow for FixedSizeListArray { - fn to_data(&self) -> ArrayData { - let data_type = self.data_type.clone().into(); - let builder = ArrayDataBuilder::new(data_type) - .len(self.len()) - .nulls(self.validity.as_ref().map(|b| b.clone().into())) - .child_data(vec![to_data(self.values.as_ref())]); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type: DataType = data.data_type().clone().into(); - let size = match data_type { - DataType::FixedSizeList(_, size) => size, - _ => unreachable!("must be FixedSizeList type"), - }; - - let mut values = from_data(&data.child_data()[0]); - values.slice(data.offset() * size, data.len() * size); - - Self { - size, - data_type, - values, - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_list/fmt.rs b/src/common/arrow/src/arrow/array/fixed_size_list/fmt.rs deleted file mode 100644 index f7ae0945fcfd..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_list/fmt.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::get_display; -use super::super::fmt::write_vec; -use super::FixedSizeListArray; - -pub fn write_value( - array: &FixedSizeListArray, - index: usize, - null: &'static str, - f: &mut W, -) -> Result { - let values = array.value(index); - let writer = |f: &mut W, index| get_display(values.as_ref(), null)(f, index); - write_vec(f, writer, None, values.len(), null, false) -} - -impl Debug for FixedSizeListArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, "None", f); - - write!(f, "FixedSizeListArray")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_list/iterator.rs b/src/common/arrow/src/arrow/array/fixed_size_list/iterator.rs deleted file mode 100644 index 7b070770ce67..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_list/iterator.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::FixedSizeListArray; -use crate::arrow::array::Array; -use crate::arrow::array::ArrayAccessor; -use crate::arrow::array::ArrayValuesIter; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; - -unsafe impl<'a> ArrayAccessor<'a> for FixedSizeListArray { - type Item = Box; - - #[inline] - unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) - } - - #[inline] - fn len(&self) -> usize { - self.len() - } -} - -/// Iterator of values of a [`FixedSizeListArray`]. -pub type FixedSizeListValuesIter<'a> = ArrayValuesIter<'a, FixedSizeListArray>; - -type ZipIter<'a> = ZipValidity, FixedSizeListValuesIter<'a>, BitmapIter<'a>>; - -impl<'a> IntoIterator for &'a FixedSizeListArray { - type Item = Option>; - type IntoIter = ZipIter<'a>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> FixedSizeListArray { - /// Returns an iterator of `Option>` - pub fn iter(&'a self) -> ZipIter<'a> { - ZipValidity::new_with_validity(FixedSizeListValuesIter::new(self), self.validity()) - } - - /// Returns an iterator of `Box` - pub fn values_iter(&'a self) -> FixedSizeListValuesIter<'a> { - FixedSizeListValuesIter::new(self) - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_list/mod.rs b/src/common/arrow/src/arrow/array/fixed_size_list/mod.rs deleted file mode 100644 index 2fbb4c63405f..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_list/mod.rs +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::new_empty_array; -use super::new_null_array; -use super::Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod iterator; -mod mutable; -pub use mutable::*; - -/// The Arrow's equivalent to an immutable `Vec>` where `T` is an Arrow type. -/// Cloning and slicing this struct is `O(1)`. -#[derive(Clone)] -pub struct FixedSizeListArray { - size: usize, /* this is redundant with `data_type`, but useful to not have to deconstruct the data_type. */ - data_type: DataType, - values: Box, - validity: Option, -} - -impl FixedSizeListArray { - /// Creates a new [`FixedSizeListArray`]. - /// - /// # Errors - /// This function returns an error iff: - /// * The `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::FixedSizeList`] - /// * The `data_type`'s inner field's data type is not equal to `values.data_type`. - /// * The length of `values` is not a multiple of `size` in `data_type` - /// * the validity's length is not equal to `values.len() / size`. - pub fn try_new( - data_type: DataType, - values: Box, - validity: Option, - ) -> Result { - let (child, size) = Self::try_child_and_size(&data_type)?; - - let child_data_type = &child.data_type; - let values_data_type = values.data_type(); - if child_data_type != values_data_type { - return Err(Error::oos(format!( - "FixedSizeListArray's child's DataType must match. However, the expected DataType is {child_data_type:?} while it got {values_data_type:?}." - ))); - } - - if values.len() % size != 0 { - return Err(Error::oos(format!( - "values (of len {}) must be a multiple of size ({}) in FixedSizeListArray.", - values.len(), - size - ))); - } - let len = values.len() / size; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != len) - { - return Err(Error::oos( - "validity mask length must be equal to the number of values divided by size", - )); - } - - Ok(Self { - size, - data_type, - values, - validity, - }) - } - - /// Alias to `Self::try_new(...).unwrap()` - pub fn new(data_type: DataType, values: Box, validity: Option) -> Self { - Self::try_new(data_type, values, validity).unwrap() - } - - /// Returns the size (number of elements per slot) of this [`FixedSizeListArray`]. - pub const fn size(&self) -> usize { - self.size - } - - /// Returns a new empty [`FixedSizeListArray`]. - pub fn new_empty(data_type: DataType) -> Self { - let values = new_empty_array(Self::get_child_and_size(&data_type).0.data_type().clone()); - Self::new(data_type, values, None) - } - - /// Returns a new null [`FixedSizeListArray`]. - pub fn new_null(data_type: DataType, length: usize) -> Self { - let (field, size) = Self::get_child_and_size(&data_type); - - let values = new_null_array(field.data_type().clone(), length * size); - Self::new(data_type, values, Some(Bitmap::new_zeroed(length))) - } -} - -// must use -impl FixedSizeListArray { - /// Slices this [`FixedSizeListArray`]. - /// # Implementation - /// This operation is `O(1)`. - /// # Panics - /// panics iff `offset + length > self.len()` - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new Buffer cannot exceed the existing length" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Slices this [`FixedSizeListArray`]. - /// # Implementation - /// This operation is `O(1)`. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.values - .slice_unchecked(offset * self.size, length * self.size); - } - - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); -} - -// accessors -impl FixedSizeListArray { - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.values.len() / self.size - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The optional validity. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - /// Returns the inner array. - #[allow(clippy::borrowed_box)] - pub fn values(&self) -> &Box { - &self.values - } - - /// Returns the `Vec` at position `i`. - /// # Panic: - /// panics iff `i >= self.len()` - #[inline] - pub fn value(&self, i: usize) -> Box { - self.values.sliced(i * self.size, self.size) - } - - /// Returns the `Vec` at position `i`. - /// # Safety - /// Caller must ensure that `i < self.len()` - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> Box { - self.values.sliced_unchecked(i * self.size, self.size) - } - - /// Returns the element at index `i` or `None` if it is null - /// # Panics - /// iff `i >= self.len()` - #[inline] - pub fn get(&self, i: usize) -> Option> { - if !self.is_null(i) { - // soundness: Array::is_null panics if i >= self.len - unsafe { Some(self.value_unchecked(i)) } - } else { - None - } - } -} - -impl FixedSizeListArray { - pub(crate) fn try_child_and_size(data_type: &DataType) -> Result<(&Field, usize), Error> { - match data_type.to_logical_type() { - DataType::FixedSizeList(child, size) => { - if *size == 0 { - return Err(Error::oos("FixedSizeBinaryArray expects a positive size")); - } - Ok((child.as_ref(), *size)) - } - _ => Err(Error::oos( - "FixedSizeListArray expects DataType::FixedSizeList", - )), - } - } - - pub(crate) fn get_child_and_size(data_type: &DataType) -> (&Field, usize) { - Self::try_child_and_size(data_type).unwrap() - } - - /// Returns a [`DataType`] consistent with [`FixedSizeListArray`]. - pub fn default_datatype(data_type: DataType, size: usize) -> DataType { - let field = Box::new(Field::new("item", data_type, true)); - DataType::FixedSizeList(field, size) - } -} - -impl Array for FixedSizeListArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/fixed_size_list/mutable.rs b/src/common/arrow/src/arrow/array/fixed_size_list/mutable.rs deleted file mode 100644 index 5cb678a169f0..000000000000 --- a/src/common/arrow/src/arrow/array/fixed_size_list/mutable.rs +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::FixedSizeListArray; -use crate::arrow::array::physical_binary::extend_validity; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::PushUnchecked; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// The mutable version of [`FixedSizeListArray`]. -#[derive(Debug, Clone)] -pub struct MutableFixedSizeListArray { - data_type: DataType, - size: usize, - values: M, - validity: Option, -} - -impl From> for FixedSizeListArray { - fn from(mut other: MutableFixedSizeListArray) -> Self { - FixedSizeListArray::new( - other.data_type, - other.values.as_box(), - other.validity.map(|x| x.into()), - ) - } -} - -impl MutableFixedSizeListArray { - /// Creates a new [`MutableFixedSizeListArray`] from a [`MutableArray`] and size. - pub fn new(values: M, size: usize) -> Self { - let data_type = FixedSizeListArray::default_datatype(values.data_type().clone(), size); - Self::new_from(values, data_type, size) - } - - /// Creates a new [`MutableFixedSizeListArray`] from a [`MutableArray`] and size. - pub fn new_with_field(values: M, name: &str, nullable: bool, size: usize) -> Self { - let data_type = DataType::FixedSizeList( - Box::new(Field::new(name, values.data_type().clone(), nullable)), - size, - ); - Self::new_from(values, data_type, size) - } - - /// Creates a new [`MutableFixedSizeListArray`] from a [`MutableArray`], [`DataType`] and size. - pub fn new_from(values: M, data_type: DataType, size: usize) -> Self { - assert_eq!(values.len(), 0); - match data_type { - DataType::FixedSizeList(..) => (), - _ => panic!("data type must be FixedSizeList (got {data_type:?})"), - }; - Self { - size, - data_type, - values, - validity: None, - } - } - - /// Returns the size (number of elements per slot) of this [`FixedSizeListArray`]. - pub const fn size(&self) -> usize { - self.size - } - - /// The length of this array - pub fn len(&self) -> usize { - self.values.len() / self.size - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The inner values - pub fn values(&self) -> &M { - &self.values - } - - /// The values as a mutable reference - pub fn mut_values(&mut self) -> &mut M { - &mut self.values - } - - fn init_validity(&mut self) { - let len = self.values.len() / self.size; - - let mut validity = MutableBitmap::new(); - validity.extend_constant(len, true); - validity.set(len - 1, false); - self.validity = Some(validity) - } - - #[inline] - /// Needs to be called when a valid value was extended to this array. - /// This is a relatively low level function, prefer `try_push` when you can. - pub fn try_push_valid(&mut self) -> Result<()> { - if self.values.len() % self.size != 0 { - return Err(Error::Overflow); - }; - if let Some(validity) = &mut self.validity { - validity.push(true) - } - Ok(()) - } - - #[inline] - /// Needs to be called when a valid value was extended to this array. - /// This is a relatively low level function, prefer `try_push` when you can. - pub fn push_valid(&mut self) { - if let Some(validity) = &mut self.validity { - validity.push(true) - } - } - - #[inline] - fn push_null(&mut self) { - (0..self.size).for_each(|_| self.values.push_null()); - match &mut self.validity { - Some(validity) => validity.push(false), - None => self.init_validity(), - } - } - - /// Reserves `additional` slots. - pub fn reserve(&mut self, additional: usize) { - self.values.reserve(additional); - if let Some(x) = self.validity.as_mut() { - x.reserve(additional) - } - } - - /// Shrinks the capacity of the [`MutableFixedSizeListArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - if let Some(validity) = &mut self.validity { - validity.shrink_to_fit() - } - } -} - -impl MutableArray for MutableFixedSizeListArray { - fn len(&self) -> usize { - self.values.len() / self.size - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - FixedSizeListArray::new( - self.data_type.clone(), - self.values.as_box(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .boxed() - } - - fn as_arc(&mut self) -> Arc { - FixedSizeListArray::new( - self.data_type.clone(), - self.values.as_box(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .arced() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - #[inline] - fn push_null(&mut self) { - (0..self.size).for_each(|_| { - self.values.push_null(); - }); - if let Some(validity) = &mut self.validity { - validity.push(false) - } else { - self.init_validity() - } - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl TryExtend> for MutableFixedSizeListArray -where - M: MutableArray + TryExtend>, - I: IntoIterator>, -{ - #[inline] - fn try_extend>>(&mut self, iter: II) -> Result<()> { - for items in iter { - self.try_push(items)?; - } - Ok(()) - } -} - -impl TryPush> for MutableFixedSizeListArray -where - M: MutableArray + TryExtend>, - I: IntoIterator>, -{ - #[inline] - fn try_push(&mut self, item: Option) -> Result<()> { - if let Some(items) = item { - self.values.try_extend(items)?; - self.try_push_valid()?; - } else { - self.push_null(); - } - Ok(()) - } -} - -impl PushUnchecked> for MutableFixedSizeListArray -where - M: MutableArray + Extend>, - I: IntoIterator>, -{ - /// # Safety - /// The caller must ensure that the `I` iterates exactly over `size` - /// items, where `size` is the fixed size width. - #[inline] - unsafe fn push_unchecked(&mut self, item: Option) { - if let Some(items) = item { - self.values.extend(items); - self.push_valid(); - } else { - self.push_null(); - } - } -} - -impl TryExtendFromSelf for MutableFixedSizeListArray -where M: MutableArray + TryExtendFromSelf -{ - fn try_extend_from_self(&mut self, other: &Self) -> Result<()> { - extend_validity(self.len(), &mut self.validity, &other.validity); - - self.values.try_extend_from_self(&other.values) - } -} diff --git a/src/common/arrow/src/arrow/array/fmt.rs b/src/common/arrow/src/arrow/array/fmt.rs deleted file mode 100644 index 6dd39f7ed36a..000000000000 --- a/src/common/arrow/src/arrow/array/fmt.rs +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Result; -use std::fmt::Write; - -use super::Array; -use crate::arrow::bitmap::Bitmap; - -/// Returns a function that writes the value of the element of `array` -/// at position `index` to a [`Write`], -/// writing `null` in the null slots. -#[allow(clippy::type_complexity)] -pub fn get_value_display<'a, F: Write + 'a>( - array: &'a dyn Array, - null: &'static str, -) -> Box Result + 'a> { - use crate::arrow::datatypes::PhysicalType::*; - match array.data_type().to_physical_type() { - Null => Box::new(move |f, _| write!(f, "{null}")), - Boolean => Box::new(|f, index| { - super::boolean::fmt::write_value(array.as_any().downcast_ref().unwrap(), index, f) - }), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - let writer = super::primitive::fmt::get_write_value::<$T, _>( - array.as_any().downcast_ref().unwrap(), - ); - Box::new(move |f, index| writer(f, index)) - }), - Binary => Box::new(|f, index| { - super::binary::fmt::write_value::( - array.as_any().downcast_ref().unwrap(), - index, - f, - ) - }), - FixedSizeBinary => Box::new(|f, index| { - super::fixed_size_binary::fmt::write_value( - array.as_any().downcast_ref().unwrap(), - index, - f, - ) - }), - LargeBinary => Box::new(|f, index| { - super::binary::fmt::write_value::( - array.as_any().downcast_ref().unwrap(), - index, - f, - ) - }), - Utf8 => Box::new(|f, index| { - super::utf8::fmt::write_value::( - array.as_any().downcast_ref().unwrap(), - index, - f, - ) - }), - LargeUtf8 => Box::new(|f, index| { - super::utf8::fmt::write_value::( - array.as_any().downcast_ref().unwrap(), - index, - f, - ) - }), - List => Box::new(move |f, index| { - super::list::fmt::write_value::( - array.as_any().downcast_ref().unwrap(), - index, - null, - f, - ) - }), - FixedSizeList => Box::new(move |f, index| { - super::fixed_size_list::fmt::write_value( - array.as_any().downcast_ref().unwrap(), - index, - null, - f, - ) - }), - LargeList => Box::new(move |f, index| { - super::list::fmt::write_value::( - array.as_any().downcast_ref().unwrap(), - index, - null, - f, - ) - }), - Struct => Box::new(move |f, index| { - super::struct_::fmt::write_value(array.as_any().downcast_ref().unwrap(), index, null, f) - }), - Union => Box::new(move |f, index| { - super::union::fmt::write_value(array.as_any().downcast_ref().unwrap(), index, null, f) - }), - Map => Box::new(move |f, index| { - super::map::fmt::write_value(array.as_any().downcast_ref().unwrap(), index, null, f) - }), - BinaryView => Box::new(move |f, index| { - super::binview::fmt::write_value::<[u8], _>( - array.as_any().downcast_ref().unwrap(), - index, - f, - ) - }), - Utf8View => Box::new(move |f, index| { - super::binview::fmt::write_value::( - array.as_any().downcast_ref().unwrap(), - index, - f, - ) - }), - Dictionary(key_type) => match_integer_type!(key_type, |$T| { - Box::new(move |f, index| { - super::dictionary::fmt::write_value::<$T,_>(array.as_any().downcast_ref().unwrap(), index, null, f) - }) - }), - } -} - -/// Returns a function that writes the element of `array` -/// at position `index` to a [`Write`], writing `null` to the null slots. -#[allow(clippy::type_complexity)] -pub fn get_display<'a, F: Write + 'a>( - array: &'a dyn Array, - null: &'static str, -) -> Box Result + 'a> { - let value_display = get_value_display(array, null); - Box::new(move |f, row| { - if array.is_null(row) { - f.write_str(null) - } else { - value_display(f, row) - } - }) -} - -pub fn write_vec( - f: &mut F, - d: D, - validity: Option<&Bitmap>, - len: usize, - null: &'static str, - new_lines: bool, -) -> Result -where - D: Fn(&mut F, usize) -> Result, - F: Write, -{ - f.write_char('[')?; - write_list(f, d, validity, len, null, new_lines)?; - f.write_char(']')?; - Ok(()) -} - -fn write_list( - f: &mut F, - d: D, - validity: Option<&Bitmap>, - len: usize, - null: &'static str, - new_lines: bool, -) -> Result -where - D: Fn(&mut F, usize) -> Result, - F: Write, -{ - for index in 0..len { - if index != 0 { - f.write_char(',')?; - f.write_char(if new_lines { '\n' } else { ' ' })?; - } - if let Some(val) = validity { - if val.get_bit(index) { - d(f, index) - } else { - write!(f, "{null}") - } - } else { - d(f, index) - }?; - } - Ok(()) -} - -pub fn write_map( - f: &mut F, - d: D, - validity: Option<&Bitmap>, - len: usize, - null: &'static str, - new_lines: bool, -) -> Result -where - D: Fn(&mut F, usize) -> Result, - F: Write, -{ - f.write_char('{')?; - write_list(f, d, validity, len, null, new_lines)?; - f.write_char('}')?; - Ok(()) -} diff --git a/src/common/arrow/src/arrow/array/growable/binary.rs b/src/common/arrow/src/arrow/array/growable/binary.rs deleted file mode 100644 index 664f1558a9d2..000000000000 --- a/src/common/arrow/src/arrow/array/growable/binary.rs +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::utils::build_extend_null_bits; -use super::utils::extend_offset_values; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; - -/// Concrete [`Growable`] for the [`BinaryArray`]. -pub struct GrowableBinary<'a, O: Offset> { - arrays: Vec<&'a BinaryArray>, - data_type: DataType, - validity: MutableBitmap, - values: Vec, - offsets: Offsets, - extend_null_bits: Vec>, -} - -impl<'a, O: Offset> GrowableBinary<'a, O> { - /// Creates a new [`GrowableBinary`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new(arrays: Vec<&'a BinaryArray>, mut use_validity: bool, capacity: usize) -> Self { - let data_type = arrays[0].data_type().clone(); - - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if !use_validity & arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - Self { - arrays, - data_type, - values: Vec::with_capacity(0), - offsets: Offsets::with_capacity(capacity), - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - } - } - - fn to(&mut self) -> BinaryArray { - let data_type = self.data_type.clone(); - let validity = std::mem::take(&mut self.validity); - let offsets = std::mem::take(&mut self.offsets); - let values = std::mem::take(&mut self.values); - - BinaryArray::::new(data_type, offsets.into(), values.into(), validity.into()) - } -} - -impl<'a, O: Offset> Growable<'a> for GrowableBinary<'a, O> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - - let array = self.arrays[index]; - let offsets = array.offsets(); - let values = array.values(); - - self.offsets - .try_extend_from_slice(offsets, start, len) - .unwrap(); - - // values - extend_offset_values::(&mut self.values, offsets.buffer(), values, start, len); - } - - fn extend_validity(&mut self, additional: usize) { - self.offsets.extend_constant(additional); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - self.offsets.len() - 1 - } - - fn as_arc(&mut self) -> Arc { - self.to().arced() - } - - fn as_box(&mut self) -> Box { - self.to().boxed() - } -} - -impl<'a, O: Offset> From> for BinaryArray { - fn from(val: GrowableBinary<'a, O>) -> Self { - BinaryArray::::new( - val.data_type, - val.offsets.into(), - val.values.into(), - val.validity.into(), - ) - } -} diff --git a/src/common/arrow/src/arrow/array/growable/binview.rs b/src/common/arrow/src/arrow/array/growable/binview.rs deleted file mode 100644 index 6fa525471e82..000000000000 --- a/src/common/arrow/src/arrow/array/growable/binview.rs +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::hash::Hash; -use std::hash::Hasher; -use std::sync::Arc; - -use indexmap::IndexSet; - -use crate::arrow::array::growable::utils::extend_validity; -use crate::arrow::array::growable::utils::prepare_validity; -use crate::arrow::array::growable::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::BinaryViewArrayGeneric; -use crate::arrow::array::View; -use crate::arrow::array::ViewType; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; - -pub type ArrowIndexSet = IndexSet; - -struct BufferKey<'a> { - inner: &'a Buffer, -} - -impl Hash for BufferKey<'_> { - fn hash(&self, state: &mut H) { - state.write_u64(self.inner.data_ptr() as u64) - } -} - -impl PartialEq for BufferKey<'_> { - #[inline] - fn eq(&self, other: &Self) -> bool { - self.inner.data_ptr() == other.inner.data_ptr() - } -} - -impl Eq for BufferKey<'_> {} - -/// Concrete [`Growable`] for the [`BinaryArray`]. -pub struct GrowableBinaryViewArray<'a, T: ViewType + ?Sized> { - arrays: Vec<&'a BinaryViewArrayGeneric>, - data_type: DataType, - validity: Option, - views: Vec, - // We need to use a set/hashmap to deduplicate - // A growable can be called with many chunks from self. - buffers: ArrowIndexSet>, - total_bytes_len: usize, - total_buffer_len: usize, -} - -impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> { - /// Creates a new [`GrowableBinaryViewArray`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new( - arrays: Vec<&'a BinaryViewArrayGeneric>, - mut use_validity: bool, - capacity: usize, - ) -> Self { - let data_type = arrays[0].data_type().clone(); - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if !use_validity & arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let buffers = arrays - .iter() - .flat_map(|array| { - array - .data_buffers() - .as_ref() - .iter() - .map(|buf| BufferKey { inner: buf }) - }) - .collect::>(); - - let total_buffer_len = buffers.iter().map(|v| v.inner.len()).sum(); - Self { - arrays, - data_type, - validity: prepare_validity(use_validity, capacity), - views: Vec::with_capacity(capacity), - buffers, - total_bytes_len: 0, - total_buffer_len, - } - } - - fn to(&mut self) -> BinaryViewArrayGeneric { - let views = std::mem::take(&mut self.views); - let buffers = std::mem::take(&mut self.buffers); - let validity = self.validity.take(); - BinaryViewArrayGeneric::::new_unchecked( - self.data_type.clone(), - views.into(), - Arc::from( - buffers - .into_iter() - .map(|buf| buf.inner.clone()) - .collect::>(), - ), - validity.map(|v| v.into()), - self.total_bytes_len, - self.total_buffer_len, - ) - .maybe_gc() - } - - /// # Safety - /// doesn't check bounds - pub unsafe fn extend_unchecked(&mut self, index: usize, start: usize, len: usize) { - let array = *self.arrays.get_unchecked(index); - let local_buffers = array.data_buffers(); - - extend_validity(&mut self.validity, array, start, len); - - let range = start..start + len; - - self.views - .extend(array.views().get_unchecked(range).iter().map(|view| { - let mut view = *view; - let len = view.length as usize; - self.total_bytes_len += len; - - if len > 12 { - let buffer = local_buffers.get_unchecked(view.buffer_idx as usize); - let key = BufferKey { inner: buffer }; - let idx = self.buffers.get_full(&key).unwrap_unchecked().0; - - view.buffer_idx = idx as u32; - } - view - })); - } - - #[inline] - /// Ignores the buffers and doesn't update the view. This is only correct in a filter. - /// # Safety - /// doesn't check bounds - pub unsafe fn extend_unchecked_no_buffers(&mut self, index: usize, start: usize, len: usize) { - let array = *self.arrays.get_unchecked(index); - - extend_validity(&mut self.validity, array, start, len); - - let range = start..start + len; - - self.views - .extend(array.views().get_unchecked(range).iter().map(|view| { - let len = view.length as usize; - self.total_bytes_len += len; - - *view - })) - } -} - -impl<'a, T: ViewType + ?Sized> Growable<'a> for GrowableBinaryViewArray<'a, T> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - unsafe { self.extend_unchecked(index, start, len) } - } - - fn extend_validity(&mut self, additional: usize) { - self.views - .extend(std::iter::repeat(View::default()).take(additional)); - if let Some(validity) = &mut self.validity { - validity.extend_constant(additional, false); - } - } - - #[inline] - fn len(&self) -> usize { - self.views.len() - } - - fn as_arc(&mut self) -> Arc { - self.to().arced() - } - - fn as_box(&mut self) -> Box { - self.to().boxed() - } -} - -impl<'a, T: ViewType + ?Sized> From> for BinaryViewArrayGeneric { - fn from(val: GrowableBinaryViewArray<'a, T>) -> Self { - BinaryViewArrayGeneric::::new_unchecked( - val.data_type, - val.views.into(), - Arc::from( - val.buffers - .into_iter() - .map(|buf| buf.inner.clone()) - .collect::>(), - ), - val.validity.map(|v| v.into()), - val.total_bytes_len, - val.total_buffer_len, - ) - .maybe_gc() - } -} diff --git a/src/common/arrow/src/arrow/array/growable/boolean.rs b/src/common/arrow/src/arrow/array/growable/boolean.rs deleted file mode 100644 index a7019b5bb142..000000000000 --- a/src/common/arrow/src/arrow/array/growable/boolean.rs +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::utils::build_extend_null_bits; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::BooleanArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; - -/// Concrete [`Growable`] for the [`BooleanArray`]. -pub struct GrowableBoolean<'a> { - arrays: Vec<&'a BooleanArray>, - data_type: DataType, - validity: MutableBitmap, - values: MutableBitmap, - extend_null_bits: Vec>, -} - -impl<'a> GrowableBoolean<'a> { - /// Creates a new [`GrowableBoolean`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new(arrays: Vec<&'a BooleanArray>, mut use_validity: bool, capacity: usize) -> Self { - let data_type = arrays[0].data_type().clone(); - - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if !use_validity & arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - Self { - arrays, - data_type, - values: MutableBitmap::with_capacity(capacity), - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - } - } - - fn to(&mut self) -> BooleanArray { - let validity = std::mem::take(&mut self.validity); - let values = std::mem::take(&mut self.values); - - BooleanArray::new(self.data_type.clone(), values.into(), validity.into()) - } -} - -impl<'a> Growable<'a> for GrowableBoolean<'a> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - - let array = self.arrays[index]; - let values = array.values(); - - let (slice, offset, _) = values.as_slice(); - // safety: invariant offset + length <= slice.len() - unsafe { - self.values - .extend_from_slice_unchecked(slice, start + offset, len); - } - } - - fn extend_validity(&mut self, additional: usize) { - self.values.extend_constant(additional, false); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - self.values.len() - } - - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a> From> for BooleanArray { - fn from(val: GrowableBoolean<'a>) -> Self { - BooleanArray::new(val.data_type, val.values.into(), val.validity.into()) - } -} diff --git a/src/common/arrow/src/arrow/array/growable/dictionary.rs b/src/common/arrow/src/arrow/array/growable/dictionary.rs deleted file mode 100644 index de487e9a6858..000000000000 --- a/src/common/arrow/src/arrow/array/growable/dictionary.rs +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::make_growable; -use super::utils::build_extend_null_bits; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryArray; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; - -/// Concrete [`Growable`] for the [`DictionaryArray`]. -/// # Implementation -/// This growable does not perform collision checks and instead concatenates -/// the values of each [`DictionaryArray`] one after the other. -pub struct GrowableDictionary<'a, K: DictionaryKey> { - data_type: DataType, - keys_values: Vec<&'a [K]>, - key_values: Vec, - key_validity: MutableBitmap, - offsets: Vec, - values: Box, - extend_null_bits: Vec>, -} - -fn concatenate_values( - arrays_keys: &[&PrimitiveArray], - arrays_values: &[&dyn Array], - capacity: usize, -) -> (Box, Vec) { - let mut mutable = make_growable(arrays_values, false, capacity); - let mut offsets = Vec::with_capacity(arrays_keys.len() + 1); - offsets.push(0); - for (i, values) in arrays_values.iter().enumerate() { - mutable.extend(i, 0, values.len()); - offsets.push(offsets[i] + values.len()); - } - (mutable.as_box(), offsets) -} - -impl<'a, T: DictionaryKey> GrowableDictionary<'a, T> { - /// Creates a new [`GrowableDictionary`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new(arrays: &[&'a DictionaryArray], mut use_validity: bool, capacity: usize) -> Self { - let data_type = arrays[0].data_type().clone(); - - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let arrays_keys = arrays.iter().map(|array| array.keys()).collect::>(); - let keys_values = arrays_keys - .iter() - .map(|array| array.values().as_slice()) - .collect::>(); - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(array.keys(), use_validity)) - .collect(); - - let arrays_values = arrays - .iter() - .map(|array| array.values().as_ref()) - .collect::>(); - - let (values, offsets) = concatenate_values(&arrays_keys, &arrays_values, capacity); - - Self { - data_type, - offsets, - values, - keys_values, - key_values: Vec::with_capacity(capacity), - key_validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - } - } - - #[inline] - fn to(&mut self) -> DictionaryArray { - let validity = std::mem::take(&mut self.key_validity); - let key_values = std::mem::take(&mut self.key_values); - - #[cfg(debug_assertions)] - { - crate::arrow::array::specification::check_indexes(&key_values, self.values.len()) - .unwrap(); - } - let keys = - PrimitiveArray::::new(T::PRIMITIVE.into(), key_values.into(), validity.into()); - - // Safety - the invariant of this struct ensures that this is up-held - unsafe { - DictionaryArray::::try_new_unchecked( - self.data_type.clone(), - keys, - self.values.clone(), - ) - .unwrap() - } - } -} - -impl<'a, T: DictionaryKey> Growable<'a> for GrowableDictionary<'a, T> { - #[inline] - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.key_validity, start, len); - - let values = &self.keys_values[index][start..start + len]; - let offset = self.offsets[index]; - self.key_values.extend( - values - .iter() - // `.unwrap_or(0)` because this operation does not check for null values, which may contain any key. - .map(|x| { - let x: usize = offset + (*x).try_into().unwrap_or(0); - let x: T = match x.try_into() { - Ok(key) => key, - // todo: convert this to an error. - Err(_) => { - panic!("The maximum key is too small") - } - }; - x - }), - ); - } - - #[inline] - fn len(&self) -> usize { - self.key_values.len() - } - - #[inline] - fn extend_validity(&mut self, additional: usize) { - self.key_values - .resize(self.key_values.len() + additional, T::default()); - self.key_validity.extend_constant(additional, false); - } - - #[inline] - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - #[inline] - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a, T: DictionaryKey> From> for DictionaryArray { - #[inline] - fn from(mut val: GrowableDictionary<'a, T>) -> Self { - val.to() - } -} diff --git a/src/common/arrow/src/arrow/array/growable/fixed_binary.rs b/src/common/arrow/src/arrow/array/growable/fixed_binary.rs deleted file mode 100644 index 1247ce2b6e3d..000000000000 --- a/src/common/arrow/src/arrow/array/growable/fixed_binary.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::utils::build_extend_null_bits; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::FixedSizeBinaryArray; -use crate::arrow::bitmap::MutableBitmap; - -/// Concrete [`Growable`] for the [`FixedSizeBinaryArray`]. -pub struct GrowableFixedSizeBinary<'a> { - arrays: Vec<&'a FixedSizeBinaryArray>, - validity: MutableBitmap, - values: Vec, - extend_null_bits: Vec>, - size: usize, // just a cache -} - -impl<'a> GrowableFixedSizeBinary<'a> { - /// Creates a new [`GrowableFixedSizeBinary`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new( - arrays: Vec<&'a FixedSizeBinaryArray>, - mut use_validity: bool, - capacity: usize, - ) -> Self { - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - let size = FixedSizeBinaryArray::get_size(arrays[0].data_type()); - Self { - arrays, - values: Vec::with_capacity(0), - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - size, - } - } - - fn to(&mut self) -> FixedSizeBinaryArray { - let validity = std::mem::take(&mut self.validity); - let values = std::mem::take(&mut self.values); - - FixedSizeBinaryArray::new( - self.arrays[0].data_type().clone(), - values.into(), - validity.into(), - ) - } -} - -impl<'a> Growable<'a> for GrowableFixedSizeBinary<'a> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - - let array = self.arrays[index]; - let values = array.values(); - - self.values - .extend_from_slice(&values[start * self.size..start * self.size + len * self.size]); - } - - fn extend_validity(&mut self, additional: usize) { - self.values - .extend_from_slice(&vec![0; self.size * additional]); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - self.values.len() / self.size - } - - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a> From> for FixedSizeBinaryArray { - fn from(val: GrowableFixedSizeBinary<'a>) -> Self { - FixedSizeBinaryArray::new( - val.arrays[0].data_type().clone(), - val.values.into(), - val.validity.into(), - ) - } -} diff --git a/src/common/arrow/src/arrow/array/growable/fixed_size_list.rs b/src/common/arrow/src/arrow/array/growable/fixed_size_list.rs deleted file mode 100644 index 923fb56a6ca3..000000000000 --- a/src/common/arrow/src/arrow/array/growable/fixed_size_list.rs +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::make_growable; -use super::utils::build_extend_null_bits; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::FixedSizeListArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; - -/// Concrete [`Growable`] for the [`FixedSizeListArray`]. -pub struct GrowableFixedSizeList<'a> { - arrays: Vec<&'a FixedSizeListArray>, - validity: MutableBitmap, - values: Box + 'a>, - extend_null_bits: Vec>, - size: usize, -} - -impl<'a> GrowableFixedSizeList<'a> { - /// Creates a new [`GrowableFixedSizeList`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new( - arrays: Vec<&'a FixedSizeListArray>, - mut use_validity: bool, - capacity: usize, - ) -> Self { - assert!(!arrays.is_empty()); - - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if !use_validity & arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let size = - if let DataType::FixedSizeList(_, size) = &arrays[0].data_type().to_logical_type() { - *size - } else { - unreachable!("`GrowableFixedSizeList` expects `DataType::FixedSizeList`") - }; - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - let inner = arrays - .iter() - .map(|array| array.values().as_ref()) - .collect::>(); - let values = make_growable(&inner, use_validity, 0); - - Self { - arrays, - values, - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - size, - } - } - - fn to(&mut self) -> FixedSizeListArray { - let validity = std::mem::take(&mut self.validity); - let values = self.values.as_box(); - - FixedSizeListArray::new(self.arrays[0].data_type().clone(), values, validity.into()) - } -} - -impl<'a> Growable<'a> for GrowableFixedSizeList<'a> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - self.values - .extend(index, start * self.size, len * self.size); - } - - fn extend_validity(&mut self, additional: usize) { - self.values.extend_validity(additional * self.size); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - self.values.len() / self.size - } - - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a> From> for FixedSizeListArray { - fn from(val: GrowableFixedSizeList<'a>) -> Self { - let mut values = val.values; - let values = values.as_box(); - - Self::new( - val.arrays[0].data_type().clone(), - values, - val.validity.into(), - ) - } -} diff --git a/src/common/arrow/src/arrow/array/growable/list.rs b/src/common/arrow/src/arrow/array/growable/list.rs deleted file mode 100644 index 75c57b6ed818..000000000000 --- a/src/common/arrow/src/arrow/array/growable/list.rs +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::make_growable; -use super::utils::build_extend_null_bits; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::ListArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; - -fn extend_offset_values( - growable: &mut GrowableList<'_, O>, - index: usize, - start: usize, - len: usize, -) { - let array = growable.arrays[index]; - let offsets = array.offsets(); - - growable - .offsets - .try_extend_from_slice(offsets, start, len) - .unwrap(); - - let end = offsets.buffer()[start + len].to_usize(); - let start = offsets.buffer()[start].to_usize(); - let len = end - start; - growable.values.extend(index, start, len); -} - -/// Concrete [`Growable`] for the [`ListArray`]. -pub struct GrowableList<'a, O: Offset> { - arrays: Vec<&'a ListArray>, - validity: MutableBitmap, - values: Box + 'a>, - offsets: Offsets, - extend_null_bits: Vec>, -} - -impl<'a, O: Offset> GrowableList<'a, O> { - /// Creates a new [`GrowableList`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new(arrays: Vec<&'a ListArray>, mut use_validity: bool, capacity: usize) -> Self { - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if !use_validity & arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - let inner = arrays - .iter() - .map(|array| array.values().as_ref()) - .collect::>(); - let values = make_growable(&inner, use_validity, 0); - - Self { - arrays, - offsets: Offsets::with_capacity(capacity), - values, - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - } - } - - fn to(&mut self) -> ListArray { - let validity = std::mem::take(&mut self.validity); - let offsets = std::mem::take(&mut self.offsets); - let values = self.values.as_box(); - - ListArray::::new( - self.arrays[0].data_type().clone(), - offsets.into(), - values, - validity.into(), - ) - } -} - -impl<'a, O: Offset> Growable<'a> for GrowableList<'a, O> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - extend_offset_values::(self, index, start, len); - } - - fn extend_validity(&mut self, additional: usize) { - self.offsets.extend_constant(additional); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - self.offsets.len() - 1 - } - - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a, O: Offset> From> for ListArray { - fn from(mut val: GrowableList<'a, O>) -> Self { - val.to() - } -} diff --git a/src/common/arrow/src/arrow/array/growable/map.rs b/src/common/arrow/src/arrow/array/growable/map.rs deleted file mode 100644 index cc3b1226267e..000000000000 --- a/src/common/arrow/src/arrow/array/growable/map.rs +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::make_growable; -use super::utils::build_extend_null_bits; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::MapArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::offset::Offsets; - -fn extend_offset_values(growable: &mut GrowableMap<'_>, index: usize, start: usize, len: usize) { - let array = growable.arrays[index]; - let offsets = array.offsets(); - - growable - .offsets - .try_extend_from_slice(offsets, start, len) - .unwrap(); - - let end = offsets.buffer()[start + len] as usize; - let start = offsets.buffer()[start] as usize; - let len = end - start; - growable.values.extend(index, start, len); -} - -/// Concrete [`Growable`] for the [`MapArray`]. -pub struct GrowableMap<'a> { - arrays: Vec<&'a MapArray>, - validity: MutableBitmap, - values: Box + 'a>, - offsets: Offsets, - extend_null_bits: Vec>, -} - -impl<'a> GrowableMap<'a> { - /// Creates a new [`GrowableMap`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new(arrays: Vec<&'a MapArray>, mut use_validity: bool, capacity: usize) -> Self { - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if !use_validity & arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - let inner = arrays - .iter() - .map(|array| array.field().as_ref()) - .collect::>(); - let values = make_growable(&inner, use_validity, 0); - - Self { - arrays, - offsets: Offsets::with_capacity(capacity), - values, - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - } - } - - fn to(&mut self) -> MapArray { - let validity = std::mem::take(&mut self.validity); - let offsets = std::mem::take(&mut self.offsets); - let values = self.values.as_box(); - - MapArray::new( - self.arrays[0].data_type().clone(), - offsets.into(), - values, - validity.into(), - ) - } -} - -impl<'a> Growable<'a> for GrowableMap<'a> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - extend_offset_values(self, index, start, len); - } - - fn extend_validity(&mut self, additional: usize) { - self.offsets.extend_constant(additional); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - self.offsets.len() - 1 - } - - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a> From> for MapArray { - fn from(mut val: GrowableMap<'a>) -> Self { - val.to() - } -} diff --git a/src/common/arrow/src/arrow/array/growable/mod.rs b/src/common/arrow/src/arrow/array/growable/mod.rs deleted file mode 100644 index 8730567827eb..000000000000 --- a/src/common/arrow/src/arrow/array/growable/mod.rs +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains the trait [`Growable`] and corresponding concreate implementations, one per concrete array, -//! that offer the ability to create a new [`Array`] out of slices of existing [`Array`]s. - -use std::sync::Arc; - -use crate::arrow::array::*; -use crate::arrow::datatypes::*; - -mod binary; -pub use binary::GrowableBinary; -mod union; -pub use union::GrowableUnion; -mod boolean; -pub use boolean::GrowableBoolean; -mod fixed_binary; -pub use fixed_binary::GrowableFixedSizeBinary; -mod null; -pub use null::GrowableNull; -mod primitive; -pub use primitive::GrowablePrimitive; -mod list; -pub use list::GrowableList; -mod map; -pub use map::GrowableMap; -mod structure; -pub use structure::GrowableStruct; -mod fixed_size_list; -pub use fixed_size_list::GrowableFixedSizeList; -mod utf8; -pub use utf8::GrowableUtf8; -mod dictionary; -pub use dictionary::GrowableDictionary; - -mod binview; -mod utils; - -/// Describes a struct that can be extended from slices of other pre-existing [`Array`]s. -/// This is used in operations where a new array is built out of other arrays, such -/// as filter and concatenation. -pub trait Growable<'a> { - /// Extends this [`Growable`] with elements from the bounded [`Array`] at index `index` from - /// a slice starting at `start` and length `len`. - /// # Panic - /// This function panics if the range is out of bounds, i.e. if `start + len >= array.len()`. - fn extend(&mut self, index: usize, start: usize, len: usize); - - /// Extends this [`Growable`] with null elements, disregarding the bound arrays - fn extend_validity(&mut self, additional: usize); - - /// The current length of the [`Growable`]. - fn len(&self) -> usize; - - /// Returns `true` if the length of the [`Growable`] is 0. - fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Converts this [`Growable`] to an [`Arc`], thereby finishing the mutation. - /// Self will be empty after such operation. - fn as_arc(&mut self) -> Arc { - self.as_box().into() - } - - /// Converts this [`Growable`] to an [`Box`], thereby finishing the mutation. - /// Self will be empty after such operation - fn as_box(&mut self) -> Box; -} - -macro_rules! dyn_growable { - ($ty:ty, $arrays:expr, $use_validity:expr, $capacity:expr) => {{ - let arrays = $arrays - .iter() - .map(|array| array.as_any().downcast_ref().unwrap()) - .collect::>(); - Box::new(<$ty>::new(arrays, $use_validity, $capacity)) - }}; -} - -/// Creates a new [`Growable`] from an arbitrary number of [`Array`]s. -/// # Panics -/// This function panics iff -/// * the arrays do not have the same [`DataType`]. -/// * `arrays.is_empty()`. -pub fn make_growable<'a>( - arrays: &[&'a dyn Array], - use_validity: bool, - capacity: usize, -) -> Box + 'a> { - assert!(!arrays.is_empty()); - let data_type = arrays[0].data_type(); - - use PhysicalType::*; - match data_type.to_physical_type() { - Null => Box::new(null::GrowableNull::new(data_type.clone())), - Boolean => dyn_growable!(boolean::GrowableBoolean, arrays, use_validity, capacity), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - dyn_growable!(primitive::GrowablePrimitive::<$T>, arrays, use_validity, capacity) - }), - Utf8 => dyn_growable!(utf8::GrowableUtf8::, arrays, use_validity, capacity), - LargeUtf8 => dyn_growable!(utf8::GrowableUtf8::, arrays, use_validity, capacity), - Binary => dyn_growable!( - binary::GrowableBinary::, - arrays, - use_validity, - capacity - ), - LargeBinary => dyn_growable!( - binary::GrowableBinary::, - arrays, - use_validity, - capacity - ), - FixedSizeBinary => dyn_growable!( - fixed_binary::GrowableFixedSizeBinary, - arrays, - use_validity, - capacity - ), - List => dyn_growable!(list::GrowableList::, arrays, use_validity, capacity), - LargeList => dyn_growable!(list::GrowableList::, arrays, use_validity, capacity), - Struct => dyn_growable!(structure::GrowableStruct, arrays, use_validity, capacity), - FixedSizeList => dyn_growable!( - fixed_size_list::GrowableFixedSizeList, - arrays, - use_validity, - capacity - ), - Union => { - let arrays = arrays - .iter() - .map(|array| array.as_any().downcast_ref().unwrap()) - .collect::>(); - Box::new(union::GrowableUnion::new(arrays, capacity)) - } - Map => dyn_growable!(map::GrowableMap, arrays, use_validity, capacity), - BinaryView => { - dyn_growable!( - binview::GrowableBinaryViewArray::<[u8]>, - arrays, - use_validity, - capacity - ) - } - Utf8View => { - dyn_growable!( - binview::GrowableBinaryViewArray::, - arrays, - use_validity, - capacity - ) - } - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - let arrays = arrays - .iter() - .map(|array| { - array - .as_any() - .downcast_ref::>() - .unwrap() - }) - .collect::>(); - Box::new(dictionary::GrowableDictionary::<$T>::new( - &arrays, - use_validity, - capacity, - )) - }) - } - } -} diff --git a/src/common/arrow/src/arrow/array/growable/null.rs b/src/common/arrow/src/arrow/array/growable/null.rs deleted file mode 100644 index fb163329ef09..000000000000 --- a/src/common/arrow/src/arrow/array/growable/null.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::NullArray; -use crate::arrow::datatypes::DataType; - -/// Concrete [`Growable`] for the [`NullArray`]. -pub struct GrowableNull { - data_type: DataType, - length: usize, -} - -impl Default for GrowableNull { - fn default() -> Self { - Self::new(DataType::Null) - } -} - -impl GrowableNull { - /// Creates a new [`GrowableNull`]. - pub fn new(data_type: DataType) -> Self { - Self { - data_type, - length: 0, - } - } -} - -impl<'a> Growable<'a> for GrowableNull { - fn extend(&mut self, _: usize, _: usize, len: usize) { - self.length += len; - } - - fn extend_validity(&mut self, additional: usize) { - self.length += additional; - } - - #[inline] - fn len(&self) -> usize { - self.length - } - - fn as_arc(&mut self) -> Arc { - Arc::new(NullArray::new(self.data_type.clone(), self.length)) - } - - fn as_box(&mut self) -> Box { - Box::new(NullArray::new(self.data_type.clone(), self.length)) - } -} - -impl From for NullArray { - fn from(val: GrowableNull) -> Self { - NullArray::new(val.data_type, val.length) - } -} diff --git a/src/common/arrow/src/arrow/array/growable/primitive.rs b/src/common/arrow/src/arrow/array/growable/primitive.rs deleted file mode 100644 index 1c4e0d2721c4..000000000000 --- a/src/common/arrow/src/arrow/array/growable/primitive.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::utils::build_extend_null_bits; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::types::NativeType; - -/// Concrete [`Growable`] for the [`PrimitiveArray`]. -pub struct GrowablePrimitive<'a, T: NativeType> { - data_type: DataType, - arrays: Vec<&'a [T]>, - validity: MutableBitmap, - values: Vec, - extend_null_bits: Vec>, -} - -impl<'a, T: NativeType> GrowablePrimitive<'a, T> { - /// Creates a new [`GrowablePrimitive`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new( - arrays: Vec<&'a PrimitiveArray>, - mut use_validity: bool, - capacity: usize, - ) -> Self { - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if !use_validity & arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let data_type = arrays[0].data_type().clone(); - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - let arrays = arrays - .iter() - .map(|array| array.values().as_slice()) - .collect::>(); - - Self { - data_type, - arrays, - values: Vec::with_capacity(capacity), - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - } - } - - #[inline] - fn to(&mut self) -> PrimitiveArray { - let validity = std::mem::take(&mut self.validity); - let values = std::mem::take(&mut self.values); - - PrimitiveArray::::new(self.data_type.clone(), values.into(), validity.into()) - } -} - -impl<'a, T: NativeType> Growable<'a> for GrowablePrimitive<'a, T> { - #[inline] - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - - let values = self.arrays[index]; - self.values.extend_from_slice(&values[start..start + len]); - } - - #[inline] - fn extend_validity(&mut self, additional: usize) { - self.values - .resize(self.values.len() + additional, T::default()); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - self.values.len() - } - - #[inline] - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - #[inline] - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a, T: NativeType> From> for PrimitiveArray { - #[inline] - fn from(val: GrowablePrimitive<'a, T>) -> Self { - PrimitiveArray::::new(val.data_type, val.values.into(), val.validity.into()) - } -} diff --git a/src/common/arrow/src/arrow/array/growable/structure.rs b/src/common/arrow/src/arrow/array/growable/structure.rs deleted file mode 100644 index 6386ce52a442..000000000000 --- a/src/common/arrow/src/arrow/array/growable/structure.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::make_growable; -use super::utils::build_extend_null_bits; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::StructArray; -use crate::arrow::bitmap::MutableBitmap; - -/// Concrete [`Growable`] for the [`StructArray`]. -pub struct GrowableStruct<'a> { - arrays: Vec<&'a StructArray>, - validity: MutableBitmap, - values: Vec + 'a>>, - extend_null_bits: Vec>, -} - -impl<'a> GrowableStruct<'a> { - /// Creates a new [`GrowableStruct`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new(arrays: Vec<&'a StructArray>, mut use_validity: bool, capacity: usize) -> Self { - assert!(!arrays.is_empty()); - - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - let arrays = arrays - .iter() - .map(|array| array.as_any().downcast_ref::().unwrap()) - .collect::>(); - - // ([field1, field2], [field3, field4]) -> ([field1, field3], [field2, field3]) - let values = (0..arrays[0].values().len()) - .map(|i| { - make_growable( - &arrays - .iter() - .map(|x| x.values()[i].as_ref()) - .collect::>(), - use_validity, - capacity, - ) - }) - .collect::>>(); - - Self { - arrays, - values, - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - } - } - - fn to(&mut self) -> StructArray { - let validity = std::mem::take(&mut self.validity); - let values = std::mem::take(&mut self.values); - let values = values.into_iter().map(|mut x| x.as_box()).collect(); - - StructArray::new(self.arrays[0].data_type().clone(), values, validity.into()) - } -} - -impl<'a> Growable<'a> for GrowableStruct<'a> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - - let array = self.arrays[index]; - if array.null_count() == 0 { - self.values - .iter_mut() - .for_each(|child| child.extend(index, start, len)) - } else { - (start..start + len).for_each(|i| { - if array.is_valid(i) { - self.values - .iter_mut() - .for_each(|child| child.extend(index, i, 1)) - } else { - self.values - .iter_mut() - .for_each(|child| child.extend_validity(1)) - } - }) - } - } - - fn extend_validity(&mut self, additional: usize) { - self.values - .iter_mut() - .for_each(|child| child.extend_validity(additional)); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - // All children should have the same indexing, so just use the first - // one. If we don't have children, we might still have a validity - // array, so use that. - if let Some(child) = self.values.first() { - child.len() - } else { - self.validity.len() - } - } - - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a> From> for StructArray { - fn from(val: GrowableStruct<'a>) -> Self { - let values = val.values.into_iter().map(|mut x| x.as_box()).collect(); - - StructArray::new( - val.arrays[0].data_type().clone(), - values, - val.validity.into(), - ) - } -} diff --git a/src/common/arrow/src/arrow/array/growable/union.rs b/src/common/arrow/src/arrow/array/growable/union.rs deleted file mode 100644 index 18271a6cad5b..000000000000 --- a/src/common/arrow/src/arrow/array/growable/union.rs +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::make_growable; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::UnionArray; - -/// Concrete [`Growable`] for the [`UnionArray`]. -pub struct GrowableUnion<'a> { - arrays: Vec<&'a UnionArray>, - types: Vec, - offsets: Option>, - fields: Vec + 'a>>, -} - -impl<'a> GrowableUnion<'a> { - /// Creates a new [`GrowableUnion`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// Panics iff - /// * `arrays` is empty. - /// * any of the arrays has a different - pub fn new(arrays: Vec<&'a UnionArray>, capacity: usize) -> Self { - let first = arrays[0].data_type(); - assert!(arrays.iter().all(|x| x.data_type() == first)); - - let has_offsets = arrays[0].offsets().is_some(); - - let fields = (0..arrays[0].fields().len()) - .map(|i| { - make_growable( - &arrays - .iter() - .map(|x| x.fields()[i].as_ref()) - .collect::>(), - false, - capacity, - ) - }) - .collect::>>(); - - Self { - arrays, - fields, - offsets: if has_offsets { - Some(Vec::with_capacity(capacity)) - } else { - None - }, - types: Vec::with_capacity(capacity), - } - } - - fn to(&mut self) -> UnionArray { - let types = std::mem::take(&mut self.types); - let fields = std::mem::take(&mut self.fields); - let offsets = std::mem::take(&mut self.offsets); - let fields = fields.into_iter().map(|mut x| x.as_box()).collect(); - - UnionArray::new( - self.arrays[0].data_type().clone(), - types.into(), - fields, - offsets.map(|x| x.into()), - ) - } -} - -impl<'a> Growable<'a> for GrowableUnion<'a> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - let array = self.arrays[index]; - - let types = &array.types()[start..start + len]; - self.types.extend(types); - if let Some(x) = self.offsets.as_mut() { - let offsets = &array.offsets().unwrap()[start..start + len]; - - // in a dense union, each slot has its own offset. We extend the fields accordingly. - for (&type_, &offset) in types.iter().zip(offsets.iter()) { - let field = &mut self.fields[type_ as usize]; - // The offset for the element that is about to be extended is the current length - // of the child field of the corresponding type. Note that this may be very - // different than the original offset from the array we are extending from as - // it is a function of the previous extensions to this child. - x.push(field.len() as i32); - field.extend(index, offset as usize, 1); - } - } else { - // in a sparse union, every field has the same length => extend all fields equally - self.fields - .iter_mut() - .for_each(|field| field.extend(index, start, len)) - } - } - - fn extend_validity(&mut self, _additional: usize) {} - - #[inline] - fn len(&self) -> usize { - self.types.len() - } - - fn as_arc(&mut self) -> Arc { - self.to().arced() - } - - fn as_box(&mut self) -> Box { - self.to().boxed() - } -} - -impl<'a> From> for UnionArray { - fn from(val: GrowableUnion<'a>) -> Self { - let fields = val.fields.into_iter().map(|mut x| x.as_box()).collect(); - - UnionArray::new( - val.arrays[0].data_type().clone(), - val.types.into(), - fields, - val.offsets.map(|x| x.into()), - ) - } -} diff --git a/src/common/arrow/src/arrow/array/growable/utf8.rs b/src/common/arrow/src/arrow/array/growable/utf8.rs deleted file mode 100644 index 61810310e610..000000000000 --- a/src/common/arrow/src/arrow/array/growable/utf8.rs +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::utils::build_extend_null_bits; -use super::utils::extend_offset_values; -use super::utils::ExtendNullBits; -use super::Growable; -use crate::arrow::array::Array; -use crate::arrow::array::Utf8Array; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; - -/// Concrete [`Growable`] for the [`Utf8Array`]. -pub struct GrowableUtf8<'a, O: Offset> { - arrays: Vec<&'a Utf8Array>, - validity: MutableBitmap, - values: Vec, - offsets: Offsets, - extend_null_bits: Vec>, -} - -impl<'a, O: Offset> GrowableUtf8<'a, O> { - /// Creates a new [`GrowableUtf8`] bound to `arrays` with a pre-allocated `capacity`. - /// # Panics - /// If `arrays` is empty. - pub fn new(arrays: Vec<&'a Utf8Array>, mut use_validity: bool, capacity: usize) -> Self { - // if any of the arrays has nulls, insertions from any array requires setting bits - // as there is at least one array with nulls. - if arrays.iter().any(|array| array.null_count() > 0) { - use_validity = true; - }; - - let extend_null_bits = arrays - .iter() - .map(|array| build_extend_null_bits(*array, use_validity)) - .collect(); - - Self { - arrays: arrays.to_vec(), - values: Vec::with_capacity(0), - offsets: Offsets::with_capacity(capacity), - validity: MutableBitmap::with_capacity(capacity), - extend_null_bits, - } - } - - fn to(&mut self) -> Utf8Array { - let validity = std::mem::take(&mut self.validity); - let offsets = std::mem::take(&mut self.offsets); - let values = std::mem::take(&mut self.values); - - #[cfg(debug_assertions)] - { - crate::arrow::array::specification::try_check_utf8(&offsets, &values).unwrap(); - } - - unsafe { - Utf8Array::::try_new_unchecked( - self.arrays[0].data_type().clone(), - offsets.into(), - values.into(), - validity.into(), - ) - .unwrap() - } - } -} - -impl<'a, O: Offset> Growable<'a> for GrowableUtf8<'a, O> { - fn extend(&mut self, index: usize, start: usize, len: usize) { - (self.extend_null_bits[index])(&mut self.validity, start, len); - - let array = self.arrays[index]; - let offsets = array.offsets(); - let values = array.values(); - - self.offsets - .try_extend_from_slice(offsets, start, len) - .unwrap(); - - // values - extend_offset_values::(&mut self.values, offsets.as_slice(), values, start, len); - } - - fn extend_validity(&mut self, additional: usize) { - self.offsets.extend_constant(additional); - self.validity.extend_constant(additional, false); - } - - #[inline] - fn len(&self) -> usize { - self.offsets.len() - 1 - } - - fn as_arc(&mut self) -> Arc { - Arc::new(self.to()) - } - - fn as_box(&mut self) -> Box { - Box::new(self.to()) - } -} - -impl<'a, O: Offset> From> for Utf8Array { - fn from(mut val: GrowableUtf8<'a, O>) -> Self { - val.to() - } -} diff --git a/src/common/arrow/src/arrow/array/growable/utils.rs b/src/common/arrow/src/arrow/array/growable/utils.rs deleted file mode 100644 index 5536113321ae..000000000000 --- a/src/common/arrow/src/arrow/array/growable/utils.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::offset::Offset; - -// function used to extend nulls from arrays. This function's lifetime is bound to the array -// because it reads nulls from it. -pub(super) type ExtendNullBits<'a> = Box; - -pub(super) fn build_extend_null_bits(array: &dyn Array, use_validity: bool) -> ExtendNullBits { - if let Some(bitmap) = array.validity() { - Box::new(move |validity, start, len| { - debug_assert!(start + len <= bitmap.len()); - let (slice, offset, _) = bitmap.as_slice(); - // safety: invariant offset + length <= slice.len() - unsafe { - validity.extend_from_slice_unchecked(slice, start + offset, len); - } - }) - } else if use_validity { - Box::new(|validity, _, len| { - validity.extend_constant(len, true); - }) - } else { - Box::new(|_, _, _| {}) - } -} - -pub(super) fn prepare_validity(use_validity: bool, capacity: usize) -> Option { - if use_validity { - Some(MutableBitmap::with_capacity(capacity)) - } else { - None - } -} - -#[inline] -pub(super) fn extend_offset_values( - buffer: &mut Vec, - offsets: &[O], - values: &[u8], - start: usize, - len: usize, -) { - let start_values = offsets[start].to_usize(); - let end_values = offsets[start + len].to_usize(); - let new_values = &values[start_values..end_values]; - buffer.extend_from_slice(new_values); -} - -pub(super) fn extend_validity( - mutable_validity: &mut Option, - array: &dyn Array, - start: usize, - len: usize, -) { - if let Some(mutable_validity) = mutable_validity { - match array.validity() { - None => mutable_validity.extend_constant(len, true), - Some(validity) => { - debug_assert!(start + len <= validity.len()); - let (slice, offset, _) = validity.as_slice(); - // safety: invariant offset + length <= slice.len() - unsafe { - mutable_validity.extend_from_slice_unchecked(slice, start + offset, len); - } - } - } - } -} diff --git a/src/common/arrow/src/arrow/array/indexable.rs b/src/common/arrow/src/arrow/array/indexable.rs deleted file mode 100644 index 9b7d80a6261e..000000000000 --- a/src/common/arrow/src/arrow/array/indexable.rs +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::borrow::Borrow; - -use crate::arrow::array::MutableArray; -use crate::arrow::array::MutableBinaryArray; -use crate::arrow::array::MutableBinaryValuesArray; -use crate::arrow::array::MutableBooleanArray; -use crate::arrow::array::MutableFixedSizeBinaryArray; -use crate::arrow::array::MutablePrimitiveArray; -use crate::arrow::array::MutableUtf8Array; -use crate::arrow::array::MutableUtf8ValuesArray; -use crate::arrow::offset::Offset; -use crate::arrow::types::NativeType; - -/// Trait for arrays that can be indexed directly to extract a value. -pub trait Indexable { - /// The type of the element at index `i`; may be a reference type or a value type. - type Value<'a>: Borrow - where Self: 'a; - - type Type: ?Sized; - - /// Returns the element at index `i`. - /// # Panic - /// May panic if `i >= self.len()`. - fn value_at(&self, index: usize) -> Self::Value<'_>; - - /// Returns the element at index `i`. - /// # Safety - /// Assumes that the `i < self.len`. - #[inline] - unsafe fn value_unchecked_at(&self, index: usize) -> Self::Value<'_> { - self.value_at(index) - } -} - -pub trait AsIndexed { - fn as_indexed(&self) -> &M::Type; -} - -impl Indexable for MutableBooleanArray { - type Value<'a> = bool; - type Type = bool; - - #[inline] - fn value_at(&self, i: usize) -> Self::Value<'_> { - self.values().get(i) - } -} - -impl AsIndexed for bool { - #[inline] - fn as_indexed(&self) -> &bool { - self - } -} - -impl Indexable for MutableBinaryArray { - type Value<'a> = &'a [u8]; - type Type = [u8]; - - #[inline] - fn value_at(&self, i: usize) -> Self::Value<'_> { - // TODO: add .value() / .value_unchecked() to MutableBinaryArray? - assert!(i < self.len()); - unsafe { self.value_unchecked_at(i) } - } - - #[inline] - unsafe fn value_unchecked_at(&self, i: usize) -> Self::Value<'_> { - // TODO: add .value() / .value_unchecked() to MutableBinaryArray? - // soundness: the invariant of the function - let (start, end) = self.offsets().start_end_unchecked(i); - // soundness: the invariant of the struct - self.values().get_unchecked(start..end) - } -} - -impl AsIndexed> for &[u8] { - #[inline] - fn as_indexed(&self) -> &[u8] { - self - } -} - -impl Indexable for MutableBinaryValuesArray { - type Value<'a> = &'a [u8]; - type Type = [u8]; - - #[inline] - fn value_at(&self, i: usize) -> Self::Value<'_> { - self.value(i) - } - - #[inline] - unsafe fn value_unchecked_at(&self, i: usize) -> Self::Value<'_> { - self.value_unchecked(i) - } -} - -impl AsIndexed> for &[u8] { - #[inline] - fn as_indexed(&self) -> &[u8] { - self - } -} - -impl Indexable for MutableFixedSizeBinaryArray { - type Value<'a> = &'a [u8]; - type Type = [u8]; - - #[inline] - fn value_at(&self, i: usize) -> Self::Value<'_> { - self.value(i) - } - - #[inline] - unsafe fn value_unchecked_at(&self, i: usize) -> Self::Value<'_> { - // soundness: the invariant of the struct - self.value_unchecked(i) - } -} - -impl AsIndexed for &[u8] { - #[inline] - fn as_indexed(&self) -> &[u8] { - self - } -} - -// TODO: should NativeType derive from Hash? -impl Indexable for MutablePrimitiveArray { - type Value<'a> = T; - type Type = T; - - #[inline] - fn value_at(&self, i: usize) -> Self::Value<'_> { - assert!(i < self.len()); - // TODO: add Length trait? (for both Array and MutableArray) - unsafe { self.value_unchecked_at(i) } - } - - #[inline] - unsafe fn value_unchecked_at(&self, i: usize) -> Self::Value<'_> { - *self.values().get_unchecked(i) - } -} - -impl AsIndexed> for T { - #[inline] - fn as_indexed(&self) -> &T { - self - } -} - -impl Indexable for MutableUtf8Array { - type Value<'a> = &'a str; - type Type = str; - - #[inline] - fn value_at(&self, i: usize) -> Self::Value<'_> { - self.value(i) - } - - #[inline] - unsafe fn value_unchecked_at(&self, i: usize) -> Self::Value<'_> { - self.value_unchecked(i) - } -} - -impl> AsIndexed> for V { - #[inline] - fn as_indexed(&self) -> &str { - self.as_ref() - } -} - -impl Indexable for MutableUtf8ValuesArray { - type Value<'a> = &'a str; - type Type = str; - - #[inline] - fn value_at(&self, i: usize) -> Self::Value<'_> { - self.value(i) - } - - #[inline] - unsafe fn value_unchecked_at(&self, i: usize) -> Self::Value<'_> { - self.value_unchecked(i) - } -} - -impl> AsIndexed> for V { - #[inline] - fn as_indexed(&self) -> &str { - self.as_ref() - } -} diff --git a/src/common/arrow/src/arrow/array/list/data.rs b/src/common/arrow/src/arrow/array/list/data.rs deleted file mode 100644 index 63e3d90ea3cb..000000000000 --- a/src/common/arrow/src/arrow/array/list/data.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::from_data; -use crate::arrow::array::to_data; -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::ListArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::offset::Offset; -use crate::arrow::offset::OffsetsBuffer; - -impl Arrow2Arrow for ListArray { - fn to_data(&self) -> ArrayData { - let data_type = self.data_type.clone().into(); - - let builder = ArrayDataBuilder::new(data_type) - .len(self.len()) - .buffers(vec![self.offsets.clone().into_inner().into()]) - .nulls(self.validity.as_ref().map(|b| b.clone().into())) - .child_data(vec![to_data(self.values.as_ref())]); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type = data.data_type().clone().into(); - if data.is_empty() { - // Handle empty offsets - return Self::new_empty(data_type); - } - - let mut offsets = unsafe { OffsetsBuffer::new_unchecked(data.buffers()[0].clone().into()) }; - offsets.slice(data.offset(), data.len() + 1); - - Self { - data_type, - offsets, - values: from_data(&data.child_data()[0]), - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/list/fmt.rs b/src/common/arrow/src/arrow/array/list/fmt.rs deleted file mode 100644 index da20dd15ab60..000000000000 --- a/src/common/arrow/src/arrow/array/list/fmt.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::get_display; -use super::super::fmt::write_vec; -use super::ListArray; -use crate::arrow::offset::Offset; - -pub fn write_value( - array: &ListArray, - index: usize, - null: &'static str, - f: &mut W, -) -> Result { - let values = array.value(index); - let writer = |f: &mut W, index| get_display(values.as_ref(), null)(f, index); - write_vec(f, writer, None, values.len(), null, false) -} - -impl Debug for ListArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, "None", f); - - let head = if O::IS_LARGE { - "LargeListArray" - } else { - "ListArray" - }; - write!(f, "{head}")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/list/iterator.rs b/src/common/arrow/src/arrow/array/list/iterator.rs deleted file mode 100644 index eacb205bc636..000000000000 --- a/src/common/arrow/src/arrow/array/list/iterator.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::ListArray; -use crate::arrow::array::Array; -use crate::arrow::array::ArrayAccessor; -use crate::arrow::array::ArrayValuesIter; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::offset::Offset; - -unsafe impl<'a, O: Offset> ArrayAccessor<'a> for ListArray { - type Item = Box; - - #[inline] - unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) - } - - #[inline] - fn len(&self) -> usize { - self.len() - } -} - -/// Iterator of values of a [`ListArray`]. -pub type ListValuesIter<'a, O> = ArrayValuesIter<'a, ListArray>; - -type ZipIter<'a, O> = ZipValidity, ListValuesIter<'a, O>, BitmapIter<'a>>; - -impl<'a, O: Offset> IntoIterator for &'a ListArray { - type Item = Option>; - type IntoIter = ZipIter<'a, O>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a, O: Offset> ListArray { - /// Returns an iterator of `Option>` - pub fn iter(&'a self) -> ZipIter<'a, O> { - ZipValidity::new_with_validity(ListValuesIter::new(self), self.validity.as_ref()) - } - - /// Returns an iterator of `Box` - pub fn values_iter(&'a self) -> ListValuesIter<'a, O> { - ListValuesIter::new(self) - } -} - -struct Iter>> { - current: i32, - offsets: std::vec::IntoIter, - values: I, -} - -impl> + Clone> Iterator for Iter { - type Item = Option>>; - - fn next(&mut self) -> Option { - let next = self.offsets.next(); - next.map(|next| { - let length = next - self.current; - let iter = self - .values - .clone() - .skip(self.current as usize) - .take(length as usize); - self.current = next; - Some(iter) - }) - } -} diff --git a/src/common/arrow/src/arrow/array/list/mod.rs b/src/common/arrow/src/arrow/array/list/mod.rs deleted file mode 100644 index db68a7d03033..000000000000 --- a/src/common/arrow/src/arrow/array/list/mod.rs +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::new_empty_array; -use super::specification::try_check_offsets_bounds; -use super::Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::offset::OffsetsBuffer; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod iterator; -pub use iterator::*; -mod mutable; -pub use mutable::*; - -/// An [`Array`] semantically equivalent to `Vec>>>` with Arrow's in-memory. -#[derive(Clone)] -pub struct ListArray { - data_type: DataType, - offsets: OffsetsBuffer, - values: Box, - validity: Option, -} - -impl ListArray { - /// Creates a new [`ListArray`]. - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either [`crate::arrow::datatypes::PhysicalType::List`] or [`crate::arrow::datatypes::PhysicalType::LargeList`]. - /// * The `data_type`'s inner field's data type is not equal to `values.data_type`. - /// # Implementation - /// This function is `O(1)` - pub fn try_new( - data_type: DataType, - offsets: OffsetsBuffer, - values: Box, - validity: Option, - ) -> Result { - try_check_offsets_bounds(&offsets, values.len())?; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != offsets.len_proxy()) - { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - - let child_data_type = Self::try_get_child(&data_type)?.data_type(); - let values_data_type = values.data_type(); - if child_data_type != values_data_type { - return Err(Error::oos(format!( - "ListArray's child's DataType must match. However, the expected DataType is {child_data_type:?} while it got {values_data_type:?}." - ))); - } - - Ok(Self { - data_type, - offsets, - values, - validity, - }) - } - - /// Creates a new [`ListArray`]. - /// - /// # Panics - /// This function panics iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either [`crate::arrow::datatypes::PhysicalType::List`] or [`crate::arrow::datatypes::PhysicalType::LargeList`]. - /// * The `data_type`'s inner field's data type is not equal to `values.data_type`. - /// # Implementation - /// This function is `O(1)` - pub fn new( - data_type: DataType, - offsets: OffsetsBuffer, - values: Box, - validity: Option, - ) -> Self { - Self::try_new(data_type, offsets, values, validity).unwrap() - } - - /// Returns a new empty [`ListArray`]. - pub fn new_empty(data_type: DataType) -> Self { - let values = new_empty_array(Self::get_child_type(&data_type).clone()); - Self::new(data_type, OffsetsBuffer::default(), values, None) - } - - /// Returns a new null [`ListArray`]. - #[inline] - pub fn new_null(data_type: DataType, length: usize) -> Self { - let child = Self::get_child_type(&data_type).clone(); - Self::new( - data_type, - Offsets::new_zeroed(length).into(), - new_empty_array(child), - Some(Bitmap::new_zeroed(length)), - ) - } -} - -impl ListArray { - /// Slices this [`ListArray`]. - /// # Panics - /// panics iff `offset + length > self.len()` - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new Buffer cannot exceed the existing length" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Slices this [`ListArray`]. - /// # Safety - /// The caller must ensure that `offset + length < self.len()`. - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.offsets.slice_unchecked(offset, length + 1); - } - - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); -} - -// Accessors -impl ListArray { - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.offsets.len_proxy() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the element at index `i` - /// # Panic - /// Panics iff `i >= self.len()` - #[inline] - pub fn value(&self, i: usize) -> Box { - assert!(i < self.len()); - // Safety: invariant of this function - unsafe { self.value_unchecked(i) } - } - - /// Returns the element at index `i` as &str - /// # Safety - /// Assumes that the `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> Box { - // safety: the invariant of the function - let (start, end) = self.offsets.start_end_unchecked(i); - let length = end - start; - - // safety: the invariant of the struct - self.values.sliced_unchecked(start, length) - } - - /// The optional validity. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - /// The offsets [`Buffer`]. - #[inline] - pub fn offsets(&self) -> &OffsetsBuffer { - &self.offsets - } - - /// The values. - #[inline] - #[allow(clippy::borrowed_box)] - pub fn values(&self) -> &Box { - &self.values - } -} - -impl ListArray { - /// Returns a default [`DataType`]: inner field is named "item" and is nullable - pub fn default_datatype(data_type: DataType) -> DataType { - let field = Box::new(Field::new("item", data_type, true)); - if O::IS_LARGE { - DataType::LargeList(field) - } else { - DataType::List(field) - } - } - - /// Returns a the inner [`Field`] - /// # Panics - /// Panics iff the logical type is not consistent with this struct. - pub fn get_child_field(data_type: &DataType) -> &Field { - Self::try_get_child(data_type).unwrap() - } - - /// Returns a the inner [`Field`] - /// # Errors - /// Panics iff the logical type is not consistent with this struct. - pub fn try_get_child(data_type: &DataType) -> Result<&Field, Error> { - if O::IS_LARGE { - match data_type.to_logical_type() { - DataType::LargeList(child) => Ok(child.as_ref()), - _ => Err(Error::oos("ListArray expects DataType::LargeList")), - } - } else { - match data_type.to_logical_type() { - DataType::List(child) => Ok(child.as_ref()), - _ => Err(Error::oos("ListArray expects DataType::List")), - } - } - } - - /// Returns a the inner [`DataType`] - /// # Panics - /// Panics iff the logical type is not consistent with this struct. - pub fn get_child_type(data_type: &DataType) -> &DataType { - Self::get_child_field(data_type).data_type() - } -} - -impl Array for ListArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/list/mutable.rs b/src/common/arrow/src/arrow/array/list/mutable.rs deleted file mode 100644 index 4834b84c4bff..000000000000 --- a/src/common/arrow/src/arrow/array/list/mutable.rs +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::ListArray; -use crate::arrow::array::physical_binary::extend_validity; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::trusted_len::TrustedLen; - -/// The mutable version of [`ListArray`]. -#[derive(Debug, Clone)] -pub struct MutableListArray { - data_type: DataType, - offsets: Offsets, - values: M, - validity: Option, -} - -impl MutableListArray { - /// Creates a new empty [`MutableListArray`]. - pub fn new() -> Self { - let values = M::default(); - let data_type = ListArray::::default_datatype(values.data_type().clone()); - Self::new_from(values, data_type, 0) - } - - /// Creates a new [`MutableListArray`] with a capacity. - pub fn with_capacity(capacity: usize) -> Self { - let values = M::default(); - let data_type = ListArray::::default_datatype(values.data_type().clone()); - - let offsets = Offsets::::with_capacity(capacity); - Self { - data_type, - offsets, - values, - validity: None, - } - } -} - -impl Default for MutableListArray { - fn default() -> Self { - Self::new() - } -} - -impl From> for ListArray { - fn from(mut other: MutableListArray) -> Self { - ListArray::new( - other.data_type, - other.offsets.into(), - other.values.as_box(), - other.validity.map(|x| x.into()), - ) - } -} - -impl TryExtend> for MutableListArray -where - O: Offset, - M: MutableArray + TryExtend>, - I: IntoIterator>, -{ - fn try_extend>>(&mut self, iter: II) -> Result<()> { - let iter = iter.into_iter(); - self.reserve(iter.size_hint().0); - for items in iter { - self.try_push(items)?; - } - Ok(()) - } -} - -impl TryPush> for MutableListArray -where - O: Offset, - M: MutableArray + TryExtend>, - I: IntoIterator>, -{ - #[inline] - fn try_push(&mut self, item: Option) -> Result<()> { - if let Some(items) = item { - let values = self.mut_values(); - values.try_extend(items)?; - self.try_push_valid()?; - } else { - self.push_null(); - } - Ok(()) - } -} - -impl TryExtendFromSelf for MutableListArray -where - O: Offset, - M: MutableArray + TryExtendFromSelf, -{ - fn try_extend_from_self(&mut self, other: &Self) -> Result<()> { - extend_validity(self.len(), &mut self.validity, &other.validity); - - self.values.try_extend_from_self(&other.values)?; - self.offsets.try_extend_from_self(&other.offsets) - } -} - -impl MutableListArray { - /// Creates a new [`MutableListArray`] from a [`MutableArray`] and capacity. - pub fn new_from(values: M, data_type: DataType, capacity: usize) -> Self { - let offsets = Offsets::::with_capacity(capacity); - assert_eq!(values.len(), 0); - ListArray::::get_child_field(&data_type); - Self { - data_type, - offsets, - values, - validity: None, - } - } - - /// Creates a new [`MutableListArray`] from a [`MutableArray`]. - pub fn new_with_field(values: M, name: &str, nullable: bool) -> Self { - let field = Box::new(Field::new(name, values.data_type().clone(), nullable)); - let data_type = if O::IS_LARGE { - DataType::LargeList(field) - } else { - DataType::List(field) - }; - Self::new_from(values, data_type, 0) - } - - /// Creates a new [`MutableListArray`] from a [`MutableArray`] and capacity. - pub fn new_with_capacity(values: M, capacity: usize) -> Self { - let data_type = ListArray::::default_datatype(values.data_type().clone()); - Self::new_from(values, data_type, capacity) - } - - /// Creates a new [`MutableListArray`] from a [`MutableArray`], [`Offsets`] and - /// [`MutableBitmap`]. - pub fn new_from_mutable( - values: M, - offsets: Offsets, - validity: Option, - ) -> Self { - assert_eq!(values.len(), offsets.last().to_usize()); - let data_type = ListArray::::default_datatype(values.data_type().clone()); - Self { - data_type, - offsets, - values, - validity, - } - } - - #[inline] - /// Needs to be called when a valid value was extended to this array. - /// This is a relatively low level function, prefer `try_push` when you can. - pub fn try_push_valid(&mut self) -> Result<()> { - let total_length = self.values.len(); - let offset = self.offsets.last().to_usize(); - let length = total_length - .checked_sub(offset) - .ok_or_else(|| Error::Overflow)?; - - self.offsets.try_push_usize(length)?; - if let Some(validity) = &mut self.validity { - validity.push(true) - } - Ok(()) - } - - #[inline] - fn push_null(&mut self) { - self.offsets.extend_constant(1); - match &mut self.validity { - Some(validity) => validity.push(false), - None => self.init_validity(), - } - } - - /// Expand this array, using elements from the underlying backing array. - /// Assumes the expansion begins at the highest previous offset, or zero if - /// this [`MutableListArray`] is currently empty. - /// - /// Panics if: - /// - the new offsets are not in monotonic increasing order. - /// - any new offset is not in bounds of the backing array. - /// - the passed iterator has no upper bound. - pub fn try_extend_from_lengths(&mut self, iterator: II) -> Result<()> - where II: TrustedLen> + Clone { - self.offsets - .try_extend_from_lengths(iterator.clone().map(|x| x.unwrap_or_default()))?; - if let Some(validity) = &mut self.validity { - validity.extend_from_trusted_len_iter(iterator.map(|x| x.is_some())) - } - assert_eq!(self.offsets.last().to_usize(), self.values.len()); - Ok(()) - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.offsets.len_proxy() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The values - pub fn mut_values(&mut self) -> &mut M { - &mut self.values - } - - /// The offsets - pub fn offsets(&self) -> &Offsets { - &self.offsets - } - - /// The values - pub fn values(&self) -> &M { - &self.values - } - - fn init_validity(&mut self) { - let len = self.offsets.len_proxy(); - - let mut validity = MutableBitmap::with_capacity(self.offsets.capacity()); - validity.extend_constant(len, true); - validity.set(len - 1, false); - self.validity = Some(validity) - } - - /// Converts itself into an [`Array`]. - pub fn into_arc(self) -> Arc { - let a: ListArray = self.into(); - Arc::new(a) - } - - /// converts itself into [`Box`] - pub fn into_box(self) -> Box { - let a: ListArray = self.into(); - Box::new(a) - } - - /// Reserves `additional` slots. - pub fn reserve(&mut self, additional: usize) { - self.offsets.reserve(additional); - if let Some(x) = self.validity.as_mut() { - x.reserve(additional) - } - } - - /// Shrinks the capacity of the [`MutableListArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - self.offsets.shrink_to_fit(); - if let Some(validity) = &mut self.validity { - validity.shrink_to_fit() - } - } -} - -impl MutableArray for MutableListArray { - fn len(&self) -> usize { - MutableListArray::len(self) - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - ListArray::new( - self.data_type.clone(), - std::mem::take(&mut self.offsets).into(), - self.values.as_box(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .boxed() - } - - fn as_arc(&mut self) -> Arc { - ListArray::new( - self.data_type.clone(), - std::mem::take(&mut self.offsets).into(), - self.values.as_box(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .arced() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - #[inline] - fn push_null(&mut self) { - self.push_null() - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit(); - } -} diff --git a/src/common/arrow/src/arrow/array/map/data.rs b/src/common/arrow/src/arrow/array/map/data.rs deleted file mode 100644 index a2bf97827ba5..000000000000 --- a/src/common/arrow/src/arrow/array/map/data.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::from_data; -use crate::arrow::array::to_data; -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::MapArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::offset::OffsetsBuffer; - -impl Arrow2Arrow for MapArray { - fn to_data(&self) -> ArrayData { - let data_type = self.data_type.clone().into(); - - let builder = ArrayDataBuilder::new(data_type) - .len(self.len()) - .buffers(vec![self.offsets.clone().into_inner().into()]) - .nulls(self.validity.as_ref().map(|b| b.clone().into())) - .child_data(vec![to_data(self.field.as_ref())]); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type = data.data_type().clone().into(); - if data.is_empty() { - // Handle empty offsets - return Self::new_empty(data_type); - } - - let mut offsets = unsafe { OffsetsBuffer::new_unchecked(data.buffers()[0].clone().into()) }; - offsets.slice(data.offset(), data.len() + 1); - - Self { - data_type: data.data_type().clone().into(), - offsets, - field: from_data(&data.child_data()[0]), - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/map/fmt.rs b/src/common/arrow/src/arrow/array/map/fmt.rs deleted file mode 100644 index e486cfd9fddb..000000000000 --- a/src/common/arrow/src/arrow/array/map/fmt.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::get_display; -use super::super::fmt::write_vec; -use super::MapArray; - -pub fn write_value( - array: &MapArray, - index: usize, - null: &'static str, - f: &mut W, -) -> Result { - let values = array.value(index); - let writer = |f: &mut W, index| get_display(values.as_ref(), null)(f, index); - write_vec(f, writer, None, values.len(), null, false) -} - -impl Debug for MapArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, "None", f); - - write!(f, "MapArray")?; - write_vec(f, writer, self.validity.as_ref(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/map/iterator.rs b/src/common/arrow/src/arrow/array/map/iterator.rs deleted file mode 100644 index 0a75024a436c..000000000000 --- a/src/common/arrow/src/arrow/array/map/iterator.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::MapArray; -use crate::arrow::array::Array; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::trusted_len::TrustedLen; - -/// Iterator of values of an [`ListArray`]. -#[derive(Clone, Debug)] -pub struct MapValuesIter<'a> { - array: &'a MapArray, - index: usize, - end: usize, -} - -impl<'a> MapValuesIter<'a> { - #[inline] - pub fn new(array: &'a MapArray) -> Self { - Self { - array, - index: 0, - end: array.len(), - } - } -} - -impl<'a> Iterator for MapValuesIter<'a> { - type Item = Box; - - #[inline] - fn next(&mut self) -> Option { - if self.index == self.end { - return None; - } - let old = self.index; - self.index += 1; - // Safety: - // self.end is maximized by the length of the array - Some(unsafe { self.array.value_unchecked(old) }) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.end - self.index, Some(self.end - self.index)) - } -} - -unsafe impl<'a> TrustedLen for MapValuesIter<'a> {} - -impl<'a> DoubleEndedIterator for MapValuesIter<'a> { - #[inline] - fn next_back(&mut self) -> Option { - if self.index == self.end { - None - } else { - self.end -= 1; - // Safety: - // self.end is maximized by the length of the array - Some(unsafe { self.array.value_unchecked(self.end) }) - } - } -} - -impl<'a> IntoIterator for &'a MapArray { - type Item = Option>; - type IntoIter = ZipValidity, MapValuesIter<'a>, BitmapIter<'a>>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> MapArray { - /// Returns an iterator of `Option>` - pub fn iter(&'a self) -> ZipValidity, MapValuesIter<'a>, BitmapIter<'a>> { - ZipValidity::new_with_validity(MapValuesIter::new(self), self.validity()) - } - - /// Returns an iterator of `Box` - pub fn values_iter(&'a self) -> MapValuesIter<'a> { - MapValuesIter::new(self) - } -} diff --git a/src/common/arrow/src/arrow/array/map/mod.rs b/src/common/arrow/src/arrow/array/map/mod.rs deleted file mode 100644 index 193084293f4b..000000000000 --- a/src/common/arrow/src/arrow/array/map/mod.rs +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::new_empty_array; -use super::specification::try_check_offsets_bounds; -use super::Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; -use crate::arrow::offset::OffsetsBuffer; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod iterator; - -/// An array representing a (key, value), both of arbitrary logical types. -#[derive(Clone)] -pub struct MapArray { - data_type: DataType, - // invariant: field.len() == offsets.len() - offsets: OffsetsBuffer, - field: Box, - // invariant: offsets.len() - 1 == Bitmap::len() - validity: Option, -} - -impl MapArray { - /// Returns a new [`MapArray`]. - /// # Errors - /// This function errors iff: - /// * The last offset is not equal to the field' length - /// * The `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::Map`] - /// * The fields' `data_type` is not equal to the inner field of `data_type` - /// * The validity is not `None` and its length is different from `offsets.len() - 1`. - pub fn try_new( - data_type: DataType, - offsets: OffsetsBuffer, - field: Box, - validity: Option, - ) -> Result { - try_check_offsets_bounds(&offsets, field.len())?; - - let inner_field = Self::try_get_field(&data_type)?; - if let DataType::Struct(inner) = inner_field.data_type() { - if inner.len() != 2 { - return Err(Error::InvalidArgumentError( - "MapArray's inner `Struct` must have 2 fields (keys and maps)".to_string(), - )); - } - } else { - return Err(Error::InvalidArgumentError( - "MapArray expects `DataType::Struct` as its inner logical type".to_string(), - )); - } - if field.data_type() != inner_field.data_type() { - return Err(Error::InvalidArgumentError( - "MapArray expects `field.data_type` to match its inner DataType".to_string(), - )); - } - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != offsets.len_proxy()) - { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - - Ok(Self { - data_type, - field, - offsets, - validity, - }) - } - - /// Creates a new [`MapArray`]. - /// # Panics - /// * The last offset is not equal to the field' length. - /// * The `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::Map`], - /// * The validity is not `None` and its length is different from `offsets.len() - 1`. - pub fn new( - data_type: DataType, - offsets: OffsetsBuffer, - field: Box, - validity: Option, - ) -> Self { - Self::try_new(data_type, offsets, field, validity).unwrap() - } - - /// Returns a new null [`MapArray`] of `length`. - pub fn new_null(data_type: DataType, length: usize) -> Self { - let field = new_empty_array(Self::get_field(&data_type).data_type().clone()); - Self::new( - data_type, - vec![0i32; 1 + length].try_into().unwrap(), - field, - Some(Bitmap::new_zeroed(length)), - ) - } - - /// Returns a new empty [`MapArray`]. - pub fn new_empty(data_type: DataType) -> Self { - let field = new_empty_array(Self::get_field(&data_type).data_type().clone()); - Self::new(data_type, OffsetsBuffer::default(), field, None) - } -} - -impl MapArray { - /// Returns a slice of this [`MapArray`]. - /// # Panics - /// panics iff `offset + length > self.len()` - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new Buffer cannot exceed the existing length" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Returns a slice of this [`MapArray`]. - /// # Safety - /// The caller must ensure that `offset + length < self.len()`. - #[inline] - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.offsets.slice_unchecked(offset, length + 1); - } - - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); - - pub(crate) fn try_get_field(data_type: &DataType) -> Result<&Field, Error> { - if let DataType::Map(field, _) = data_type.to_logical_type() { - Ok(field.as_ref()) - } else { - Err(Error::oos( - "The data_type's logical type must be DataType::Map", - )) - } - } - - pub(crate) fn get_field(data_type: &DataType) -> &Field { - Self::try_get_field(data_type).unwrap() - } -} - -// Accessors -impl MapArray { - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.offsets.len_proxy() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// returns the offsets - #[inline] - pub fn offsets(&self) -> &OffsetsBuffer { - &self.offsets - } - - /// Returns the field (guaranteed to be a `Struct`) - #[inline] - #[allow(clippy::borrowed_box)] - pub fn field(&self) -> &Box { - &self.field - } - - /// Returns the element at index `i`. - #[inline] - pub fn value(&self, i: usize) -> Box { - assert!(i < self.len()); - unsafe { self.value_unchecked(i) } - } - - /// Returns the element at index `i`. - /// # Safety - /// Assumes that the `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> Box { - // soundness: the invariant of the function - let (start, end) = self.offsets.start_end_unchecked(i); - let length = end - start; - - // soundness: the invariant of the struct - self.field.sliced_unchecked(start, length) - } -} - -impl Array for MapArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/mod.rs b/src/common/arrow/src/arrow/array/mod.rs deleted file mode 100644 index fc72f7e21518..000000000000 --- a/src/common/arrow/src/arrow/array/mod.rs +++ /dev/null @@ -1,842 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains the [`Array`] and [`MutableArray`] trait objects declaring arrays, -//! as well as concrete arrays (such as [`Utf8Array`] and [`MutableUtf8Array`]). -//! -//! Fixed-length containers with optional values -//! that are laid in memory according to the Arrow specification. -//! Each array type has its own `struct`. The following are the main array types: -//! -//! * [`PrimitiveArray`] and [`MutablePrimitiveArray`], an array of values with a fixed length such as integers, floats, etc. -//! * [`BooleanArray`] and [`MutableBooleanArray`], an array of boolean values (stored as a bitmap) -//! * [`Utf8Array`] and [`MutableUtf8Array`], an array of variable length utf8 values -//! * [`BinaryArray`] and [`MutableBinaryArray`], an array of opaque variable length values -//! * [`ListArray`] and [`MutableListArray`], an array of arrays (e.g. `[[1, 2], None, [], [None]]`) -//! * [`StructArray`] and [`MutableStructArray`], an array of arrays identified by a string (e.g. `{"a": [1, 2], "b": [true, false]}`) -//! -//! All immutable arrays implement the trait object [`Array`] and that can be downcasted -//! to a concrete struct based on [`PhysicalType`](crate::arrow::datatypes::PhysicalType) available from [`Array::data_type`]. -//! All immutable arrays are backed by [`Buffer`](crate::arrow::buffer::Buffer) and thus cloning and slicing them is `O(1)`. -//! -//! Most arrays contain a [`MutableArray`] counterpart that is neither cloneable nor sliceable, but -//! can be operated in-place. -use std::any::Any; -use std::sync::Arc; - -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; - -mod physical_binary; - -/// A trait representing an immutable Arrow array. Arrow arrays are trait objects -/// that are infallibly downcasted to concrete types according to the [`Array::data_type`]. -pub trait Array: Send + Sync + dyn_clone::DynClone + 'static { - /// Converts itself to a reference of [`Any`], which enables downcasting to concrete types. - fn as_any(&self) -> &dyn Any; - - /// Converts itself to a mutable reference of [`Any`], which enables mutable downcasting to concrete types. - fn as_any_mut(&mut self) -> &mut dyn Any; - - /// The length of the [`Array`]. Every array has a length corresponding to the number of - /// elements (slots). - fn len(&self) -> usize; - - /// whether the array is empty - fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The [`DataType`] of the [`Array`]. In combination with [`Array::as_any`], this can be - /// used to downcast trait objects (`dyn Array`) to concrete arrays. - fn data_type(&self) -> &DataType; - - /// The validity of the [`Array`]: every array has an optional [`Bitmap`] that, when available - /// specifies whether the array slot is valid or not (null). - /// When the validity is [`None`], all slots are valid. - fn validity(&self) -> Option<&Bitmap>; - - /// The number of null slots on this [`Array`]. - /// # Implementation - /// This is `O(1)` since the number of null elements is pre-computed. - #[inline] - fn null_count(&self) -> usize { - if self.data_type() == &DataType::Null { - return self.len(); - }; - self.validity() - .as_ref() - .map(|x| x.unset_bits()) - .unwrap_or(0) - } - - /// Returns whether slot `i` is null. - /// # Panic - /// Panics iff `i >= self.len()`. - #[inline] - fn is_null(&self, i: usize) -> bool { - assert!(i < self.len()); - unsafe { self.is_null_unchecked(i) } - } - - /// Returns whether slot `i` is null. - /// # Safety - /// The caller must ensure `i < self.len()` - #[inline] - unsafe fn is_null_unchecked(&self, i: usize) -> bool { - self.validity() - .as_ref() - .map(|x| !x.get_bit_unchecked(i)) - .unwrap_or(false) - } - - /// Returns whether slot `i` is valid. - /// # Panic - /// Panics iff `i >= self.len()`. - #[inline] - fn is_valid(&self, i: usize) -> bool { - !self.is_null(i) - } - - /// Slices this [`Array`]. - /// # Implementation - /// This operation is `O(1)` over `len`. - /// # Panic - /// This function panics iff `offset + length > self.len()`. - fn slice(&mut self, offset: usize, length: usize); - - /// Slices the [`Array`]. - /// # Implementation - /// This operation is `O(1)`. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()` - unsafe fn slice_unchecked(&mut self, offset: usize, length: usize); - - /// Returns a slice of this [`Array`]. - /// # Implementation - /// This operation is `O(1)` over `len`. - /// # Panic - /// This function panics iff `offset + length > self.len()`. - #[must_use] - fn sliced(&self, offset: usize, length: usize) -> Box { - let mut new = self.to_boxed(); - new.slice(offset, length); - new - } - - /// Returns a slice of this [`Array`]. - /// # Implementation - /// This operation is `O(1)` over `len`, as it amounts to increase two ref counts - /// and moving the struct to the heap. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()` - #[must_use] - unsafe fn sliced_unchecked(&self, offset: usize, length: usize) -> Box { - let mut new = self.to_boxed(); - new.slice_unchecked(offset, length); - new - } - - /// Clones this [`Array`] with a new new assigned bitmap. - /// # Panic - /// This function panics iff `validity.len() != self.len()`. - fn with_validity(&self, validity: Option) -> Box; - - /// Clone a `&dyn Array` to an owned `Box`. - fn to_boxed(&self) -> Box; -} - -dyn_clone::clone_trait_object!(Array); - -/// A trait describing an array with a backing store that can be preallocated to -/// a given size. -pub(crate) trait Container { - /// Create this array with a given capacity. - fn with_capacity(capacity: usize) -> Self - where Self: Sized; -} - -/// A trait describing a mutable array; i.e. an array whose values can be changed. -/// Mutable arrays cannot be cloned but can be mutated in place, -/// thereby making them useful to perform numeric operations without allocations. -/// As in [`Array`], concrete arrays (such as [`MutablePrimitiveArray`]) implement how they are mutated. -pub trait MutableArray: std::fmt::Debug + Send + Sync { - /// The [`DataType`] of the array. - fn data_type(&self) -> &DataType; - - /// The length of the array. - fn len(&self) -> usize; - - /// Whether the array is empty. - fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The optional validity of the array. - fn validity(&self) -> Option<&MutableBitmap>; - - /// Convert itself to an (immutable) [`Array`]. - fn as_box(&mut self) -> Box; - - /// Convert itself to an (immutable) atomically reference counted [`Array`]. - // This provided implementation has an extra allocation as it first - // boxes `self`, then converts the box into an `Arc`. Implementors may wish - // to avoid an allocation by skipping the box completely. - fn as_arc(&mut self) -> std::sync::Arc { - self.as_box().into() - } - - /// Convert to `Any`, to enable dynamic casting. - fn as_any(&self) -> &dyn Any; - - /// Convert to mutable `Any`, to enable dynamic casting. - fn as_mut_any(&mut self) -> &mut dyn Any; - - /// Adds a new null element to the array. - fn push_null(&mut self); - - /// Whether `index` is valid / set. - /// # Panic - /// Panics if `index >= self.len()`. - #[inline] - fn is_valid(&self, index: usize) -> bool { - self.validity() - .as_ref() - .map(|x| x.get(index)) - .unwrap_or(true) - } - - /// Reserves additional slots to its capacity. - fn reserve(&mut self, additional: usize); - - /// Shrink the array to fit its length. - fn shrink_to_fit(&mut self); -} - -impl MutableArray for Box { - fn len(&self) -> usize { - self.as_ref().len() - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.as_ref().validity() - } - - fn as_box(&mut self) -> Box { - self.as_mut().as_box() - } - - fn as_arc(&mut self) -> Arc { - self.as_mut().as_arc() - } - - fn data_type(&self) -> &DataType { - self.as_ref().data_type() - } - - fn as_any(&self) -> &dyn std::any::Any { - self.as_ref().as_any() - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self.as_mut().as_mut_any() - } - - #[inline] - fn push_null(&mut self) { - self.as_mut().push_null() - } - - fn shrink_to_fit(&mut self) { - self.as_mut().shrink_to_fit(); - } - - fn reserve(&mut self, additional: usize) { - self.as_mut().reserve(additional); - } -} - -macro_rules! general_dyn { - ($array:expr, $ty:ty, $f:expr) => {{ - let array = $array.as_any().downcast_ref::<$ty>().unwrap(); - ($f)(array) - }}; -} - -macro_rules! fmt_dyn { - ($array:expr, $ty:ty, $f:expr) => {{ - let mut f = |x: &$ty| x.fmt($f); - general_dyn!($array, $ty, f) - }}; -} - -#[macro_export] -macro_rules! match_integer_type {( - $key_type:expr, | $_:tt $T:ident | $($body:tt)* -) => ({ - macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )} - use $crate::arrow::datatypes::IntegerType::*; - match $key_type { - Int8 => __with_ty__! { i8 }, - Int16 => __with_ty__! { i16 }, - Int32 => __with_ty__! { i32 }, - Int64 => __with_ty__! { i64 }, - UInt8 => __with_ty__! { u8 }, - UInt16 => __with_ty__! { u16 }, - UInt32 => __with_ty__! { u32 }, - UInt64 => __with_ty__! { u64 }, - } -})} - -#[macro_export] -macro_rules! with_match_primitive_type {( - $key_type:expr, | $_:tt $T:ident | $($body:tt)* -) => ({ - macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )} - use $crate::arrow::datatypes::PrimitiveType::*; - use $crate::arrow::types::{days_ms, months_days_ns, f16, i256}; - match $key_type { - Int8 => __with_ty__! { i8 }, - Int16 => __with_ty__! { i16 }, - Int32 => __with_ty__! { i32 }, - Int64 => __with_ty__! { i64 }, - Int128 => __with_ty__! { i128 }, - Int256 => __with_ty__! { i256 }, - DaysMs => __with_ty__! { days_ms }, - MonthDayNano => __with_ty__! { months_days_ns }, - UInt8 => __with_ty__! { u8 }, - UInt16 => __with_ty__! { u16 }, - UInt32 => __with_ty__! { u32 }, - UInt64 => __with_ty__! { u64 }, - Float16 => __with_ty__! { f16 }, - Float32 => __with_ty__! { f32 }, - Float64 => __with_ty__! { f64 }, - _ => panic!("Do not support primitive `{:?}`", $key_type) - } -})} - -impl std::fmt::Debug for dyn Array + '_ { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - use crate::arrow::datatypes::PhysicalType::*; - match self.data_type().to_physical_type() { - Null => fmt_dyn!(self, NullArray, f), - Boolean => fmt_dyn!(self, BooleanArray, f), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - fmt_dyn!(self, PrimitiveArray<$T>, f) - }), - BinaryView => fmt_dyn!(self, BinaryViewArray, f), - Utf8View => fmt_dyn!(self, Utf8ViewArray, f), - Binary => fmt_dyn!(self, BinaryArray, f), - LargeBinary => fmt_dyn!(self, BinaryArray, f), - FixedSizeBinary => fmt_dyn!(self, FixedSizeBinaryArray, f), - Utf8 => fmt_dyn!(self, Utf8Array::, f), - LargeUtf8 => fmt_dyn!(self, Utf8Array::, f), - List => fmt_dyn!(self, ListArray::, f), - LargeList => fmt_dyn!(self, ListArray::, f), - FixedSizeList => fmt_dyn!(self, FixedSizeListArray, f), - Struct => fmt_dyn!(self, StructArray, f), - Union => fmt_dyn!(self, UnionArray, f), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - fmt_dyn!(self, DictionaryArray::<$T>, f) - }) - } - Map => fmt_dyn!(self, MapArray, f), - } - } -} - -/// Creates a new [`Array`] with a [`Array::len`] of 0. -pub fn new_empty_array(data_type: DataType) -> Box { - use crate::arrow::datatypes::PhysicalType::*; - match data_type.to_physical_type() { - Null => Box::new(NullArray::new_empty(data_type)), - Boolean => Box::new(BooleanArray::new_empty(data_type)), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - Box::new(PrimitiveArray::<$T>::new_empty(data_type)) - }), - Binary => Box::new(BinaryArray::::new_empty(data_type)), - LargeBinary => Box::new(BinaryArray::::new_empty(data_type)), - FixedSizeBinary => Box::new(FixedSizeBinaryArray::new_empty(data_type)), - Utf8 => Box::new(Utf8Array::::new_empty(data_type)), - LargeUtf8 => Box::new(Utf8Array::::new_empty(data_type)), - List => Box::new(ListArray::::new_empty(data_type)), - LargeList => Box::new(ListArray::::new_empty(data_type)), - FixedSizeList => Box::new(FixedSizeListArray::new_empty(data_type)), - Struct => Box::new(StructArray::new_empty(data_type)), - Union => Box::new(UnionArray::new_empty(data_type)), - Map => Box::new(MapArray::new_empty(data_type)), - Utf8View => Box::new(Utf8ViewArray::new_empty(data_type)), - BinaryView => Box::new(BinaryViewArray::new_empty(data_type)), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - Box::new(DictionaryArray::<$T>::new_empty(data_type)) - }) - } - } -} - -/// Creates a new [`Array`] of [`DataType`] `data_type` and `length`. -/// The array is guaranteed to have [`Array::null_count`] equal to [`Array::len`] -/// for all types except Union, which does not have a validity. -pub fn new_null_array(data_type: DataType, length: usize) -> Box { - use crate::arrow::datatypes::PhysicalType::*; - match data_type.to_physical_type() { - Null => Box::new(NullArray::new_null(data_type, length)), - Boolean => Box::new(BooleanArray::new_null(data_type, length)), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - Box::new(PrimitiveArray::<$T>::new_null(data_type, length)) - }), - Binary => Box::new(BinaryArray::::new_null(data_type, length)), - LargeBinary => Box::new(BinaryArray::::new_null(data_type, length)), - FixedSizeBinary => Box::new(FixedSizeBinaryArray::new_null(data_type, length)), - Utf8 => Box::new(Utf8Array::::new_null(data_type, length)), - LargeUtf8 => Box::new(Utf8Array::::new_null(data_type, length)), - List => Box::new(ListArray::::new_null(data_type, length)), - LargeList => Box::new(ListArray::::new_null(data_type, length)), - FixedSizeList => Box::new(FixedSizeListArray::new_null(data_type, length)), - Struct => Box::new(StructArray::new_null(data_type, length)), - Union => Box::new(UnionArray::new_null(data_type, length)), - Map => Box::new(MapArray::new_null(data_type, length)), - BinaryView => Box::new(BinaryViewArray::new_null(data_type, length)), - Utf8View => Box::new(Utf8ViewArray::new_null(data_type, length)), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - Box::new(DictionaryArray::<$T>::new_null(data_type, length)) - }) - } - } -} - -/// Trait providing bi-directional conversion between arrow2 [`Array`] and arrow-rs [`ArrayData`] -/// -/// [`ArrayData`]: arrow_data::ArrayData -#[cfg(feature = "arrow")] -pub trait Arrow2Arrow: Array { - /// Convert this [`Array`] into [`ArrayData`] - fn to_data(&self) -> arrow_data::ArrayData; - - /// Create this [`Array`] from [`ArrayData`] - fn from_data(data: &arrow_data::ArrayData) -> Self; -} - -#[cfg(feature = "arrow")] -macro_rules! to_data_dyn { - ($array:expr, $ty:ty) => {{ - let f = |x: &$ty| x.to_data(); - general_dyn!($array, $ty, f) - }}; -} - -#[cfg(feature = "arrow")] -impl From> for arrow_array::ArrayRef { - fn from(value: Box) -> Self { - value.as_ref().into() - } -} - -#[cfg(feature = "arrow")] -impl From<&dyn Array> for arrow_array::ArrayRef { - fn from(value: &dyn Array) -> Self { - arrow_array::make_array(to_data(value)) - } -} - -#[cfg(feature = "arrow")] -impl From for Box { - fn from(value: arrow_array::ArrayRef) -> Self { - value.as_ref().into() - } -} - -#[cfg(feature = "arrow")] -impl From<&dyn arrow_array::Array> for Box { - fn from(value: &dyn arrow_array::Array) -> Self { - from_data(&value.to_data()) - } -} - -/// Convert an arrow2 [`Array`] to [`arrow_data::ArrayData`] -#[cfg(feature = "arrow")] -pub fn to_data(array: &dyn Array) -> arrow_data::ArrayData { - use crate::arrow::datatypes::PhysicalType::*; - match array.data_type().to_physical_type() { - Null => to_data_dyn!(array, NullArray), - Boolean => to_data_dyn!(array, BooleanArray), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - to_data_dyn!(array, PrimitiveArray<$T>) - }), - Binary => to_data_dyn!(array, BinaryArray), - LargeBinary => to_data_dyn!(array, BinaryArray), - FixedSizeBinary => to_data_dyn!(array, FixedSizeBinaryArray), - Utf8 => to_data_dyn!(array, Utf8Array::), - LargeUtf8 => to_data_dyn!(array, Utf8Array::), - List => to_data_dyn!(array, ListArray::), - LargeList => to_data_dyn!(array, ListArray::), - FixedSizeList => to_data_dyn!(array, FixedSizeListArray), - Struct => to_data_dyn!(array, StructArray), - Union => to_data_dyn!(array, UnionArray), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - to_data_dyn!(array, DictionaryArray::<$T>) - }) - } - Map => to_data_dyn!(array, MapArray), - BinaryView => to_data_dyn!(array, BinaryViewArray), - Utf8View => to_data_dyn!(array, Utf8ViewArray), - } -} - -/// Convert an [`arrow_data::ArrayData`] to arrow2 [`Array`] -#[cfg(feature = "arrow")] -pub fn from_data(data: &arrow_data::ArrayData) -> Box { - use crate::arrow::datatypes::PhysicalType::*; - let data_type: DataType = data.data_type().clone().into(); - match data_type.to_physical_type() { - Null => Box::new(NullArray::from_data(data)), - Boolean => Box::new(BooleanArray::from_data(data)), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - Box::new(PrimitiveArray::<$T>::from_data(data)) - }), - Binary => Box::new(BinaryArray::::from_data(data)), - LargeBinary => Box::new(BinaryArray::::from_data(data)), - FixedSizeBinary => Box::new(FixedSizeBinaryArray::from_data(data)), - Utf8 => Box::new(Utf8Array::::from_data(data)), - LargeUtf8 => Box::new(Utf8Array::::from_data(data)), - List => Box::new(ListArray::::from_data(data)), - LargeList => Box::new(ListArray::::from_data(data)), - FixedSizeList => Box::new(FixedSizeListArray::from_data(data)), - Struct => Box::new(StructArray::from_data(data)), - Union => Box::new(UnionArray::from_data(data)), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - Box::new(DictionaryArray::<$T>::from_data(data)) - }) - } - Map => Box::new(MapArray::from_data(data)), - BinaryView => Box::new(BinaryViewArray::from_data(data)), - Utf8View => Box::new(Utf8ViewArray::from_data(data)), - } -} - -macro_rules! clone_dyn { - ($array:expr, $ty:ty) => {{ - let f = |x: &$ty| Box::new(x.clone()); - general_dyn!($array, $ty, f) - }}; -} - -// macro implementing `sliced` and `sliced_unchecked` -macro_rules! impl_sliced { - () => { - /// Returns this array sliced. - /// # Implementation - /// This function is `O(1)`. - /// # Panics - /// iff `offset + length > self.len()`. - #[inline] - #[must_use] - pub fn sliced(self, offset: usize, length: usize) -> Self { - assert!( - offset + length <= self.len(), - "the offset of the new Buffer cannot exceed the existing length" - ); - unsafe { self.sliced_unchecked(offset, length) } - } - - /// Returns this array sliced. - /// # Implementation - /// This function is `O(1)`. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - #[inline] - #[must_use] - pub unsafe fn sliced_unchecked(mut self, offset: usize, length: usize) -> Self { - self.slice_unchecked(offset, length); - self - } - }; -} - -// macro implementing `with_validity` and `set_validity` -macro_rules! impl_mut_validity { - () => { - /// Returns this array with a new validity. - /// # Panic - /// Panics iff `validity.len() != self.len()`. - #[must_use] - #[inline] - pub fn with_validity(mut self, validity: Option) -> Self { - self.set_validity(validity); - self - } - - /// Sets the validity of this array. - /// # Panics - /// This function panics iff `values.len() != self.len()`. - #[inline] - pub fn set_validity(&mut self, validity: Option) { - if matches!(&validity, Some(bitmap) if bitmap.len() != self.len()) { - panic!("validity must be equal to the array's length") - } - self.validity = validity; - } - } -} - -// macro implementing `with_validity`, `set_validity` and `apply_validity` for mutable arrays -macro_rules! impl_mutable_array_mut_validity { - () => { - /// Returns this array with a new validity. - /// # Panic - /// Panics iff `validity.len() != self.len()`. - #[must_use] - #[inline] - pub fn with_validity(mut self, validity: Option) -> Self { - self.set_validity(validity); - self - } - - /// Sets the validity of this array. - /// # Panics - /// This function panics iff `values.len() != self.len()`. - #[inline] - pub fn set_validity(&mut self, validity: Option) { - if matches!(&validity, Some(bitmap) if bitmap.len() != self.len()) { - panic!("validity must be equal to the array's length") - } - self.validity = validity; - } - - /// Applies a function `f` to the validity of this array. - /// - /// This is an API to leverage clone-on-write - /// # Panics - /// This function panics if the function `f` modifies the length of the [`Bitmap`]. - #[inline] - pub fn apply_validity MutableBitmap>(&mut self, f: F) { - if let Some(validity) = std::mem::take(&mut self.validity) { - self.set_validity(Some(f(validity))) - } - } - - } -} - -// macro implementing `boxed` and `arced` -macro_rules! impl_into_array { - () => { - /// Boxes this array into a [`Box`]. - pub fn boxed(self) -> Box { - Box::new(self) - } - - /// Arcs this array into a [`std::sync::Arc`]. - pub fn arced(self) -> std::sync::Arc { - std::sync::Arc::new(self) - } - }; -} - -// macro implementing common methods of trait `Array` -macro_rules! impl_common_array { - () => { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn as_any_mut(&mut self) -> &mut dyn std::any::Any { - self - } - - #[inline] - fn len(&self) -> usize { - self.len() - } - - #[inline] - fn data_type(&self) -> &DataType { - &self.data_type - } - - #[inline] - fn slice(&mut self, offset: usize, length: usize) { - self.slice(offset, length); - } - - #[inline] - unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.slice_unchecked(offset, length); - } - - #[inline] - fn to_boxed(&self) -> Box { - Box::new(self.clone()) - } - }; -} - -/// Clones a dynamic [`Array`]. -/// # Implementation -/// This operation is `O(1)` over `len`, as it amounts to increase two ref counts -/// and moving the concrete struct under a `Box`. -pub fn clone(array: &dyn Array) -> Box { - use crate::arrow::datatypes::PhysicalType::*; - match array.data_type().to_physical_type() { - Null => clone_dyn!(array, NullArray), - Boolean => clone_dyn!(array, BooleanArray), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - clone_dyn!(array, PrimitiveArray<$T>) - }), - Binary => clone_dyn!(array, BinaryArray), - LargeBinary => clone_dyn!(array, BinaryArray), - FixedSizeBinary => clone_dyn!(array, FixedSizeBinaryArray), - Utf8 => clone_dyn!(array, Utf8Array::), - LargeUtf8 => clone_dyn!(array, Utf8Array::), - List => clone_dyn!(array, ListArray::), - LargeList => clone_dyn!(array, ListArray::), - FixedSizeList => clone_dyn!(array, FixedSizeListArray), - Struct => clone_dyn!(array, StructArray), - Union => clone_dyn!(array, UnionArray), - Map => clone_dyn!(array, MapArray), - BinaryView => clone_dyn!(array, BinaryViewArray), - Utf8View => clone_dyn!(array, Utf8ViewArray), - Dictionary(key_type) => { - match_integer_type!(key_type, |$T| { - clone_dyn!(array, DictionaryArray::<$T>) - }) - } - } -} - -// see https://users.rust-lang.org/t/generic-for-dyn-a-or-box-dyn-a-or-arc-dyn-a/69430/3 -// for details -impl<'a> AsRef<(dyn Array + 'a)> for dyn Array { - fn as_ref(&self) -> &(dyn Array + 'a) { - self - } -} - -mod binary; -mod binview; - -mod boolean; -mod dictionary; -mod fixed_size_binary; -mod fixed_size_list; -mod list; -mod map; -mod null; -mod primitive; -mod specification; -mod struct_; -mod union; -mod utf8; - -mod equal; - -mod fmt; -#[doc(hidden)] -pub mod indexable; -mod iterator; - -pub mod growable; -pub mod ord; - -pub use binary::BinaryArray; -pub use binary::BinaryValueIter; -pub use binary::MutableBinaryArray; -pub use binary::MutableBinaryValuesArray; -pub use binview::BinaryViewArray; -pub use binview::BinaryViewArrayGeneric; -pub use binview::BinaryViewValueIter; -pub use binview::MutableBinaryViewArray; -pub use binview::Utf8ViewArray; -pub use binview::View; -pub use binview::ViewType; -pub use boolean::BooleanArray; -pub use boolean::MutableBooleanArray; -pub use dictionary::DictionaryArray; -pub use dictionary::DictionaryKey; -pub use dictionary::MutableDictionaryArray; -pub use equal::equal; -pub use fixed_size_binary::FixedSizeBinaryArray; -pub use fixed_size_binary::MutableFixedSizeBinaryArray; -pub use fixed_size_list::FixedSizeListArray; -pub use fixed_size_list::MutableFixedSizeListArray; -pub use fmt::get_display; -pub use fmt::get_value_display; -pub(crate) use iterator::ArrayAccessor; -pub use iterator::ArrayValuesIter; -pub use list::ListArray; -pub use list::ListValuesIter; -pub use list::MutableListArray; -pub use map::MapArray; -pub use null::MutableNullArray; -pub use null::NullArray; -pub use primitive::*; -pub use struct_::MutableStructArray; -pub use struct_::StructArray; -pub use union::UnionArray; -pub use utf8::MutableUtf8Array; -pub use utf8::MutableUtf8ValuesArray; -pub use utf8::Utf8Array; -pub use utf8::Utf8ValuesIter; - -/// A trait describing the ability of a struct to create itself from a iterator. -/// This is similar to [`Extend`], but accepted the creation to error. -pub trait TryExtend { - /// Fallible version of [`Extend::extend`]. - fn try_extend>(&mut self, iter: I) -> Result<()>; -} - -/// A trait describing the ability of a struct to receive new items. -pub trait TryPush { - /// Tries to push a new element. - fn try_push(&mut self, item: A) -> Result<()>; -} - -/// A trait describing the ability of a struct to receive new items. -pub trait PushUnchecked { - /// Push a new element that holds the invariants of the struct. - /// # Safety - /// The items must uphold the invariants of the struct - /// Read the specific implementation of the trait to understand what these are. - unsafe fn push_unchecked(&mut self, item: A); -} - -/// A trait describing the ability of a struct to extend from a reference of itself. -/// Specialization of [`TryExtend`]. -pub trait TryExtendFromSelf { - /// Tries to extend itself with elements from `other`, failing only on overflow. - fn try_extend_from_self(&mut self, other: &Self) -> Result<()>; -} - -/// Trait that [`BinaryArray`] and [`Utf8Array`] implement for the purposes of DRY. -/// # Safety -/// The implementer must ensure that -/// 1. `offsets.len() > 0` -/// 2. `offsets[i] >= offsets[i-1] for all i` -/// 3. `offsets[i] < values.len() for all i` -pub unsafe trait GenericBinaryArray: Array { - /// The values of the array - fn values(&self) -> &[u8]; - /// The offsets of the array - fn offsets(&self) -> &[O]; -} diff --git a/src/common/arrow/src/arrow/array/null.rs b/src/common/arrow/src/arrow/array/null.rs deleted file mode 100644 index 22617e47fa3d..000000000000 --- a/src/common/arrow/src/arrow/array/null.rs +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::any::Any; - -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; - -/// The concrete [`Array`] of [`DataType::Null`]. -#[derive(Clone)] -pub struct NullArray { - data_type: DataType, - length: usize, -} - -impl NullArray { - /// Returns a new [`NullArray`]. - /// # Errors - /// This function errors iff: - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to [`crate::arrow::datatypes::PhysicalType::Null`]. - pub fn try_new(data_type: DataType, length: usize) -> Result { - if data_type.to_physical_type() != PhysicalType::Null { - return Err(Error::oos( - "NullArray can only be initialized with a DataType whose physical type is Boolean", - )); - } - - Ok(Self { data_type, length }) - } - - /// Returns a new [`NullArray`]. - /// # Panics - /// This function errors iff: - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to [`crate::arrow::datatypes::PhysicalType::Null`]. - pub fn new(data_type: DataType, length: usize) -> Self { - Self::try_new(data_type, length).unwrap() - } - - /// Returns a new empty [`NullArray`]. - pub fn new_empty(data_type: DataType) -> Self { - Self::new(data_type, 0) - } - - /// Returns a new [`NullArray`]. - pub fn new_null(data_type: DataType, length: usize) -> Self { - Self::new(data_type, length) - } - - impl_sliced!(); - impl_into_array!(); -} - -impl NullArray { - /// Returns a slice of the [`NullArray`]. - /// # Panic - /// This function panics iff `offset + length > self.len()`. - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new array cannot exceed the arrays' length" - ); - unsafe { self.slice_unchecked(offset, length) }; - } - - /// Returns a slice of the [`NullArray`]. - /// # Safety - /// The caller must ensure that `offset + length < self.len()`. - pub unsafe fn slice_unchecked(&mut self, _offset: usize, length: usize) { - self.length = length; - } - - #[inline] - fn len(&self) -> usize { - self.length - } -} - -impl Array for NullArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - None - } - - fn with_validity(&self, _: Option) -> Box { - panic!("cannot set validity of a null array") - } -} - -#[derive(Debug)] -/// A distinct type to disambiguate -/// clashing methods -pub struct MutableNullArray { - inner: NullArray, -} - -impl MutableNullArray { - /// Returns a new [`MutableNullArray`]. - /// # Panics - /// This function errors iff: - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to [`crate::arrow::datatypes::PhysicalType::Null`]. - pub fn new(data_type: DataType, length: usize) -> Self { - let inner = NullArray::try_new(data_type, length).unwrap(); - Self { inner } - } -} - -impl From for NullArray { - fn from(value: MutableNullArray) -> Self { - value.inner - } -} - -impl MutableArray for MutableNullArray { - fn data_type(&self) -> &DataType { - &DataType::Null - } - - fn len(&self) -> usize { - self.inner.length - } - - fn validity(&self) -> Option<&MutableBitmap> { - None - } - - fn as_box(&mut self) -> Box { - self.inner.clone().boxed() - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn Any { - self - } - - fn push_null(&mut self) { - self.inner.length += 1; - } - - fn reserve(&mut self, _additional: usize) { - // no-op - } - - fn shrink_to_fit(&mut self) { - // no-op - } -} - -impl std::fmt::Debug for NullArray { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "NullArray({})", self.len()) - } -} - -#[cfg(feature = "arrow")] -mod arrow { - use arrow_data::ArrayData; - use arrow_data::ArrayDataBuilder; - - use super::*; - impl NullArray { - /// Convert this array into [`arrow_data::ArrayData`] - pub fn to_data(&self) -> ArrayData { - let builder = ArrayDataBuilder::new(arrow_schema::DataType::Null).len(self.len()); - - // Safety: safe by construction - unsafe { builder.build_unchecked() } - } - - /// Create this array from [`ArrayData`] - pub fn from_data(data: &ArrayData) -> Self { - Self::new(DataType::Null, data.len()) - } - } -} diff --git a/src/common/arrow/src/arrow/array/ord.rs b/src/common/arrow/src/arrow/array/ord.rs deleted file mode 100644 index abb594792a6f..000000000000 --- a/src/common/arrow/src/arrow/array/ord.rs +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains functions and function factories to order values within arrays. - -use std::cmp::Ordering; - -use crate::arrow::array::*; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::types::NativeType; - -/// Compare the values at two arbitrary indices in two arrays. -pub type DynComparator = Box Ordering + Send + Sync>; - -/// implements comparison using IEEE 754 total ordering for f32 -// Original implementation from https://doc.rust-lang.org/std/primitive.f32.html#method.total_cmp -// TODO to change to use std when it becomes stable -#[inline] -pub fn total_cmp_f32(l: &f32, r: &f32) -> std::cmp::Ordering { - let mut left = l.to_bits() as i32; - let mut right = r.to_bits() as i32; - - left ^= (((left >> 31) as u32) >> 1) as i32; - right ^= (((right >> 31) as u32) >> 1) as i32; - - left.cmp(&right) -} - -/// implements comparison using IEEE 754 total ordering for f64 -// Original implementation from https://doc.rust-lang.org/std/primitive.f64.html#method.total_cmp -// TODO to change to use std when it becomes stable -#[inline] -pub fn total_cmp_f64(l: &f64, r: &f64) -> std::cmp::Ordering { - let mut left = l.to_bits() as i64; - let mut right = r.to_bits() as i64; - - left ^= (((left >> 63) as u64) >> 1) as i64; - right ^= (((right >> 63) as u64) >> 1) as i64; - - left.cmp(&right) -} - -/// Total order of all native types whose Rust implementation -/// that support total order. -#[inline] -pub fn total_cmp(l: &T, r: &T) -> std::cmp::Ordering -where T: NativeType + Ord { - l.cmp(r) -} - -fn compare_primitives(left: &dyn Array, right: &dyn Array) -> DynComparator { - let left = left - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - let right = right - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - Box::new(move |i, j| total_cmp(&left.value(i), &right.value(j))) -} - -fn compare_boolean(left: &dyn Array, right: &dyn Array) -> DynComparator { - let left = left - .as_any() - .downcast_ref::() - .unwrap() - .clone(); - let right = right - .as_any() - .downcast_ref::() - .unwrap() - .clone(); - Box::new(move |i, j| left.value(i).cmp(&right.value(j))) -} - -fn compare_f32(left: &dyn Array, right: &dyn Array) -> DynComparator { - let left = left - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - let right = right - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - Box::new(move |i, j| total_cmp_f32(&left.value(i), &right.value(j))) -} - -fn compare_f64(left: &dyn Array, right: &dyn Array) -> DynComparator { - let left = left - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - let right = right - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - Box::new(move |i, j| total_cmp_f64(&left.value(i), &right.value(j))) -} - -fn compare_string(left: &dyn Array, right: &dyn Array) -> DynComparator { - let left = left - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - let right = right - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - Box::new(move |i, j| left.value(i).cmp(right.value(j))) -} - -fn compare_binary(left: &dyn Array, right: &dyn Array) -> DynComparator { - let left = left - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - let right = right - .as_any() - .downcast_ref::>() - .unwrap() - .clone(); - Box::new(move |i, j| left.value(i).cmp(right.value(j))) -} - -fn compare_dict(left: &DictionaryArray, right: &DictionaryArray) -> Result -where K: DictionaryKey { - let left_keys = left.keys().values().clone(); - let right_keys = right.keys().values().clone(); - - let comparator = build_compare(left.values().as_ref(), right.values().as_ref())?; - - Ok(Box::new(move |i: usize, j: usize| { - // safety: all dictionaries keys are guaranteed to be castable to usize - let key_left = unsafe { left_keys[i].as_usize() }; - let key_right = unsafe { right_keys[j].as_usize() }; - (comparator)(key_left, key_right) - })) -} - -macro_rules! dyn_dict { - ($key:ty, $lhs:expr, $rhs:expr) => {{ - let lhs = $lhs.as_any().downcast_ref().unwrap(); - let rhs = $rhs.as_any().downcast_ref().unwrap(); - compare_dict::<$key>(lhs, rhs)? - }}; -} - -/// returns a comparison function that compares values at two different slots -/// between two [`Array`]. -/// # Example -/// ``` -/// use arrow2::array::ord::build_compare; -/// use arrow2::array::PrimitiveArray; -/// -/// # fn main() -> arrow2::error::Result<()> { -/// let array1 = PrimitiveArray::from_slice([1, 2]); -/// let array2 = PrimitiveArray::from_slice([3, 4]); -/// -/// let cmp = build_compare(&array1, &array2)?; -/// -/// // 1 (index 0 of array1) is smaller than 4 (index 1 of array2) -/// assert_eq!(std::cmp::Ordering::Less, (cmp)(0, 1)); -/// # Ok(()) -/// # } -/// ``` -/// # Error -/// The arrays' [`DataType`] must be equal and the types must have a natural order. -// This is a factory of comparisons. -pub fn build_compare(left: &dyn Array, right: &dyn Array) -> Result { - use DataType::*; - use IntervalUnit::*; - use TimeUnit::*; - Ok(match (left.data_type(), right.data_type()) { - (a, b) if a != b => { - return Err(Error::InvalidArgumentError( - "Can't compare arrays of different types".to_string(), - )); - } - (Boolean, Boolean) => compare_boolean(left, right), - (UInt8, UInt8) => compare_primitives::(left, right), - (UInt16, UInt16) => compare_primitives::(left, right), - (UInt32, UInt32) => compare_primitives::(left, right), - (UInt64, UInt64) => compare_primitives::(left, right), - (Int8, Int8) => compare_primitives::(left, right), - (Int16, Int16) => compare_primitives::(left, right), - (Int32, Int32) - | (Date32, Date32) - | (Time32(Second), Time32(Second)) - | (Time32(Millisecond), Time32(Millisecond)) - | (Interval(YearMonth), Interval(YearMonth)) => compare_primitives::(left, right), - (Int64, Int64) - | (Date64, Date64) - | (Time64(Microsecond), Time64(Microsecond)) - | (Time64(Nanosecond), Time64(Nanosecond)) - | (Timestamp(Second, None), Timestamp(Second, None)) - | (Timestamp(Millisecond, None), Timestamp(Millisecond, None)) - | (Timestamp(Microsecond, None), Timestamp(Microsecond, None)) - | (Timestamp(Nanosecond, None), Timestamp(Nanosecond, None)) - | (Duration(Second), Duration(Second)) - | (Duration(Millisecond), Duration(Millisecond)) - | (Duration(Microsecond), Duration(Microsecond)) - | (Duration(Nanosecond), Duration(Nanosecond)) => compare_primitives::(left, right), - (Float32, Float32) => compare_f32(left, right), - (Float64, Float64) => compare_f64(left, right), - (Decimal(_, _), Decimal(_, _)) => compare_primitives::(left, right), - (Utf8, Utf8) => compare_string::(left, right), - (LargeUtf8, LargeUtf8) => compare_string::(left, right), - (Binary, Binary) => compare_binary::(left, right), - (LargeBinary, LargeBinary) => compare_binary::(left, right), - (Dictionary(key_type_lhs, ..), Dictionary(key_type_rhs, ..)) => { - match (key_type_lhs, key_type_rhs) { - (IntegerType::UInt8, IntegerType::UInt8) => dyn_dict!(u8, left, right), - (IntegerType::UInt16, IntegerType::UInt16) => dyn_dict!(u16, left, right), - (IntegerType::UInt32, IntegerType::UInt32) => dyn_dict!(u32, left, right), - (IntegerType::UInt64, IntegerType::UInt64) => dyn_dict!(u64, left, right), - (IntegerType::Int8, IntegerType::Int8) => dyn_dict!(i8, left, right), - (IntegerType::Int16, IntegerType::Int16) => dyn_dict!(i16, left, right), - (IntegerType::Int32, IntegerType::Int32) => dyn_dict!(i32, left, right), - (IntegerType::Int64, IntegerType::Int64) => dyn_dict!(i64, left, right), - (lhs, _) => { - return Err(Error::InvalidArgumentError(format!( - "Dictionaries do not support keys of type {lhs:?}" - ))); - } - } - } - (lhs, _) => { - return Err(Error::InvalidArgumentError(format!( - "The data type type {lhs:?} has no natural order" - ))); - } - }) -} diff --git a/src/common/arrow/src/arrow/array/physical_binary.rs b/src/common/arrow/src/arrow/array/physical_binary.rs deleted file mode 100644 index 4d522e1ac5fd..000000000000 --- a/src/common/arrow/src/arrow/array/physical_binary.rs +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; - -/// # Safety -/// The caller must ensure that `iterator` is `TrustedLen`. -#[inline] -#[allow(clippy::type_complexity)] -pub(crate) unsafe fn try_trusted_len_unzip( - iterator: I, -) -> std::result::Result<(Option, Offsets, Vec), E> -where - O: Offset, - P: AsRef<[u8]>, - I: Iterator, E>>, -{ - let (_, upper) = iterator.size_hint(); - let len = upper.expect("trusted_len_unzip requires an upper limit"); - - let mut null = MutableBitmap::with_capacity(len); - let mut offsets = Vec::::with_capacity(len + 1); - let mut values = Vec::::new(); - - let mut length = O::default(); - let mut dst = offsets.as_mut_ptr(); - std::ptr::write(dst, length); - dst = dst.add(1); - for item in iterator { - if let Some(item) = item? { - null.push_unchecked(true); - let s = item.as_ref(); - length += O::from_usize(s.len()).unwrap(); - values.extend_from_slice(s); - } else { - null.push_unchecked(false); - }; - - std::ptr::write(dst, length); - dst = dst.add(1); - } - assert_eq!( - dst.offset_from(offsets.as_ptr()) as usize, - len + 1, - "Trusted iterator length was not accurately reported" - ); - offsets.set_len(len + 1); - - Ok((null.into(), Offsets::new_unchecked(offsets), values)) -} - -/// Creates [`MutableBitmap`] and two [`Vec`]s from an iterator of `Option`. -/// The first buffer corresponds to a offset buffer, the second one -/// corresponds to a values buffer. -/// # Safety -/// The caller must ensure that `iterator` is `TrustedLen`. -#[inline] -pub(crate) unsafe fn trusted_len_unzip( - iterator: I, -) -> (Option, Offsets, Vec) -where - O: Offset, - P: AsRef<[u8]>, - I: Iterator>, -{ - let (_, upper) = iterator.size_hint(); - let len = upper.expect("trusted_len_unzip requires an upper limit"); - - let mut offsets = Offsets::::with_capacity(len); - let mut values = Vec::::new(); - let mut validity = MutableBitmap::new(); - - extend_from_trusted_len_iter(&mut offsets, &mut values, &mut validity, iterator); - - let validity = if validity.unset_bits() > 0 { - Some(validity) - } else { - None - }; - - (validity, offsets, values) -} - -/// Creates two [`Buffer`]s from an iterator of `&[u8]`. -/// The first buffer corresponds to a offset buffer, the second to a values buffer. -/// # Safety -/// The caller must ensure that `iterator` is [`TrustedLen`]. -#[inline] -pub(crate) unsafe fn trusted_len_values_iter(iterator: I) -> (Offsets, Vec) -where - O: Offset, - P: AsRef<[u8]>, - I: Iterator, -{ - let (_, upper) = iterator.size_hint(); - let len = upper.expect("trusted_len_unzip requires an upper limit"); - - let mut offsets = Offsets::::with_capacity(len); - let mut values = Vec::::new(); - - extend_from_trusted_len_values_iter(&mut offsets, &mut values, iterator); - - (offsets, values) -} - -// Populates `offsets` and `values` [`Vec`]s with information extracted -// from the incoming `iterator`. -// # Safety -// The caller must ensure the `iterator` is [`TrustedLen`] -#[inline] -pub(crate) unsafe fn extend_from_trusted_len_values_iter( - offsets: &mut Offsets, - values: &mut Vec, - iterator: I, -) where - O: Offset, - P: AsRef<[u8]>, - I: Iterator, -{ - let lengths = iterator.map(|item| { - let s = item.as_ref(); - // Push new entries for both `values` and `offsets` buffer - values.extend_from_slice(s); - s.len() - }); - offsets.try_extend_from_lengths(lengths).unwrap(); -} - -// Populates `offsets` and `values` [`Vec`]s with information extracted -// from the incoming `iterator`. -// the return value indicates how many items were added. -#[inline] -pub(crate) fn extend_from_values_iter( - offsets: &mut Offsets, - values: &mut Vec, - iterator: I, -) -> usize -where - O: Offset, - P: AsRef<[u8]>, - I: Iterator, -{ - let (size_hint, _) = iterator.size_hint(); - - offsets.reserve(size_hint); - - let start_index = offsets.len_proxy(); - - for item in iterator { - let bytes = item.as_ref(); - values.extend_from_slice(bytes); - offsets.try_push_usize(bytes.len()).unwrap(); - } - offsets.len_proxy() - start_index -} - -// Populates `offsets`, `values`, and `validity` [`Vec`]s with -// information extracted from the incoming `iterator`. -// -// # Safety -// The caller must ensure that `iterator` is [`TrustedLen`] -#[inline] -pub(crate) unsafe fn extend_from_trusted_len_iter( - offsets: &mut Offsets, - values: &mut Vec, - validity: &mut MutableBitmap, - iterator: I, -) where - O: Offset, - P: AsRef<[u8]>, - I: Iterator>, -{ - let (_, upper) = iterator.size_hint(); - let additional = upper.expect("extend_from_trusted_len_iter requires an upper limit"); - - offsets.reserve(additional); - validity.reserve(additional); - - let lengths = iterator.map(|item| { - if let Some(item) = item { - let bytes = item.as_ref(); - values.extend_from_slice(bytes); - validity.push_unchecked(true); - bytes.len() - } else { - validity.push_unchecked(false); - 0 - } - }); - offsets.try_extend_from_lengths(lengths).unwrap(); -} - -/// Creates two [`Vec`]s from an iterator of `&[u8]`. -/// The first buffer corresponds to a offset buffer, the second to a values buffer. -#[inline] -pub(crate) fn values_iter(iterator: I) -> (Offsets, Vec) -where - O: Offset, - P: AsRef<[u8]>, - I: Iterator, -{ - let (lower, _) = iterator.size_hint(); - - let mut offsets = Offsets::::with_capacity(lower); - let mut values = Vec::::new(); - - for item in iterator { - let s = item.as_ref(); - values.extend_from_slice(s); - offsets.try_push_usize(s.len()).unwrap(); - } - (offsets, values) -} - -/// Extends `validity` with all items from `other` -pub(crate) fn extend_validity( - length: usize, - validity: &mut Option, - other: &Option, -) { - if let Some(other) = other { - if let Some(validity) = validity { - let slice = other.as_slice(); - // safety: invariant offset + length <= slice.len() - unsafe { validity.extend_from_slice_unchecked(slice, 0, other.len()) } - } else { - let mut new_validity = MutableBitmap::from_len_set(length); - new_validity.extend_from_slice(other.as_slice(), 0, other.len()); - *validity = Some(new_validity); - } - } -} diff --git a/src/common/arrow/src/arrow/array/primitive/data.rs b/src/common/arrow/src/arrow/array/primitive/data.rs deleted file mode 100644 index eb14aafa1ea6..000000000000 --- a/src/common/arrow/src/arrow/array/primitive/data.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::types::NativeType; - -impl Arrow2Arrow for PrimitiveArray { - fn to_data(&self) -> ArrayData { - let data_type = self.data_type.clone().into(); - - let builder = ArrayDataBuilder::new(data_type) - .len(self.len()) - .buffers(vec![self.values.clone().into()]) - .nulls(self.validity.as_ref().map(|b| b.clone().into())); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type = data.data_type().clone().into(); - - let mut values: Buffer = data.buffers()[0].clone().into(); - values.slice(data.offset(), data.len()); - - Self { - data_type, - values, - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/primitive/fmt.rs b/src/common/arrow/src/arrow/array/primitive/fmt.rs deleted file mode 100644 index ada429022bb6..000000000000 --- a/src/common/arrow/src/arrow/array/primitive/fmt.rs +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::PrimitiveArray; -use crate::arrow::array::fmt::write_vec; -use crate::arrow::array::Array; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::temporal_conversions; -use crate::arrow::types::days_ms; -use crate::arrow::types::i256; -use crate::arrow::types::months_days_ns; -use crate::arrow::types::NativeType; - -macro_rules! dyn_primitive { - ($array:expr, $ty:ty, $expr:expr) => {{ - let array = ($array as &dyn Array) - .as_any() - .downcast_ref::>() - .unwrap(); - Box::new(move |f, index| write!(f, "{}", $expr(array.value(index)))) - }}; -} - -#[allow(clippy::type_complexity)] -pub fn get_write_value<'a, T: NativeType, F: Write>( - array: &'a PrimitiveArray, -) -> Box Result + 'a> { - use crate::arrow::datatypes::DataType::*; - match array.data_type().to_logical_type() { - Int8 => Box::new(|f, index| write!(f, "{}", array.value(index))), - Int16 => Box::new(|f, index| write!(f, "{}", array.value(index))), - Int32 => Box::new(|f, index| write!(f, "{}", array.value(index))), - Int64 => Box::new(|f, index| write!(f, "{}", array.value(index))), - UInt8 => Box::new(|f, index| write!(f, "{}", array.value(index))), - UInt16 => Box::new(|f, index| write!(f, "{}", array.value(index))), - UInt32 => Box::new(|f, index| write!(f, "{}", array.value(index))), - UInt64 => Box::new(|f, index| write!(f, "{}", array.value(index))), - Float16 => unreachable!(), - Float32 => Box::new(|f, index| write!(f, "{}", array.value(index))), - Float64 => Box::new(|f, index| write!(f, "{}", array.value(index))), - Date32 => { - dyn_primitive!(array, i32, temporal_conversions::date32_to_date) - } - Date64 => { - dyn_primitive!(array, i64, temporal_conversions::date64_to_date) - } - Time32(TimeUnit::Second) => { - dyn_primitive!(array, i32, temporal_conversions::time32s_to_time) - } - Time32(TimeUnit::Millisecond) => { - dyn_primitive!(array, i32, temporal_conversions::time32ms_to_time) - } - Time32(_) => unreachable!(), // remaining are not valid - Time64(TimeUnit::Microsecond) => { - dyn_primitive!(array, i64, temporal_conversions::time64us_to_time) - } - Time64(TimeUnit::Nanosecond) => { - dyn_primitive!(array, i64, temporal_conversions::time64ns_to_time) - } - Time64(_) => unreachable!(), // remaining are not valid - Timestamp(time_unit, tz) => { - if let Some(tz) = tz { - let timezone = temporal_conversions::parse_offset(tz); - match timezone { - Ok(timezone) => { - dyn_primitive!(array, i64, |time| { - temporal_conversions::timestamp_to_datetime(time, *time_unit, &timezone) - }) - } - #[cfg(feature = "chrono-tz")] - Err(_) => { - let timezone = temporal_conversions::parse_offset_tz(tz); - match timezone { - Ok(timezone) => dyn_primitive!(array, i64, |time| { - temporal_conversions::timestamp_to_datetime( - time, *time_unit, &timezone, - ) - }), - Err(_) => { - let tz = tz.clone(); - Box::new(move |f, index| { - write!(f, "{} ({})", array.value(index), tz) - }) - } - } - } - #[cfg(not(feature = "chrono-tz"))] - _ => { - let tz = tz.clone(); - Box::new(move |f, index| write!(f, "{} ({})", array.value(index), tz)) - } - } - } else { - dyn_primitive!(array, i64, |time| { - temporal_conversions::timestamp_to_naive_datetime(time, *time_unit) - }) - } - } - Interval(IntervalUnit::YearMonth) => { - dyn_primitive!(array, i32, |x| format!("{x}m")) - } - Interval(IntervalUnit::DayTime) => { - dyn_primitive!(array, days_ms, |x: days_ms| format!( - "{}d{}ms", - x.days(), - x.milliseconds() - )) - } - Interval(IntervalUnit::MonthDayNano) => { - dyn_primitive!(array, months_days_ns, |x: months_days_ns| format!( - "{}m{}d{}ns", - x.months(), - x.days(), - x.ns() - )) - } - Duration(TimeUnit::Second) => dyn_primitive!(array, i64, |x| format!("{x}s")), - Duration(TimeUnit::Millisecond) => dyn_primitive!(array, i64, |x| format!("{x}ms")), - Duration(TimeUnit::Microsecond) => dyn_primitive!(array, i64, |x| format!("{x}us")), - Duration(TimeUnit::Nanosecond) => dyn_primitive!(array, i64, |x| format!("{x}ns")), - Decimal(_, scale) => { - // The number 999.99 has a precision of 5 and scale of 2 - let scale = *scale as u32; - let factor = 10i128.pow(scale); - let display = move |x: i128| { - let base = x / factor; - let decimals = (x - base * factor).abs(); - format!("{base}.{decimals}") - }; - dyn_primitive!(array, i128, display) - } - Decimal256(_, scale) => { - let scale = *scale as u32; - let factor = (ethnum::I256::ONE * 10).pow(scale); - let display = move |x: i256| { - let base = x.0 / factor; - let decimals = (x.0 - base * factor).abs(); - format!("{base}.{decimals}") - }; - dyn_primitive!(array, i256, display) - } - _ => unreachable!(), - } -} - -impl Debug for PrimitiveArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = get_write_value(self); - - write!(f, "{:?}", self.data_type())?; - write_vec(f, &*writer, self.validity(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/primitive/from_natural.rs b/src/common/arrow/src/arrow/array/primitive/from_natural.rs deleted file mode 100644 index a8528351d1ae..000000000000 --- a/src/common/arrow/src/arrow/array/primitive/from_natural.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; - -use super::MutablePrimitiveArray; -use super::PrimitiveArray; -use crate::arrow::types::NativeType; - -impl]>> From

for PrimitiveArray { - fn from(slice: P) -> Self { - MutablePrimitiveArray::::from(slice).into() - } -} - -impl>> FromIterator for PrimitiveArray { - fn from_iter>(iter: I) -> Self { - MutablePrimitiveArray::::from_iter(iter).into() - } -} diff --git a/src/common/arrow/src/arrow/array/primitive/iterator.rs b/src/common/arrow/src/arrow/array/primitive/iterator.rs deleted file mode 100644 index 598564b96b23..000000000000 --- a/src/common/arrow/src/arrow/array/primitive/iterator.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::MutablePrimitiveArray; -use super::PrimitiveArray; -use crate::arrow::array::MutableArray; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::IntoIter as BitmapIntoIter; -use crate::arrow::buffer::IntoIter; -use crate::arrow::types::NativeType; - -impl IntoIterator for PrimitiveArray { - type Item = Option; - type IntoIter = ZipValidity, BitmapIntoIter>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - let (_, values, validity) = self.into_inner(); - let values = values.into_iter(); - let validity = - validity.and_then(|validity| (validity.unset_bits() > 0).then(|| validity.into_iter())); - ZipValidity::new(values, validity) - } -} - -impl<'a, T: NativeType> IntoIterator for &'a PrimitiveArray { - type Item = Option<&'a T>; - type IntoIter = ZipValidity<&'a T, std::slice::Iter<'a, T>, BitmapIter<'a>>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a, T: NativeType> MutablePrimitiveArray { - /// Returns an iterator over `Option` - #[inline] - pub fn iter(&'a self) -> ZipValidity<&'a T, std::slice::Iter<'a, T>, BitmapIter<'a>> { - ZipValidity::new( - self.values().iter(), - self.validity().as_ref().map(|x| x.iter()), - ) - } - - /// Returns an iterator of `T` - #[inline] - pub fn values_iter(&'a self) -> std::slice::Iter<'a, T> { - self.values().iter() - } -} diff --git a/src/common/arrow/src/arrow/array/primitive/mod.rs b/src/common/arrow/src/arrow/array/primitive/mod.rs deleted file mode 100644 index f9fdc0176e71..000000000000 --- a/src/common/arrow/src/arrow/array/primitive/mod.rs +++ /dev/null @@ -1,541 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use either::Either; - -use super::Array; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::*; -use crate::arrow::error::Error; -use crate::arrow::trusted_len::TrustedLen; -use crate::arrow::types::days_ms; -use crate::arrow::types::f16; -use crate::arrow::types::i256; -use crate::arrow::types::months_days_ns; -use crate::arrow::types::NativeType; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod from_natural; -mod iterator; -mod mutable; -pub use mutable::*; - -/// A [`PrimitiveArray`] is Arrow's semantically equivalent of an immutable `Vec>` where -/// T is [`NativeType`] (e.g. [`i32`]). It implements [`Array`]. -/// -/// One way to think about a [`PrimitiveArray`] is `(DataType, Arc>, Option>>)` -/// where: -/// * the first item is the array's logical type -/// * the second is the immutable values -/// * the third is the immutable validity (whether a value is null or not as a bitmap). -/// -/// The size of this struct is `O(1)`, as all data is stored behind an [`std::sync::Arc`]. -/// # Example -/// ``` -/// use arrow2::array::PrimitiveArray; -/// use arrow2::bitmap::Bitmap; -/// use arrow2::buffer::Buffer; -/// -/// let array = PrimitiveArray::from([Some(1i32), None, Some(10)]); -/// assert_eq!(array.value(0), 1); -/// assert_eq!(array.iter().collect::>(), vec![ -/// Some(&1i32), -/// None, -/// Some(&10) -/// ]); -/// assert_eq!(array.values_iter().copied().collect::>(), vec![ -/// 1, 0, 10 -/// ]); -/// // the underlying representation -/// assert_eq!(array.values(), &Buffer::from(vec![1i32, 0, 10])); -/// assert_eq!(array.validity(), Some(&Bitmap::from([true, false, true]))); -/// ``` -#[derive(Clone)] -pub struct PrimitiveArray { - data_type: DataType, - values: Buffer, - validity: Option, -} - -pub(super) fn check( - data_type: &DataType, - values: &[T], - validity_len: Option, -) -> Result<(), Error> { - if validity_len.map_or(false, |len| len != values.len()) { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - - if data_type.to_physical_type() != PhysicalType::Primitive(T::PRIMITIVE) { - return Err(Error::oos( - "PrimitiveArray can only be initialized with a DataType whose physical type is Primitive", - )); - } - Ok(()) -} - -impl PrimitiveArray { - /// The canonical method to create a [`PrimitiveArray`] out of its internal components. - /// # Implementation - /// This function is `O(1)`. - /// - /// # Errors - /// This function errors iff: - /// * The validity is not `None` and its length is different from `values`'s length - /// * The `data_type`'s [`PhysicalType`] is not equal to [`PhysicalType::Primitive(T::PRIMITIVE)`] - pub fn try_new( - data_type: DataType, - values: Buffer, - validity: Option, - ) -> Result { - check(&data_type, &values, validity.as_ref().map(|v| v.len()))?; - Ok(Self { - data_type, - values, - validity, - }) - } - - /// Returns a new [`PrimitiveArray`] with a different logical type. - /// - /// This function is useful to assign a different [`DataType`] to the array. - /// Used to change the arrays' logical type (see example). - /// # Example - /// ``` - /// use arrow2::array::Int32Array; - /// use arrow2::datatypes::DataType; - /// - /// let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Date32); - /// assert_eq!( - /// format!("{:?}", array), - /// "Date32[1970-01-02, None, 1970-01-03]" - /// ); - /// ``` - /// # Panics - /// Panics iff the `data_type`'s [`PhysicalType`] is not equal to [`PhysicalType::Primitive(T::PRIMITIVE)`] - #[inline] - #[must_use] - pub fn to(self, data_type: DataType) -> Self { - check( - &data_type, - &self.values, - self.validity.as_ref().map(|v| v.len()), - ) - .unwrap(); - Self { - data_type, - values: self.values, - validity: self.validity, - } - } - - /// Creates a (non-null) [`PrimitiveArray`] from a vector of values. - /// This function is `O(1)`. - /// # Examples - /// ``` - /// use arrow2::array::PrimitiveArray; - /// - /// let array = PrimitiveArray::from_vec(vec![1, 2, 3]); - /// assert_eq!(format!("{:?}", array), "Int32[1, 2, 3]"); - /// ``` - pub fn from_vec(values: Vec) -> Self { - Self::new(T::PRIMITIVE.into(), values.into(), None) - } - - /// Returns an iterator over the values and validity, `Option<&T>`. - #[inline] - pub fn iter(&self) -> ZipValidity<&T, std::slice::Iter, BitmapIter> { - ZipValidity::new_with_validity(self.values().iter(), self.validity()) - } - - /// Returns an iterator of the values, `&T`, ignoring the arrays' validity. - #[inline] - pub fn values_iter(&self) -> std::slice::Iter { - self.values().iter() - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.values.len() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The values [`Buffer`]. - /// Values on null slots are undetermined (they can be anything). - #[inline] - pub fn values(&self) -> &Buffer { - &self.values - } - - /// Returns the optional validity. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - /// Returns the arrays' [`DataType`]. - #[inline] - pub fn data_type(&self) -> &DataType { - &self.data_type - } - - /// Returns the value at slot `i`. - /// - /// Equivalent to `self.values()[i]`. The value of a null slot is undetermined (it can be anything). - /// # Panic - /// This function panics iff `i >= self.len`. - #[inline] - pub fn value(&self, i: usize) -> T { - assert!(i < self.values.len(), "i is out of bounds"); - self.values[i] - } - - /// Returns the value at index `i`. - /// The value on null slots is undetermined (it can be anything). - /// # Safety - /// Caller must be sure that `i < self.len()` - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> T { - *self.values.get_unchecked(i) - } - - /// Returns the element at index `i` or `None` if it is null - /// # Panics - /// iff `i >= self.len()` - #[inline] - pub fn get(&self, i: usize) -> Option { - if !self.is_null(i) { - // soundness: Array::is_null panics if i >= self.len - unsafe { Some(self.value_unchecked(i)) } - } else { - None - } - } - - /// Slices this [`PrimitiveArray`] by an offset and length. - /// # Implementation - /// This operation is `O(1)`. - #[inline] - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "offset + length may not exceed length of array" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Slices this [`PrimitiveArray`] by an offset and length. - /// # Implementation - /// This operation is `O(1)`. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - #[inline] - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.values.slice_unchecked(offset, length); - } - - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); - - /// Returns this [`PrimitiveArray`] with new values. - /// # Panics - /// This function panics iff `values.len() != self.len()`. - #[must_use] - pub fn with_values(mut self, values: Buffer) -> Self { - self.set_values(values); - self - } - - /// Update the values of this [`PrimitiveArray`]. - /// # Panics - /// This function panics iff `values.len() != self.len()`. - pub fn set_values(&mut self, values: Buffer) { - assert_eq!( - values.len(), - self.len(), - "values' length must be equal to this arrays' length" - ); - self.values = values; - } - - /// Applies a function `f` to the validity of this array. - /// - /// This is an API to leverage clone-on-write - /// # Panics - /// This function panics if the function `f` modifies the length of the [`Bitmap`]. - pub fn apply_validity Bitmap>(&mut self, f: F) { - if let Some(validity) = std::mem::take(&mut self.validity) { - self.set_validity(Some(f(validity))) - } - } - - /// Returns an option of a mutable reference to the values of this [`PrimitiveArray`]. - pub fn get_mut_values(&mut self) -> Option<&mut [T]> { - self.values.get_mut_slice() - } - - /// Returns its internal representation - #[must_use] - pub fn into_inner(self) -> (DataType, Buffer, Option) { - let Self { - data_type, - values, - validity, - } = self; - (data_type, values, validity) - } - - /// Creates a `[PrimitiveArray]` from its internal representation. - /// This is the inverted from `[PrimitiveArray::into_inner]` - pub fn from_inner( - data_type: DataType, - values: Buffer, - validity: Option, - ) -> Result { - check(&data_type, &values, validity.as_ref().map(|v| v.len()))?; - Ok(unsafe { Self::from_inner_unchecked(data_type, values, validity) }) - } - - /// Creates a `[PrimitiveArray]` from its internal representation. - /// This is the inverted from `[PrimitiveArray::into_inner]` - /// - /// # Safety - /// Callers must ensure all invariants of this struct are upheld. - pub unsafe fn from_inner_unchecked( - data_type: DataType, - values: Buffer, - validity: Option, - ) -> Self { - Self { - data_type, - values, - validity, - } - } - - /// Try to convert this [`PrimitiveArray`] to a [`MutablePrimitiveArray`] via copy-on-write semantics. - /// - /// A [`PrimitiveArray`] is backed by a [`Buffer`] and [`Bitmap`] which are essentially `Arc>`. - /// This function returns a [`MutablePrimitiveArray`] (via [`std::sync::Arc::get_mut`]) iff both values - /// and validity have not been cloned / are unique references to their underlying vectors. - /// - /// This function is primarily used to re-use memory regions. - #[must_use] - pub fn into_mut(self) -> Either> { - use Either::*; - - if let Some(bitmap) = self.validity { - match bitmap.into_mut() { - Left(bitmap) => Left(PrimitiveArray::new( - self.data_type, - self.values, - Some(bitmap), - )), - Right(mutable_bitmap) => match self.values.into_mut() { - Right(values) => Right( - MutablePrimitiveArray::try_new( - self.data_type, - values, - Some(mutable_bitmap), - ) - .unwrap(), - ), - Left(values) => Left(PrimitiveArray::new( - self.data_type, - values, - Some(mutable_bitmap.into()), - )), - }, - } - } else { - match self.values.into_mut() { - Right(values) => { - Right(MutablePrimitiveArray::try_new(self.data_type, values, None).unwrap()) - } - Left(values) => Left(PrimitiveArray::new(self.data_type, values, None)), - } - } - } - - /// Returns a new empty (zero-length) [`PrimitiveArray`]. - pub fn new_empty(data_type: DataType) -> Self { - Self::new(data_type, Buffer::new(), None) - } - - /// Returns a new [`PrimitiveArray`] where all slots are null / `None`. - #[inline] - pub fn new_null(data_type: DataType, length: usize) -> Self { - Self::new( - data_type, - vec![T::default(); length].into(), - Some(Bitmap::new_zeroed(length)), - ) - } - - /// Creates a (non-null) [`PrimitiveArray`] from an iterator of values. - /// # Implementation - /// This does not assume that the iterator has a known length. - pub fn from_values>(iter: I) -> Self { - Self::new(T::PRIMITIVE.into(), Vec::::from_iter(iter).into(), None) - } - - /// Creates a (non-null) [`PrimitiveArray`] from a slice of values. - /// # Implementation - /// This is essentially a memcopy and is thus `O(N)` - pub fn from_slice>(slice: P) -> Self { - Self::new( - T::PRIMITIVE.into(), - Vec::::from(slice.as_ref()).into(), - None, - ) - } - - /// Creates a (non-null) [`PrimitiveArray`] from a [`TrustedLen`] of values. - /// # Implementation - /// This does not assume that the iterator has a known length. - pub fn from_trusted_len_values_iter>(iter: I) -> Self { - MutablePrimitiveArray::::from_trusted_len_values_iter(iter).into() - } - - /// Creates a new [`PrimitiveArray`] from an iterator over values - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - pub unsafe fn from_trusted_len_values_iter_unchecked>(iter: I) -> Self { - MutablePrimitiveArray::::from_trusted_len_values_iter_unchecked(iter).into() - } - - /// Creates a [`PrimitiveArray`] from a [`TrustedLen`] of optional values. - pub fn from_trusted_len_iter>>(iter: I) -> Self { - MutablePrimitiveArray::::from_trusted_len_iter(iter).into() - } - - /// Creates a [`PrimitiveArray`] from an iterator of optional values. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - pub unsafe fn from_trusted_len_iter_unchecked>>(iter: I) -> Self { - MutablePrimitiveArray::::from_trusted_len_iter_unchecked(iter).into() - } - - /// Alias for `Self::try_new(..).unwrap()`. - /// # Panics - /// This function errors iff: - /// * The validity is not `None` and its length is different from `values`'s length - /// * The `data_type`'s [`PhysicalType`] is not equal to [`PhysicalType::Primitive`]. - pub fn new(data_type: DataType, values: Buffer, validity: Option) -> Self { - Self::try_new(data_type, values, validity).unwrap() - } -} - -impl Array for PrimitiveArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} - -/// A type definition [`PrimitiveArray`] for `i8` -pub type Int8Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `i16` -pub type Int16Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `i32` -pub type Int32Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `i64` -pub type Int64Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `i128` -pub type Int128Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `i256` -pub type Int256Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for [`days_ms`] -pub type DaysMsArray = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for [`months_days_ns`] -pub type MonthsDaysNsArray = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `f16` -pub type Float16Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `f32` -pub type Float32Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `f64` -pub type Float64Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `u8` -pub type UInt8Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `u16` -pub type UInt16Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `u32` -pub type UInt32Array = PrimitiveArray; -/// A type definition [`PrimitiveArray`] for `u64` -pub type UInt64Array = PrimitiveArray; - -/// A type definition [`MutablePrimitiveArray`] for `i8` -pub type Int8Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `i16` -pub type Int16Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `i32` -pub type Int32Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `i64` -pub type Int64Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `i128` -pub type Int128Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `i256` -pub type Int256Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for [`days_ms`] -pub type DaysMsVec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for [`months_days_ns`] -pub type MonthsDaysNsVec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `f16` -pub type Float16Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `f32` -pub type Float32Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `f64` -pub type Float64Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `u8` -pub type UInt8Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `u16` -pub type UInt16Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `u32` -pub type UInt32Vec = MutablePrimitiveArray; -/// A type definition [`MutablePrimitiveArray`] for `u64` -pub type UInt64Vec = MutablePrimitiveArray; - -impl Default for PrimitiveArray { - fn default() -> Self { - PrimitiveArray::new(T::PRIMITIVE.into(), Default::default(), None) - } -} diff --git a/src/common/arrow/src/arrow/array/primitive/mutable.rs b/src/common/arrow/src/arrow/array/primitive/mutable.rs deleted file mode 100644 index defc9c38319b..000000000000 --- a/src/common/arrow/src/arrow/array/primitive/mutable.rs +++ /dev/null @@ -1,682 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; -use std::sync::Arc; - -use super::check; -use super::PrimitiveArray; -use crate::arrow::array::physical_binary::extend_validity; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::trusted_len::TrustedLen; -use crate::arrow::types::NativeType; - -/// The Arrow's equivalent to `Vec>` where `T` is byte-size (e.g. `i32`). -/// Converting a [`MutablePrimitiveArray`] into a [`PrimitiveArray`] is `O(1)`. -#[derive(Debug, Clone)] -pub struct MutablePrimitiveArray { - data_type: DataType, - values: Vec, - validity: Option, -} - -impl From> for PrimitiveArray { - fn from(other: MutablePrimitiveArray) -> Self { - let validity = other.validity.and_then(|x| { - let bitmap: Bitmap = x.into(); - if bitmap.unset_bits() == 0 { - None - } else { - Some(bitmap) - } - }); - - PrimitiveArray::::new(other.data_type, other.values.into(), validity) - } -} - -impl]>> From

for MutablePrimitiveArray { - fn from(slice: P) -> Self { - Self::from_trusted_len_iter(slice.as_ref().iter().map(|x| x.as_ref())) - } -} - -impl MutablePrimitiveArray { - /// Creates a new empty [`MutablePrimitiveArray`]. - pub fn new() -> Self { - Self::with_capacity(0) - } - - /// Creates a new [`MutablePrimitiveArray`] with a capacity. - pub fn with_capacity(capacity: usize) -> Self { - Self::with_capacity_from(capacity, T::PRIMITIVE.into()) - } - - /// The canonical method to create a [`MutablePrimitiveArray`] out of its internal components. - /// # Implementation - /// This function is `O(1)`. - /// - /// # Errors - /// This function errors iff: - /// * The validity is not `None` and its length is different from `values`'s length - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to [`crate::arrow::datatypes::PhysicalType::Primitive(T::PRIMITIVE)`] - pub fn try_new( - data_type: DataType, - values: Vec, - validity: Option, - ) -> Result { - check(&data_type, &values, validity.as_ref().map(|x| x.len()))?; - Ok(Self { - data_type, - values, - validity, - }) - } - - /// Extract the low-end APIs from the [`MutablePrimitiveArray`]. - pub fn into_inner(self) -> (DataType, Vec, Option) { - (self.data_type, self.values, self.validity) - } - - /// Applies a function `f` to the values of this array, cloning the values - /// iff they are being shared with others - /// - /// This is an API to use clone-on-write - /// # Implementation - /// This function is `O(f)` if the data is not being shared, and `O(N) + O(f)` - /// if it is being shared (since it results in a `O(N)` memcopy). - /// # Panics - /// This function panics iff `f` panics - pub fn apply_values(&mut self, f: F) { - f(&mut self.values); - } -} - -impl Default for MutablePrimitiveArray { - fn default() -> Self { - Self::new() - } -} - -impl From for MutablePrimitiveArray { - fn from(data_type: DataType) -> Self { - assert!(data_type.to_physical_type().eq_primitive(T::PRIMITIVE)); - Self { - data_type, - values: Vec::::new(), - validity: None, - } - } -} - -impl MutablePrimitiveArray { - /// Creates a new [`MutablePrimitiveArray`] from a capacity and [`DataType`]. - pub fn with_capacity_from(capacity: usize, data_type: DataType) -> Self { - assert!(data_type.to_physical_type().eq_primitive(T::PRIMITIVE)); - Self { - data_type, - values: Vec::::with_capacity(capacity), - validity: None, - } - } - - /// Reserves `additional` entries. - pub fn reserve(&mut self, additional: usize) { - self.values.reserve(additional); - if let Some(x) = self.validity.as_mut() { - x.reserve(additional) - } - } - - /// Adds a new value to the array. - #[inline] - pub fn push(&mut self, value: Option) { - match value { - Some(value) => { - self.values.push(value); - match &mut self.validity { - Some(validity) => validity.push(true), - None => {} - } - } - None => { - self.values.push(T::default()); - match &mut self.validity { - Some(validity) => validity.push(false), - None => { - self.init_validity(); - } - } - } - } - } - - /// Pop a value from the array. - /// Note if the values is empty, this method will return None. - pub fn pop(&mut self) -> Option { - let value = self.values.pop()?; - self.validity - .as_mut() - .map(|x| x.pop()?.then_some(value)) - .unwrap_or_else(|| Some(value)) - } - - /// Extends the [`MutablePrimitiveArray`] with a constant - #[inline] - pub fn extend_constant(&mut self, additional: usize, value: Option) { - if let Some(value) = value { - self.values.resize(self.values.len() + additional, value); - if let Some(validity) = &mut self.validity { - validity.extend_constant(additional, true) - } - } else { - if let Some(validity) = &mut self.validity { - validity.extend_constant(additional, false) - } else { - let mut validity = MutableBitmap::with_capacity(self.values.capacity()); - validity.extend_constant(self.len(), true); - validity.extend_constant(additional, false); - self.validity = Some(validity) - } - self.values - .resize(self.values.len() + additional, T::default()); - } - } - - /// Extends the [`MutablePrimitiveArray`] from an iterator of trusted len. - #[inline] - pub fn extend_trusted_len(&mut self, iterator: I) - where - P: std::borrow::Borrow, - I: TrustedLen>, - { - unsafe { self.extend_trusted_len_unchecked(iterator) } - } - - /// Extends the [`MutablePrimitiveArray`] from an iterator of trusted len. - /// # Safety - /// The iterator must be trusted len. - #[inline] - pub unsafe fn extend_trusted_len_unchecked(&mut self, iterator: I) - where - P: std::borrow::Borrow, - I: Iterator>, - { - if let Some(validity) = self.validity.as_mut() { - extend_trusted_len_unzip(iterator, validity, &mut self.values) - } else { - let mut validity = MutableBitmap::new(); - validity.extend_constant(self.len(), true); - extend_trusted_len_unzip(iterator, &mut validity, &mut self.values); - self.validity = Some(validity); - } - } - /// Extends the [`MutablePrimitiveArray`] from an iterator of values of trusted len. - /// This differs from `extend_trusted_len` which accepts in iterator of optional values. - #[inline] - pub fn extend_trusted_len_values(&mut self, iterator: I) - where I: TrustedLen { - unsafe { self.extend_trusted_len_values_unchecked(iterator) } - } - - /// Extends the [`MutablePrimitiveArray`] from an iterator of values of trusted len. - /// This differs from `extend_trusted_len_unchecked` which accepts in iterator of optional values. - /// # Safety - /// The iterator must be trusted len. - #[inline] - pub unsafe fn extend_trusted_len_values_unchecked(&mut self, iterator: I) - where I: Iterator { - self.values.extend(iterator); - self.update_all_valid(); - } - - #[inline] - /// Extends the [`MutablePrimitiveArray`] from a slice - pub fn extend_from_slice(&mut self, items: &[T]) { - self.values.extend_from_slice(items); - self.update_all_valid(); - } - - fn update_all_valid(&mut self) { - // get len before mutable borrow - let len = self.len(); - if let Some(validity) = self.validity.as_mut() { - validity.extend_constant(len - validity.len(), true); - } - } - - fn init_validity(&mut self) { - let mut validity = MutableBitmap::with_capacity(self.values.capacity()); - validity.extend_constant(self.len(), true); - validity.set(self.len() - 1, false); - self.validity = Some(validity) - } - - /// Changes the arrays' [`DataType`], returning a new [`MutablePrimitiveArray`]. - /// Use to change the logical type without changing the corresponding physical Type. - /// # Implementation - /// This operation is `O(1)`. - #[inline] - pub fn to(self, data_type: DataType) -> Self { - Self::try_new(data_type, self.values, self.validity).unwrap() - } - - /// Converts itself into an [`Array`]. - pub fn into_arc(self) -> Arc { - let a: PrimitiveArray = self.into(); - Arc::new(a) - } - - /// Shrinks the capacity of the [`MutablePrimitiveArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - if let Some(validity) = &mut self.validity { - validity.shrink_to_fit() - } - } - - /// Returns the capacity of this [`MutablePrimitiveArray`]. - pub fn capacity(&self) -> usize { - self.values.capacity() - } -} - -/// Accessors -impl MutablePrimitiveArray { - /// Returns its values. - pub fn values(&self) -> &Vec { - &self.values - } - - /// Returns a mutable slice of values. - pub fn values_mut_slice(&mut self) -> &mut [T] { - self.values.as_mut_slice() - } -} - -/// Setters -impl MutablePrimitiveArray { - /// Sets position `index` to `value`. - /// Note that if it is the first time a null appears in this array, - /// this initializes the validity bitmap (`O(N)`). - /// # Panic - /// Panics iff index is larger than or equal to `self.len()`. - pub fn set(&mut self, index: usize, value: Option) { - assert!(index < self.len()); - // Safety: - // we just checked bounds - unsafe { self.set_unchecked(index, value) } - } - - /// Sets position `index` to `value`. - /// Note that if it is the first time a null appears in this array, - /// this initializes the validity bitmap (`O(N)`). - /// # Safety - /// Caller must ensure `index < self.len()` - pub unsafe fn set_unchecked(&mut self, index: usize, value: Option) { - *self.values.get_unchecked_mut(index) = value.unwrap_or_default(); - - if value.is_none() && self.validity.is_none() { - // When the validity is None, all elements so far are valid. When one of the elements is set to null, - // the validity must be initialized. - let mut validity = MutableBitmap::new(); - validity.extend_constant(self.len(), true); - self.validity = Some(validity); - } - if let Some(x) = self.validity.as_mut() { - x.set_unchecked(index, value.is_some()) - } - } - - /// Sets the validity. - /// # Panic - /// Panics iff the validity's len is not equal to the existing values' length. - pub fn set_validity(&mut self, validity: Option) { - if let Some(validity) = &validity { - assert_eq!(self.values.len(), validity.len()) - } - self.validity = validity; - } - - /// Sets values. - /// # Panic - /// Panics iff the values' length is not equal to the existing validity's len. - pub fn set_values(&mut self, values: Vec) { - assert_eq!(values.len(), self.values.len()); - self.values = values; - } -} - -impl Extend> for MutablePrimitiveArray { - fn extend>>(&mut self, iter: I) { - let iter = iter.into_iter(); - self.reserve(iter.size_hint().0); - iter.for_each(|x| self.push(x)) - } -} - -impl TryExtend> for MutablePrimitiveArray { - /// This is infalible and is implemented for consistency with all other types - fn try_extend>>(&mut self, iter: I) -> Result<(), Error> { - self.extend(iter); - Ok(()) - } -} - -impl TryPush> for MutablePrimitiveArray { - /// This is infalible and is implemented for consistency with all other types - fn try_push(&mut self, item: Option) -> Result<(), Error> { - self.push(item); - Ok(()) - } -} - -impl MutableArray for MutablePrimitiveArray { - fn len(&self) -> usize { - self.values.len() - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - PrimitiveArray::new( - self.data_type.clone(), - std::mem::take(&mut self.values).into(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .boxed() - } - - fn as_arc(&mut self) -> Arc { - PrimitiveArray::new( - self.data_type.clone(), - std::mem::take(&mut self.values).into(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .arced() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn push_null(&mut self) { - self.push(None) - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl MutablePrimitiveArray { - /// Creates a [`MutablePrimitiveArray`] from a slice of values. - pub fn from_slice>(slice: P) -> Self { - Self::from_trusted_len_values_iter(slice.as_ref().iter().copied()) - } - - /// Creates a [`MutablePrimitiveArray`] from an iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: std::borrow::Borrow, - I: Iterator>, - { - let (validity, values) = trusted_len_unzip(iterator); - - Self { - data_type: T::PRIMITIVE.into(), - values, - validity, - } - } - - /// Creates a [`MutablePrimitiveArray`] from a [`TrustedLen`]. - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: std::borrow::Borrow, - I: TrustedLen>, - { - unsafe { Self::from_trusted_len_iter_unchecked(iterator) } - } - - /// Creates a [`MutablePrimitiveArray`] from an fallible iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn try_from_trusted_len_iter_unchecked( - iter: I, - ) -> std::result::Result - where - P: std::borrow::Borrow, - I: IntoIterator, E>>, - { - let iterator = iter.into_iter(); - - let (validity, values) = try_trusted_len_unzip(iterator)?; - - Ok(Self { - data_type: T::PRIMITIVE.into(), - values, - validity, - }) - } - - /// Creates a [`MutablePrimitiveArray`] from an fallible iterator of trusted length. - #[inline] - pub fn try_from_trusted_len_iter(iterator: I) -> std::result::Result - where - P: std::borrow::Borrow, - I: TrustedLen, E>>, - { - unsafe { Self::try_from_trusted_len_iter_unchecked(iterator) } - } - - /// Creates a new [`MutablePrimitiveArray`] out an iterator over values - pub fn from_trusted_len_values_iter>(iter: I) -> Self { - Self { - data_type: T::PRIMITIVE.into(), - values: iter.collect(), - validity: None, - } - } - - /// Creates a (non-null) [`MutablePrimitiveArray`] from a vector of values. - /// This does not have memcopy and is the fastest way to create a [`PrimitiveArray`]. - pub fn from_vec(values: Vec) -> Self { - Self::try_new(T::PRIMITIVE.into(), values, None).unwrap() - } - - /// Creates a new [`MutablePrimitiveArray`] from an iterator over values - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - pub unsafe fn from_trusted_len_values_iter_unchecked>(iter: I) -> Self { - Self { - data_type: T::PRIMITIVE.into(), - values: iter.collect(), - validity: None, - } - } -} - -impl>> FromIterator - for MutablePrimitiveArray -{ - fn from_iter>(iter: I) -> Self { - let iter = iter.into_iter(); - let (lower, _) = iter.size_hint(); - - let mut validity = MutableBitmap::with_capacity(lower); - - let values: Vec = iter - .map(|item| { - if let Some(a) = item.borrow() { - validity.push(true); - *a - } else { - validity.push(false); - T::default() - } - }) - .collect(); - - let validity = Some(validity); - - Self { - data_type: T::PRIMITIVE.into(), - values, - validity, - } - } -} - -/// Extends a [`MutableBitmap`] and a [`Vec`] from an iterator of `Option`. -/// The first buffer corresponds to a bitmap buffer, the second one -/// corresponds to a values buffer. -/// # Safety -/// The caller must ensure that `iterator` is `TrustedLen`. -#[inline] -pub(crate) unsafe fn extend_trusted_len_unzip( - iterator: I, - validity: &mut MutableBitmap, - buffer: &mut Vec, -) where - T: NativeType, - P: std::borrow::Borrow, - I: Iterator>, -{ - let (_, upper) = iterator.size_hint(); - let additional = upper.expect("trusted_len_unzip requires an upper limit"); - - validity.reserve(additional); - let values = iterator.map(|item| { - if let Some(item) = item { - validity.push_unchecked(true); - *item.borrow() - } else { - validity.push_unchecked(false); - T::default() - } - }); - buffer.extend(values); -} - -/// Creates a [`MutableBitmap`] and a [`Vec`] from an iterator of `Option`. -/// The first buffer corresponds to a bitmap buffer, the second one -/// corresponds to a values buffer. -/// # Safety -/// The caller must ensure that `iterator` is `TrustedLen`. -#[inline] -pub(crate) unsafe fn trusted_len_unzip(iterator: I) -> (Option, Vec) -where - T: NativeType, - P: std::borrow::Borrow, - I: Iterator>, -{ - let mut validity = MutableBitmap::new(); - let mut buffer = Vec::::new(); - - extend_trusted_len_unzip(iterator, &mut validity, &mut buffer); - - let validity = Some(validity); - - (validity, buffer) -} - -/// # Safety -/// The caller must ensure that `iterator` is `TrustedLen`. -#[inline] -pub(crate) unsafe fn try_trusted_len_unzip( - iterator: I, -) -> std::result::Result<(Option, Vec), E> -where - T: NativeType, - P: std::borrow::Borrow, - I: Iterator, E>>, -{ - let (_, upper) = iterator.size_hint(); - let len = upper.expect("trusted_len_unzip requires an upper limit"); - - let mut null = MutableBitmap::with_capacity(len); - let mut buffer = Vec::::with_capacity(len); - - let mut dst = buffer.as_mut_ptr(); - for item in iterator { - let item = if let Some(item) = item? { - null.push(true); - *item.borrow() - } else { - null.push(false); - T::default() - }; - std::ptr::write(dst, item); - dst = dst.add(1); - } - assert_eq!( - dst.offset_from(buffer.as_ptr()) as usize, - len, - "Trusted iterator length was not accurately reported" - ); - buffer.set_len(len); - null.set_len(len); - - let validity = Some(null); - - Ok((validity, buffer)) -} - -impl PartialEq for MutablePrimitiveArray { - fn eq(&self, other: &Self) -> bool { - self.iter().eq(other.iter()) - } -} - -impl TryExtendFromSelf for MutablePrimitiveArray { - fn try_extend_from_self(&mut self, other: &Self) -> Result<(), Error> { - extend_validity(self.len(), &mut self.validity, &other.validity); - - let slice = other.values.as_slice(); - self.values.extend_from_slice(slice); - Ok(()) - } -} diff --git a/src/common/arrow/src/arrow/array/specification.rs b/src/common/arrow/src/arrow/array/specification.rs deleted file mode 100644 index 21fd72437fbc..000000000000 --- a/src/common/arrow/src/arrow/array/specification.rs +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::DictionaryKey; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::offset::OffsetsBuffer; - -/// Helper trait to support `Offset` and `OffsetBuffer` -pub(crate) trait OffsetsContainer { - fn last(&self) -> usize; - fn as_slice(&self) -> &[O]; -} - -impl OffsetsContainer for OffsetsBuffer { - #[inline] - fn last(&self) -> usize { - self.last().to_usize() - } - - #[inline] - fn as_slice(&self) -> &[O] { - self.buffer() - } -} - -impl OffsetsContainer for Offsets { - #[inline] - fn last(&self) -> usize { - self.last().to_usize() - } - - #[inline] - fn as_slice(&self) -> &[O] { - self.as_slice() - } -} - -pub(crate) fn try_check_offsets_bounds>( - offsets: &C, - values_len: usize, -) -> Result<()> { - if offsets.last() > values_len { - Err(Error::oos("offsets must not exceed the values length")) - } else { - Ok(()) - } -} - -/// # Error -/// * any offset is larger or equal to `values_len`. -/// * any slice of `values` between two consecutive pairs from `offsets` is invalid `utf8`, or -pub(crate) fn try_check_utf8>( - offsets: &C, - values: &[u8], -) -> Result<()> { - if offsets.as_slice().len() == 1 { - return Ok(()); - } - - try_check_offsets_bounds(offsets, values.len())?; - - if values.is_ascii() { - Ok(()) - } else { - simdutf8::basic::from_utf8(values)?; - - // offsets can be == values.len() - // find first offset from the end that is smaller - // Example: - // values.len() = 10 - // offsets = [0, 5, 10, 10] - let offsets = offsets.as_slice(); - let last = offsets - .iter() - .enumerate() - .skip(1) - .rev() - .find_map(|(i, offset)| (offset.to_usize() < values.len()).then_some(i)); - - let last = if let Some(last) = last { - // following the example: last = 1 (offset = 5) - last - } else { - // given `l = values.len()`, this branch is hit iff either: - // * `offsets = [0, l, l, ...]`, which was covered by `from_utf8(values)` above - // * `offsets = [0]`, which never happens because offsets.as_slice().len() == 1 is short-circuited above - return Ok(()); - }; - - // truncate to relevant offsets. Note: `=last` because last was computed skipping the first item - // following the example: starts = [0, 5] - let starts = unsafe { offsets.get_unchecked(..=last) }; - - let mut any_invalid = false; - for start in starts { - let start = start.to_usize(); - - // Safety: `try_check_offsets_bounds` just checked for bounds - let b = *unsafe { values.get_unchecked(start) }; - - // A valid code-point iff it does not start with 0b10xxxxxx - // Bit-magic taken from `std::str::is_char_boundary` - if (b as i8) < -0x40 { - any_invalid = true - } - } - if any_invalid { - return Err(Error::oos("Non-valid char boundary detected")); - } - Ok(()) - } -} - -/// Check dictionary indexes without checking usize conversion. -/// # Safety -/// The caller must ensure that `K::as_usize` always succeeds. -pub(crate) unsafe fn check_indexes_unchecked( - keys: &[K], - len: usize, -) -> Result<()> { - let mut invalid = false; - - // this loop is auto-vectorized - keys.iter().for_each(|k| { - if k.as_usize() > len { - invalid = true; - } - }); - - if invalid { - let key = keys.iter().map(|k| k.as_usize()).max().unwrap(); - Err(Error::oos(format!( - "One of the dictionary keys is {key} but it must be < than the length of the dictionary values, which is {len}" - ))) - } else { - Ok(()) - } -} - -pub fn check_indexes(keys: &[K], len: usize) -> Result<()> -where K: std::fmt::Debug + Copy + TryInto { - keys.iter().try_for_each(|key| { - let key: usize = (*key) - .try_into() - .map_err(|_| Error::oos(format!("The dictionary key must fit in a `usize`, but {key:?} does not")))?; - if key >= len { - Err(Error::oos(format!("One of the dictionary keys is {key} but it must be < than the length of the dictionary values, which is {len}"))) - } else { - Ok(()) - } - }) -} - -#[cfg(test)] -mod tests { - use proptest::prelude::*; - - use super::*; - - pub(crate) fn binary_strategy() -> impl Strategy> { - prop::collection::vec(any::(), 1..100) - } - - proptest! { - // a bit expensive, feel free to run it when changing the code above - // #![proptest_config(ProptestConfig::with_cases(100000))] - #[test] - #[cfg_attr(miri, ignore)] // miri and proptest do not work well - fn check_utf8_validation(values in binary_strategy()) { - - for offset in 0..values.len() - 1 { - let offsets = vec![0, offset as i32, values.len() as i32].try_into().unwrap(); - - let mut is_valid = std::str::from_utf8(&values[..offset]).is_ok(); - is_valid &= std::str::from_utf8(&values[offset..]).is_ok(); - - assert_eq!(try_check_utf8::>(&offsets, &values).is_ok(), is_valid) - } - } - } -} diff --git a/src/common/arrow/src/arrow/array/struct_/data.rs b/src/common/arrow/src/arrow/array/struct_/data.rs deleted file mode 100644 index 7027b028dc78..000000000000 --- a/src/common/arrow/src/arrow/array/struct_/data.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::from_data; -use crate::arrow::array::to_data; -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::StructArray; -use crate::arrow::bitmap::Bitmap; - -impl Arrow2Arrow for StructArray { - fn to_data(&self) -> ArrayData { - let data_type = self.data_type.clone().into(); - - let builder = ArrayDataBuilder::new(data_type) - .len(self.len()) - .nulls(self.validity.as_ref().map(|b| b.clone().into())) - .child_data(self.values.iter().map(|x| to_data(x.as_ref())).collect()); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type = data.data_type().clone().into(); - - Self { - data_type, - values: data.child_data().iter().map(from_data).collect(), - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/struct_/fmt.rs b/src/common/arrow/src/arrow/array/struct_/fmt.rs deleted file mode 100644 index 480293936897..000000000000 --- a/src/common/arrow/src/arrow/array/struct_/fmt.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::get_display; -use super::super::fmt::write_map; -use super::super::fmt::write_vec; -use super::StructArray; - -pub fn write_value( - array: &StructArray, - index: usize, - null: &'static str, - f: &mut W, -) -> Result { - let writer = |f: &mut W, _index| { - for (i, (field, column)) in array.fields().iter().zip(array.values()).enumerate() { - if i != 0 { - write!(f, ", ")?; - } - let writer = get_display(column.as_ref(), null); - write!(f, "{}: ", field.name)?; - writer(f, index)?; - } - Ok(()) - }; - - write_map(f, writer, None, 1, null, false) -} - -impl Debug for StructArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, "None", f); - - write!(f, "StructArray")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/struct_/iterator.rs b/src/common/arrow/src/arrow/array/struct_/iterator.rs deleted file mode 100644 index d694ed715f18..000000000000 --- a/src/common/arrow/src/arrow/array/struct_/iterator.rs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::StructArray; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::scalar::new_scalar; -use crate::arrow::scalar::Scalar; -use crate::arrow::trusted_len::TrustedLen; - -pub struct StructValueIter<'a> { - array: &'a StructArray, - index: usize, - end: usize, -} - -impl<'a> StructValueIter<'a> { - #[inline] - pub fn new(array: &'a StructArray) -> Self { - Self { - array, - index: 0, - end: array.len(), - } - } -} - -impl<'a> Iterator for StructValueIter<'a> { - type Item = Vec>; - - #[inline] - fn next(&mut self) -> Option { - if self.index == self.end { - return None; - } - let old = self.index; - self.index += 1; - - // Safety: - // self.end is maximized by the length of the array - Some( - self.array - .values() - .iter() - .map(|v| new_scalar(v.as_ref(), old)) - .collect(), - ) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.end - self.index, Some(self.end - self.index)) - } -} - -unsafe impl<'a> TrustedLen for StructValueIter<'a> {} - -impl<'a> DoubleEndedIterator for StructValueIter<'a> { - #[inline] - fn next_back(&mut self) -> Option { - if self.index == self.end { - None - } else { - self.end -= 1; - - // Safety: - // self.end is maximized by the length of the array - Some( - self.array - .values() - .iter() - .map(|v| new_scalar(v.as_ref(), self.end)) - .collect(), - ) - } - } -} - -type ValuesIter<'a> = StructValueIter<'a>; -type ZipIter<'a> = ZipValidity>, ValuesIter<'a>, BitmapIter<'a>>; - -impl<'a> IntoIterator for &'a StructArray { - type Item = Option>>; - type IntoIter = ZipIter<'a>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> StructArray { - /// Returns an iterator of `Option>` - pub fn iter(&'a self) -> ZipIter<'a> { - ZipValidity::new_with_validity(StructValueIter::new(self), self.validity()) - } - - /// Returns an iterator of `Box` - pub fn values_iter(&'a self) -> ValuesIter<'a> { - StructValueIter::new(self) - } -} diff --git a/src/common/arrow/src/arrow/array/struct_/mod.rs b/src/common/arrow/src/arrow/array/struct_/mod.rs deleted file mode 100644 index b213f401bfd7..000000000000 --- a/src/common/arrow/src/arrow/array/struct_/mod.rs +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::new_empty_array; -use super::new_null_array; -use super::Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod iterator; -mod mutable; -pub use mutable::*; - -/// A [`StructArray`] is a nested [`Array`] with an optional validity representing -/// multiple [`Array`] with the same number of rows. -/// # Example -/// ``` -/// use arrow2::array::*; -/// use arrow2::datatypes::*; -/// let boolean = BooleanArray::from_slice(&[false, false, true, true]).boxed(); -/// let int = Int32Array::from_slice(&[42, 28, 19, 31]).boxed(); -/// -/// let fields = vec![ -/// Field::new("b", DataType::Boolean, false), -/// Field::new("c", DataType::Int32, false), -/// ]; -/// -/// let array = StructArray::new(DataType::Struct(fields), vec![boolean, int], None); -/// ``` -#[derive(Clone)] -pub struct StructArray { - data_type: DataType, - values: Vec>, - validity: Option, -} - -impl StructArray { - /// Returns a new [`StructArray`]. - /// # Errors - /// This function errors iff: - /// * `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::Struct`]. - /// * the children of `data_type` are empty - /// * the values's len is different from children's length - /// * any of the values's data type is different from its corresponding children' data type - /// * any element of values has a different length than the first element - /// * the validity's length is not equal to the length of the first element - pub fn try_new( - data_type: DataType, - values: Vec>, - validity: Option, - ) -> Result { - let fields = Self::try_get_fields(&data_type)?; - if fields.is_empty() { - return Err(Error::oos("A StructArray must contain at least one field")); - } - if fields.len() != values.len() { - return Err(Error::oos( - "A StructArray must have a number of fields in its DataType equal to the number of child values", - )); - } - - fields - .iter().map(|a| &a.data_type) - .zip(values.iter().map(|a| a.data_type())) - .enumerate() - .try_for_each(|(index, (data_type, child))| { - if data_type != child { - Err(Error::oos(format!( - "The children DataTypes of a StructArray must equal the children data types. - However, the field {index} has data type {data_type:?} but the value has data type {child:?}" - ))) - } else { - Ok(()) - } - })?; - - let len = values[0].len(); - values - .iter() - .map(|a| a.len()) - .enumerate() - .try_for_each(|(index, a_len)| { - if a_len != len { - Err(Error::oos(format!( - "The children must have an equal number of values. - However, the values at index {index} have a length of {a_len}, which is different from values at index 0, {len}." - ))) - } else { - Ok(()) - } - })?; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != len) - { - return Err(Error::oos( - "The validity length of a StructArray must match its number of elements", - )); - } - - Ok(Self { - data_type, - values, - validity, - }) - } - - /// Returns a new [`StructArray`] - /// # Panics - /// This function panics iff: - /// * `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::Struct`]. - /// * the children of `data_type` are empty - /// * the values's len is different from children's length - /// * any of the values's data type is different from its corresponding children' data type - /// * any element of values has a different length than the first element - /// * the validity's length is not equal to the length of the first element - pub fn new(data_type: DataType, values: Vec>, validity: Option) -> Self { - Self::try_new(data_type, values, validity).unwrap() - } - - /// Creates an empty [`StructArray`]. - pub fn new_empty(data_type: DataType) -> Self { - if let DataType::Struct(fields) = &data_type.to_logical_type() { - let values = fields - .iter() - .map(|field| new_empty_array(field.data_type().clone())) - .collect(); - Self::new(data_type, values, None) - } else { - panic!("StructArray must be initialized with DataType::Struct"); - } - } - - /// Creates a null [`StructArray`] of length `length`. - pub fn new_null(data_type: DataType, length: usize) -> Self { - if let DataType::Struct(fields) = &data_type { - let values = fields - .iter() - .map(|field| new_null_array(field.data_type().clone(), length)) - .collect(); - Self::new(data_type, values, Some(Bitmap::new_zeroed(length))) - } else { - panic!("StructArray must be initialized with DataType::Struct"); - } - } -} - -// must use -impl StructArray { - /// Deconstructs the [`StructArray`] into its individual components. - #[must_use] - pub fn into_data(self) -> (Vec, Vec>, Option) { - let Self { - data_type, - values, - validity, - } = self; - let fields = if let DataType::Struct(fields) = data_type { - fields - } else { - unreachable!() - }; - (fields, values, validity) - } - - /// Slices this [`StructArray`]. - /// # Panics - /// * `offset + length` must be smaller than `self.len()`. - /// # Implementation - /// This operation is `O(F)` where `F` is the number of fields. - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "offset + length may not exceed length of array" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Slices this [`StructArray`]. - /// # Implementation - /// This operation is `O(F)` where `F` is the number of fields. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.values - .iter_mut() - .for_each(|x| x.slice_unchecked(offset, length)); - } - - impl_sliced!(); - - impl_mut_validity!(); - - impl_into_array!(); -} - -// Accessors -impl StructArray { - #[inline] - fn len(&self) -> usize { - self.values[0].len() - } - - /// The optional validity. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - /// Returns the values of this [`StructArray`]. - pub fn values(&self) -> &[Box] { - &self.values - } - - /// Returns the fields of this [`StructArray`]. - pub fn fields(&self) -> &[Field] { - Self::get_fields(&self.data_type) - } -} - -impl StructArray { - /// Returns the fields the `DataType::Struct`. - pub(crate) fn try_get_fields(data_type: &DataType) -> Result<&[Field], Error> { - match data_type.to_logical_type() { - DataType::Struct(fields) => Ok(fields), - _ => Err(Error::oos( - "Struct array must be created with a DataType whose physical type is Struct", - )), - } - } - - /// Returns the fields the `DataType::Struct`. - pub fn get_fields(data_type: &DataType) -> &[Field] { - Self::try_get_fields(data_type).unwrap() - } -} - -impl Array for StructArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} diff --git a/src/common/arrow/src/arrow/array/struct_/mutable.rs b/src/common/arrow/src/arrow/array/struct_/mutable.rs deleted file mode 100644 index 988ad72d2a4e..000000000000 --- a/src/common/arrow/src/arrow/array/struct_/mutable.rs +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::StructArray; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; - -/// Converting a [`MutableStructArray`] into a [`StructArray`] is `O(1)`. -#[derive(Debug)] -pub struct MutableStructArray { - data_type: DataType, - values: Vec>, - validity: Option, -} - -fn check( - data_type: &DataType, - values: &[Box], - validity: Option, -) -> Result<(), Error> { - let fields = StructArray::try_get_fields(data_type)?; - if fields.is_empty() { - return Err(Error::oos("A StructArray must contain at least one field")); - } - if fields.len() != values.len() { - return Err(Error::oos( - "A StructArray must have a number of fields in its DataType equal to the number of child values", - )); - } - - fields - .iter().map(|a| &a.data_type) - .zip(values.iter().map(|a| a.data_type())) - .enumerate() - .try_for_each(|(index, (data_type, child))| { - if data_type != child { - Err(Error::oos(format!( - "The children DataTypes of a StructArray must equal the children data types. - However, the field {index} has data type {data_type:?} but the value has data type {child:?}" - ))) - } else { - Ok(()) - } - })?; - - let len = values[0].len(); - values - .iter() - .map(|a| a.len()) - .enumerate() - .try_for_each(|(index, a_len)| { - if a_len != len { - Err(Error::oos(format!( - "The children must have an equal number of values. - However, the values at index {index} have a length of {a_len}, which is different from values at index 0, {len}." - ))) - } else { - Ok(()) - } - })?; - - if validity.map_or(false, |validity| validity != len) { - return Err(Error::oos( - "The validity length of a StructArray must match its number of elements", - )); - } - Ok(()) -} - -impl From for StructArray { - fn from(other: MutableStructArray) -> Self { - let validity = if other.validity.as_ref().map(|x| x.unset_bits()).unwrap_or(0) > 0 { - other.validity.map(|x| x.into()) - } else { - None - }; - - StructArray::new( - other.data_type, - other.values.into_iter().map(|mut v| v.as_box()).collect(), - validity, - ) - } -} - -impl MutableStructArray { - /// Creates a new [`MutableStructArray`]. - pub fn new(data_type: DataType, values: Vec>) -> Self { - Self::try_new(data_type, values, None).unwrap() - } - - /// Create a [`MutableStructArray`] out of low-end APIs. - /// # Errors - /// This function errors iff: - /// * `data_type` is not [`DataType::Struct`] - /// * The inner types of `data_type` are not equal to those of `values` - /// * `validity` is not `None` and its length is different from the `values`'s length - pub fn try_new( - data_type: DataType, - values: Vec>, - validity: Option, - ) -> Result { - check(&data_type, &values, validity.as_ref().map(|x| x.len()))?; - Ok(Self { - data_type, - values, - validity, - }) - } - - /// Extract the low-end APIs from the [`MutableStructArray`]. - pub fn into_inner(self) -> (DataType, Vec>, Option) { - (self.data_type, self.values, self.validity) - } - - /// The mutable values - pub fn mut_values(&mut self) -> &mut Vec> { - &mut self.values - } - - /// The values - pub fn values(&self) -> &Vec> { - &self.values - } - - /// Return the `i`th child array. - pub fn value(&mut self, i: usize) -> Option<&mut A> { - self.values[i].as_mut_any().downcast_mut::() - } -} - -impl MutableStructArray { - /// Reserves `additional` entries. - pub fn reserve(&mut self, additional: usize) { - for v in &mut self.values { - v.reserve(additional); - } - if let Some(x) = self.validity.as_mut() { - x.reserve(additional) - } - } - - /// Call this once for each "row" of children you push. - pub fn push(&mut self, valid: bool) { - match &mut self.validity { - Some(validity) => validity.push(valid), - None => match valid { - true => (), - false => self.init_validity(), - }, - }; - } - - fn push_null(&mut self) { - for v in &mut self.values { - v.push_null(); - } - self.push(false); - } - - fn init_validity(&mut self) { - let mut validity = MutableBitmap::with_capacity(self.values.capacity()); - let len = self.len(); - if len > 0 { - validity.extend_constant(len, true); - validity.set(len - 1, false); - } - self.validity = Some(validity) - } - - /// Converts itself into an [`Array`]. - pub fn into_arc(self) -> Arc { - let a: StructArray = self.into(); - Arc::new(a) - } - - /// Shrinks the capacity of the [`MutableStructArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - for v in &mut self.values { - v.shrink_to_fit(); - } - if let Some(validity) = self.validity.as_mut() { - validity.shrink_to_fit() - } - } -} - -impl MutableArray for MutableStructArray { - fn len(&self) -> usize { - self.values.first().map(|v| v.len()).unwrap_or(0) - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - StructArray::new( - self.data_type.clone(), - std::mem::take(&mut self.values) - .into_iter() - .map(|mut v| v.as_box()) - .collect(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .boxed() - } - - fn as_arc(&mut self) -> Arc { - StructArray::new( - self.data_type.clone(), - std::mem::take(&mut self.values) - .into_iter() - .map(|mut v| v.as_box()) - .collect(), - std::mem::take(&mut self.validity).map(|x| x.into()), - ) - .arced() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn push_null(&mut self) { - self.push_null() - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional) - } -} diff --git a/src/common/arrow/src/arrow/array/union/data.rs b/src/common/arrow/src/arrow/array/union/data.rs deleted file mode 100644 index 3738952ae636..000000000000 --- a/src/common/arrow/src/arrow/array/union/data.rs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::from_data; -use crate::arrow::array::to_data; -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::UnionArray; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; - -impl Arrow2Arrow for UnionArray { - fn to_data(&self) -> ArrayData { - let data_type = arrow_schema::DataType::from(self.data_type.clone()); - let len = self.len(); - - let builder = match self.offsets.clone() { - Some(offsets) => ArrayDataBuilder::new(data_type) - .len(len) - .buffers(vec![self.types.clone().into(), offsets.into()]) - .child_data(self.fields.iter().map(|x| to_data(x.as_ref())).collect()), - None => ArrayDataBuilder::new(data_type) - .len(len) - .buffers(vec![self.types.clone().into()]) - .child_data( - self.fields - .iter() - .map(|x| to_data(x.as_ref()).slice(self.offset, len)) - .collect(), - ), - }; - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type: DataType = data.data_type().clone().into(); - - let fields = data.child_data().iter().map(from_data).collect(); - let buffers = data.buffers(); - let mut types: Buffer = buffers[0].clone().into(); - types.slice(data.offset(), data.len()); - let offsets = match buffers.len() == 2 { - true => { - let mut offsets: Buffer = buffers[1].clone().into(); - offsets.slice(data.offset(), data.len()); - Some(offsets) - } - false => None, - }; - - // Map from type id to array index - let map = match &data_type { - DataType::Union(_, Some(ids), _) => { - let mut map = [0; 127]; - for (pos, &id) in ids.iter().enumerate() { - map[id as usize] = pos; - } - Some(map) - } - DataType::Union(_, None, _) => None, - _ => unreachable!("must be Union type"), - }; - - Self { - types, - map, - fields, - offsets, - data_type, - offset: data.offset(), - } - } -} diff --git a/src/common/arrow/src/arrow/array/union/fmt.rs b/src/common/arrow/src/arrow/array/union/fmt.rs deleted file mode 100644 index 19c94dc2b415..000000000000 --- a/src/common/arrow/src/arrow/array/union/fmt.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::get_display; -use super::super::fmt::write_vec; -use super::UnionArray; - -pub fn write_value( - array: &UnionArray, - index: usize, - null: &'static str, - f: &mut W, -) -> Result { - let (field, index) = array.index(index); - - get_display(array.fields()[field].as_ref(), null)(f, index) -} - -impl Debug for UnionArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, "None", f); - - write!(f, "UnionArray")?; - write_vec(f, writer, None, self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/union/iterator.rs b/src/common/arrow/src/arrow/array/union/iterator.rs deleted file mode 100644 index 2f1c7e9299d6..000000000000 --- a/src/common/arrow/src/arrow/array/union/iterator.rs +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::UnionArray; -use crate::arrow::scalar::Scalar; -use crate::arrow::trusted_len::TrustedLen; - -#[derive(Debug, Clone)] -pub struct UnionIter<'a> { - array: &'a UnionArray, - current: usize, -} - -impl<'a> UnionIter<'a> { - #[inline] - pub fn new(array: &'a UnionArray) -> Self { - Self { array, current: 0 } - } -} - -impl<'a> Iterator for UnionIter<'a> { - type Item = Box; - - #[inline] - fn next(&mut self) -> Option { - if self.current == self.array.len() { - None - } else { - let old = self.current; - self.current += 1; - Some(unsafe { self.array.value_unchecked(old) }) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = self.array.len() - self.current; - (len, Some(len)) - } -} - -impl<'a> IntoIterator for &'a UnionArray { - type Item = Box; - type IntoIter = UnionIter<'a>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> UnionArray { - /// constructs a new iterator - #[inline] - pub fn iter(&'a self) -> UnionIter<'a> { - UnionIter::new(self) - } -} - -impl<'a> std::iter::ExactSizeIterator for UnionIter<'a> {} - -unsafe impl<'a> TrustedLen for UnionIter<'a> {} diff --git a/src/common/arrow/src/arrow/array/union/mod.rs b/src/common/arrow/src/arrow/array/union/mod.rs deleted file mode 100644 index 2bf2947d5ece..000000000000 --- a/src/common/arrow/src/arrow/array/union/mod.rs +++ /dev/null @@ -1,402 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::new_empty_array; -use super::new_null_array; -use super::Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::UnionMode; -use crate::arrow::error::Error; -use crate::arrow::scalar::new_scalar; -use crate::arrow::scalar::Scalar; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod iterator; - -type UnionComponents<'a> = (&'a [Field], Option<&'a [i32]>, UnionMode); - -/// [`UnionArray`] represents an array whose each slot can contain different values. -// How to read a value at slot i: -// ``` -// let index = self.types()[i] as usize; -// let field = self.fields()[index]; -// let offset = self.offsets().map(|x| x[index]).unwrap_or(i); -// let field = field.as_any().downcast to correct type; -// let value = field.value(offset); -// ``` -#[derive(Clone)] -pub struct UnionArray { - // Invariant: every item in `types` is `> 0 && < fields.len()` - types: Buffer, - // Invariant: `map.len() == fields.len()` - // Invariant: every item in `map` is `> 0 && < fields.len()` - map: Option<[usize; 127]>, - fields: Vec>, - // Invariant: when set, `offsets.len() == types.len()` - offsets: Option>, - data_type: DataType, - offset: usize, -} - -impl UnionArray { - /// Returns a new [`UnionArray`]. - /// # Errors - /// This function errors iff: - /// * `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::Union`]. - /// * the fields's len is different from the `data_type`'s children's length - /// * The number of `fields` is larger than `i8::MAX` - /// * any of the values's data type is different from its corresponding children' data type - pub fn try_new( - data_type: DataType, - types: Buffer, - fields: Vec>, - offsets: Option>, - ) -> Result { - let (f, ids, mode) = Self::try_get_all(&data_type)?; - - if f.len() != fields.len() { - return Err(Error::oos( - "The number of `fields` must equal the number of children fields in DataType::Union", - )); - }; - let number_of_fields: i8 = fields - .len() - .try_into() - .map_err(|_| Error::oos("The number of `fields` cannot be larger than i8::MAX"))?; - - f - .iter().map(|a| a.data_type()) - .zip(fields.iter().map(|a| a.data_type())) - .enumerate() - .try_for_each(|(index, (data_type, child))| { - if data_type != child { - Err(Error::oos(format!( - "The children DataTypes of a UnionArray must equal the children data types. - However, the field {index} has data type {data_type:?} but the value has data type {child:?}" - ))) - } else { - Ok(()) - } - })?; - - if let Some(offsets) = &offsets { - if offsets.len() != types.len() { - return Err(Error::oos( - "In a UnionArray, the offsets' length must be equal to the number of types", - )); - } - } - if offsets.is_none() != mode.is_sparse() { - return Err(Error::oos( - "In a sparse UnionArray, the offsets must be set (and vice-versa)", - )); - } - - // build hash - let map = if let Some(&ids) = ids.as_ref() { - if ids.len() != fields.len() { - return Err(Error::oos( - "In a union, when the ids are set, their length must be equal to the number of fields", - )); - } - - // example: - // * types = [5, 7, 5, 7, 7, 7, 5, 7, 7, 5, 5] - // * ids = [5, 7] - // => hash = [0, 0, 0, 0, 0, 0, 1, 0, ...] - let mut hash = [0; 127]; - - for (pos, &id) in ids.iter().enumerate() { - if !(0..=127).contains(&id) { - return Err(Error::oos( - "In a union, when the ids are set, every id must belong to [0, 128[", - )); - } - hash[id as usize] = pos; - } - - types.iter().try_for_each(|&type_| { - if type_ < 0 { - return Err(Error::oos("In a union, when the ids are set, every type must be >= 0")); - } - let id = hash[type_ as usize]; - if id >= fields.len() { - Err(Error::oos("In a union, when the ids are set, each id must be smaller than the number of fields.")) - } else { - Ok(()) - } - })?; - - Some(hash) - } else { - // Safety: every type in types is smaller than number of fields - let mut is_valid = true; - for &type_ in types.iter() { - if type_ < 0 || type_ >= number_of_fields { - is_valid = false - } - } - if !is_valid { - return Err(Error::oos( - "Every type in `types` must be larger than 0 and smaller than the number of fields.", - )); - } - - None - }; - - Ok(Self { - data_type, - map, - fields, - offsets, - types, - offset: 0, - }) - } - - /// Returns a new [`UnionArray`]. - /// # Panics - /// This function panics iff: - /// * `data_type`'s physical type is not [`crate::arrow::datatypes::PhysicalType::Union`]. - /// * the fields's len is different from the `data_type`'s children's length - /// * any of the values's data type is different from its corresponding children' data type - pub fn new( - data_type: DataType, - types: Buffer, - fields: Vec>, - offsets: Option>, - ) -> Self { - Self::try_new(data_type, types, fields, offsets).unwrap() - } - - /// Creates a new null [`UnionArray`]. - pub fn new_null(data_type: DataType, length: usize) -> Self { - if let DataType::Union(f, _, mode) = &data_type { - let fields = f - .iter() - .map(|x| new_null_array(x.data_type().clone(), length)) - .collect(); - - let offsets = if mode.is_sparse() { - None - } else { - Some((0..length as i32).collect::>().into()) - }; - - // all from the same field - let types = vec![0i8; length].into(); - - Self::new(data_type, types, fields, offsets) - } else { - panic!("Union struct must be created with the corresponding Union DataType") - } - } - - /// Creates a new empty [`UnionArray`]. - pub fn new_empty(data_type: DataType) -> Self { - if let DataType::Union(f, _, mode) = data_type.to_logical_type() { - let fields = f - .iter() - .map(|x| new_empty_array(x.data_type().clone())) - .collect(); - - let offsets = if mode.is_sparse() { - None - } else { - Some(Buffer::default()) - }; - - Self { - data_type, - map: None, - fields, - offsets, - types: Buffer::new(), - offset: 0, - } - } else { - panic!("Union struct must be created with the corresponding Union DataType") - } - } -} - -impl UnionArray { - /// Returns a slice of this [`UnionArray`]. - /// # Implementation - /// This operation is `O(F)` where `F` is the number of fields. - /// # Panic - /// This function panics iff `offset + length > self.len()`. - #[inline] - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new array cannot exceed the existing length" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Returns a slice of this [`UnionArray`]. - /// # Implementation - /// This operation is `O(F)` where `F` is the number of fields. - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - #[inline] - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - debug_assert!(offset + length <= self.len()); - - self.types.slice_unchecked(offset, length); - if let Some(offsets) = self.offsets.as_mut() { - offsets.slice_unchecked(offset, length) - } - self.offset += offset; - } - - impl_sliced!(); - impl_into_array!(); -} - -impl UnionArray { - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.types.len() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The optional offsets. - pub fn offsets(&self) -> Option<&Buffer> { - self.offsets.as_ref() - } - - /// The fields. - pub fn fields(&self) -> &Vec> { - &self.fields - } - - /// The types. - pub fn types(&self) -> &Buffer { - &self.types - } - - #[inline] - unsafe fn field_slot_unchecked(&self, index: usize) -> usize { - self.offsets() - .as_ref() - .map(|x| *x.get_unchecked(index) as usize) - .unwrap_or(index + self.offset) - } - - /// Returns the index and slot of the field to select from `self.fields`. - #[inline] - pub fn index(&self, index: usize) -> (usize, usize) { - assert!(index < self.len()); - unsafe { self.index_unchecked(index) } - } - - /// Returns the index and slot of the field to select from `self.fields`. - /// The first value is guaranteed to be `< self.fields().len()` - /// # Safety - /// This function is safe iff `index < self.len`. - #[inline] - pub unsafe fn index_unchecked(&self, index: usize) -> (usize, usize) { - debug_assert!(index < self.len()); - // Safety: assumption of the function - let type_ = unsafe { *self.types.get_unchecked(index) }; - // Safety: assumption of the struct - let type_ = self - .map - .as_ref() - .map(|map| unsafe { *map.get_unchecked(type_ as usize) }) - .unwrap_or(type_ as usize); - // Safety: assumption of the function - let index = self.field_slot_unchecked(index); - (type_, index) - } - - /// Returns the slot `index` as a [`Scalar`]. - /// # Panics - /// iff `index >= self.len()` - pub fn value(&self, index: usize) -> Box { - assert!(index < self.len()); - unsafe { self.value_unchecked(index) } - } - - /// Returns the slot `index` as a [`Scalar`]. - /// # Safety - /// This function is safe iff `i < self.len`. - pub unsafe fn value_unchecked(&self, index: usize) -> Box { - debug_assert!(index < self.len()); - let (type_, index) = self.index_unchecked(index); - // Safety: assumption of the struct - debug_assert!(type_ < self.fields.len()); - let field = self.fields.get_unchecked(type_).as_ref(); - new_scalar(field, index) - } -} - -impl Array for UnionArray { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - None - } - - fn with_validity(&self, _: Option) -> Box { - panic!("cannot set validity of a union array") - } -} - -impl UnionArray { - fn try_get_all(data_type: &DataType) -> Result { - match data_type.to_logical_type() { - DataType::Union(fields, ids, mode) => { - Ok((fields, ids.as_ref().map(|x| x.as_ref()), *mode)) - } - _ => Err(Error::oos( - "The UnionArray requires a logical type of DataType::Union", - )), - } - } - - fn get_all(data_type: &DataType) -> (&[Field], Option<&[i32]>, UnionMode) { - Self::try_get_all(data_type).unwrap() - } - - /// Returns all fields from [`DataType::Union`]. - /// # Panic - /// Panics iff `data_type`'s logical type is not [`DataType::Union`]. - pub fn get_fields(data_type: &DataType) -> &[Field] { - Self::get_all(data_type).0 - } - - /// Returns whether the [`DataType::Union`] is sparse or not. - /// # Panic - /// Panics iff `data_type`'s logical type is not [`DataType::Union`]. - pub fn is_sparse(data_type: &DataType) -> bool { - Self::get_all(data_type).2.is_sparse() - } -} diff --git a/src/common/arrow/src/arrow/array/utf8/data.rs b/src/common/arrow/src/arrow/array/utf8/data.rs deleted file mode 100644 index 63ff538fe214..000000000000 --- a/src/common/arrow/src/arrow/array/utf8/data.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_data::ArrayData; -use arrow_data::ArrayDataBuilder; - -use crate::arrow::array::Arrow2Arrow; -use crate::arrow::array::Utf8Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::offset::Offset; -use crate::arrow::offset::OffsetsBuffer; - -impl Arrow2Arrow for Utf8Array { - fn to_data(&self) -> ArrayData { - let data_type = self.data_type().clone().into(); - let builder = ArrayDataBuilder::new(data_type) - .len(self.offsets().len_proxy()) - .buffers(vec![ - self.offsets.clone().into_inner().into(), - self.values.clone().into(), - ]) - .nulls(self.validity.as_ref().map(|b| b.clone().into())); - - // Safety: Array is valid - unsafe { builder.build_unchecked() } - } - - fn from_data(data: &ArrayData) -> Self { - let data_type = data.data_type().clone().into(); - if data.is_empty() { - // Handle empty offsets - return Self::new_empty(data_type); - } - - let buffers = data.buffers(); - - // Safety: ArrayData is valid - let mut offsets = unsafe { OffsetsBuffer::new_unchecked(buffers[0].clone().into()) }; - offsets.slice(data.offset(), data.len() + 1); - - Self { - data_type, - offsets, - values: buffers[1].clone().into(), - validity: data.nulls().map(|n| Bitmap::from_null_buffer(n.clone())), - } - } -} diff --git a/src/common/arrow/src/arrow/array/utf8/fmt.rs b/src/common/arrow/src/arrow/array/utf8/fmt.rs deleted file mode 100644 index 835c8349805e..000000000000 --- a/src/common/arrow/src/arrow/array/utf8/fmt.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::fmt::Result; -use std::fmt::Write; - -use super::super::fmt::write_vec; -use super::Utf8Array; -use crate::arrow::offset::Offset; - -pub fn write_value(array: &Utf8Array, index: usize, f: &mut W) -> Result { - write!(f, "{}", array.value(index)) -} - -impl Debug for Utf8Array { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, f); - - let head = if O::IS_LARGE { - "LargeUtf8Array" - } else { - "Utf8Array" - }; - write!(f, "{head}")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) - } -} diff --git a/src/common/arrow/src/arrow/array/utf8/from.rs b/src/common/arrow/src/arrow/array/utf8/from.rs deleted file mode 100644 index 7a30a6919db0..000000000000 --- a/src/common/arrow/src/arrow/array/utf8/from.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; - -use super::MutableUtf8Array; -use super::Utf8Array; -use crate::arrow::offset::Offset; - -impl> FromIterator> for Utf8Array { - #[inline] - fn from_iter>>(iter: I) -> Self { - MutableUtf8Array::::from_iter(iter).into() - } -} diff --git a/src/common/arrow/src/arrow/array/utf8/iterator.rs b/src/common/arrow/src/arrow/array/utf8/iterator.rs deleted file mode 100644 index db20f331121e..000000000000 --- a/src/common/arrow/src/arrow/array/utf8/iterator.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::MutableUtf8Array; -use super::MutableUtf8ValuesArray; -use super::Utf8Array; -use crate::arrow::array::ArrayAccessor; -use crate::arrow::array::ArrayValuesIter; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::offset::Offset; - -unsafe impl<'a, O: Offset> ArrayAccessor<'a> for Utf8Array { - type Item = &'a str; - - #[inline] - unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) - } - - #[inline] - fn len(&self) -> usize { - self.len() - } -} - -/// Iterator of values of an [`Utf8Array`]. -pub type Utf8ValuesIter<'a, O> = ArrayValuesIter<'a, Utf8Array>; - -impl<'a, O: Offset> IntoIterator for &'a Utf8Array { - type Item = Option<&'a str>; - type IntoIter = ZipValidity<&'a str, Utf8ValuesIter<'a, O>, BitmapIter<'a>>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -unsafe impl<'a, O: Offset> ArrayAccessor<'a> for MutableUtf8Array { - type Item = &'a str; - - #[inline] - unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) - } - - #[inline] - fn len(&self) -> usize { - self.len() - } -} - -/// Iterator of values of an [`MutableUtf8ValuesArray`]. -pub type MutableUtf8ValuesIter<'a, O> = ArrayValuesIter<'a, MutableUtf8ValuesArray>; - -impl<'a, O: Offset> IntoIterator for &'a MutableUtf8Array { - type Item = Option<&'a str>; - type IntoIter = ZipValidity<&'a str, MutableUtf8ValuesIter<'a, O>, BitmapIter<'a>>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -unsafe impl<'a, O: Offset> ArrayAccessor<'a> for MutableUtf8ValuesArray { - type Item = &'a str; - - #[inline] - unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) - } - - #[inline] - fn len(&self) -> usize { - self.len() - } -} - -impl<'a, O: Offset> IntoIterator for &'a MutableUtf8ValuesArray { - type Item = &'a str; - type IntoIter = ArrayValuesIter<'a, MutableUtf8ValuesArray>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} diff --git a/src/common/arrow/src/arrow/array/utf8/mod.rs b/src/common/arrow/src/arrow/array/utf8/mod.rs deleted file mode 100644 index 8010b5118189..000000000000 --- a/src/common/arrow/src/arrow/array/utf8/mod.rs +++ /dev/null @@ -1,581 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use either::Either; - -use super::specification::try_check_offsets_bounds; -use super::specification::try_check_utf8; -use super::Array; -use super::GenericBinaryArray; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::offset::OffsetsBuffer; -use crate::arrow::trusted_len::TrustedLen; - -#[cfg(feature = "arrow")] -mod data; - -pub(super) mod fmt; -mod from; -mod iterator; -mod mutable; -mod mutable_values; -pub use iterator::*; -pub use mutable::*; -pub use mutable_values::MutableUtf8ValuesArray; - -// Auxiliary struct to allow presenting &str as [u8] to a generic function -pub(super) struct StrAsBytes

(P); -impl> AsRef<[u8]> for StrAsBytes { - #[inline(always)] - fn as_ref(&self) -> &[u8] { - self.0.as_ref().as_bytes() - } -} - -/// A [`Utf8Array`] is arrow's semantic equivalent of an immutable `Vec>`. -/// Cloning and slicing this struct is `O(1)`. -/// # Example -/// ``` -/// use arrow2::array::Utf8Array; -/// use arrow2::bitmap::Bitmap; -/// use arrow2::buffer::Buffer; -/// # fn main() { -/// let array = Utf8Array::::from([Some("hi"), None, Some("there")]); -/// assert_eq!(array.value(0), "hi"); -/// assert_eq!(array.iter().collect::>(), vec![ -/// Some("hi"), -/// None, -/// Some("there") -/// ]); -/// assert_eq!(array.values_iter().collect::>(), vec![ -/// "hi", "", "there" -/// ]); -/// // the underlying representation -/// assert_eq!(array.validity(), Some(&Bitmap::from([true, false, true]))); -/// assert_eq!(array.values(), &Buffer::from(b"hithere".to_vec())); -/// assert_eq!( -/// array.offsets().buffer(), -/// &Buffer::from(vec![0, 2, 2, 2 + 5]) -/// ); -/// # } -/// ``` -/// -/// # Generic parameter -/// The generic parameter [`Offset`] can only be `i32` or `i64` and tradeoffs maximum array length with -/// memory usage: -/// * the sum of lengths of all elements cannot exceed `Offset::MAX` -/// * the total size of the underlying data is `array.len() * size_of::() + sum of lengths of all elements` -/// -/// # Safety -/// The following invariants hold: -/// * Two consecutives `offsets` casted (`as`) to `usize` are valid slices of `values`. -/// * A slice of `values` taken from two consecutives `offsets` is valid `utf8`. -/// * `len` is equal to `validity.len()`, when defined. -#[derive(Clone)] -pub struct Utf8Array { - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, -} - -// constructors -impl Utf8Array { - /// Returns a [`Utf8Array`] created from its internal representation. - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Utf8` or `LargeUtf8`. - /// * The `values` between two consecutive `offsets` are not valid utf8 - /// # Implementation - /// This function is `O(N)` - checking utf8 is `O(N)` - pub fn try_new( - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, - ) -> Result { - try_check_utf8(&offsets, &values)?; - if validity - .as_ref() - .map_or(false, |validity| validity.len() != offsets.len_proxy()) - { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - - if data_type.to_physical_type() != Self::default_data_type().to_physical_type() { - return Err(Error::oos( - "Utf8Array can only be initialized with DataType::Utf8 or DataType::LargeUtf8", - )); - } - - Ok(Self { - data_type, - offsets, - values, - validity, - }) - } - - /// Returns a [`Utf8Array`] from a slice of `&str`. - /// - /// A convenience method that uses [`Self::from_trusted_len_values_iter`]. - pub fn from_slice, P: AsRef<[T]>>(slice: P) -> Self { - Self::from_trusted_len_values_iter(slice.as_ref().iter()) - } - - /// Returns a new [`Utf8Array`] from a slice of `&str`. - /// - /// A convenience method that uses [`Self::from_trusted_len_iter`]. - // Note: this can't be `impl From` because Rust does not allow double `AsRef` on it. - pub fn from, P: AsRef<[Option]>>(slice: P) -> Self { - MutableUtf8Array::::from(slice).into() - } - - /// Returns an iterator of `Option<&str>` - pub fn iter(&self) -> ZipValidity<&str, Utf8ValuesIter, BitmapIter> { - ZipValidity::new_with_validity(self.values_iter(), self.validity()) - } - - /// Returns an iterator of `&str` - pub fn values_iter(&self) -> Utf8ValuesIter { - Utf8ValuesIter::new(self) - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.offsets.len_proxy() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the value of the element at index `i`, ignoring the array's validity. - /// # Panic - /// This function panics iff `i >= self.len`. - #[inline] - pub fn value(&self, i: usize) -> &str { - assert!(i < self.len()); - unsafe { self.value_unchecked(i) } - } - - /// Returns the value of the element at index `i`, ignoring the array's validity. - /// # Safety - /// This function is safe iff `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> &str { - // soundness: the invariant of the function - let (start, end) = self.offsets.start_end_unchecked(i); - - // soundness: the invariant of the struct - let slice = self.values.get_unchecked(start..end); - - // soundness: the invariant of the struct - std::str::from_utf8_unchecked(slice) - } - - /// Returns the element at index `i` or `None` if it is null - /// # Panics - /// iff `i >= self.len()` - #[inline] - pub fn get(&self, i: usize) -> Option<&str> { - if !self.is_null(i) { - // soundness: Array::is_null panics if i >= self.len - unsafe { Some(self.value_unchecked(i)) } - } else { - None - } - } - - /// Returns the [`DataType`] of this array. - #[inline] - pub fn data_type(&self) -> &DataType { - &self.data_type - } - - /// Returns the values of this [`Utf8Array`]. - #[inline] - pub fn values(&self) -> &Buffer { - &self.values - } - - /// Returns the offsets of this [`Utf8Array`]. - #[inline] - pub fn offsets(&self) -> &OffsetsBuffer { - &self.offsets - } - - /// The optional validity. - #[inline] - pub fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - /// Slices this [`Utf8Array`]. - /// # Implementation - /// This function is `O(1)`. - /// # Panics - /// iff `offset + length > self.len()`. - pub fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new array cannot exceed the arrays' length" - ); - unsafe { self.slice_unchecked(offset, length) } - } - - /// Slices this [`Utf8Array`]. - /// # Implementation - /// This function is `O(1)` - /// # Safety - /// The caller must ensure that `offset + length <= self.len()`. - pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.validity.as_mut().and_then(|bitmap| { - bitmap.slice_unchecked(offset, length); - (bitmap.unset_bits() > 0).then_some(bitmap) - }); - self.offsets.slice_unchecked(offset, length + 1); - } - - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); - - /// Returns its internal representation - #[must_use] - pub fn into_inner(self) -> (DataType, OffsetsBuffer, Buffer, Option) { - let Self { - data_type, - offsets, - values, - validity, - } = self; - (data_type, offsets, values, validity) - } - - /// Try to convert this `Utf8Array` to a `MutableUtf8Array` - #[must_use] - pub fn into_mut(self) -> Either> { - use Either::*; - if let Some(bitmap) = self.validity { - match bitmap.into_mut() { - // Safety: invariants are preserved - Left(bitmap) => Left(unsafe { - Utf8Array::new_unchecked( - self.data_type, - self.offsets, - self.values, - Some(bitmap), - ) - }), - Right(mutable_bitmap) => match (self.values.into_mut(), self.offsets.into_mut()) { - (Left(values), Left(offsets)) => { - // Safety: invariants are preserved - Left(unsafe { - Utf8Array::new_unchecked( - self.data_type, - offsets, - values, - Some(mutable_bitmap.into()), - ) - }) - } - (Left(values), Right(offsets)) => { - // Safety: invariants are preserved - Left(unsafe { - Utf8Array::new_unchecked( - self.data_type, - offsets.into(), - values, - Some(mutable_bitmap.into()), - ) - }) - } - (Right(values), Left(offsets)) => { - // Safety: invariants are preserved - Left(unsafe { - Utf8Array::new_unchecked( - self.data_type, - offsets, - values.into(), - Some(mutable_bitmap.into()), - ) - }) - } - (Right(values), Right(offsets)) => Right(unsafe { - MutableUtf8Array::new_unchecked( - self.data_type, - offsets, - values, - Some(mutable_bitmap), - ) - }), - }, - } - } else { - match (self.values.into_mut(), self.offsets.into_mut()) { - (Left(values), Left(offsets)) => { - Left(unsafe { Utf8Array::new_unchecked(self.data_type, offsets, values, None) }) - } - (Left(values), Right(offsets)) => Left(unsafe { - Utf8Array::new_unchecked(self.data_type, offsets.into(), values, None) - }), - (Right(values), Left(offsets)) => Left(unsafe { - Utf8Array::new_unchecked(self.data_type, offsets, values.into(), None) - }), - (Right(values), Right(offsets)) => Right(unsafe { - MutableUtf8Array::new_unchecked(self.data_type, offsets, values, None) - }), - } - } - } - - /// Returns a new empty [`Utf8Array`]. - /// - /// The array is guaranteed to have no elements nor validity. - #[inline] - pub fn new_empty(data_type: DataType) -> Self { - unsafe { Self::new_unchecked(data_type, OffsetsBuffer::new(), Buffer::new(), None) } - } - - /// Returns a new [`Utf8Array`] whose all slots are null / `None`. - #[inline] - pub fn new_null(data_type: DataType, length: usize) -> Self { - Self::new( - data_type, - Offsets::new_zeroed(length).into(), - Buffer::new(), - Some(Bitmap::new_zeroed(length)), - ) - } - - /// Returns a default [`DataType`] of this array, which depends on the generic parameter `O`: `DataType::Utf8` or `DataType::LargeUtf8` - pub fn default_data_type() -> DataType { - if O::IS_LARGE { - DataType::LargeUtf8 - } else { - DataType::Utf8 - } - } - - /// Creates a new [`Utf8Array`] without checking for offsets monotinicity nor utf8-validity - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Utf8` or `LargeUtf8`. - /// # Safety - /// This function is unsound iff: - /// * The `values` between two consecutive `offsets` are not valid utf8 - /// # Implementation - /// This function is `O(1)` - pub unsafe fn try_new_unchecked( - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, - ) -> Result { - try_check_offsets_bounds(&offsets, values.len())?; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != offsets.len_proxy()) - { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - - if data_type.to_physical_type() != Self::default_data_type().to_physical_type() { - return Err(Error::oos( - "Utf8Array can only be initialized with DataType::Utf8 or DataType::LargeUtf8", - )); - } - - Ok(Self { - data_type, - offsets, - values, - validity, - }) - } - - /// Creates a new [`Utf8Array`]. - /// # Panics - /// This function panics iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Utf8` or `LargeUtf8`. - /// * The `values` between two consecutive `offsets` are not valid utf8 - /// # Implementation - /// This function is `O(N)` - checking utf8 is `O(N)` - pub fn new( - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, - ) -> Self { - Self::try_new(data_type, offsets, values, validity).unwrap() - } - - /// Creates a new [`Utf8Array`] without checking for offsets monotinicity. - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Utf8` or `LargeUtf8`. - /// # Safety - /// This function is unsound iff: - /// * the offsets are not monotonically increasing - /// * The `values` between two consecutive `offsets` are not valid utf8 - /// # Implementation - /// This function is `O(1)` - pub unsafe fn new_unchecked( - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, - ) -> Self { - Self::try_new_unchecked(data_type, offsets, values, validity).unwrap() - } - - /// Returns a (non-null) [`Utf8Array`] created from a [`TrustedLen`] of `&str`. - /// # Implementation - /// This function is `O(N)` - #[inline] - pub fn from_trusted_len_values_iter, I: TrustedLen>( - iterator: I, - ) -> Self { - MutableUtf8Array::::from_trusted_len_values_iter(iterator).into() - } - - /// Creates a new [`Utf8Array`] from a [`Iterator`] of `&str`. - pub fn from_iter_values, I: Iterator>(iterator: I) -> Self { - MutableUtf8Array::::from_iter_values(iterator).into() - } - - /// Creates a [`Utf8Array`] from an iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: AsRef, - I: Iterator>, - { - MutableUtf8Array::::from_trusted_len_iter_unchecked(iterator).into() - } - - /// Creates a [`Utf8Array`] from an iterator of trusted length. - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: AsRef, - I: TrustedLen>, - { - MutableUtf8Array::::from_trusted_len_iter(iterator).into() - } - - /// Creates a [`Utf8Array`] from an falible iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn try_from_trusted_len_iter_unchecked( - iterator: I, - ) -> std::result::Result - where - P: AsRef, - I: IntoIterator, E>>, - { - MutableUtf8Array::::try_from_trusted_len_iter_unchecked(iterator).map(|x| x.into()) - } - - /// Creates a [`Utf8Array`] from an fallible iterator of trusted length. - #[inline] - pub fn try_from_trusted_len_iter(iter: I) -> std::result::Result - where - P: AsRef, - I: TrustedLen, E>>, - { - MutableUtf8Array::::try_from_trusted_len_iter(iter).map(|x| x.into()) - } - - /// Applies a function `f` to the validity of this array. - /// - /// This is an API to leverage clone-on-write - /// # Panics - /// This function panics if the function `f` modifies the length of the [`Bitmap`]. - pub fn apply_validity Bitmap>(&mut self, f: F) { - if let Some(validity) = std::mem::take(&mut self.validity) { - self.set_validity(Some(f(validity))) - } - } -} - -impl Array for Utf8Array { - impl_common_array!(); - - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() - } - - #[inline] - fn with_validity(&self, validity: Option) -> Box { - Box::new(self.clone().with_validity(validity)) - } -} - -unsafe impl GenericBinaryArray for Utf8Array { - #[inline] - fn values(&self) -> &[u8] { - self.values() - } - - #[inline] - fn offsets(&self) -> &[O] { - self.offsets().buffer() - } -} - -impl Default for Utf8Array { - fn default() -> Self { - let data_type = if O::IS_LARGE { - DataType::LargeUtf8 - } else { - DataType::Utf8 - }; - Utf8Array::new(data_type, Default::default(), Default::default(), None) - } -} diff --git a/src/common/arrow/src/arrow/array/utf8/mutable.rs b/src/common/arrow/src/arrow/array/utf8/mutable.rs deleted file mode 100644 index c2b855407a1e..000000000000 --- a/src/common/arrow/src/arrow/array/utf8/mutable.rs +++ /dev/null @@ -1,581 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; -use std::sync::Arc; - -use super::MutableUtf8ValuesArray; -use super::MutableUtf8ValuesIter; -use super::StrAsBytes; -use super::Utf8Array; -use crate::arrow::array::physical_binary::*; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::trusted_len::TrustedLen; - -/// A [`MutableArray`] that builds a [`Utf8Array`]. It differs -/// from [`MutableUtf8ValuesArray`] in that it can build nullable [`Utf8Array`]s. -#[derive(Debug, Clone)] -pub struct MutableUtf8Array { - values: MutableUtf8ValuesArray, - validity: Option, -} - -impl From> for Utf8Array { - fn from(other: MutableUtf8Array) -> Self { - let validity = other.validity.and_then(|x| { - let validity: Option = x.into(); - validity - }); - let array: Utf8Array = other.values.into(); - array.with_validity(validity) - } -} - -impl Default for MutableUtf8Array { - fn default() -> Self { - Self::new() - } -} - -impl MutableUtf8Array { - /// Initializes a new empty [`MutableUtf8Array`]. - pub fn new() -> Self { - Self { - values: Default::default(), - validity: None, - } - } - - /// Returns a [`MutableUtf8Array`] created from its internal representation. - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * the validity's length is not equal to `offsets.len()`. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Utf8` or `LargeUtf8`. - /// * The `values` between two consecutive `offsets` are not valid utf8 - /// # Implementation - /// This function is `O(N)` - checking utf8 is `O(N)` - pub fn try_new( - data_type: DataType, - offsets: Offsets, - values: Vec, - validity: Option, - ) -> Result { - let values = MutableUtf8ValuesArray::try_new(data_type, offsets, values)?; - - if validity - .as_ref() - .map_or(false, |validity| validity.len() != values.len()) - { - return Err(Error::oos( - "validity's length must be equal to the number of values", - )); - } - - Ok(Self { values, validity }) - } - - /// Create a [`MutableUtf8Array`] out of low-end APIs. - /// # Safety - /// The caller must ensure that every value between offsets is a valid utf8. - /// # Panics - /// This function panics iff: - /// * The `offsets` and `values` are inconsistent - /// * The validity is not `None` and its length is different from `offsets`'s length minus one. - pub unsafe fn new_unchecked( - data_type: DataType, - offsets: Offsets, - values: Vec, - validity: Option, - ) -> Self { - let values = MutableUtf8ValuesArray::new_unchecked(data_type, offsets, values); - if let Some(ref validity) = validity { - assert_eq!(values.len(), validity.len()); - } - Self { values, validity } - } - - /// Creates a new [`MutableUtf8Array`] from a slice of optional `&[u8]`. - // Note: this can't be `impl From` because Rust does not allow double `AsRef` on it. - pub fn from, P: AsRef<[Option]>>(slice: P) -> Self { - Self::from_trusted_len_iter(slice.as_ref().iter().map(|x| x.as_ref())) - } - - fn default_data_type() -> DataType { - Utf8Array::::default_data_type() - } - - /// Initializes a new [`MutableUtf8Array`] with a pre-allocated capacity of slots. - pub fn with_capacity(capacity: usize) -> Self { - Self::with_capacities(capacity, 0) - } - - /// Initializes a new [`MutableUtf8Array`] with a pre-allocated capacity of slots and values. - pub fn with_capacities(capacity: usize, values: usize) -> Self { - Self { - values: MutableUtf8ValuesArray::with_capacities(capacity, values), - validity: None, - } - } - - /// Reserves `additional` elements and `additional_values` on the values buffer. - pub fn reserve(&mut self, additional: usize, additional_values: usize) { - self.values.reserve(additional, additional_values); - if let Some(x) = self.validity.as_mut() { - x.reserve(additional) - } - } - - /// Reserves `additional` elements and `additional_values` on the values buffer. - pub fn capacity(&self) -> usize { - self.values.capacity() - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.values.len() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Pushes a new element to the array. - /// # Panic - /// This operation panics iff the length of all values (in bytes) exceeds `O` maximum value. - #[inline] - pub fn push>(&mut self, value: Option) { - self.try_push(value).unwrap() - } - - /// Returns the value of the element at index `i`, ignoring the array's validity. - /// # Safety - /// This function is safe iff `i < self.len`. - #[inline] - pub fn value(&self, i: usize) -> &str { - self.values.value(i) - } - - /// Returns the value of the element at index `i`, ignoring the array's validity. - /// # Safety - /// This function is safe iff `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> &str { - self.values.value_unchecked(i) - } - - /// Pop the last entry from [`MutableUtf8Array`]. - /// This function returns `None` iff this array is empty. - pub fn pop(&mut self) -> Option { - let value = self.values.pop()?; - self.validity - .as_mut() - .map(|x| x.pop()?.then_some(())) - .unwrap_or_else(|| Some(())) - .map(|_| value) - } - - fn init_validity(&mut self) { - let mut validity = MutableBitmap::with_capacity(self.values.capacity()); - validity.extend_constant(self.len(), true); - validity.set(self.len() - 1, false); - self.validity = Some(validity); - } - - /// Returns an iterator of `Option<&str>` - pub fn iter(&self) -> ZipValidity<&str, MutableUtf8ValuesIter, BitmapIter> { - ZipValidity::new(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) - } - - /// Converts itself into an [`Array`]. - pub fn into_arc(self) -> Arc { - let a: Utf8Array = self.into(); - Arc::new(a) - } - - /// Shrinks the capacity of the [`MutableUtf8Array`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - if let Some(validity) = &mut self.validity { - validity.shrink_to_fit() - } - } - - /// Extract the low-end APIs from the [`MutableUtf8Array`]. - pub fn into_data(self) -> (DataType, Offsets, Vec, Option) { - let (data_type, offsets, values) = self.values.into_inner(); - (data_type, offsets, values, self.validity) - } - - /// Returns an iterator of `&str` - pub fn values_iter(&self) -> MutableUtf8ValuesIter { - self.values.iter() - } - - /// Sets the validity. - /// # Panic - /// Panics iff the validity's len is not equal to the existing values' length. - pub fn set_validity(&mut self, validity: Option) { - if let Some(validity) = &validity { - assert_eq!(self.values.len(), validity.len()) - } - self.validity = validity; - } - - /// Applies a function `f` to the validity of this array. - /// - /// This is an API to leverage clone-on-write - /// # Panics - /// This function panics if the function `f` modifies the length of the [`Bitmap`]. - pub fn apply_validity MutableBitmap>(&mut self, f: F) { - if let Some(validity) = std::mem::take(&mut self.validity) { - self.set_validity(Some(f(validity))) - } - } -} - -impl MutableUtf8Array { - /// returns its values. - pub fn values(&self) -> &Vec { - self.values.values() - } - - /// returns its offsets. - pub fn offsets(&self) -> &Offsets { - self.values.offsets() - } -} - -impl MutableArray for MutableUtf8Array { - fn len(&self) -> usize { - self.len() - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - let array: Utf8Array = std::mem::take(self).into(); - array.boxed() - } - - fn as_arc(&mut self) -> Arc { - let array: Utf8Array = std::mem::take(self).into(); - array.arced() - } - - fn data_type(&self) -> &DataType { - if O::IS_LARGE { - &DataType::LargeUtf8 - } else { - &DataType::Utf8 - } - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - #[inline] - fn push_null(&mut self) { - self.push::<&str>(None) - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional, 0) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl> FromIterator> for MutableUtf8Array { - fn from_iter>>(iter: I) -> Self { - Self::try_from_iter(iter).unwrap() - } -} - -impl MutableUtf8Array { - /// Extends the [`MutableUtf8Array`] from an iterator of values of trusted len. - /// This differs from `extended_trusted_len` which accepts iterator of optional values. - #[inline] - pub fn extend_trusted_len_values(&mut self, iterator: I) - where - P: AsRef, - I: TrustedLen, - { - unsafe { self.extend_trusted_len_values_unchecked(iterator) } - } - - /// Extends the [`MutableUtf8Array`] from an iterator of values. - /// This differs from `extended_trusted_len` which accepts iterator of optional values. - #[inline] - pub fn extend_values(&mut self, iterator: I) - where - P: AsRef, - I: Iterator, - { - let length = self.values.len(); - self.values.extend(iterator); - let additional = self.values.len() - length; - - if let Some(validity) = self.validity.as_mut() { - validity.extend_constant(additional, true); - } - } - - /// Extends the [`MutableUtf8Array`] from an iterator of values of trusted len. - /// This differs from `extended_trusted_len_unchecked` which accepts iterator of optional - /// values. - /// # Safety - /// The iterator must be trusted len. - #[inline] - pub unsafe fn extend_trusted_len_values_unchecked(&mut self, iterator: I) - where - P: AsRef, - I: Iterator, - { - let length = self.values.len(); - self.values.extend_trusted_len_unchecked(iterator); - let additional = self.values.len() - length; - - if let Some(validity) = self.validity.as_mut() { - validity.extend_constant(additional, true); - } - } - - /// Extends the [`MutableUtf8Array`] from an iterator of trusted len. - #[inline] - pub fn extend_trusted_len(&mut self, iterator: I) - where - P: AsRef, - I: TrustedLen>, - { - unsafe { self.extend_trusted_len_unchecked(iterator) } - } - - /// Extends [`MutableUtf8Array`] from an iterator of trusted len. - /// # Safety - /// The iterator must be trusted len. - #[inline] - pub unsafe fn extend_trusted_len_unchecked(&mut self, iterator: I) - where - P: AsRef, - I: Iterator>, - { - if self.validity.is_none() { - let mut validity = MutableBitmap::new(); - validity.extend_constant(self.len(), true); - self.validity = Some(validity); - } - - self.values - .extend_from_trusted_len_iter(self.validity.as_mut().unwrap(), iterator); - } - - /// Creates a [`MutableUtf8Array`] from an iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: AsRef, - I: Iterator>, - { - let iterator = iterator.map(|x| x.map(StrAsBytes)); - let (validity, offsets, values) = trusted_len_unzip(iterator); - - // soundness: P is `str` - Self::new_unchecked(Self::default_data_type(), offsets, values, validity) - } - - /// Creates a [`MutableUtf8Array`] from an iterator of trusted length. - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: AsRef, - I: TrustedLen>, - { - // soundness: I is `TrustedLen` - unsafe { Self::from_trusted_len_iter_unchecked(iterator) } - } - - /// Creates a [`MutableUtf8Array`] from an iterator of trusted length of `&str`. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_values_iter_unchecked, I: Iterator>( - iterator: I, - ) -> Self { - MutableUtf8ValuesArray::from_trusted_len_iter_unchecked(iterator).into() - } - - /// Creates a new [`MutableUtf8Array`] from a [`TrustedLen`] of `&str`. - #[inline] - pub fn from_trusted_len_values_iter, I: TrustedLen>( - iterator: I, - ) -> Self { - // soundness: I is `TrustedLen` - unsafe { Self::from_trusted_len_values_iter_unchecked(iterator) } - } - - /// Creates a new [`MutableUtf8Array`] from an iterator. - /// # Error - /// This operation errors iff the total length in bytes on the iterator exceeds `O`'s maximum value. - /// (`i32::MAX` or `i64::MAX` respectively). - fn try_from_iter, I: IntoIterator>>(iter: I) -> Result { - let iterator = iter.into_iter(); - let (lower, _) = iterator.size_hint(); - let mut array = Self::with_capacity(lower); - for item in iterator { - array.try_push(item)?; - } - Ok(array) - } - - /// Creates a [`MutableUtf8Array`] from an falible iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn try_from_trusted_len_iter_unchecked( - iterator: I, - ) -> std::result::Result - where - P: AsRef, - I: IntoIterator, E>>, - { - let iterator = iterator.into_iter(); - - let iterator = iterator.map(|x| x.map(|x| x.map(StrAsBytes))); - let (validity, offsets, values) = try_trusted_len_unzip(iterator)?; - - // soundness: P is `str` - Ok(Self::new_unchecked( - Self::default_data_type(), - offsets, - values, - validity, - )) - } - - /// Creates a [`MutableUtf8Array`] from an falible iterator of trusted length. - #[inline] - pub fn try_from_trusted_len_iter(iterator: I) -> std::result::Result - where - P: AsRef, - I: TrustedLen, E>>, - { - // soundness: I: TrustedLen - unsafe { Self::try_from_trusted_len_iter_unchecked(iterator) } - } - - /// Creates a new [`MutableUtf8Array`] from a [`Iterator`] of `&str`. - pub fn from_iter_values, I: Iterator>(iterator: I) -> Self { - MutableUtf8ValuesArray::from_iter(iterator).into() - } - - /// Extend with a fallible iterator - pub fn extend_fallible(&mut self, iter: I) -> std::result::Result<(), E> - where - E: std::error::Error, - I: IntoIterator, E>>, - T: AsRef, - { - let mut iter = iter.into_iter(); - self.reserve(iter.size_hint().0, 0); - iter.try_for_each(|x| { - self.push(x?); - Ok(()) - }) - } -} - -impl> Extend> for MutableUtf8Array { - fn extend>>(&mut self, iter: I) { - self.try_extend(iter).unwrap(); - } -} - -impl> TryExtend> for MutableUtf8Array { - fn try_extend>>(&mut self, iter: I) -> Result<()> { - let mut iter = iter.into_iter(); - self.reserve(iter.size_hint().0, 0); - iter.try_for_each(|x| self.try_push(x)) - } -} - -impl> TryPush> for MutableUtf8Array { - #[inline] - fn try_push(&mut self, value: Option) -> Result<()> { - match value { - Some(value) => { - self.values.try_push(value.as_ref())?; - - match &mut self.validity { - Some(validity) => validity.push(true), - None => {} - } - } - None => { - self.values.push(""); - match &mut self.validity { - Some(validity) => validity.push(false), - None => self.init_validity(), - } - } - } - Ok(()) - } -} - -impl PartialEq for MutableUtf8Array { - fn eq(&self, other: &Self) -> bool { - self.iter().eq(other.iter()) - } -} - -impl TryExtendFromSelf for MutableUtf8Array { - fn try_extend_from_self(&mut self, other: &Self) -> Result<()> { - extend_validity(self.len(), &mut self.validity, &other.validity); - - self.values.try_extend_from_self(&other.values) - } -} diff --git a/src/common/arrow/src/arrow/array/utf8/mutable_values.rs b/src/common/arrow/src/arrow/array/utf8/mutable_values.rs deleted file mode 100644 index c7cd89a5a5a7..000000000000 --- a/src/common/arrow/src/arrow/array/utf8/mutable_values.rs +++ /dev/null @@ -1,440 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; -use std::sync::Arc; - -use super::MutableUtf8Array; -use super::StrAsBytes; -use super::Utf8Array; -use crate::arrow::array::physical_binary::*; -use crate::arrow::array::specification::try_check_offsets_bounds; -use crate::arrow::array::specification::try_check_utf8; -use crate::arrow::array::Array; -use crate::arrow::array::ArrayValuesIter; -use crate::arrow::array::MutableArray; -use crate::arrow::array::TryExtend; -use crate::arrow::array::TryExtendFromSelf; -use crate::arrow::array::TryPush; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::trusted_len::TrustedLen; - -/// A [`MutableArray`] that builds a [`Utf8Array`]. It differs -/// from [`MutableUtf8Array`] in that it builds non-null [`Utf8Array`]. -#[derive(Debug, Clone)] -pub struct MutableUtf8ValuesArray { - data_type: DataType, - offsets: Offsets, - values: Vec, -} - -impl From> for Utf8Array { - fn from(other: MutableUtf8ValuesArray) -> Self { - // Safety: - // `MutableUtf8ValuesArray` has the same invariants as `Utf8Array` and thus - // `Utf8Array` can be safely created from `MutableUtf8ValuesArray` without checks. - unsafe { - Utf8Array::::new_unchecked( - other.data_type, - other.offsets.into(), - other.values.into(), - None, - ) - } - } -} - -impl From> for MutableUtf8Array { - fn from(other: MutableUtf8ValuesArray) -> Self { - // Safety: - // `MutableUtf8ValuesArray` has the same invariants as `MutableUtf8Array` - unsafe { - MutableUtf8Array::::new_unchecked(other.data_type, other.offsets, other.values, None) - } - } -} - -impl Default for MutableUtf8ValuesArray { - fn default() -> Self { - Self::new() - } -} - -impl MutableUtf8ValuesArray { - /// Returns an empty [`MutableUtf8ValuesArray`]. - pub fn new() -> Self { - Self { - data_type: Self::default_data_type(), - offsets: Offsets::new(), - values: Vec::::new(), - } - } - - /// Returns a [`MutableUtf8ValuesArray`] created from its internal representation. - /// - /// # Errors - /// This function returns an error iff: - /// * The last offset is not equal to the values' length. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is not equal to either `Utf8` or `LargeUtf8`. - /// * The `values` between two consecutive `offsets` are not valid utf8 - /// # Implementation - /// This function is `O(N)` - checking utf8 is `O(N)` - pub fn try_new(data_type: DataType, offsets: Offsets, values: Vec) -> Result { - try_check_utf8(&offsets, &values)?; - if data_type.to_physical_type() != Self::default_data_type().to_physical_type() { - return Err(Error::oos( - "MutableUtf8ValuesArray can only be initialized with DataType::Utf8 or DataType::LargeUtf8", - )); - } - - Ok(Self { - data_type, - offsets, - values, - }) - } - - /// Returns a [`MutableUtf8ValuesArray`] created from its internal representation. - /// - /// # Panic - /// This function does not panic iff: - /// * The last offset is equal to the values' length. - /// * The `data_type`'s [`crate::arrow::datatypes::PhysicalType`] is equal to either `Utf8` or `LargeUtf8`. - /// # Safety - /// This function is safe iff: - /// * the offsets are monotonically increasing - /// * The `values` between two consecutive `offsets` are not valid utf8 - /// # Implementation - /// This function is `O(1)` - pub unsafe fn new_unchecked(data_type: DataType, offsets: Offsets, values: Vec) -> Self { - try_check_offsets_bounds(&offsets, values.len()) - .expect("The length of the values must be equal to the last offset value"); - - if data_type.to_physical_type() != Self::default_data_type().to_physical_type() { - panic!( - "MutableUtf8ValuesArray can only be initialized with DataType::Utf8 or DataType::LargeUtf8" - ) - } - - Self { - data_type, - offsets, - values, - } - } - - /// Returns the default [`DataType`] of this container: [`DataType::Utf8`] or [`DataType::LargeUtf8`] - /// depending on the generic [`Offset`]. - pub fn default_data_type() -> DataType { - Utf8Array::::default_data_type() - } - - /// Initializes a new [`MutableUtf8ValuesArray`] with a pre-allocated capacity of items. - pub fn with_capacity(capacity: usize) -> Self { - Self::with_capacities(capacity, 0) - } - - /// Initializes a new [`MutableUtf8ValuesArray`] with a pre-allocated capacity of items and values. - pub fn with_capacities(capacity: usize, values: usize) -> Self { - Self { - data_type: Self::default_data_type(), - offsets: Offsets::::with_capacity(capacity), - values: Vec::::with_capacity(values), - } - } - - /// returns its values. - #[inline] - pub fn values(&self) -> &Vec { - &self.values - } - - /// returns its offsets. - #[inline] - pub fn offsets(&self) -> &Offsets { - &self.offsets - } - - /// Reserves `additional` elements and `additional_values` on the values. - #[inline] - pub fn reserve(&mut self, additional: usize, additional_values: usize) { - self.offsets.reserve(additional + 1); - self.values.reserve(additional_values); - } - - /// Returns the capacity in number of items - pub fn capacity(&self) -> usize { - self.offsets.capacity() - } - - /// Returns the length of this array - #[inline] - pub fn len(&self) -> usize { - self.offsets.len_proxy() - } - - /// Returns `true` if the array has a length of 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Pushes a new item to the array. - /// # Panic - /// This operation panics iff the length of all values (in bytes) exceeds `O` maximum value. - #[inline] - pub fn push>(&mut self, value: T) { - self.try_push(value).unwrap() - } - - /// Pop the last entry from [`MutableUtf8ValuesArray`]. - /// This function returns `None` iff this array is empty. - pub fn pop(&mut self) -> Option { - if self.is_empty() { - return None; - } - self.offsets.pop()?; - let start = self.offsets.last().to_usize(); - let value = self.values.split_off(start); - // Safety: utf8 is validated on initialization - Some(unsafe { String::from_utf8_unchecked(value) }) - } - - /// Returns the value of the element at index `i`. - /// # Panic - /// This function panics iff `i >= self.len`. - #[inline] - pub fn value(&self, i: usize) -> &str { - assert!(i < self.len()); - unsafe { self.value_unchecked(i) } - } - - /// Returns the value of the element at index `i`. - /// # Safety - /// This function is safe iff `i < self.len`. - #[inline] - pub unsafe fn value_unchecked(&self, i: usize) -> &str { - // soundness: the invariant of the function - let (start, end) = self.offsets.start_end(i); - - // soundness: the invariant of the struct - let slice = self.values.get_unchecked(start..end); - - // soundness: the invariant of the struct - std::str::from_utf8_unchecked(slice) - } - - /// Returns an iterator of `&str` - pub fn iter(&self) -> ArrayValuesIter { - ArrayValuesIter::new(self) - } - - /// Shrinks the capacity of the [`MutableUtf8ValuesArray`] to fit its current length. - pub fn shrink_to_fit(&mut self) { - self.values.shrink_to_fit(); - self.offsets.shrink_to_fit(); - } - - /// Extract the low-end APIs from the [`MutableUtf8ValuesArray`]. - pub fn into_inner(self) -> (DataType, Offsets, Vec) { - (self.data_type, self.offsets, self.values) - } -} - -impl MutableArray for MutableUtf8ValuesArray { - fn len(&self) -> usize { - self.len() - } - - fn validity(&self) -> Option<&MutableBitmap> { - None - } - - fn as_box(&mut self) -> Box { - let array: Utf8Array = std::mem::take(self).into(); - array.boxed() - } - - fn as_arc(&mut self) -> Arc { - let array: Utf8Array = std::mem::take(self).into(); - array.arced() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - #[inline] - fn push_null(&mut self) { - self.push::<&str>("") - } - - fn reserve(&mut self, additional: usize) { - self.reserve(additional, 0) - } - - fn shrink_to_fit(&mut self) { - self.shrink_to_fit() - } -} - -impl> FromIterator

for MutableUtf8ValuesArray { - fn from_iter>(iter: I) -> Self { - let (offsets, values) = values_iter(iter.into_iter().map(StrAsBytes)); - // soundness: T: AsRef and offsets are monotonically increasing - unsafe { Self::new_unchecked(Self::default_data_type(), offsets, values) } - } -} - -impl MutableUtf8ValuesArray { - pub(crate) unsafe fn extend_from_trusted_len_iter( - &mut self, - validity: &mut MutableBitmap, - iterator: I, - ) where - P: AsRef, - I: Iterator>, - { - let iterator = iterator.map(|x| x.map(StrAsBytes)); - extend_from_trusted_len_iter(&mut self.offsets, &mut self.values, validity, iterator); - } - - /// Extends the [`MutableUtf8ValuesArray`] from a [`TrustedLen`] - #[inline] - pub fn extend_trusted_len(&mut self, iterator: I) - where - P: AsRef, - I: TrustedLen, - { - unsafe { self.extend_trusted_len_unchecked(iterator) } - } - - /// Extends [`MutableUtf8ValuesArray`] from an iterator of trusted len. - /// # Safety - /// The iterator must be trusted len. - #[inline] - pub unsafe fn extend_trusted_len_unchecked(&mut self, iterator: I) - where - P: AsRef, - I: Iterator, - { - let iterator = iterator.map(StrAsBytes); - extend_from_trusted_len_values_iter(&mut self.offsets, &mut self.values, iterator); - } - - /// Creates a [`MutableUtf8ValuesArray`] from a [`TrustedLen`] - #[inline] - pub fn from_trusted_len_iter(iterator: I) -> Self - where - P: AsRef, - I: TrustedLen, - { - // soundness: I is `TrustedLen` - unsafe { Self::from_trusted_len_iter_unchecked(iterator) } - } - - /// Returns a new [`MutableUtf8ValuesArray`] from an iterator of trusted length. - /// # Safety - /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). - /// I.e. that `size_hint().1` correctly reports its length. - #[inline] - pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self - where - P: AsRef, - I: Iterator, - { - let iterator = iterator.map(StrAsBytes); - let (offsets, values) = trusted_len_values_iter(iterator); - - // soundness: P is `str` and offsets are monotonically increasing - Self::new_unchecked(Self::default_data_type(), offsets, values) - } - - /// Returns a new [`MutableUtf8ValuesArray`] from an iterator. - /// # Error - /// This operation errors iff the total length in bytes on the iterator exceeds `O`'s maximum value. - /// (`i32::MAX` or `i64::MAX` respectively). - pub fn try_from_iter, I: IntoIterator>(iter: I) -> Result { - let iterator = iter.into_iter(); - let (lower, _) = iterator.size_hint(); - let mut array = Self::with_capacity(lower); - for item in iterator { - array.try_push(item)?; - } - Ok(array) - } - - /// Extend with a fallible iterator - pub fn extend_fallible(&mut self, iter: I) -> std::result::Result<(), E> - where - E: std::error::Error, - I: IntoIterator>, - T: AsRef, - { - let mut iter = iter.into_iter(); - self.reserve(iter.size_hint().0, 0); - iter.try_for_each(|x| { - self.push(x?); - Ok(()) - }) - } -} - -impl> Extend for MutableUtf8ValuesArray { - fn extend>(&mut self, iter: I) { - extend_from_values_iter( - &mut self.offsets, - &mut self.values, - iter.into_iter().map(StrAsBytes), - ); - } -} - -impl> TryExtend for MutableUtf8ValuesArray { - fn try_extend>(&mut self, iter: I) -> Result<()> { - let mut iter = iter.into_iter(); - self.reserve(iter.size_hint().0, 0); - iter.try_for_each(|x| self.try_push(x)) - } -} - -impl> TryPush for MutableUtf8ValuesArray { - #[inline] - fn try_push(&mut self, value: T) -> Result<()> { - let bytes = value.as_ref().as_bytes(); - self.values.extend_from_slice(bytes); - self.offsets.try_push_usize(bytes.len()) - } -} - -impl TryExtendFromSelf for MutableUtf8ValuesArray { - fn try_extend_from_self(&mut self, other: &Self) -> Result<()> { - self.values.extend_from_slice(&other.values); - self.offsets.try_extend_from_self(&other.offsets) - } -} diff --git a/src/common/arrow/src/arrow/chunk.rs b/src/common/arrow/src/arrow/chunk.rs deleted file mode 100644 index 52ea6650e755..000000000000 --- a/src/common/arrow/src/arrow/chunk.rs +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains [`Chunk`], a container of [`Array`] where every array has the -//! same length. - -use crate::arrow::array::Array; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// A vector of trait objects of [`Array`] where every item has -/// the same length, [`Chunk::len`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Chunk> { - arrays: Vec, -} - -impl> Chunk { - /// Creates a new [`Chunk`]. - /// # Panic - /// Iff the arrays do not have the same length - pub fn new(arrays: Vec) -> Self { - Self::try_new(arrays).unwrap() - } - - /// Creates a new [`Chunk`]. - /// # Error - /// Iff the arrays do not have the same length - pub fn try_new(arrays: Vec) -> Result { - if !arrays.is_empty() { - let len = arrays.first().unwrap().as_ref().len(); - if arrays - .iter() - .map(|array| array.as_ref()) - .any(|array| array.len() != len) - { - return Err(Error::InvalidArgumentError( - "Chunk require all its arrays to have an equal number of rows".to_string(), - )); - } - } - Ok(Self { arrays }) - } - - /// returns the [`Array`]s in [`Chunk`] - pub fn arrays(&self) -> &[A] { - &self.arrays - } - - /// returns the [`Array`]s in [`Chunk`] - pub fn columns(&self) -> &[A] { - &self.arrays - } - - /// returns the number of rows of every array - pub fn len(&self) -> usize { - self.arrays - .first() - .map(|x| x.as_ref().len()) - .unwrap_or_default() - } - - /// returns whether the columns have any rows - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Consumes [`Chunk`] into its underlying arrays. - /// The arrays are guaranteed to have the same length - pub fn into_arrays(self) -> Vec { - self.arrays - } -} - -impl> From> for Vec { - fn from(c: Chunk) -> Self { - c.into_arrays() - } -} - -impl> std::ops::Deref for Chunk { - type Target = [A]; - - #[inline] - fn deref(&self) -> &[A] { - self.arrays() - } -} diff --git a/src/common/arrow/src/arrow/compute/README.md b/src/common/arrow/src/arrow/compute/README.md deleted file mode 100644 index 247a07800120..000000000000 --- a/src/common/arrow/src/arrow/compute/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Design - -This document outlines the design guide lines of this module. - -This module is composed by independent operations common in analytics. Below are some design of its principles: - -* APIs MUST return an error when either: - * The arguments are incorrect - * The execution results in a predictable error (e.g. divide by zero) - -* APIs MAY error when an operation overflows (e.g. `i32 + i32`) - -* kernels MUST NOT have side-effects - -* kernels MUST NOT take ownership of any of its arguments (i.e. everything must be a reference). - -* APIs SHOULD error when an operation on variable sized containers can overflow the maximum size of `usize`. - -* Kernels SHOULD use the arrays' logical type to decide whether kernels -can be applied on an array. For example, `Date32 + Date32` is meaningless and SHOULD NOT be implemented. - -* Kernels SHOULD be implemented via `clone`, `slice` or the `iterator` API provided by `Buffer`, `Bitmap`, `Vec` or `MutableBitmap`. - -* Kernels MUST NOT use any API to read bits other than the ones provided by `Bitmap`. - -* Implementations SHOULD aim for auto-vectorization, which is usually accomplished via `from_trusted_len_iter`. - -* Implementations MUST feature-gate any implementation that requires external dependencies - -* When a kernel accepts dynamically-typed arrays, it MUST expect them as `&dyn Array`. - -* When an API returns `&dyn Array`, it MUST return `Box`. The rational is that a `Box` is mutable, while an `Arc` is not. As such, `Box` offers the most flexible API to consumers and the compiler. Users can cast a `Box` into `Arc` via `.into()`. diff --git a/src/common/arrow/src/arrow/compute/concatenate.rs b/src/common/arrow/src/arrow/compute/concatenate.rs deleted file mode 100644 index 0990bd12c431..000000000000 --- a/src/common/arrow/src/arrow/compute/concatenate.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains the concatenate kernel -//! -//! Example: -//! -//! ``` -//! use arrow2::array::Utf8Array; -//! use arrow2::compute::concatenate::concatenate; -//! -//! let arr = concatenate(&[ -//! &Utf8Array::::from_slice(["hello", "world"]), -//! &Utf8Array::::from_slice(["!"]), -//! ]) -//! .unwrap(); -//! assert_eq!(arr.len(), 3); -//! ``` - -use crate::arrow::array::growable::make_growable; -use crate::arrow::array::Array; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -/// Concatenate multiple [Array] of the same type into a single [`Array`]. -pub fn concatenate(arrays: &[&dyn Array]) -> Result> { - if arrays.is_empty() { - return Err(Error::InvalidArgumentError( - "concat requires input of at least one array".to_string(), - )); - } - - if arrays - .iter() - .any(|array| array.data_type() != arrays[0].data_type()) - { - return Err(Error::InvalidArgumentError( - "It is not possible to concatenate arrays of different data types.".to_string(), - )); - } - - let lengths = arrays.iter().map(|array| array.len()).collect::>(); - let capacity = lengths.iter().sum(); - - let mut mutable = make_growable(arrays, false, capacity); - - for (i, len) in lengths.iter().enumerate() { - mutable.extend(i, 0, *len) - } - - Ok(mutable.as_box()) -} diff --git a/src/common/arrow/src/arrow/compute/mod.rs b/src/common/arrow/src/arrow/compute/mod.rs deleted file mode 100644 index 2b2108fb8985..000000000000 --- a/src/common/arrow/src/arrow/compute/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! contains a wide range of compute operations (e.g. -//! [`arithmetics`], [`aggregate`], -//! [`filter`], [`comparison`], and [`sort`]) -//! -//! This module's general design is -//! that each operator has two interfaces, a statically-typed version and a dynamically-typed -//! version. -//! The statically-typed version expects concrete arrays (such as [`PrimitiveArray`](crate::arrow::array::PrimitiveArray)); -//! the dynamically-typed version expects `&dyn Array` and errors if the the type is not -//! supported. -//! Some dynamically-typed operators have an auxiliary function, `can_*`, that returns -//! true if the operator can be applied to the particular `DataType`. - -#[cfg(feature = "compute_concatenate")] -#[cfg_attr(docsrs, doc(cfg(feature = "compute_concatenate")))] -pub mod concatenate; -mod utils; diff --git a/src/common/arrow/src/arrow/compute/utils.rs b/src/common/arrow/src/arrow/compute/utils.rs deleted file mode 100644 index 592feed04376..000000000000 --- a/src/common/arrow/src/arrow/compute/utils.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::error::Error; -use crate::arrow::error::Result; - -pub fn combine_validities(lhs: Option<&Bitmap>, rhs: Option<&Bitmap>) -> Option { - match (lhs, rhs) { - (Some(lhs), None) => Some(lhs.clone()), - (None, Some(rhs)) => Some(rhs.clone()), - (None, None) => None, - (Some(lhs), Some(rhs)) => Some(lhs & rhs), - } -} - -// Errors iff the two arrays have a different length. -#[inline] -pub fn check_same_len(lhs: &dyn Array, rhs: &dyn Array) -> Result<()> { - if lhs.len() != rhs.len() { - return Err(Error::InvalidArgumentError( - "Arrays must have the same length".to_string(), - )); - } - Ok(()) -} diff --git a/src/common/arrow/src/arrow/datatypes/field.rs b/src/common/arrow/src/arrow/datatypes/field.rs deleted file mode 100644 index ef852887ce7a..000000000000 --- a/src/common/arrow/src/arrow/datatypes/field.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(feature = "serde_types")] -use serde_derive::Deserialize; -#[cfg(feature = "serde_types")] -use serde_derive::Serialize; - -use super::DataType; -use super::Metadata; - -/// Represents Arrow's metadata of a "column". -/// -/// A [`Field`] is the closest representation of the traditional "column": a logical type -/// ([`DataType`]) with a name and nullability. -/// A Field has optional [`Metadata`] that can be used to annotate the field with custom metadata. -/// -/// Almost all IO in this crate uses [`Field`] to represent logical information about the data -/// to be serialized. -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] -pub struct Field { - /// Its name - pub name: String, - /// Its logical [`DataType`] - pub data_type: DataType, - /// Its nullability - pub is_nullable: bool, - /// Additional custom (opaque) metadata. - pub metadata: Metadata, -} - -impl Field { - /// Creates a new [`Field`]. - pub fn new>(name: T, data_type: DataType, is_nullable: bool) -> Self { - Field { - name: name.into(), - data_type, - is_nullable, - metadata: Default::default(), - } - } - - /// Creates a new [`Field`] with metadata. - #[inline] - pub fn with_metadata(self, metadata: Metadata) -> Self { - Self { - name: self.name, - data_type: self.data_type, - is_nullable: self.is_nullable, - metadata, - } - } - - /// Returns the [`Field`]'s [`DataType`]. - #[inline] - pub fn data_type(&self) -> &DataType { - &self.data_type - } -} - -// For databend's extension key -pub const EXTENSION_KEY: &str = "Extension"; - -#[cfg(feature = "arrow")] -impl From for arrow_schema::Field { - fn from(value: Field) -> Self { - (&value).into() - } -} - -#[cfg(feature = "arrow")] -impl From<&Field> for arrow_schema::Field { - fn from(value: &Field) -> Self { - let mut metadata = value.metadata.clone(); - let ty = if let DataType::Extension(extension_type, ty, _) = &value.data_type { - metadata.insert(EXTENSION_KEY.to_string(), extension_type.clone()); - ty.as_ref().clone() - } else { - value.data_type.clone() - }; - - Self::new(value.name.clone(), ty.into(), value.is_nullable) - .with_metadata(metadata.into_iter().collect()) - } -} - -#[cfg(feature = "arrow")] -impl From for Field { - fn from(value: arrow_schema::Field) -> Self { - (&value).into() - } -} - -#[cfg(feature = "arrow")] -impl From<&arrow_schema::Field> for Field { - fn from(value: &arrow_schema::Field) -> Self { - let mut data_type = value.data_type().clone().into(); - let mut metadata: Metadata = value - .metadata() - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - if let Some(v) = metadata.remove(EXTENSION_KEY) { - data_type = DataType::Extension(v, Box::new(data_type), None); - } - Self::new(value.name(), data_type, value.is_nullable()).with_metadata(metadata) - } -} - -#[cfg(feature = "arrow")] -impl From for Field { - fn from(value: arrow_schema::FieldRef) -> Self { - value.as_ref().into() - } -} - -#[cfg(feature = "arrow")] -impl From<&arrow_schema::FieldRef> for Field { - fn from(value: &arrow_schema::FieldRef) -> Self { - value.as_ref().into() - } -} diff --git a/src/common/arrow/src/arrow/datatypes/mod.rs b/src/common/arrow/src/arrow/datatypes/mod.rs deleted file mode 100644 index 2a0075a68573..000000000000 --- a/src/common/arrow/src/arrow/datatypes/mod.rs +++ /dev/null @@ -1,539 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![forbid(unsafe_code)] -//! Contains all metadata, such as [`PhysicalType`], [`DataType`], [`Field`] and [`Schema`]. - -mod field; -mod physical_type; -mod schema; - -use std::collections::BTreeMap; -use std::sync::Arc; - -pub use field::Field; -pub use physical_type::*; -pub use schema::Schema; -#[cfg(feature = "serde_types")] -use serde_derive::Deserialize; -#[cfg(feature = "serde_types")] -use serde_derive::Serialize; - -/// typedef for [BTreeMap] denoting [`Field`]'s and [`Schema`]'s metadata. -pub type Metadata = BTreeMap; -/// typedef for [Option<(String, Option)>] descr -pub(crate) type Extension = Option<(String, Option)>; - -/// The set of supported logical types in this crate. -/// -/// Each variant uniquely identifies a logical type, which define specific semantics to the data -/// (e.g. how it should be represented). -/// Each variant has a corresponding [`PhysicalType`], obtained via [`DataType::to_physical_type`], -/// which declares the in-memory representation of data. -/// The [`DataType::Extension`] is special in that it augments a [`DataType`] with metadata to support custom types. -/// Use `to_logical_type` to desugar such type and return its corresponding logical type. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] -pub enum DataType { - /// Null type - Null, - /// `true` and `false`. - Boolean, - /// An [`i8`] - Int8, - /// An [`i16`] - Int16, - /// An [`i32`] - Int32, - /// An [`i64`] - Int64, - /// An [`u8`] - UInt8, - /// An [`u16`] - UInt16, - /// An [`u32`] - UInt32, - /// An [`u64`] - UInt64, - /// An 16-bit float - Float16, - /// A [`f32`] - Float32, - /// A [`f64`] - Float64, - /// A [`i64`] representing a timestamp measured in [`TimeUnit`] with an optional timezone. - /// - /// Time is measured as a Unix epoch, counting the seconds from - /// 00:00:00.000 on 1 January 1970, excluding leap seconds, - /// as a 64-bit signed integer. - /// - /// The time zone is a string indicating the name of a time zone, one of: - /// - /// * As used in the Olson time zone database (the "tz database" or - /// "tzdata"), such as "America/New_York" - /// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 - /// - /// When the timezone is not specified, the timestamp is considered to have no timezone - /// and is represented _as is_ - Timestamp(TimeUnit, Option), - /// An [`i32`] representing the elapsed time since UNIX epoch (1970-01-01) - /// in days. - Date32, - /// An [`i64`] representing the elapsed time since UNIX epoch (1970-01-01) - /// in milliseconds. Values are evenly divisible by 86400000. - Date64, - /// A 32-bit time representing the elapsed time since midnight in the unit of `TimeUnit`. - /// Only [`TimeUnit::Second`] and [`TimeUnit::Millisecond`] are supported on this variant. - Time32(TimeUnit), - /// A 64-bit time representing the elapsed time since midnight in the unit of `TimeUnit`. - /// Only [`TimeUnit::Microsecond`] and [`TimeUnit::Nanosecond`] are supported on this variant. - Time64(TimeUnit), - /// Measure of elapsed time. This elapsed time is a physical duration (i.e. 1s as defined in S.I.) - Duration(TimeUnit), - /// A "calendar" interval modeling elapsed time that takes into account calendar shifts. - /// For example an interval of 1 day may represent more than 24 hours. - Interval(IntervalUnit), - /// Opaque binary data of variable length whose offsets are represented as [`i32`]. - Binary, - /// Opaque binary data of fixed size. - /// Enum parameter specifies the number of bytes per value. - FixedSizeBinary(usize), - /// Opaque binary data of variable length whose offsets are represented as [`i64`]. - LargeBinary, - /// A variable-length UTF-8 encoded string whose offsets are represented as [`i32`]. - Utf8, - /// A variable-length UTF-8 encoded string whose offsets are represented as [`i64`]. - LargeUtf8, - /// A list of some logical data type whose offsets are represented as [`i32`]. - List(Box), - /// A list of some logical data type with a fixed number of elements. - FixedSizeList(Box, usize), - /// A list of some logical data type whose offsets are represented as [`i64`]. - LargeList(Box), - /// A nested [`DataType`] with a given number of [`Field`]s. - Struct(Vec), - /// A nested datatype that can represent slots of differing types. - /// Third argument represents mode - Union(Vec, Option>, UnionMode), - /// A nested type that is represented as - /// - /// List> - /// - /// In this layout, the keys and values are each respectively contiguous. We do - /// not constrain the key and value types, so the application is responsible - /// for ensuring that the keys are hashable and unique. Whether the keys are sorted - /// may be set in the metadata for this field. - /// - /// In a field with Map type, the field has a child Struct field, which then - /// has two children: key type and the second the value type. The names of the - /// child fields may be respectively "entries", "key", and "value", but this is - /// not enforced. - /// - /// Map - /// ```text - /// - child[0] entries: Struct - /// - child[0] key: K - /// - child[1] value: V - /// ``` - /// Neither the "entries" field nor the "key" field may be nullable. - /// - /// The metadata is structured so that Arrow systems without special handling - /// for Map can make Map an alias for List. The "layout" attribute for the Map - /// field must have the same contents as a List. - Map(Box, bool), - /// A dictionary encoded array (`key_type`, `value_type`), where - /// each array element is an index of `key_type` into an - /// associated dictionary of `value_type`. - /// - /// Dictionary arrays are used to store columns of `value_type` - /// that contain many repeated values using less memory, but with - /// a higher CPU overhead for some operations. - /// - /// This type mostly used to represent low cardinality string - /// arrays or a limited set of primitive types as integers. - /// - /// The `bool` value indicates the `Dictionary` is sorted if set to `true`. - Dictionary(IntegerType, Box, bool), - /// Decimal value with precision and scale - /// precision is the number of digits in the number and - /// scale is the number of decimal places. - /// The number 999.99 has a precision of 5 and scale of 2. - Decimal(usize, usize), - /// Decimal backed by 256 bits - Decimal256(usize, usize), - /// Extension type. - Extension(String, Box, Option), - /// A binary type that inlines small values and can intern bytes. - BinaryView, - /// A string type that inlines small values and can intern strings. - Utf8View, -} - -#[cfg(feature = "arrow")] -impl From for arrow_schema::DataType { - fn from(value: DataType) -> Self { - use arrow_schema::Field as ArrowField; - use arrow_schema::UnionFields; - - match value { - DataType::Null => Self::Null, - DataType::Boolean => Self::Boolean, - DataType::Int8 => Self::Int8, - DataType::Int16 => Self::Int16, - DataType::Int32 => Self::Int32, - DataType::Int64 => Self::Int64, - DataType::UInt8 => Self::UInt8, - DataType::UInt16 => Self::UInt16, - DataType::UInt32 => Self::UInt32, - DataType::UInt64 => Self::UInt64, - DataType::Float16 => Self::Float16, - DataType::Float32 => Self::Float32, - DataType::Float64 => Self::Float64, - DataType::Timestamp(unit, tz) => Self::Timestamp(unit.into(), tz.map(Into::into)), - DataType::Date32 => Self::Date32, - DataType::Date64 => Self::Date64, - DataType::Time32(unit) => Self::Time32(unit.into()), - DataType::Time64(unit) => Self::Time64(unit.into()), - DataType::Duration(unit) => Self::Duration(unit.into()), - DataType::Interval(unit) => Self::Interval(unit.into()), - DataType::Binary => Self::Binary, - DataType::FixedSizeBinary(size) => Self::FixedSizeBinary(size as _), - DataType::LargeBinary => Self::LargeBinary, - DataType::Utf8 => Self::Utf8, - DataType::LargeUtf8 => Self::LargeUtf8, - DataType::List(f) => Self::List(Arc::new((*f).into())), - DataType::FixedSizeList(f, size) => { - Self::FixedSizeList(Arc::new((*f).into()), size as _) - } - DataType::LargeList(f) => Self::LargeList(Arc::new((*f).into())), - DataType::Struct(f) => Self::Struct(f.into_iter().map(ArrowField::from).collect()), - DataType::Union(fields, Some(ids), mode) => { - let ids = ids.into_iter().map(|x| x as _); - let fields = fields.into_iter().map(ArrowField::from); - Self::Union(UnionFields::new(ids, fields), mode.into()) - } - DataType::Union(fields, None, mode) => { - let ids = 0..fields.len() as i8; - let fields = fields.into_iter().map(ArrowField::from); - Self::Union(UnionFields::new(ids, fields), mode.into()) - } - DataType::Map(f, ordered) => Self::Map(Arc::new((*f).into()), ordered), - DataType::Dictionary(key, value, _) => Self::Dictionary( - Box::new(DataType::from(key).into()), - Box::new((*value).into()), - ), - DataType::Decimal(precision, scale) => Self::Decimal128(precision as _, scale as _), - DataType::Decimal256(precision, scale) => Self::Decimal256(precision as _, scale as _), - DataType::Extension(_, d, _) => (*d).into(), - DataType::BinaryView => Self::BinaryView, - DataType::Utf8View => Self::Utf8View, - } - } -} - -#[cfg(feature = "arrow")] -impl From for DataType { - fn from(value: arrow_schema::DataType) -> Self { - use arrow_schema::DataType; - match value { - DataType::Null => Self::Null, - DataType::Boolean => Self::Boolean, - DataType::Int8 => Self::Int8, - DataType::Int16 => Self::Int16, - DataType::Int32 => Self::Int32, - DataType::Int64 => Self::Int64, - DataType::UInt8 => Self::UInt8, - DataType::UInt16 => Self::UInt16, - DataType::UInt32 => Self::UInt32, - DataType::UInt64 => Self::UInt64, - DataType::Float16 => Self::Float16, - DataType::Float32 => Self::Float32, - DataType::Float64 => Self::Float64, - DataType::Timestamp(unit, tz) => { - Self::Timestamp(unit.into(), tz.map(|x| x.to_string())) - } - DataType::Date32 => Self::Date32, - DataType::Date64 => Self::Date64, - DataType::Time32(unit) => Self::Time32(unit.into()), - DataType::Time64(unit) => Self::Time64(unit.into()), - DataType::Duration(unit) => Self::Duration(unit.into()), - DataType::Interval(unit) => Self::Interval(unit.into()), - DataType::Binary => Self::Binary, - DataType::FixedSizeBinary(size) => Self::FixedSizeBinary(size as _), - DataType::LargeBinary => Self::LargeBinary, - DataType::Utf8 => Self::Utf8, - DataType::LargeUtf8 => Self::LargeUtf8, - DataType::List(f) => Self::List(Box::new(f.into())), - DataType::FixedSizeList(f, size) => Self::FixedSizeList(Box::new(f.into()), size as _), - DataType::LargeList(f) => Self::LargeList(Box::new(f.into())), - DataType::Struct(f) => Self::Struct(f.into_iter().map(Into::into).collect()), - DataType::Union(fields, mode) => { - let ids = fields.iter().map(|(x, _)| x as _).collect(); - let fields = fields.iter().map(|(_, f)| f.into()).collect(); - Self::Union(fields, Some(ids), mode.into()) - } - DataType::Map(f, ordered) => Self::Map(Box::new(f.into()), ordered), - DataType::Dictionary(key, value) => { - let key = match *key { - DataType::Int8 => IntegerType::Int8, - DataType::Int16 => IntegerType::Int16, - DataType::Int32 => IntegerType::Int32, - DataType::Int64 => IntegerType::Int64, - DataType::UInt8 => IntegerType::UInt8, - DataType::UInt16 => IntegerType::UInt16, - DataType::UInt32 => IntegerType::UInt32, - DataType::UInt64 => IntegerType::UInt64, - d => panic!("illegal dictionary key type: {d}"), - }; - Self::Dictionary(key, Box::new((*value).into()), false) - } - DataType::Decimal128(precision, scale) => Self::Decimal(precision as _, scale as _), - DataType::Decimal256(precision, scale) => Self::Decimal256(precision as _, scale as _), - DataType::BinaryView => Self::BinaryView, - DataType::Utf8View => Self::Utf8View, - v => panic!("{:?} encoding not supported by arrow2", v), - } - } -} - -/// Mode of [`DataType::Union`] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] -pub enum UnionMode { - /// Dense union - Dense, - /// Sparse union - Sparse, -} - -#[cfg(feature = "arrow")] -impl From for arrow_schema::UnionMode { - fn from(value: UnionMode) -> Self { - match value { - UnionMode::Dense => Self::Dense, - UnionMode::Sparse => Self::Sparse, - } - } -} - -#[cfg(feature = "arrow")] -impl From for UnionMode { - fn from(value: arrow_schema::UnionMode) -> Self { - match value { - arrow_schema::UnionMode::Dense => Self::Dense, - arrow_schema::UnionMode::Sparse => Self::Sparse, - } - } -} - -impl UnionMode { - /// Constructs a [`UnionMode::Sparse`] if the input bool is true, - /// or otherwise constructs a [`UnionMode::Dense`] - pub fn sparse(is_sparse: bool) -> Self { - if is_sparse { Self::Sparse } else { Self::Dense } - } - - /// Returns whether the mode is sparse - pub fn is_sparse(&self) -> bool { - matches!(self, Self::Sparse) - } - - /// Returns whether the mode is dense - pub fn is_dense(&self) -> bool { - matches!(self, Self::Dense) - } -} - -/// The time units defined in Arrow. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] -pub enum TimeUnit { - /// Time in seconds. - Second, - /// Time in milliseconds. - Millisecond, - /// Time in microseconds. - Microsecond, - /// Time in nanoseconds. - Nanosecond, -} - -#[cfg(feature = "arrow")] -impl From for arrow_schema::TimeUnit { - fn from(value: TimeUnit) -> Self { - match value { - TimeUnit::Nanosecond => Self::Nanosecond, - TimeUnit::Millisecond => Self::Millisecond, - TimeUnit::Microsecond => Self::Microsecond, - TimeUnit::Second => Self::Second, - } - } -} - -#[cfg(feature = "arrow")] -impl From for TimeUnit { - fn from(value: arrow_schema::TimeUnit) -> Self { - match value { - arrow_schema::TimeUnit::Nanosecond => Self::Nanosecond, - arrow_schema::TimeUnit::Millisecond => Self::Millisecond, - arrow_schema::TimeUnit::Microsecond => Self::Microsecond, - arrow_schema::TimeUnit::Second => Self::Second, - } - } -} - -/// Interval units defined in Arrow -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] -pub enum IntervalUnit { - /// The number of elapsed whole months. - YearMonth, - /// The number of elapsed days and milliseconds, - /// stored as 2 contiguous `i32` - DayTime, - /// The number of elapsed months (i32), days (i32) and nanoseconds (i64). - MonthDayNano, -} - -#[cfg(feature = "arrow")] -impl From for arrow_schema::IntervalUnit { - fn from(value: IntervalUnit) -> Self { - match value { - IntervalUnit::YearMonth => Self::YearMonth, - IntervalUnit::DayTime => Self::DayTime, - IntervalUnit::MonthDayNano => Self::MonthDayNano, - } - } -} - -#[cfg(feature = "arrow")] -impl From for IntervalUnit { - fn from(value: arrow_schema::IntervalUnit) -> Self { - match value { - arrow_schema::IntervalUnit::YearMonth => Self::YearMonth, - arrow_schema::IntervalUnit::DayTime => Self::DayTime, - arrow_schema::IntervalUnit::MonthDayNano => Self::MonthDayNano, - } - } -} - -impl DataType { - /// the [`PhysicalType`] of this [`DataType`]. - pub fn to_physical_type(&self) -> PhysicalType { - use DataType::*; - match self { - Null => PhysicalType::Null, - Boolean => PhysicalType::Boolean, - Int8 => PhysicalType::Primitive(PrimitiveType::Int8), - Int16 => PhysicalType::Primitive(PrimitiveType::Int16), - Int32 | Date32 | Time32(_) | Interval(IntervalUnit::YearMonth) => { - PhysicalType::Primitive(PrimitiveType::Int32) - } - Int64 | Date64 | Timestamp(_, _) | Time64(_) | Duration(_) => { - PhysicalType::Primitive(PrimitiveType::Int64) - } - Decimal(_, _) => PhysicalType::Primitive(PrimitiveType::Int128), - Decimal256(_, _) => PhysicalType::Primitive(PrimitiveType::Int256), - UInt8 => PhysicalType::Primitive(PrimitiveType::UInt8), - UInt16 => PhysicalType::Primitive(PrimitiveType::UInt16), - UInt32 => PhysicalType::Primitive(PrimitiveType::UInt32), - UInt64 => PhysicalType::Primitive(PrimitiveType::UInt64), - Float16 => PhysicalType::Primitive(PrimitiveType::Float16), - Float32 => PhysicalType::Primitive(PrimitiveType::Float32), - Float64 => PhysicalType::Primitive(PrimitiveType::Float64), - Interval(IntervalUnit::DayTime) => PhysicalType::Primitive(PrimitiveType::DaysMs), - Interval(IntervalUnit::MonthDayNano) => { - PhysicalType::Primitive(PrimitiveType::MonthDayNano) - } - Binary => PhysicalType::Binary, - FixedSizeBinary(_) => PhysicalType::FixedSizeBinary, - LargeBinary => PhysicalType::LargeBinary, - Utf8 => PhysicalType::Utf8, - LargeUtf8 => PhysicalType::LargeUtf8, - BinaryView => PhysicalType::BinaryView, - Utf8View => PhysicalType::Utf8View, - List(_) => PhysicalType::List, - FixedSizeList(_, _) => PhysicalType::FixedSizeList, - LargeList(_) => PhysicalType::LargeList, - Struct(_) => PhysicalType::Struct, - Union(_, _, _) => PhysicalType::Union, - Map(_, _) => PhysicalType::Map, - Dictionary(key, _, _) => PhysicalType::Dictionary(*key), - Extension(_, key, _) => key.to_physical_type(), - } - } - - /// Returns `&self` for all but [`DataType::Extension`]. For [`DataType::Extension`], - /// (recursively) returns the inner [`DataType`]. - /// Never returns the variant [`DataType::Extension`]. - pub fn to_logical_type(&self) -> &DataType { - use DataType::*; - match self { - Extension(_, key, _) => key.to_logical_type(), - _ => self, - } - } -} - -impl From for DataType { - fn from(item: IntegerType) -> Self { - match item { - IntegerType::Int8 => DataType::Int8, - IntegerType::Int16 => DataType::Int16, - IntegerType::Int32 => DataType::Int32, - IntegerType::Int64 => DataType::Int64, - IntegerType::UInt8 => DataType::UInt8, - IntegerType::UInt16 => DataType::UInt16, - IntegerType::UInt32 => DataType::UInt32, - IntegerType::UInt64 => DataType::UInt64, - } - } -} - -impl From for DataType { - fn from(item: PrimitiveType) -> Self { - match item { - PrimitiveType::Int8 => DataType::Int8, - PrimitiveType::Int16 => DataType::Int16, - PrimitiveType::Int32 => DataType::Int32, - PrimitiveType::Int64 => DataType::Int64, - PrimitiveType::UInt8 => DataType::UInt8, - PrimitiveType::UInt16 => DataType::UInt16, - PrimitiveType::UInt32 => DataType::UInt32, - PrimitiveType::UInt64 => DataType::UInt64, - PrimitiveType::Int128 => DataType::Decimal(32, 32), - PrimitiveType::Int256 => DataType::Decimal256(32, 32), - PrimitiveType::Float16 => DataType::Float16, - PrimitiveType::Float32 => DataType::Float32, - PrimitiveType::Float64 => DataType::Float64, - PrimitiveType::DaysMs => DataType::Interval(IntervalUnit::DayTime), - PrimitiveType::MonthDayNano => DataType::Interval(IntervalUnit::MonthDayNano), - PrimitiveType::UInt128 => unimplemented!(), - } - } -} - -/// typedef for [`Arc`]. -pub type SchemaRef = Arc; - -/// support get extension for metadata -pub fn get_extension(metadata: &Metadata) -> Extension { - if let Some(name) = metadata.get("ARROW:extension:name") { - let metadata = metadata.get("ARROW:extension:metadata").cloned(); - Some((name.clone(), metadata)) - } else { - None - } -} diff --git a/src/common/arrow/src/arrow/datatypes/physical_type.rs b/src/common/arrow/src/arrow/datatypes/physical_type.rs deleted file mode 100644 index 82f5eb7ee638..000000000000 --- a/src/common/arrow/src/arrow/datatypes/physical_type.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(feature = "serde_types")] -use serde_derive::Deserialize; -#[cfg(feature = "serde_types")] -use serde_derive::Serialize; - -pub use crate::arrow::types::PrimitiveType; - -/// The set of physical types: unique in-memory representations of an Arrow array. -/// A physical type has a one-to-many relationship with a [`crate::datatypes::DataType`] and -/// a one-to-one mapping to each struct in this crate that implements [`crate::array::Array`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] -pub enum PhysicalType { - /// A Null with no allocation. - Null, - /// A boolean represented as a single bit. - Boolean, - /// An array where each slot has a known compile-time size. - Primitive(PrimitiveType), - /// Opaque binary data of variable length. - Binary, - /// Opaque binary data of fixed size. - FixedSizeBinary, - /// Opaque binary data of variable length and 64-bit offsets. - LargeBinary, - /// A variable-length string in Unicode with UTF-8 encoding. - Utf8, - /// A variable-length string in Unicode with UFT-8 encoding and 64-bit offsets. - LargeUtf8, - /// A list of some data type with variable length. - List, - /// A list of some data type with fixed length. - FixedSizeList, - /// A list of some data type with variable length and 64-bit offsets. - LargeList, - /// A nested type that contains an arbitrary number of fields. - Struct, - /// A nested type that represents slots of differing types. - Union, - /// A nested type. - Map, - /// A dictionary encoded array by `IntegerType`. - Dictionary(IntegerType), - /// A binary type that inlines small values and can intern bytes. - BinaryView, - /// A string type that inlines small values and can intern strings. - Utf8View, -} - -impl PhysicalType { - /// Whether this physical type equals [`PhysicalType::Primitive`] of type `primitive`. - pub fn eq_primitive(&self, primitive: PrimitiveType) -> bool { - if let Self::Primitive(o) = self { - o == &primitive - } else { - false - } - } -} - -/// the set of valid indices types of a dictionary-encoded Array. -/// Each type corresponds to a variant of [`crate::array::DictionaryArray`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] -pub enum IntegerType { - /// A signed 8-bit integer. - Int8, - /// A signed 16-bit integer. - Int16, - /// A signed 32-bit integer. - Int32, - /// A signed 64-bit integer. - Int64, - /// An unsigned 8-bit integer. - UInt8, - /// An unsigned 16-bit integer. - UInt16, - /// An unsigned 32-bit integer. - UInt32, - /// An unsigned 64-bit integer. - UInt64, -} diff --git a/src/common/arrow/src/arrow/datatypes/schema.rs b/src/common/arrow/src/arrow/datatypes/schema.rs deleted file mode 100644 index 9a8586584f4f..000000000000 --- a/src/common/arrow/src/arrow/datatypes/schema.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(feature = "serde_types")] -use serde_derive::Deserialize; -#[cfg(feature = "serde_types")] -use serde_derive::Serialize; - -use super::Field; -use super::Metadata; - -/// An ordered sequence of [`Field`]s with associated [`Metadata`]. -/// -/// [`Schema`] is an abstraction used to read from, and write to, Arrow IPC format, -/// Apache Parquet, and Apache Avro. All these formats have a concept of a schema -/// with fields and metadata. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] -pub struct Schema { - /// The fields composing this schema. - pub fields: Vec, - /// Optional metadata. - pub metadata: Metadata, -} - -impl Schema { - /// Attaches a [`Metadata`] to [`Schema`] - #[inline] - pub fn with_metadata(self, metadata: Metadata) -> Self { - Self { - fields: self.fields, - metadata, - } - } - - /// Returns a new [`Schema`] with a subset of all fields whose `predicate` - /// evaluates to true. - pub fn filter bool>(self, predicate: F) -> Self { - let fields = self - .fields - .into_iter() - .enumerate() - .filter_map(|(index, f)| { - if (predicate)(index, &f) { - Some(f) - } else { - None - } - }) - .collect(); - - Schema { - fields, - metadata: self.metadata, - } - } -} - -impl From> for Schema { - fn from(fields: Vec) -> Self { - Self { - fields, - ..Default::default() - } - } -} diff --git a/src/common/arrow/src/arrow/mod.rs b/src/common/arrow/src/arrow/mod.rs deleted file mode 100644 index a7e59cdd435e..000000000000 --- a/src/common/arrow/src/arrow/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod bitmap; -pub mod buffer; -pub mod datatypes; -pub mod error; -pub mod offset; -pub mod trusted_len; -pub mod types; -#[macro_use] -pub mod array; -pub mod chunk; -pub mod compute; -pub mod scalar; -pub mod temporal_conversions; -pub mod util; diff --git a/src/common/arrow/src/arrow/scalar/README.md b/src/common/arrow/src/arrow/scalar/README.md deleted file mode 100644 index 2bac790873b5..000000000000 --- a/src/common/arrow/src/arrow/scalar/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Scalar API - -Design choices: - -### `Scalar` is trait object - -There are three reasons: - -* a scalar should have a small memory footprint, which an enum would not ensure given the different physical types available. -* forward-compatibility: a new entry on an `enum` is backward-incompatible -* do not expose implementation details to users (reduce the surface of the public API) - -### `Scalar` MUST contain nullability information - -This is to be aligned with the general notion of arrow's `Array`. - -This API is a companion to the `Array`, and follows the same design as `Array`. -Specifically, a `Scalar` is a trait object that can be downcasted to concrete implementations. - -Like `Array`, `Scalar` implements - -* `data_type`, which is used to perform the correct downcast -* `is_valid`, to tell whether the scalar is null or not - -### There is one implementation per arrows' physical type - -* Reduces the number of `match` that users need to write -* Allows casting of logical types without changing the underlying physical type diff --git a/src/common/arrow/src/arrow/scalar/binary.rs b/src/common/arrow/src/arrow/scalar/binary.rs deleted file mode 100644 index 55ec1ed31629..000000000000 --- a/src/common/arrow/src/arrow/scalar/binary.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Scalar; -use crate::arrow::datatypes::DataType; -use crate::arrow::offset::Offset; - -/// The [`Scalar`] implementation of binary ([`Option>`]). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BinaryScalar { - value: Option>, - phantom: std::marker::PhantomData, -} - -impl BinaryScalar { - /// Returns a new [`BinaryScalar`]. - #[inline] - pub fn new>>(value: Option

) -> Self { - Self { - value: value.map(|x| x.into()), - phantom: std::marker::PhantomData, - } - } - - /// Its value - #[inline] - pub fn value(&self) -> Option<&[u8]> { - self.value.as_ref().map(|x| x.as_ref()) - } -} - -impl>> From> for BinaryScalar { - #[inline] - fn from(v: Option

) -> Self { - Self::new(v) - } -} - -impl Scalar for BinaryScalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - self.value.is_some() - } - - #[inline] - fn data_type(&self) -> &DataType { - if O::IS_LARGE { - &DataType::LargeBinary - } else { - &DataType::Binary - } - } -} diff --git a/src/common/arrow/src/arrow/scalar/binview.rs b/src/common/arrow/src/arrow/scalar/binview.rs deleted file mode 100644 index 59aa472f29fb..000000000000 --- a/src/common/arrow/src/arrow/scalar/binview.rs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2020 Ritchie Vink -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Debug; -use std::fmt::Formatter; - -use crate::arrow::array::ViewType; -use crate::arrow::datatypes::DataType; -use crate::arrow::scalar::Scalar; - -#[derive(PartialEq, Eq)] -pub struct BinaryViewScalar { - value: Option, - phantom: std::marker::PhantomData, -} - -impl Debug for BinaryViewScalar { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "Scalar({:?})", self.value) - } -} - -impl Clone for BinaryViewScalar { - fn clone(&self) -> Self { - Self { - value: self.value.clone(), - phantom: Default::default(), - } - } -} - -impl BinaryViewScalar { - /// Returns a new [`BinaryViewScalar`] - #[inline] - pub fn new(value: Option<&T>) -> Self { - Self { - value: value.map(|x| x.into_owned()), - phantom: std::marker::PhantomData, - } - } - - /// Returns the value irrespectively of the validity. - #[allow(unused)] - #[inline] - pub fn value(&self) -> Option<&T> { - self.value.as_ref().map(|x| x.as_ref()) - } -} - -impl From> for BinaryViewScalar { - #[inline] - fn from(v: Option<&T>) -> Self { - Self::new(v) - } -} - -impl Scalar for BinaryViewScalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - self.value.is_some() - } - - #[inline] - fn data_type(&self) -> &DataType { - if T::IS_UTF8 { - &DataType::Utf8View - } else { - &DataType::BinaryView - } - } -} diff --git a/src/common/arrow/src/arrow/scalar/boolean.rs b/src/common/arrow/src/arrow/scalar/boolean.rs deleted file mode 100644 index 72cb7004eb4a..000000000000 --- a/src/common/arrow/src/arrow/scalar/boolean.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Scalar; -use crate::arrow::datatypes::DataType; - -/// The [`Scalar`] implementation of a boolean. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BooleanScalar { - value: Option, -} - -impl BooleanScalar { - /// Returns a new [`BooleanScalar`] - #[inline] - pub fn new(value: Option) -> Self { - Self { value } - } - - /// The value - #[inline] - pub fn value(&self) -> Option { - self.value - } -} - -impl Scalar for BooleanScalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - self.value.is_some() - } - - #[inline] - fn data_type(&self) -> &DataType { - &DataType::Boolean - } -} - -impl From> for BooleanScalar { - #[inline] - fn from(v: Option) -> Self { - Self::new(v) - } -} diff --git a/src/common/arrow/src/arrow/scalar/dictionary.rs b/src/common/arrow/src/arrow/scalar/dictionary.rs deleted file mode 100644 index dca7960eb7e4..000000000000 --- a/src/common/arrow/src/arrow/scalar/dictionary.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::any::Any; - -use super::Scalar; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; - -/// The [`DictionaryArray`] equivalent of [`Array`] for [`Scalar`]. -#[derive(Debug, Clone)] -pub struct DictionaryScalar { - value: Option>, - phantom: std::marker::PhantomData, - data_type: DataType, -} - -impl PartialEq for DictionaryScalar { - fn eq(&self, other: &Self) -> bool { - (self.data_type == other.data_type) && (self.value.as_ref() == other.value.as_ref()) - } -} - -impl DictionaryScalar { - /// returns a new [`DictionaryScalar`] - /// # Panics - /// iff - /// * the `data_type` is not `List` or `LargeList` (depending on this scalar's offset `O`) - /// * the child of the `data_type` is not equal to the `values` - #[inline] - pub fn new(data_type: DataType, value: Option>) -> Self { - Self { - value, - phantom: std::marker::PhantomData, - data_type, - } - } - - /// The values of the [`DictionaryScalar`] - #[allow(clippy::borrowed_box)] - pub fn value(&self) -> Option<&Box> { - self.value.as_ref() - } -} - -impl Scalar for DictionaryScalar { - fn as_any(&self) -> &dyn Any { - self - } - - fn is_valid(&self) -> bool { - self.value.is_some() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } -} diff --git a/src/common/arrow/src/arrow/scalar/equal.rs b/src/common/arrow/src/arrow/scalar/equal.rs deleted file mode 100644 index 7d813dd67e86..000000000000 --- a/src/common/arrow/src/arrow/scalar/equal.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use super::*; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::scalar::binview::BinaryViewScalar; - -impl PartialEq for dyn Scalar + '_ { - fn eq(&self, that: &dyn Scalar) -> bool { - equal(self, that) - } -} - -impl PartialEq for Arc { - fn eq(&self, that: &dyn Scalar) -> bool { - equal(&**self, that) - } -} - -impl PartialEq for Box { - fn eq(&self, that: &dyn Scalar) -> bool { - equal(&**self, that) - } -} - -macro_rules! dyn_eq { - ($ty:ty, $lhs:expr, $rhs:expr) => {{ - let lhs = $lhs.as_any().downcast_ref::<$ty>().unwrap(); - let rhs = $rhs.as_any().downcast_ref::<$ty>().unwrap(); - lhs == rhs - }}; -} - -fn equal(lhs: &dyn Scalar, rhs: &dyn Scalar) -> bool { - if lhs.data_type() != rhs.data_type() { - return false; - } - - use PhysicalType::*; - match lhs.data_type().to_physical_type() { - Null => dyn_eq!(NullScalar, lhs, rhs), - Boolean => dyn_eq!(BooleanScalar, lhs, rhs), - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - dyn_eq!(PrimitiveScalar<$T>, lhs, rhs) - }), - Utf8 => dyn_eq!(Utf8Scalar, lhs, rhs), - LargeUtf8 => dyn_eq!(Utf8Scalar, lhs, rhs), - Binary => dyn_eq!(BinaryScalar, lhs, rhs), - LargeBinary => dyn_eq!(BinaryScalar, lhs, rhs), - List => dyn_eq!(ListScalar, lhs, rhs), - LargeList => dyn_eq!(ListScalar, lhs, rhs), - Dictionary(key_type) => match_integer_type!(key_type, |$T| { - dyn_eq!(DictionaryScalar<$T>, lhs, rhs) - }), - Struct => dyn_eq!(StructScalar, lhs, rhs), - FixedSizeBinary => dyn_eq!(FixedSizeBinaryScalar, lhs, rhs), - FixedSizeList => dyn_eq!(FixedSizeListScalar, lhs, rhs), - Union => dyn_eq!(UnionScalar, lhs, rhs), - Map => dyn_eq!(MapScalar, lhs, rhs), - BinaryView => dyn_eq!(BinaryViewScalar<[u8]>, lhs, rhs), - Utf8View => dyn_eq!(BinaryViewScalar, lhs, rhs), - } -} diff --git a/src/common/arrow/src/arrow/scalar/fixed_size_binary.rs b/src/common/arrow/src/arrow/scalar/fixed_size_binary.rs deleted file mode 100644 index ad0f9edaf537..000000000000 --- a/src/common/arrow/src/arrow/scalar/fixed_size_binary.rs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Scalar; -use crate::arrow::datatypes::DataType; - -#[derive(Debug, Clone, PartialEq, Eq)] -/// The [`Scalar`] implementation of fixed size binary ([`Option>`]). -pub struct FixedSizeBinaryScalar { - value: Option>, - data_type: DataType, -} - -impl FixedSizeBinaryScalar { - /// Returns a new [`FixedSizeBinaryScalar`]. - /// # Panics - /// iff - /// * the `data_type` is not `FixedSizeBinary` - /// * the size of child binary is not equal - #[inline] - pub fn new>>(data_type: DataType, value: Option

) -> Self { - assert_eq!( - data_type.to_physical_type(), - crate::arrow::datatypes::PhysicalType::FixedSizeBinary - ); - Self { - value: value.map(|x| { - let x: Vec = x.into(); - assert_eq!( - data_type.to_logical_type(), - &DataType::FixedSizeBinary(x.len()) - ); - x.into_boxed_slice() - }), - data_type, - } - } - - /// Its value - #[inline] - pub fn value(&self) -> Option<&[u8]> { - self.value.as_ref().map(|x| x.as_ref()) - } -} - -impl Scalar for FixedSizeBinaryScalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - self.value.is_some() - } - - #[inline] - fn data_type(&self) -> &DataType { - &self.data_type - } -} diff --git a/src/common/arrow/src/arrow/scalar/fixed_size_list.rs b/src/common/arrow/src/arrow/scalar/fixed_size_list.rs deleted file mode 100644 index 1eb0990acb79..000000000000 --- a/src/common/arrow/src/arrow/scalar/fixed_size_list.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::any::Any; - -use super::Scalar; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; - -/// The scalar equivalent of [`FixedSizeListArray`]. Like [`FixedSizeListArray`], this struct holds a dynamically-typed -/// [`Array`]. The only difference is that this has only one element. -#[derive(Debug, Clone)] -pub struct FixedSizeListScalar { - values: Option>, - data_type: DataType, -} - -impl PartialEq for FixedSizeListScalar { - fn eq(&self, other: &Self) -> bool { - (self.data_type == other.data_type) - && (self.values.is_some() == other.values.is_some()) - && ((self.values.is_none()) | (self.values.as_ref() == other.values.as_ref())) - } -} - -impl FixedSizeListScalar { - /// returns a new [`FixedSizeListScalar`] - /// # Panics - /// iff - /// * the `data_type` is not `FixedSizeList` - /// * the child of the `data_type` is not equal to the `values` - /// * the size of child array is not equal - #[inline] - pub fn new(data_type: DataType, values: Option>) -> Self { - let (field, size) = FixedSizeListArray::get_child_and_size(&data_type); - let inner_data_type = field.data_type(); - let values = values.inspect(|x| { - assert_eq!(inner_data_type, x.data_type()); - assert_eq!(size, x.len()); - }); - Self { values, data_type } - } - - /// The values of the [`FixedSizeListScalar`] - #[allow(clippy::borrowed_box)] - pub fn values(&self) -> Option<&Box> { - self.values.as_ref() - } -} - -impl Scalar for FixedSizeListScalar { - fn as_any(&self) -> &dyn Any { - self - } - - fn is_valid(&self) -> bool { - self.values.is_some() - } - - fn data_type(&self) -> &DataType { - &self.data_type - } -} diff --git a/src/common/arrow/src/arrow/scalar/list.rs b/src/common/arrow/src/arrow/scalar/list.rs deleted file mode 100644 index f44a10a7d9c6..000000000000 --- a/src/common/arrow/src/arrow/scalar/list.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::any::Any; - -use super::Scalar; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::offset::Offset; - -/// The scalar equivalent of [`ListArray`]. Like [`ListArray`], this struct holds a dynamically-typed -/// [`Array`]. The only difference is that this has only one element. -#[derive(Debug, Clone)] -pub struct ListScalar { - values: Box, - is_valid: bool, - phantom: std::marker::PhantomData, - data_type: DataType, -} - -impl PartialEq for ListScalar { - fn eq(&self, other: &Self) -> bool { - (self.data_type == other.data_type) - && (self.is_valid == other.is_valid) - && ((!self.is_valid) | (self.values.as_ref() == other.values.as_ref())) - } -} - -impl ListScalar { - /// returns a new [`ListScalar`] - /// # Panics - /// iff - /// * the `data_type` is not `List` or `LargeList` (depending on this scalar's offset `O`) - /// * the child of the `data_type` is not equal to the `values` - #[inline] - pub fn new(data_type: DataType, values: Option>) -> Self { - let inner_data_type = ListArray::::get_child_type(&data_type); - let (is_valid, values) = match values { - Some(values) => { - assert_eq!(inner_data_type, values.data_type()); - (true, values) - } - None => (false, new_empty_array(inner_data_type.clone())), - }; - Self { - values, - is_valid, - phantom: std::marker::PhantomData, - data_type, - } - } - - /// The values of the [`ListScalar`] - #[allow(clippy::borrowed_box)] - pub fn values(&self) -> &Box { - &self.values - } -} - -impl Scalar for ListScalar { - fn as_any(&self) -> &dyn Any { - self - } - - fn is_valid(&self) -> bool { - self.is_valid - } - - fn data_type(&self) -> &DataType { - &self.data_type - } -} diff --git a/src/common/arrow/src/arrow/scalar/map.rs b/src/common/arrow/src/arrow/scalar/map.rs deleted file mode 100644 index f17e362acaf8..000000000000 --- a/src/common/arrow/src/arrow/scalar/map.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::any::Any; - -use super::Scalar; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; - -/// The scalar equivalent of [`MapArray`]. Like [`MapArray`], this struct holds a dynamically-typed -/// [`Array`]. The only difference is that this has only one element. -#[derive(Debug, Clone)] -pub struct MapScalar { - values: Box, - is_valid: bool, - data_type: DataType, -} - -impl PartialEq for MapScalar { - fn eq(&self, other: &Self) -> bool { - (self.data_type == other.data_type) - && (self.is_valid == other.is_valid) - && ((!self.is_valid) | (self.values.as_ref() == other.values.as_ref())) - } -} - -impl MapScalar { - /// returns a new [`MapScalar`] - /// # Panics - /// iff - /// * the `data_type` is not `Map` - /// * the child of the `data_type` is not equal to the `values` - #[inline] - pub fn new(data_type: DataType, values: Option>) -> Self { - let inner_field = MapArray::try_get_field(&data_type).unwrap(); - let inner_data_type = inner_field.data_type(); - let (is_valid, values) = match values { - Some(values) => { - assert_eq!(inner_data_type, values.data_type()); - (true, values) - } - None => (false, new_empty_array(inner_data_type.clone())), - }; - Self { - values, - is_valid, - data_type, - } - } - - /// The values of the [`MapScalar`] - #[allow(clippy::borrowed_box)] - pub fn values(&self) -> &Box { - &self.values - } -} - -impl Scalar for MapScalar { - fn as_any(&self) -> &dyn Any { - self - } - - fn is_valid(&self) -> bool { - self.is_valid - } - - fn data_type(&self) -> &DataType { - &self.data_type - } -} diff --git a/src/common/arrow/src/arrow/scalar/mod.rs b/src/common/arrow/src/arrow/scalar/mod.rs deleted file mode 100644 index 33aa056baa7a..000000000000 --- a/src/common/arrow/src/arrow/scalar/mod.rs +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! contains the [`Scalar`] trait object representing individual items of [`Array`](crate::arrow::array::Array)s, -//! as well as concrete implementations such as [`BooleanScalar`]. -use std::any::Any; - -use crate::arrow::array::*; -use crate::arrow::datatypes::*; - -mod dictionary; -pub use dictionary::*; -mod equal; -mod primitive; -pub use primitive::*; -mod utf8; -pub use utf8::*; -mod binary; -pub use binary::*; -mod boolean; -pub use boolean::*; -mod list; -pub use list::*; -mod map; -pub use map::*; -mod null; -pub use null::*; -mod struct_; -pub use struct_::*; -mod fixed_size_list; -pub use fixed_size_list::*; -mod fixed_size_binary; -pub use fixed_size_binary::*; -mod binview; -mod union; - -pub use union::UnionScalar; - -use crate::arrow::scalar::binview::BinaryViewScalar; - -/// Trait object declaring an optional value with a [`DataType`]. -/// This trait is often used in APIs that accept multiple scalar types. -pub trait Scalar: std::fmt::Debug + Send + Sync + dyn_clone::DynClone + 'static { - /// convert itself to - fn as_any(&self) -> &dyn Any; - - /// whether it is valid - fn is_valid(&self) -> bool; - - /// the logical type. - fn data_type(&self) -> &DataType; -} - -dyn_clone::clone_trait_object!(Scalar); - -macro_rules! dyn_new_utf8 { - ($array:expr, $index:expr, $type:ty) => {{ - let array = $array.as_any().downcast_ref::>().unwrap(); - let value = if array.is_valid($index) { - Some(array.value($index)) - } else { - None - }; - Box::new(Utf8Scalar::<$type>::new(value)) - }}; -} - -macro_rules! dyn_new_binary { - ($array:expr, $index:expr, $type:ty) => {{ - let array = $array - .as_any() - .downcast_ref::>() - .unwrap(); - let value = if array.is_valid($index) { - Some(array.value($index)) - } else { - None - }; - Box::new(BinaryScalar::<$type>::new(value)) - }}; -} - -macro_rules! dyn_new_binview { - ($array:expr, $index:expr, $type:ty) => {{ - let array = $array - .as_any() - .downcast_ref::>() - .unwrap(); - let value = if array.is_valid($index) { - Some(array.value($index)) - } else { - None - }; - Box::new(BinaryViewScalar::<$type>::new(value)) - }}; -} - -macro_rules! dyn_new_list { - ($array:expr, $index:expr, $type:ty) => {{ - let array = $array.as_any().downcast_ref::>().unwrap(); - let value = if array.is_valid($index) { - Some(array.value($index).into()) - } else { - None - }; - Box::new(ListScalar::<$type>::new(array.data_type().clone(), value)) - }}; -} - -/// creates a new [`Scalar`] from an [`Array`]. -pub fn new_scalar(array: &dyn Array, index: usize) -> Box { - use PhysicalType::*; - match array.data_type().to_physical_type() { - Null => Box::new(NullScalar::new()), - Boolean => { - let array = array.as_any().downcast_ref::().unwrap(); - let value = if array.is_valid(index) { - Some(array.value(index)) - } else { - None - }; - Box::new(BooleanScalar::new(value)) - } - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let value = if array.is_valid(index) { - Some(array.value(index)) - } else { - None - }; - Box::new(PrimitiveScalar::new(array.data_type().clone(), value)) - }), - Utf8 => dyn_new_utf8!(array, index, i32), - LargeUtf8 => dyn_new_utf8!(array, index, i64), - Binary => dyn_new_binary!(array, index, i32), - LargeBinary => dyn_new_binary!(array, index, i64), - BinaryView => dyn_new_binview!(array, index, [u8]), - Utf8View => dyn_new_binview!(array, index, str), - List => dyn_new_list!(array, index, i32), - LargeList => dyn_new_list!(array, index, i64), - Struct => { - let array = array.as_any().downcast_ref::().unwrap(); - if array.is_valid(index) { - let values = array - .values() - .iter() - .map(|x| new_scalar(x.as_ref(), index)) - .collect(); - Box::new(StructScalar::new(array.data_type().clone(), Some(values))) - } else { - Box::new(StructScalar::new(array.data_type().clone(), None)) - } - } - FixedSizeBinary => { - let array = array - .as_any() - .downcast_ref::() - .unwrap(); - let value = if array.is_valid(index) { - Some(array.value(index)) - } else { - None - }; - Box::new(FixedSizeBinaryScalar::new(array.data_type().clone(), value)) - } - FixedSizeList => { - let array = array.as_any().downcast_ref::().unwrap(); - let value = if array.is_valid(index) { - Some(array.value(index)) - } else { - None - }; - Box::new(FixedSizeListScalar::new(array.data_type().clone(), value)) - } - Union => { - let array = array.as_any().downcast_ref::().unwrap(); - Box::new(UnionScalar::new( - array.data_type().clone(), - array.types()[index], - array.value(index), - )) - } - Map => { - let array = array.as_any().downcast_ref::().unwrap(); - let value = if array.is_valid(index) { - Some(array.value(index)) - } else { - None - }; - Box::new(MapScalar::new(array.data_type().clone(), value)) - } - Dictionary(key_type) => match_integer_type!(key_type, |$T| { - let array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let value = if array.is_valid(index) { - Some(array.value(index).into()) - } else { - None - }; - Box::new(DictionaryScalar::<$T>::new( - array.data_type().clone(), - value, - )) - }), - } -} diff --git a/src/common/arrow/src/arrow/scalar/null.rs b/src/common/arrow/src/arrow/scalar/null.rs deleted file mode 100644 index fd0f226fe96b..000000000000 --- a/src/common/arrow/src/arrow/scalar/null.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Scalar; -use crate::arrow::datatypes::DataType; - -/// The representation of a single entry of a [`crate::arrow::array::NullArray`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct NullScalar {} - -impl NullScalar { - /// A new [`NullScalar`] - #[inline] - pub fn new() -> Self { - Self {} - } -} - -impl Default for NullScalar { - fn default() -> Self { - Self::new() - } -} - -impl Scalar for NullScalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - false - } - - #[inline] - fn data_type(&self) -> &DataType { - &DataType::Null - } -} diff --git a/src/common/arrow/src/arrow/scalar/primitive.rs b/src/common/arrow/src/arrow/scalar/primitive.rs deleted file mode 100644 index af7983df0757..000000000000 --- a/src/common/arrow/src/arrow/scalar/primitive.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Scalar; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::types::NativeType; - -/// The implementation of [`Scalar`] for primitive, semantically equivalent to [`Option`] -/// with [`DataType`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PrimitiveScalar { - value: Option, - data_type: DataType, -} - -impl PrimitiveScalar { - /// Returns a new [`PrimitiveScalar`]. - #[inline] - pub fn new(data_type: DataType, value: Option) -> Self { - if !data_type.to_physical_type().eq_primitive(T::PRIMITIVE) { - panic!( - "{:?}", - Error::InvalidArgumentError(format!( - "Type {} does not support logical type {:?}", - std::any::type_name::(), - data_type - )) - ) - } - Self { value, data_type } - } - - /// Returns the optional value. - #[inline] - pub fn value(&self) -> &Option { - &self.value - } - - /// Returns a new `PrimitiveScalar` with the same value but different [`DataType`] - /// # Panic - /// This function panics if the `data_type` is not valid for self's physical type `T`. - pub fn to(self, data_type: DataType) -> Self { - Self::new(data_type, self.value) - } -} - -impl From> for PrimitiveScalar { - #[inline] - fn from(v: Option) -> Self { - Self::new(T::PRIMITIVE.into(), v) - } -} - -impl Scalar for PrimitiveScalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - self.value.is_some() - } - - #[inline] - fn data_type(&self) -> &DataType { - &self.data_type - } -} diff --git a/src/common/arrow/src/arrow/scalar/struct_.rs b/src/common/arrow/src/arrow/scalar/struct_.rs deleted file mode 100644 index d7572d2df9e1..000000000000 --- a/src/common/arrow/src/arrow/scalar/struct_.rs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Scalar; -use crate::arrow::datatypes::DataType; - -/// A single entry of a [`crate::arrow::array::StructArray`]. -#[derive(Debug, Clone)] -pub struct StructScalar { - values: Vec>, - is_valid: bool, - data_type: DataType, -} - -impl PartialEq for StructScalar { - fn eq(&self, other: &Self) -> bool { - (self.data_type == other.data_type) - && (self.is_valid == other.is_valid) - && ((!self.is_valid) | (self.values == other.values)) - } -} - -impl StructScalar { - /// Returns a new [`StructScalar`] - #[inline] - pub fn new(data_type: DataType, values: Option>>) -> Self { - let is_valid = values.is_some(); - Self { - values: values.unwrap_or_default(), - is_valid, - data_type, - } - } - - /// Returns the values irrespectively of the validity. - #[inline] - pub fn values(&self) -> &[Box] { - &self.values - } -} - -impl Scalar for StructScalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - self.is_valid - } - - #[inline] - fn data_type(&self) -> &DataType { - &self.data_type - } -} diff --git a/src/common/arrow/src/arrow/scalar/union.rs b/src/common/arrow/src/arrow/scalar/union.rs deleted file mode 100644 index 0978d26807fd..000000000000 --- a/src/common/arrow/src/arrow/scalar/union.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Scalar; -use crate::arrow::datatypes::DataType; - -/// A single entry of a [`crate::arrow::array::UnionArray`]. -#[derive(Debug, Clone, PartialEq)] -pub struct UnionScalar { - value: Box, - type_: i8, - data_type: DataType, -} - -impl UnionScalar { - /// Returns a new [`UnionScalar`] - #[inline] - pub fn new(data_type: DataType, type_: i8, value: Box) -> Self { - Self { - value, - type_, - data_type, - } - } - - /// Returns the inner value - #[inline] - #[allow(clippy::borrowed_box)] - pub fn value(&self) -> &Box { - &self.value - } - - /// Returns the type of the union scalar - #[inline] - pub fn type_(&self) -> i8 { - self.type_ - } -} - -impl Scalar for UnionScalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - true - } - - #[inline] - fn data_type(&self) -> &DataType { - &self.data_type - } -} diff --git a/src/common/arrow/src/arrow/scalar/utf8.rs b/src/common/arrow/src/arrow/scalar/utf8.rs deleted file mode 100644 index e6a96b47436d..000000000000 --- a/src/common/arrow/src/arrow/scalar/utf8.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Scalar; -use crate::arrow::datatypes::DataType; -use crate::arrow::offset::Offset; - -/// The implementation of [`Scalar`] for utf8, semantically equivalent to [`Option`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Utf8Scalar { - value: Option, - phantom: std::marker::PhantomData, -} - -impl Utf8Scalar { - /// Returns a new [`Utf8Scalar`] - #[inline] - pub fn new>(value: Option

) -> Self { - Self { - value: value.map(|x| x.into()), - phantom: std::marker::PhantomData, - } - } - - /// Returns the value irrespectively of the validity. - #[inline] - pub fn value(&self) -> Option<&str> { - self.value.as_ref().map(|x| x.as_ref()) - } -} - -impl> From> for Utf8Scalar { - #[inline] - fn from(v: Option

) -> Self { - Self::new(v) - } -} - -impl Scalar for Utf8Scalar { - #[inline] - fn as_any(&self) -> &dyn std::any::Any { - self - } - - #[inline] - fn is_valid(&self) -> bool { - self.value.is_some() - } - - #[inline] - fn data_type(&self) -> &DataType { - if O::IS_LARGE { - &DataType::LargeUtf8 - } else { - &DataType::Utf8 - } - } -} diff --git a/src/common/arrow/src/arrow/temporal_conversions.rs b/src/common/arrow/src/arrow/temporal_conversions.rs deleted file mode 100644 index 9071634e506f..000000000000 --- a/src/common/arrow/src/arrow/temporal_conversions.rs +++ /dev/null @@ -1,578 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Conversion methods for dates and times. - -use chrono::format::parse; -use chrono::format::Parsed; -use chrono::format::StrftimeItems; -use chrono::DateTime; -use chrono::Datelike; -use chrono::Duration; -use chrono::FixedOffset; -use chrono::NaiveDate; -use chrono::NaiveDateTime; -use chrono::NaiveTime; - -use crate::arrow::array::PrimitiveArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::types::months_days_ns; - -/// Number of seconds in a day -pub const SECONDS_IN_DAY: i64 = 86_400; -/// Number of milliseconds in a second -pub const MILLISECONDS: i64 = 1_000; -/// Number of microseconds in a second -pub const MICROSECONDS: i64 = 1_000_000; -/// Number of nanoseconds in a second -pub const NANOSECONDS: i64 = 1_000_000_000; -/// Number of milliseconds in a day -pub const MILLISECONDS_IN_DAY: i64 = SECONDS_IN_DAY * MILLISECONDS; -/// Number of days between 0001-01-01 and 1970-01-01 -pub const EPOCH_DAYS_FROM_CE: i32 = 719_163; - -/// converts a `i32` representing a `date32` to [`NaiveDateTime`] -#[inline] -pub fn date32_to_datetime(v: i32) -> NaiveDateTime { - date32_to_datetime_opt(v).expect("invalid or out-of-range datetime") -} - -/// converts a `i32` representing a `date32` to [`NaiveDateTime`] -#[inline] -pub fn date32_to_datetime_opt(v: i32) -> Option { - DateTime::from_timestamp(v as i64 * SECONDS_IN_DAY, 0).map(|v| v.naive_utc()) -} - -/// converts a `i32` representing a `date32` to [`NaiveDate`] -#[inline] -pub fn date32_to_date(days: i32) -> NaiveDate { - date32_to_date_opt(days).expect("out-of-range date") -} - -/// converts a `i32` representing a `date32` to [`NaiveDate`] -#[inline] -pub fn date32_to_date_opt(days: i32) -> Option { - NaiveDate::from_num_days_from_ce_opt(EPOCH_DAYS_FROM_CE + days) -} - -/// converts a `i64` representing a `date64` to [`NaiveDateTime`] -#[inline] -pub fn date64_to_datetime(v: i64) -> NaiveDateTime { - DateTime::from_timestamp( - // extract seconds from milliseconds - v / MILLISECONDS, - // discard extracted seconds and convert milliseconds to nanoseconds - (v % MILLISECONDS * MICROSECONDS) as u32, - ) - .expect("invalid or out-of-range datetime") - .naive_utc() -} - -/// converts a `i64` representing a `date64` to [`NaiveDate`] -#[inline] -pub fn date64_to_date(milliseconds: i64) -> NaiveDate { - date64_to_datetime(milliseconds).date() -} - -/// converts a `i32` representing a `time32(s)` to [`NaiveTime`] -#[inline] -pub fn time32s_to_time(v: i32) -> NaiveTime { - NaiveTime::from_num_seconds_from_midnight_opt(v as u32, 0).expect("invalid time") -} - -/// converts a `i64` representing a `duration(s)` to [`Duration`] -#[inline] -pub fn duration_s_to_duration(v: i64) -> Duration { - Duration::seconds(v) -} - -/// converts a `i64` representing a `duration(ms)` to [`Duration`] -#[inline] -pub fn duration_ms_to_duration(v: i64) -> Duration { - Duration::milliseconds(v) -} - -/// converts a `i64` representing a `duration(us)` to [`Duration`] -#[inline] -pub fn duration_us_to_duration(v: i64) -> Duration { - Duration::microseconds(v) -} - -/// converts a `i64` representing a `duration(ns)` to [`Duration`] -#[inline] -pub fn duration_ns_to_duration(v: i64) -> Duration { - Duration::nanoseconds(v) -} - -/// converts a `i32` representing a `time32(ms)` to [`NaiveTime`] -#[inline] -pub fn time32ms_to_time(v: i32) -> NaiveTime { - let v = v as i64; - let seconds = v / MILLISECONDS; - - let milli_to_nano = 1_000_000; - let nano = (v - seconds * MILLISECONDS) * milli_to_nano; - NaiveTime::from_num_seconds_from_midnight_opt(seconds as u32, nano as u32) - .expect("invalid time") -} - -/// converts a `i64` representing a `time64(us)` to [`NaiveTime`] -#[inline] -pub fn time64us_to_time(v: i64) -> NaiveTime { - time64us_to_time_opt(v).expect("invalid time") -} - -/// converts a `i64` representing a `time64(us)` to [`NaiveTime`] -#[inline] -pub fn time64us_to_time_opt(v: i64) -> Option { - NaiveTime::from_num_seconds_from_midnight_opt( - // extract seconds from microseconds - (v / MICROSECONDS) as u32, - // discard extracted seconds and convert microseconds to - // nanoseconds - (v % MICROSECONDS * MILLISECONDS) as u32, - ) -} - -/// converts a `i64` representing a `time64(ns)` to [`NaiveTime`] -#[inline] -pub fn time64ns_to_time(v: i64) -> NaiveTime { - time64ns_to_time_opt(v).expect("invalid time") -} - -/// converts a `i64` representing a `time64(ns)` to [`NaiveTime`] -#[inline] -pub fn time64ns_to_time_opt(v: i64) -> Option { - NaiveTime::from_num_seconds_from_midnight_opt( - // extract seconds from nanoseconds - (v / NANOSECONDS) as u32, - // discard extracted seconds - (v % NANOSECONDS) as u32, - ) -} - -/// converts a `i64` representing a `timestamp(s)` to [`NaiveDateTime`] -#[inline] -pub fn timestamp_s_to_datetime(seconds: i64) -> NaiveDateTime { - timestamp_s_to_datetime_opt(seconds).expect("invalid or out-of-range datetime") -} - -/// converts a `i64` representing a `timestamp(s)` to [`NaiveDateTime`] -#[inline] -pub fn timestamp_s_to_datetime_opt(seconds: i64) -> Option { - DateTime::from_timestamp(seconds, 0).map(|v| v.naive_utc()) -} - -/// converts a `i64` representing a `timestamp(ms)` to [`NaiveDateTime`] -#[inline] -pub fn timestamp_ms_to_datetime(v: i64) -> NaiveDateTime { - timestamp_ms_to_datetime_opt(v).expect("invalid or out-of-range datetime") -} - -/// converts a `i64` representing a `timestamp(ms)` to [`NaiveDateTime`] -#[inline] -pub fn timestamp_ms_to_datetime_opt(v: i64) -> Option { - let t = if v >= 0 { - DateTime::from_timestamp( - // extract seconds from milliseconds - v / MILLISECONDS, - // discard extracted seconds and convert milliseconds to nanoseconds - (v % MILLISECONDS * MICROSECONDS) as u32, - ) - } else { - let secs_rem = (v / MILLISECONDS, v % MILLISECONDS); - if secs_rem.1 == 0 { - // whole/integer seconds; no adjustment required - DateTime::from_timestamp(secs_rem.0, 0) - } else { - // negative values with fractional seconds require 'div_floor' rounding behaviour. - // (which isn't yet stabilised: https://github.com/rust-lang/rust/issues/88581) - DateTime::from_timestamp( - secs_rem.0 - 1, - (NANOSECONDS + (v % MILLISECONDS * MICROSECONDS)) as u32, - ) - } - }; - - t.map(|v| v.naive_utc()) -} - -/// converts a `i64` representing a `timestamp(us)` to [`NaiveDateTime`] -#[inline] -pub fn timestamp_us_to_datetime(v: i64) -> NaiveDateTime { - timestamp_us_to_datetime_opt(v).expect("invalid or out-of-range datetime") -} - -/// converts a `i64` representing a `timestamp(us)` to [`NaiveDateTime`] -#[inline] -pub fn timestamp_us_to_datetime_opt(v: i64) -> Option { - let t = if v >= 0 { - DateTime::from_timestamp( - // extract seconds from microseconds - v / MICROSECONDS, - // discard extracted seconds and convert microseconds to nanoseconds - (v % MICROSECONDS * MILLISECONDS) as u32, - ) - } else { - let secs_rem = (v / MICROSECONDS, v % MICROSECONDS); - if secs_rem.1 == 0 { - // whole/integer seconds; no adjustment required - DateTime::from_timestamp(secs_rem.0, 0) - } else { - // negative values with fractional seconds require 'div_floor' rounding behaviour. - // (which isn't yet stabilised: https://github.com/rust-lang/rust/issues/88581) - DateTime::from_timestamp( - secs_rem.0 - 1, - (NANOSECONDS + (v % MICROSECONDS * MILLISECONDS)) as u32, - ) - } - }; - - t.map(|v| v.naive_utc()) -} - -/// converts a `i64` representing a `timestamp(ns)` to [`NaiveDateTime`] -#[inline] -pub fn timestamp_ns_to_datetime(v: i64) -> NaiveDateTime { - timestamp_ns_to_datetime_opt(v).expect("invalid or out-of-range datetime") -} - -/// converts a `i64` representing a `timestamp(ns)` to [`NaiveDateTime`] -#[inline] -pub fn timestamp_ns_to_datetime_opt(v: i64) -> Option { - let t = if v >= 0 { - DateTime::from_timestamp( - // extract seconds from nanoseconds - v / NANOSECONDS, - // discard extracted seconds - (v % NANOSECONDS) as u32, - ) - } else { - let secs_rem = (v / NANOSECONDS, v % NANOSECONDS); - if secs_rem.1 == 0 { - // whole/integer seconds; no adjustment required - DateTime::from_timestamp(secs_rem.0, 0) - } else { - // negative values with fractional seconds require 'div_floor' rounding behaviour. - // (which isn't yet stabilised: https://github.com/rust-lang/rust/issues/88581) - DateTime::from_timestamp(secs_rem.0 - 1, (NANOSECONDS + (v % NANOSECONDS)) as u32) - } - }; - - t.map(|v| v.naive_utc()) -} - -/// Converts a timestamp in `time_unit` and `timezone` into [`chrono::DateTime`]. -#[inline] -pub fn timestamp_to_naive_datetime(timestamp: i64, time_unit: TimeUnit) -> chrono::NaiveDateTime { - match time_unit { - TimeUnit::Second => timestamp_s_to_datetime(timestamp), - TimeUnit::Millisecond => timestamp_ms_to_datetime(timestamp), - TimeUnit::Microsecond => timestamp_us_to_datetime(timestamp), - TimeUnit::Nanosecond => timestamp_ns_to_datetime(timestamp), - } -} - -/// Converts a timestamp in `time_unit` and `timezone` into [`chrono::DateTime`]. -#[inline] -pub fn timestamp_to_datetime( - timestamp: i64, - time_unit: TimeUnit, - timezone: &T, -) -> chrono::DateTime { - timezone.from_utc_datetime(×tamp_to_naive_datetime(timestamp, time_unit)) -} - -/// Calculates the scale factor between two TimeUnits. The function returns the -/// scale that should multiply the TimeUnit "b" to have the same time scale as -/// the TimeUnit "a". -pub fn timeunit_scale(a: TimeUnit, b: TimeUnit) -> f64 { - match (a, b) { - (TimeUnit::Second, TimeUnit::Second) => 1.0, - (TimeUnit::Second, TimeUnit::Millisecond) => 0.001, - (TimeUnit::Second, TimeUnit::Microsecond) => 0.000_001, - (TimeUnit::Second, TimeUnit::Nanosecond) => 0.000_000_001, - (TimeUnit::Millisecond, TimeUnit::Second) => 1_000.0, - (TimeUnit::Millisecond, TimeUnit::Millisecond) => 1.0, - (TimeUnit::Millisecond, TimeUnit::Microsecond) => 0.001, - (TimeUnit::Millisecond, TimeUnit::Nanosecond) => 0.000_001, - (TimeUnit::Microsecond, TimeUnit::Second) => 1_000_000.0, - (TimeUnit::Microsecond, TimeUnit::Millisecond) => 1_000.0, - (TimeUnit::Microsecond, TimeUnit::Microsecond) => 1.0, - (TimeUnit::Microsecond, TimeUnit::Nanosecond) => 0.001, - (TimeUnit::Nanosecond, TimeUnit::Second) => 1_000_000_000.0, - (TimeUnit::Nanosecond, TimeUnit::Millisecond) => 1_000_000.0, - (TimeUnit::Nanosecond, TimeUnit::Microsecond) => 1_000.0, - (TimeUnit::Nanosecond, TimeUnit::Nanosecond) => 1.0, - } -} - -/// Parses an offset of the form `"+WX:YZ"` or `"UTC"` into [`FixedOffset`]. -/// # Errors -/// If the offset is not in any of the allowed forms. -pub fn parse_offset(offset: &str) -> Result { - if offset == "UTC" { - return Ok(FixedOffset::east_opt(0).expect("FixedOffset::east out of bounds")); - } - let error = "timezone offset must be of the form [-]00:00"; - - let mut a = offset.split(':'); - let first = a - .next() - .map(Ok) - .unwrap_or_else(|| Err(Error::InvalidArgumentError(error.to_string())))?; - let last = a - .next() - .map(Ok) - .unwrap_or_else(|| Err(Error::InvalidArgumentError(error.to_string())))?; - let hours: i32 = first - .parse() - .map_err(|_| Error::InvalidArgumentError(error.to_string()))?; - let minutes: i32 = last - .parse() - .map_err(|_| Error::InvalidArgumentError(error.to_string()))?; - - Ok(FixedOffset::east_opt(hours * 60 * 60 + minutes * 60) - .expect("FixedOffset::east out of bounds")) -} - -/// Parses `value` to `Option` consistent with the Arrow's definition of timestamp with timezone. -/// `tz` must be built from `timezone` (either via [`parse_offset`] or `chrono-tz`). -#[inline] -pub fn utf8_to_timestamp_ns_scalar( - value: &str, - fmt: &str, - tz: &T, -) -> Option { - utf8_to_timestamp_scalar(value, fmt, tz, &TimeUnit::Nanosecond) -} - -/// Parses `value` to `Option` consistent with the Arrow's definition of timestamp with timezone. -/// `tz` must be built from `timezone` (either via [`parse_offset`] or `chrono-tz`). -/// Returns in scale `tz` of `TimeUnit`. -#[inline] -pub fn utf8_to_timestamp_scalar( - value: &str, - fmt: &str, - tz: &T, - tu: &TimeUnit, -) -> Option { - let mut parsed = Parsed::new(); - let fmt = StrftimeItems::new(fmt); - let r = parse(&mut parsed, value, fmt).ok(); - if r.is_some() { - parsed - .to_datetime() - .map(|x| x.naive_utc()) - .map(|x| tz.from_utc_datetime(&x)) - .map(|x| match tu { - TimeUnit::Second => x.timestamp(), - TimeUnit::Millisecond => x.timestamp_millis(), - TimeUnit::Microsecond => x.timestamp_micros(), - TimeUnit::Nanosecond => x.timestamp_nanos_opt().unwrap(), - }) - .ok() - } else { - None - } -} - -/// Parses `value` to `Option` consistent with the Arrow's definition of timestamp without timezone. -#[inline] -pub fn utf8_to_naive_timestamp_ns_scalar(value: &str, fmt: &str) -> Option { - utf8_to_naive_timestamp_scalar(value, fmt, &TimeUnit::Nanosecond) -} - -/// Parses `value` to `Option` consistent with the Arrow's definition of timestamp without timezone. -/// Returns in scale `tz` of `TimeUnit`. -#[inline] -pub fn utf8_to_naive_timestamp_scalar(value: &str, fmt: &str, tu: &TimeUnit) -> Option { - let fmt = StrftimeItems::new(fmt); - let mut parsed = Parsed::new(); - parse(&mut parsed, value, fmt.clone()).ok(); - parsed - .to_naive_datetime_with_offset(0) - .map(|x| match tu { - TimeUnit::Second => x.and_utc().timestamp(), - TimeUnit::Millisecond => x.and_utc().timestamp_millis(), - TimeUnit::Microsecond => x.and_utc().timestamp_micros(), - TimeUnit::Nanosecond => x.and_utc().timestamp_nanos_opt().unwrap(), - }) - .ok() -} - -fn utf8_to_timestamp_ns_impl( - array: &Utf8Array, - fmt: &str, - timezone: String, - tz: T, -) -> PrimitiveArray { - let iter = array - .iter() - .map(|x| x.and_then(|x| utf8_to_timestamp_ns_scalar(x, fmt, &tz))); - - PrimitiveArray::from_trusted_len_iter(iter) - .to(DataType::Timestamp(TimeUnit::Nanosecond, Some(timezone))) -} - -/// Parses `value` to a [`chrono_tz::Tz`] with the Arrow's definition of timestamp with a timezone. -#[cfg(feature = "chrono-tz")] -#[cfg_attr(docsrs, doc(cfg(feature = "chrono-tz")))] -pub fn parse_offset_tz(timezone: &str) -> Result { - timezone.parse::().map_err(|_| { - Error::InvalidArgumentError(format!("timezone \"{timezone}\" cannot be parsed")) - }) -} - -#[cfg(feature = "chrono-tz")] -#[cfg_attr(docsrs, doc(cfg(feature = "chrono-tz")))] -fn chrono_tz_utf_to_timestamp_ns( - array: &Utf8Array, - fmt: &str, - timezone: String, -) -> Result> { - let tz = parse_offset_tz(&timezone)?; - Ok(utf8_to_timestamp_ns_impl(array, fmt, timezone, tz)) -} - -#[cfg(not(feature = "chrono-tz"))] -fn chrono_tz_utf_to_timestamp_ns( - _: &Utf8Array, - _: &str, - timezone: String, -) -> Result> { - Err(Error::InvalidArgumentError(format!( - "timezone \"{timezone}\" cannot be parsed (feature chrono-tz is not active)", - ))) -} - -/// Parses a [`Utf8Array`] to a timeozone-aware timestamp, i.e. [`PrimitiveArray`] with type `Timestamp(Nanosecond, Some(timezone))`. -/// -/// # Implementation -/// -/// * parsed values with timezone other than `timezone` are converted to `timezone`. -/// * parsed values without timezone are null. Use [`utf8_to_naive_timestamp_ns`] to parse naive timezones. -/// * Null elements remain null; non-parsable elements are null. -/// -/// The feature `"chrono-tz"` enables IANA and zoneinfo formats for `timezone`. -/// -/// # Error -/// -/// This function errors iff `timezone` is not parsable to an offset. -pub fn utf8_to_timestamp_ns( - array: &Utf8Array, - fmt: &str, - timezone: String, -) -> Result> { - let tz = parse_offset(timezone.as_str()); - - if let Ok(tz) = tz { - Ok(utf8_to_timestamp_ns_impl(array, fmt, timezone, tz)) - } else { - chrono_tz_utf_to_timestamp_ns(array, fmt, timezone) - } -} - -/// Parses a [`Utf8Array`] to naive timestamp, i.e. -/// [`PrimitiveArray`] with type `Timestamp(Nanosecond, None)`. -/// Timezones are ignored. -/// Null elements remain null; non-parsable elements are set to null. -pub fn utf8_to_naive_timestamp_ns( - array: &Utf8Array, - fmt: &str, -) -> PrimitiveArray { - let iter = array - .iter() - .map(|x| x.and_then(|x| utf8_to_naive_timestamp_ns_scalar(x, fmt))); - - PrimitiveArray::from_trusted_len_iter(iter).to(DataType::Timestamp(TimeUnit::Nanosecond, None)) -} - -fn add_month(year: i32, month: u32, months: i32) -> chrono::NaiveDate { - let new_year = (year * 12 + (month - 1) as i32 + months) / 12; - let new_month = (year * 12 + (month - 1) as i32 + months) % 12 + 1; - chrono::NaiveDate::from_ymd_opt(new_year, new_month as u32, 1) - .expect("invalid or out-of-range date") -} - -fn get_days_between_months(year: i32, month: u32, months: i32) -> i64 { - add_month(year, month, months) - .signed_duration_since( - chrono::NaiveDate::from_ymd_opt(year, month, 1).expect("invalid or out-of-range date"), - ) - .num_days() -} - -/// Adds an `interval` to a `timestamp` in `time_unit` units without timezone. -#[inline] -pub fn add_naive_interval(timestamp: i64, time_unit: TimeUnit, interval: months_days_ns) -> i64 { - // convert seconds to a DateTime of a given offset. - let datetime = match time_unit { - TimeUnit::Second => timestamp_s_to_datetime(timestamp), - TimeUnit::Millisecond => timestamp_ms_to_datetime(timestamp), - TimeUnit::Microsecond => timestamp_us_to_datetime(timestamp), - TimeUnit::Nanosecond => timestamp_ns_to_datetime(timestamp), - }; - - // compute the number of days in the interval, which depends on the particular year and month (leap days) - let delta_days = get_days_between_months(datetime.year(), datetime.month(), interval.months()) - + interval.days() as i64; - - // add; no leap hours are considered - let new_datetime_tz = datetime - + chrono::Duration::nanoseconds(delta_days * 24 * 60 * 60 * 1_000_000_000 + interval.ns()); - - // convert back to the target unit - match time_unit { - TimeUnit::Second => new_datetime_tz.and_utc().timestamp_millis() / 1000, - TimeUnit::Millisecond => new_datetime_tz.and_utc().timestamp_millis(), - TimeUnit::Microsecond => new_datetime_tz.and_utc().timestamp_nanos_opt().unwrap() / 1000, - TimeUnit::Nanosecond => new_datetime_tz.and_utc().timestamp_nanos_opt().unwrap(), - } -} - -/// Adds an `interval` to a `timestamp` in `time_unit` units and timezone `timezone`. -#[inline] -pub fn add_interval( - timestamp: i64, - time_unit: TimeUnit, - interval: months_days_ns, - timezone: &T, -) -> i64 { - // convert seconds to a DateTime of a given offset. - let datetime_tz = timestamp_to_datetime(timestamp, time_unit, timezone); - - // compute the number of days in the interval, which depends on the particular year and month (leap days) - let delta_days = - get_days_between_months(datetime_tz.year(), datetime_tz.month(), interval.months()) - + interval.days() as i64; - - // add; tz will take care of leap hours - let new_datetime_tz = datetime_tz - + chrono::Duration::nanoseconds(delta_days * 24 * 60 * 60 * 1_000_000_000 + interval.ns()); - - // convert back to the target unit - match time_unit { - TimeUnit::Second => new_datetime_tz.timestamp_millis() / 1000, - TimeUnit::Millisecond => new_datetime_tz.timestamp_millis(), - TimeUnit::Microsecond => new_datetime_tz.timestamp_nanos_opt().unwrap() / 1000, - TimeUnit::Nanosecond => new_datetime_tz.timestamp_nanos_opt().unwrap(), - } -} diff --git a/src/common/arrow/src/arrow/trusted_len.rs b/src/common/arrow/src/arrow/trusted_len.rs deleted file mode 100644 index 06465ab8cd34..000000000000 --- a/src/common/arrow/src/arrow/trusted_len.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Declares [`TrustedLen`]. -use std::slice::Iter; - -/// An iterator of known, fixed size. -/// A trait denoting Rusts' unstable [TrustedLen](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html). -/// This is re-defined here and implemented for some iterators until `std::iter::TrustedLen` -/// is stabilized. -/// -/// # Safety -/// This trait must only be implemented when the contract is upheld. -/// Consumers of this trait must inspect Iterator::size_hint()’s upper bound. -pub unsafe trait TrustedLen: Iterator {} - -unsafe impl TrustedLen for Iter<'_, T> {} - -unsafe impl B> TrustedLen for std::iter::Map {} - -unsafe impl<'a, I, T: 'a> TrustedLen for std::iter::Copied -where - I: TrustedLen, - T: Copy, -{ -} -unsafe impl<'a, I, T: 'a> TrustedLen for std::iter::Cloned -where - I: TrustedLen, - T: Clone, -{ -} - -unsafe impl TrustedLen for std::iter::Enumerate where I: TrustedLen {} - -unsafe impl TrustedLen for std::iter::Zip -where - A: TrustedLen, - B: TrustedLen, -{ -} - -unsafe impl TrustedLen for std::slice::ChunksExact<'_, T> {} - -unsafe impl TrustedLen for std::slice::Windows<'_, T> {} - -unsafe impl TrustedLen for std::iter::Chain -where - A: TrustedLen, - B: TrustedLen, -{ -} - -unsafe impl TrustedLen for std::iter::Once {} - -unsafe impl TrustedLen for std::vec::IntoIter {} - -unsafe impl TrustedLen for std::iter::Repeat {} -unsafe impl A> TrustedLen for std::iter::RepeatWith {} -unsafe impl TrustedLen for std::iter::Take {} diff --git a/src/common/arrow/src/arrow/util/bench_util.rs b/src/common/arrow/src/arrow/util/bench_util.rs deleted file mode 100644 index 169168af6339..000000000000 --- a/src/common/arrow/src/arrow/util/bench_util.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Utilities for benchmarking - -use rand::distributions::Alphanumeric; -use rand::distributions::Distribution; -use rand::distributions::Standard; -use rand::rngs::StdRng; -use rand::Rng; -use rand::SeedableRng; - -use crate::arrow::array::*; -use crate::arrow::offset::Offset; -use crate::arrow::types::NativeType; - -/// Returns fixed seedable RNG -pub fn seedable_rng() -> StdRng { - StdRng::seed_from_u64(42) -} - -/// Creates an random (but fixed-seeded) array of a given size and null density -pub fn create_primitive_array(size: usize, null_density: f32) -> PrimitiveArray -where - T: NativeType, - Standard: Distribution, -{ - let mut rng = seedable_rng(); - - (0..size) - .map(|_| { - if rng.gen::() < null_density { - None - } else { - Some(rng.gen()) - } - }) - .collect::>() -} - -/// Creates a new [`PrimitiveArray`] from random values with a pre-set seed. -pub fn create_primitive_array_with_seed( - size: usize, - null_density: f32, - seed: u64, -) -> PrimitiveArray -where - T: NativeType, - Standard: Distribution, -{ - let mut rng = StdRng::seed_from_u64(seed); - - (0..size) - .map(|_| { - if rng.gen::() < null_density { - None - } else { - Some(rng.gen()) - } - }) - .collect::>() -} - -/// Creates an random (but fixed-seeded) array of a given size and null density -pub fn create_boolean_array(size: usize, null_density: f32, true_density: f32) -> BooleanArray -where Standard: Distribution { - let mut rng = seedable_rng(); - (0..size) - .map(|_| { - if rng.gen::() < null_density { - None - } else { - let value = rng.gen::() < true_density; - Some(value) - } - }) - .collect() -} - -/// Creates an random (but fixed-seeded) [`Utf8Array`] of a given length, number of characters and null density. -pub fn create_string_array( - length: usize, - size: usize, - null_density: f32, - seed: u64, -) -> Utf8Array { - let mut rng = StdRng::seed_from_u64(seed); - - (0..length) - .map(|_| { - if rng.gen::() < null_density { - None - } else { - let value = (&mut rng) - .sample_iter(&Alphanumeric) - .take(size) - .map(char::from) - .collect::(); - Some(value) - } - }) - .collect() -} diff --git a/src/common/arrow/src/arrow/util/mod.rs b/src/common/arrow/src/arrow/util/mod.rs deleted file mode 100644 index 2e6c349ef88e..000000000000 --- a/src/common/arrow/src/arrow/util/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Misc utilities used in different places in the crate. - -pub mod bench_util; diff --git a/src/common/arrow/src/native/nested.rs b/src/common/arrow/src/native/nested.rs deleted file mode 100644 index 87c7dd49ffda..000000000000 --- a/src/common/arrow/src/native/nested.rs +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::arrow::array::Array; -use crate::arrow::array::FixedSizeListArray; -use crate::arrow::array::ListArray; -use crate::arrow::array::MapArray; -use crate::arrow::array::StructArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::Offset; -use crate::arrow::offset::Offsets; -use crate::arrow::offset::OffsetsBuffer; - -/// Descriptor of nested information of a field -#[derive(Debug, Clone, PartialEq)] -pub enum Nested { - /// A primitive array - Primitive(usize, bool, Option), - /// a list - List(ListNested), - /// a list - LargeList(ListNested), - /// A struct array - Struct(usize, bool, Option), -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ListNested { - pub is_nullable: bool, - pub offsets: OffsetsBuffer, - pub validity: Option, -} - -impl ListNested { - pub fn new(offsets: OffsetsBuffer, validity: Option, is_nullable: bool) -> Self { - Self { - is_nullable, - offsets, - validity, - } - } -} - -pub type NestedState = Vec; - -impl Nested { - pub fn length(&self) -> usize { - match self { - Nested::Primitive(len, _, _) => *len, - Nested::List(l) => l.offsets.len_proxy(), - Nested::LargeList(l) => l.offsets.len_proxy(), - Nested::Struct(len, _, _) => *len, - } - } - - pub fn is_nullable(&self) -> bool { - match self { - Nested::Primitive(_, b, _) => *b, - Nested::List(l) => l.is_nullable, - Nested::LargeList(l) => l.is_nullable, - Nested::Struct(_, b, _) => *b, - } - } - - pub fn inner(&self) -> (Buffer, &Option) { - match self { - Nested::Primitive(_, _, v) => (Buffer::new(), v), - Nested::List(l) => { - let start = l.offsets.first(); - let buffer = l - .offsets - .buffer() - .iter() - .map(|x| (*x - start) as i64) - .collect(); - (buffer, &l.validity) - } - Nested::LargeList(l) => { - let start = l.offsets.first(); - let buffer = if *start == 0 { - l.offsets.buffer().clone() - } else { - l.offsets.buffer().iter().map(|x| *x - start).collect() - }; - (buffer, &l.validity) - } - Nested::Struct(_, _, v) => (Buffer::new(), v), - } - } - - pub fn validity(&self) -> &Option { - match self { - Nested::Primitive(_, _, v) => v, - Nested::List(l) => &l.validity, - Nested::LargeList(l) => &l.validity, - Nested::Struct(_, _, v) => v, - } - } - - pub fn is_list(&self) -> bool { - matches!(self, Nested::List(_) | Nested::LargeList(_)) - } -} - -/// Constructs the necessary `Vec>` to write the rep and def levels of `array` to parquet -pub fn to_nested(array: &dyn Array, f: &Field) -> Result>> { - let mut nested = vec![]; - - to_nested_recursive(array, f, &mut nested, vec![])?; - Ok(nested) -} - -pub fn is_nested_type(t: &DataType) -> bool { - matches!( - t, - DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_) | DataType::Map(_, _) - ) -} - -/// Slices the [`Array`] to `Box` and `Vec`. -pub fn slice_nest_array( - primitive_array: &mut dyn Array, - nested: &mut [Nested], - mut current_offset: usize, - mut current_length: usize, -) { - for nested in nested.iter_mut() { - match nested { - Nested::LargeList(l_nested) => { - l_nested.offsets.slice(current_offset, current_length + 1); - if let Some(validity) = l_nested.validity.as_mut() { - validity.slice(current_offset, current_length) - }; - - current_length = l_nested.offsets.range() as usize; - current_offset = *l_nested.offsets.first() as usize; - } - Nested::List(l_nested) => { - l_nested.offsets.slice(current_offset, current_length + 1); - if let Some(validity) = l_nested.validity.as_mut() { - validity.slice(current_offset, current_length) - }; - - current_length = l_nested.offsets.range() as usize; - current_offset = *l_nested.offsets.first() as usize; - } - Nested::Struct(length, _, validity) => { - *length = current_length; - if let Some(validity) = validity.as_mut() { - validity.slice(current_offset, current_length) - }; - } - Nested::Primitive(length, _, validity) => { - *length = current_length; - if let Some(validity) = validity.as_mut() { - validity.slice(current_offset, current_length) - }; - primitive_array.slice(current_offset, current_length); - } - } - } -} - -fn to_nested_recursive( - array: &dyn Array, - f: &Field, - nested: &mut Vec>, - mut parents: Vec, -) -> Result<()> { - use PhysicalType::*; - let lt = f.data_type.to_logical_type(); - let nullable = f.is_nullable; - match array.data_type().to_physical_type() { - Struct => { - let array = array.as_any().downcast_ref::().unwrap(); - parents.push(Nested::Struct( - array.len(), - nullable, - array.validity().cloned(), - )); - - if let DataType::Struct(fs) = lt { - for (array, f) in array.values().iter().zip(fs.iter()) { - to_nested_recursive(array.as_ref(), f, nested, parents.clone())?; - } - } else { - return Err(Error::InvalidArgumentError( - "DataType type must be a group for a struct array".to_string(), - )); - } - } - List => { - let array = array.as_any().downcast_ref::>().unwrap(); - - if let DataType::List(fs) = lt { - parents.push(Nested::List(ListNested::new( - array.offsets().clone(), - array.validity().cloned(), - nullable, - ))); - to_nested_recursive(array.values().as_ref(), fs.as_ref(), nested, parents)?; - } else { - return Err(Error::InvalidArgumentError( - "DataType type must be a group for a List array".to_string(), - )); - } - } - LargeList => { - let array = array.as_any().downcast_ref::>().unwrap(); - if let DataType::LargeList(fs) = lt { - parents.push(Nested::LargeList(ListNested::::new( - array.offsets().clone(), - array.validity().cloned(), - nullable, - ))); - to_nested_recursive(array.values().as_ref(), fs.as_ref(), nested, parents)?; - } else { - return Err(Error::InvalidArgumentError( - "DataType type must be a group for a LargeList array".to_string(), - )); - } - } - Map => { - let array = array.as_any().downcast_ref::().unwrap(); - if let DataType::Map(fs, _) = lt { - parents.push(Nested::List(ListNested::new( - array.offsets().clone(), - array.validity().cloned(), - nullable, - ))); - to_nested_recursive(array.field().as_ref(), fs.as_ref(), nested, parents)?; - } else { - return Err(Error::InvalidArgumentError( - "DataType type must be a group for a LargeList array".to_string(), - )); - } - } - _ => { - parents.push(Nested::Primitive( - array.len(), - nullable, - array.validity().cloned(), - )); - nested.push(parents); - } - } - Ok(()) -} - -/// Convert [`Array`] to `Vec<&dyn Array>` leaves in DFS order. -pub fn to_leaves(array: &dyn Array) -> Vec<&dyn Array> { - let mut leaves = vec![]; - to_leaves_recursive(array, &mut leaves); - leaves -} - -fn to_leaves_recursive<'a>(array: &'a dyn Array, leaves: &mut Vec<&'a dyn Array>) { - use PhysicalType::*; - match array.data_type().to_physical_type() { - Struct => { - let array = array.as_any().downcast_ref::().unwrap(); - array - .values() - .iter() - .for_each(|a| to_leaves_recursive(a.as_ref(), leaves)); - } - List => { - let array = array.as_any().downcast_ref::>().unwrap(); - to_leaves_recursive(array.values().as_ref(), leaves); - } - LargeList => { - let array = array.as_any().downcast_ref::>().unwrap(); - to_leaves_recursive(array.values().as_ref(), leaves); - } - Map => { - let array = array.as_any().downcast_ref::().unwrap(); - to_leaves_recursive(array.field().as_ref(), leaves); - } - Null | Boolean | Primitive(_) | Binary | FixedSizeBinary | LargeBinary | Utf8 - | LargeUtf8 | Dictionary(_) | BinaryView | Utf8View => leaves.push(array), - other => todo!("Writing {:?} to native not yet implemented", other), - } -} - -/// The initial info of nested data types. -/// The initial info of nested data types. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum InitNested { - /// Primitive data types - Primitive(bool), - /// List data types - List(bool), - /// Struct data types - Struct(bool), -} - -impl InitNested { - pub fn is_nullable(&self) -> bool { - match self { - InitNested::Primitive(b) => *b, - InitNested::List(b) => *b, - InitNested::Struct(b) => *b, - } - } -} - -/// Creates a new [`ListArray`] or [`FixedSizeListArray`]. -pub fn create_list( - data_type: DataType, - nested: &mut NestedState, - values: Box, -) -> Box { - let n = nested.pop().unwrap(); - let (offsets, validity) = n.inner(); - match data_type.to_logical_type() { - DataType::List(_) => { - let offsets = offsets.iter().map(|x| *x as i32).collect::>(); - let offsets: Offsets = offsets - .try_into() - .expect("i64 offsets do not fit in i32 offsets"); - - Box::new(ListArray::::new( - data_type, - OffsetsBuffer::from(offsets), - values, - validity.clone(), - )) - } - DataType::LargeList(_) => Box::new(ListArray::::new( - data_type, - unsafe { OffsetsBuffer::new_unchecked(offsets) }, - values, - validity.clone(), - )), - DataType::FixedSizeList(_, _) => { - Box::new(FixedSizeListArray::new(data_type, values, validity.clone())) - } - _ => unreachable!(), - } -} - -/// Creates a new [`MapArray`]. -pub fn create_map( - data_type: DataType, - nested: &mut NestedState, - values: Box, -) -> Box { - let n = nested.pop().unwrap(); - let (offsets, validity) = n.inner(); - match data_type.to_logical_type() { - DataType::Map(_, _) => { - let offsets = offsets.iter().map(|x| *x as i32).collect::>(); - - let offsets: Offsets = offsets - .try_into() - .expect("i64 offsets do not fit in i32 offsets"); - - Box::new(MapArray::new( - data_type, - offsets.into(), - values, - validity.clone(), - )) - } - _ => unreachable!(), - } -} - -pub fn create_struct( - fields: Vec, - nested: &mut Vec, - values: Vec>, -) -> (NestedState, Box) { - let mut nest = nested.pop().unwrap(); - let n = nest.pop().unwrap(); - let (_, validity) = n.inner(); - - ( - nest, - Box::new(StructArray::new( - DataType::Struct(fields), - values, - validity.clone(), - )), - ) -} diff --git a/src/common/arrow/src/native/read/array/binary.rs b/src/common/arrow/src/native/read/array/binary.rs deleted file mode 100644 index 31fc3981a460..000000000000 --- a/src/common/arrow/src/native/read/array/binary.rs +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Cursor; -use std::marker::PhantomData; - -use crate::arrow::array::Array; -use crate::arrow::array::BinaryArray; -use crate::arrow::array::Utf8Array; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::offset::OffsetsBuffer; -use crate::arrow::types::Offset; -use crate::native::compression::binary::decompress_binary; -use crate::native::nested::InitNested; -use crate::native::nested::NestedState; -use crate::native::read::read_basic::*; -use crate::native::read::BufReader; -use crate::native::read::NativeReadBuf; -use crate::native::read::PageIterator; -use crate::native::PageMeta; - -#[derive(Debug)] -pub struct BinaryNestedIter -where - I: Iterator)>> + PageIterator + Send + Sync, - O: Offset, -{ - iter: I, - data_type: DataType, - init: Vec, - scratch: Vec, - _phantom: PhantomData, -} - -impl BinaryNestedIter -where - I: Iterator)>> + PageIterator + Send + Sync, - O: Offset, -{ - pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { - Self { - iter, - data_type, - init, - scratch: vec![], - _phantom: PhantomData, - } - } -} - -impl BinaryNestedIter -where - I: Iterator)>> + PageIterator + Send + Sync, - O: Offset, -{ - fn deserialize( - &mut self, - num_values: u64, - buffer: Vec, - ) -> Result<(NestedState, Box)> { - let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); - let length = num_values as usize; - let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; - let mut offsets: Vec = Vec::with_capacity(length + 1); - let mut values = Vec::with_capacity(0); - - decompress_binary( - &mut reader, - length, - &mut offsets, - &mut values, - &mut self.scratch, - )?; - - let array = try_new_binary_array( - self.data_type.clone(), - unsafe { OffsetsBuffer::new_unchecked(offsets.into()) }, - values.into(), - validity, - )?; - Ok((nested, array)) - } -} - -impl Iterator for BinaryNestedIter -where - I: Iterator)>> + PageIterator + Send + Sync, - O: Offset, -{ - type Item = Result<(NestedState, Box)>; - - fn nth(&mut self, n: usize) -> Option { - match self.iter.nth(n) { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), - Some(Err(err)) => Some(Result::Err(err)), - None => None, - } - } -} - -pub fn read_nested_binary( - reader: &mut R, - data_type: DataType, - init: Vec, - page_metas: Vec, -) -> Result)>> { - let mut scratch = vec![]; - - let mut results = Vec::with_capacity(page_metas.len()); - - for page_meta in page_metas { - let num_values = page_meta.num_values as usize; - let (nested, validity) = read_nested(reader, &init, num_values)?; - let mut offsets: Vec = Vec::with_capacity(num_values + 1); - let mut values = Vec::with_capacity(0); - - decompress_binary(reader, num_values, &mut offsets, &mut values, &mut scratch)?; - - let array = try_new_binary_array( - data_type.clone(), - unsafe { OffsetsBuffer::new_unchecked(offsets.into()) }, - values.into(), - validity, - )?; - results.push((nested, array)); - } - Ok(results) -} - -fn try_new_binary_array( - data_type: DataType, - offsets: OffsetsBuffer, - values: Buffer, - validity: Option, -) -> Result> { - if matches!(data_type, DataType::Utf8 | DataType::LargeUtf8) { - let array = - Utf8Array::::try_new(data_type, offsets, values, validity).map_err(|err| { - Error::External( - "Encountered invalid utf8 data for string type, \ - if you were reading column with string type from a table, \ - it's recommended to alter the column type to `BINARY`.\n\ - Example: `ALTER TABLE MODIFY COLUMN BINARY;`" - .to_string(), - Box::new(err), - ) - })?; - Ok(Box::new(array) as Box) - } else { - let array = BinaryArray::::try_new(data_type, offsets, values, validity)?; - Ok(Box::new(array) as Box) - } -} diff --git a/src/common/arrow/src/native/read/batch_read.rs b/src/common/arrow/src/native/read/batch_read.rs deleted file mode 100644 index c8d70e33a8b9..000000000000 --- a/src/common/arrow/src/native/read/batch_read.rs +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::array::*; -use super::NativeReadBuf; -use crate::arrow::array::*; -use crate::arrow::compute::concatenate::concatenate; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::native::nested::create_list; -use crate::native::nested::create_map; -use crate::native::nested::create_struct; -use crate::native::nested::InitNested; -use crate::native::nested::NestedState; -use crate::native::util::n_columns; -use crate::native::PageMeta; - -pub fn read_nested( - mut readers: Vec, - field: Field, - mut init: Vec, - mut page_metas: Vec>, -) -> Result)>> { - use PhysicalType::*; - - Ok(match field.data_type().to_physical_type() { - Null => unimplemented!(), - Boolean => { - init.push(InitNested::Primitive(field.is_nullable)); - read_nested_boolean( - &mut readers.pop().unwrap(), - field.data_type().clone(), - init, - page_metas.pop().unwrap(), - )? - } - Primitive(primitive) => with_match_integer_double_type!(primitive, - |$T| { - init.push(InitNested::Primitive(field.is_nullable)); - read_nested_integer::<$T, _>( - &mut readers.pop().unwrap(), - field.data_type().clone(), - init, - page_metas.pop().unwrap(), - )? - }, - |$T| { - init.push(InitNested::Primitive(field.is_nullable)); - read_nested_primitive::<$T, _>( - &mut readers.pop().unwrap(), - field.data_type().clone(), - init, - page_metas.pop().unwrap(), - )? - } - ), - Binary | Utf8 => { - init.push(InitNested::Primitive(field.is_nullable)); - read_nested_binary::( - &mut readers.pop().unwrap(), - field.data_type().clone(), - init, - page_metas.pop().unwrap(), - )? - } - - BinaryView | Utf8View => { - init.push(InitNested::Primitive(field.is_nullable)); - read_nested_view_array::<_>( - &mut readers.pop().unwrap(), - field.data_type().clone(), - init, - page_metas.pop().unwrap(), - )? - } - - LargeBinary | LargeUtf8 => { - init.push(InitNested::Primitive(field.is_nullable)); - read_nested_binary::( - &mut readers.pop().unwrap(), - field.data_type().clone(), - init, - page_metas.pop().unwrap(), - )? - } - - FixedSizeBinary => unimplemented!(), - _ => match field.data_type().to_logical_type() { - DataType::List(inner) - | DataType::LargeList(inner) - | DataType::FixedSizeList(inner, _) => { - init.push(InitNested::List(field.is_nullable)); - let results = read_nested(readers, inner.as_ref().clone(), init, page_metas)?; - let mut arrays = Vec::with_capacity(results.len()); - for (mut nested, values) in results { - let array = create_list(field.data_type().clone(), &mut nested, values); - arrays.push((nested, array)); - } - arrays - } - DataType::Map(inner, _) => { - init.push(InitNested::List(field.is_nullable)); - let results = read_nested(readers, inner.as_ref().clone(), init, page_metas)?; - let mut arrays = Vec::with_capacity(results.len()); - for (mut nested, values) in results { - let array = create_map(field.data_type().clone(), &mut nested, values); - arrays.push((nested, array)); - } - arrays - } - DataType::Struct(fields) => { - let mut results = fields - .iter() - .map(|f| { - let mut init = init.clone(); - init.push(InitNested::Struct(field.is_nullable)); - let n = n_columns(&f.data_type); - let readers = readers.drain(..n).collect(); - let page_metas = page_metas.drain(..n).collect(); - read_nested(readers, f.clone(), init, page_metas) - }) - .collect::>>()?; - let mut arrays = Vec::with_capacity(results[0].len()); - while !results[0].is_empty() { - let mut nesteds = Vec::with_capacity(fields.len()); - let mut values = Vec::with_capacity(fields.len()); - for result in results.iter_mut() { - let (nested, value) = result.pop().unwrap(); - nesteds.push(nested); - values.push(value); - } - let array = create_struct(fields.clone(), &mut nesteds, values); - arrays.push(array); - } - arrays.reverse(); - arrays - } - _ => unreachable!(), - }, - }) -} - -/// Read all pages of column at once. -pub fn batch_read_array( - readers: Vec, - field: Field, - page_metas: Vec>, -) -> Result> { - let results = read_nested(readers, field, vec![], page_metas)?; - let arrays: Vec<&dyn Array> = results.iter().map(|(_, v)| v.as_ref()).collect(); - let array = concatenate(&arrays).unwrap(); - Ok(array) -} diff --git a/src/common/arrow/src/native/read/deserialize.rs b/src/common/arrow/src/native/read/deserialize.rs deleted file mode 100644 index 094daa18a307..000000000000 --- a/src/common/arrow/src/native/read/deserialize.rs +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::array::*; -use super::PageIterator; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::native::nested::InitNested; -use crate::native::nested::NestedState; -use crate::native::util::n_columns; - -/// [`DynIter`] is an iterator adapter adds a custom `nth` method implementation. -pub struct DynIter<'a, V> { - iter: Box + Send + Sync + 'a>, -} - -impl<'a, V> Iterator for DynIter<'a, V> { - type Item = V; - - fn next(&mut self) -> Option { - self.iter.next() - } - - fn nth(&mut self, n: usize) -> Option { - self.iter.nth(n) - } -} - -impl<'a, V> DynIter<'a, V> { - pub fn new(iter: I) -> Self - where I: Iterator + Send + Sync + 'a { - Self { - iter: Box::new(iter), - } - } -} - -pub type ArrayIter<'a> = DynIter<'a, Result>>; - -/// [`NestedIter`] is a wrapper iterator used to remove the `NestedState` from inner iterator -/// and return only the `Box` -#[derive(Debug)] -pub struct NestedIter -where I: Iterator)>> + Send + Sync -{ - iter: I, -} - -impl NestedIter -where I: Iterator)>> + Send + Sync -{ - pub fn new(iter: I) -> Self { - Self { iter } - } -} - -impl Iterator for NestedIter -where I: Iterator)>> + Send + Sync -{ - type Item = Result>; - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((_, item))) => Some(Ok(item)), - Some(Err(err)) => Some(Err(err)), - None => None, - } - } - - fn nth(&mut self, n: usize) -> Option { - match self.iter.nth(n) { - Some(Ok((_, item))) => Some(Ok(item)), - Some(Err(err)) => Some(Err(err)), - None => None, - } - } -} - -pub type NestedIters<'a> = DynIter<'a, Result<(NestedState, Box)>>; - -fn deserialize_nested<'a, I>( - mut readers: Vec, - field: Field, - mut init: Vec, -) -> Result> -where - I: Iterator)>> + PageIterator + Send + Sync + 'a, -{ - use PhysicalType::*; - - Ok(match field.data_type().to_physical_type() { - Null => unimplemented!(), - Boolean => { - init.push(InitNested::Primitive(field.is_nullable)); - DynIter::new(BooleanNestedIter::new( - readers.pop().unwrap(), - field.data_type().clone(), - init, - )) - } - Primitive(primitive) => with_match_integer_double_type!(primitive, - |$I| { - init.push(InitNested::Primitive(field.is_nullable)); - DynIter::new(IntegerNestedIter::<_, $I>::new( - readers.pop().unwrap(), - field.data_type().clone(), - init, - )) - }, - |$T| { - init.push(InitNested::Primitive(field.is_nullable)); - DynIter::new(DoubleNestedIter::<_, $T>::new( - readers.pop().unwrap(), - field.data_type().clone(), - init, - )) - } - ), - Binary | Utf8 => { - init.push(InitNested::Primitive(field.is_nullable)); - DynIter::new(BinaryNestedIter::<_, i32>::new( - readers.pop().unwrap(), - field.data_type().clone(), - init, - )) - } - BinaryView | Utf8View => { - init.push(InitNested::Primitive(field.is_nullable)); - DynIter::new(ViewArrayNestedIter::<_>::new( - readers.pop().unwrap(), - field.data_type().clone(), - init, - )) - } - LargeBinary | LargeUtf8 => { - init.push(InitNested::Primitive(field.is_nullable)); - DynIter::new(BinaryNestedIter::<_, i64>::new( - readers.pop().unwrap(), - field.data_type().clone(), - init, - )) - } - - FixedSizeBinary => unimplemented!(), - _ => match field.data_type().to_logical_type() { - DataType::List(inner) - | DataType::LargeList(inner) - | DataType::FixedSizeList(inner, _) => { - init.push(InitNested::List(field.is_nullable)); - let iter = deserialize_nested(readers, inner.as_ref().clone(), init)?; - DynIter::new(ListIterator::new(iter, field.clone())) - } - DataType::Map(inner, _) => { - init.push(InitNested::List(field.is_nullable)); - let iter = deserialize_nested(readers, inner.as_ref().clone(), init)?; - DynIter::new(MapIterator::new(iter, field.clone())) - } - DataType::Struct(fields) => { - let columns = fields - .iter() - .rev() - .map(|f| { - let mut init = init.clone(); - init.push(InitNested::Struct(field.is_nullable)); - let n = n_columns(&f.data_type); - let readers = readers.drain(readers.len() - n..).collect(); - deserialize_nested(readers, f.clone(), init) - }) - .collect::>>()?; - let columns = columns.into_iter().rev().collect(); - DynIter::new(StructIterator::new(columns, fields.clone())) - } - _ => unreachable!(), - }, - }) -} - -/// An iterator adapter that maps [`PageIterator`]s into an iterator of [`Array`]s. -pub fn column_iter_to_arrays<'a, I>( - readers: Vec, - field: Field, - init: Vec, -) -> Result> -where - I: Iterator)>> + PageIterator + Send + Sync + 'a, -{ - let iter = deserialize_nested(readers, field, init)?; - let nested_iter = NestedIter::new(iter); - Ok(DynIter::new(nested_iter)) -} diff --git a/src/common/arrow/src/native/util/mod.rs b/src/common/arrow/src/native/util/mod.rs deleted file mode 100644 index a6953a428635..000000000000 --- a/src/common/arrow/src/native/util/mod.rs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[allow(dead_code)] -mod bit_util; -mod byte_writer; -#[allow(dead_code)] -pub mod env; -pub mod memory; - -pub use bit_util::*; -pub use byte_writer::ByteWriter; - -use crate::arrow::datatypes::DataType; - -#[macro_export] -macro_rules! with_match_integer_double_type { - ( - $key_type:expr, | $_:tt $I:ident | $body_integer:tt, | $__ :tt $T:ident | $body_primitive:tt -) => {{ - macro_rules! __with_ty__ { - ( $_ $I:ident ) => { - $body_integer - }; - } - macro_rules! __with_ty_double__ { - ( $_ $T:ident ) => { - $body_primitive - }; - } - use $crate::arrow::datatypes::PrimitiveType::*; - use $crate::arrow::types::i256; - match $key_type { - Int8 => __with_ty__! { i8 }, - Int16 => __with_ty__! { i16 }, - Int32 => __with_ty__! { i32 }, - Int64 => __with_ty__! { i64 }, - Int128 => __with_ty__! { i128 }, - Int256 => __with_ty__! { i256 }, - UInt8 => __with_ty__! { u8 }, - UInt16 => __with_ty__! { u16 }, - UInt32 => __with_ty__! { u32 }, - UInt64 => __with_ty__! { u64 }, - - Float32 => __with_ty_double__! { f32 }, - Float64 => __with_ty_double__! { f64 }, - Float16 => unreachable! {}, - DaysMs => unreachable!(), - MonthDayNano => unreachable!(), - UInt128 => unimplemented!(), - } - }}; -} - -/// Returns the number of (parquet) columns that a [`DataType`] contains. -pub fn n_columns(data_type: &DataType) -> usize { - use crate::arrow::datatypes::PhysicalType::*; - match data_type.to_physical_type() { - Null | Boolean | Primitive(_) | Binary | FixedSizeBinary | LargeBinary | Utf8 - | Dictionary(_) | LargeUtf8 | BinaryView | Utf8View => 1, - List | FixedSizeList | LargeList => { - let a = data_type.to_logical_type(); - if let DataType::List(inner) = a { - n_columns(&inner.data_type) - } else if let DataType::LargeList(inner) = a { - n_columns(&inner.data_type) - } else if let DataType::FixedSizeList(inner, _) = a { - n_columns(&inner.data_type) - } else { - unreachable!() - } - } - Map => { - let a = data_type.to_logical_type(); - if let DataType::Map(inner, _) = a { - n_columns(&inner.data_type) - } else { - unreachable!() - } - } - Struct => { - if let DataType::Struct(fields) = data_type.to_logical_type() { - fields.iter().map(|inner| n_columns(&inner.data_type)).sum() - } else { - unreachable!() - } - } - _ => todo!(), - } -} diff --git a/src/common/arrow/src/native/write/primitive.rs b/src/common/arrow/src/native/write/primitive.rs deleted file mode 100644 index 5f7f741eb74f..000000000000 --- a/src/common/arrow/src/native/write/primitive.rs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -use super::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::arrow::types::i256; -use crate::arrow::types::NativeType; -use crate::native::compression::double::compress_double; -use crate::native::compression::integer::compress_integer; - -pub(crate) fn write_primitive( - w: &mut W, - array: &PrimitiveArray, - write_options: WriteOptions, - scratch: &mut Vec, -) -> Result<()> { - scratch.clear(); - // compress_integer(array, write_options, scratch)?; - match T::PRIMITIVE { - crate::arrow::types::PrimitiveType::Int8 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::Int16 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::Int32 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::Int64 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::UInt8 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::UInt16 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::UInt32 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::UInt64 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::Int128 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::Int256 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - compress_integer(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::Float32 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - - compress_double(array, write_options, scratch)?; - } - crate::arrow::types::PrimitiveType::Float64 => { - let array: &PrimitiveArray = array.as_any().downcast_ref().unwrap(); - - compress_double(array, write_options, scratch)?; - } - - crate::arrow::types::PrimitiveType::Float16 => unimplemented!(), - crate::arrow::types::PrimitiveType::DaysMs => unimplemented!(), - crate::arrow::types::PrimitiveType::MonthDayNano => unimplemented!(), - crate::arrow::types::PrimitiveType::UInt128 => unimplemented!(), - } - w.write_all(scratch.as_slice())?; - Ok(()) -} diff --git a/src/common/arrow/src/native/write/serialize.rs b/src/common/arrow/src/native/write/serialize.rs deleted file mode 100644 index 35bae7636280..000000000000 --- a/src/common/arrow/src/native/write/serialize.rs +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::Write; - -use super::boolean::write_bitmap; -use super::WriteOptions; -use crate::arrow::array::*; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::native::nested::Nested; -use crate::native::util::encode_bool; -use crate::native::write::binary::write_binary; -use crate::native::write::primitive::write_primitive; -use crate::native::write::view::write_view; -use crate::with_match_primitive_type; - -/// Writes an [`Array`] to the file -pub fn write( - w: &mut W, - array: &dyn Array, - nested: &[Nested], - write_options: WriteOptions, - scratch: &mut Vec, -) -> Result<()> { - use PhysicalType::*; - write_nest_info::(w, nested)?; - match array.data_type().to_physical_type() { - Null => {} - Boolean => { - let array: &BooleanArray = array.as_any().downcast_ref().unwrap(); - write_bitmap::(w, array, write_options, scratch)? - } - Primitive(primitive) => with_match_primitive_type!(primitive, |$T| { - let array: &PrimitiveArray<$T> = array.as_any().downcast_ref().unwrap(); - write_primitive::<$T, W>(w, array, write_options, scratch)?; - }), - Binary => { - let array: &BinaryArray = array.as_any().downcast_ref().unwrap(); - write_binary::(w, array, write_options, scratch)?; - } - LargeBinary => { - let array: &BinaryArray = array.as_any().downcast_ref().unwrap(); - write_binary::(w, array, write_options, scratch)?; - } - Utf8 => { - let binary_array: &Utf8Array = array.as_any().downcast_ref().unwrap(); - let binary_array = BinaryArray::new( - DataType::Binary, - binary_array.offsets().clone(), - binary_array.values().clone(), - binary_array.validity().cloned(), - ); - write_binary::(w, &binary_array, write_options, scratch)?; - } - LargeUtf8 => { - let binary_array: &Utf8Array = array.as_any().downcast_ref().unwrap(); - - let binary_array = BinaryArray::new( - DataType::LargeBinary, - binary_array.offsets().clone(), - binary_array.values().clone(), - binary_array.validity().cloned(), - ); - write_binary::(w, &binary_array, write_options, scratch)?; - } - BinaryView => { - let array: &BinaryViewArray = array.as_any().downcast_ref().unwrap(); - write_view::(w, array, write_options, scratch)?; - } - Utf8View => { - let array: &Utf8ViewArray = array.as_any().downcast_ref().unwrap(); - let array = array.clone().to_binview(); - write_view::(w, &array, write_options, scratch)?; - } - Struct => unreachable!(), - List => unreachable!(), - FixedSizeList => unreachable!(), - Dictionary(_key_type) => unreachable!(), - Union => unreachable!(), - Map => unreachable!(), - _ => todo!(), - } - - Ok(()) -} - -fn write_nest_info(w: &mut W, nesteds: &[Nested]) -> Result<()> { - let is_simple = nesteds.len() == 1; - - if is_simple { - let nest = nesteds.last().unwrap(); - - if nest.is_nullable() { - let (_, validity) = nest.inner(); - if let Some(bitmap) = validity { - w.write_all(&(bitmap.len() as u32).to_le_bytes())?; - let (s, offset, _) = bitmap.as_slice(); - if offset == 0 { - w.write_all(s)?; - } else { - encode_bool(w, bitmap.iter())?; - } - } else { - w.write_all(&0u32.to_le_bytes())?; - } - } - } else { - for nested in nesteds { - let (values, validity) = nested.inner(); - - if nested.is_nullable() { - if let Some(bitmap) = validity { - w.write_all(&(bitmap.len() as u32).to_le_bytes())?; - let (s, offset, _) = bitmap.as_slice(); - if offset == 0 { - w.write_all(s)?; - } else { - encode_bool(w, bitmap.iter())?; - } - } else { - w.write_all(&0u32.to_le_bytes())?; - } - } - - if nested.is_list() { - w.write_all(&(values.len() as u32).to_le_bytes())?; - let input_buf: &[u8] = bytemuck::cast_slice(&values); - w.write_all(input_buf)?; - } - } - } - - Ok(()) -} diff --git a/src/common/arrow/src/schema_projection.rs b/src/common/arrow/src/schema_projection.rs deleted file mode 100644 index cebf58d36103..000000000000 --- a/src/common/arrow/src/schema_projection.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::BTreeMap; - -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::Schema; - -/// Project a [`Schema`] by picking the fields at the given indices. -pub fn project(schema: &Schema, indices: &[usize]) -> Schema { - let fields = indices - .iter() - .map(|idx| schema.fields[*idx].clone()) - .collect::>(); - Schema::with_metadata(fields.into(), schema.metadata.clone()) -} - -/// Project a [`Schema`] with inner columns by path. -pub fn inner_project(schema: &Schema, path_indices: &BTreeMap>) -> Schema { - let paths: Vec> = path_indices.values().cloned().collect(); - let fields = paths - .iter() - .map(|path| traverse_paths(&schema.fields, path)) - .collect::>(); - Schema::with_metadata(fields.into(), schema.metadata.clone()) -} - -fn traverse_paths(fields: &[Field], path: &[usize]) -> Field { - assert!(!path.is_empty()); - let field = &fields[path[0]]; - if path.len() == 1 { - return field.clone(); - } - if let DataType::Struct(inner_fields) = field.data_type() { - let fields = inner_fields - .iter() - .map(|inner| { - let inner_name = format!("{}:{}", field.name, inner.name.to_lowercase()); - Field { - name: inner_name, - ..inner.clone() - } - }) - .collect::>(); - return traverse_paths(&fields, &path[1..]); - } - unreachable!("Unable to get field paths. Fields: {:?}", fields); -} diff --git a/src/common/arrow/tests/it/arrow/array/binary/mod.rs b/src/common/arrow/tests/it/arrow/array/binary/mod.rs deleted file mode 100644 index 7f6d349f359a..000000000000 --- a/src/common/arrow/tests/it/arrow/array/binary/mod.rs +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::BinaryArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::error::Result; -use databend_common_arrow::arrow::offset::OffsetsBuffer; - -mod mutable; -mod mutable_values; -mod to_mutable; - -#[test] -fn basics() { - let data = vec![Some(b"hello".to_vec()), None, Some(b"hello2".to_vec())]; - - let array: BinaryArray = data.into_iter().collect(); - - assert_eq!(array.value(0), b"hello"); - assert_eq!(array.value(1), b""); - assert_eq!(array.value(2), b"hello2"); - assert_eq!(unsafe { array.value_unchecked(2) }, b"hello2"); - assert_eq!(array.values().as_slice(), b"hellohello2"); - assert_eq!(array.offsets().as_slice(), &[0, 5, 5, 11]); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = BinaryArray::::new( - DataType::Binary, - array.offsets().clone(), - array.values().clone(), - array.validity().cloned(), - ); - assert_eq!(array, array2); - - let array = array.sliced(1, 2); - assert_eq!(array.value(0), b""); - assert_eq!(array.value(1), b"hello2"); - // note how this keeps everything: the offsets were sliced - assert_eq!(array.values().as_slice(), b"hellohello2"); - assert_eq!(array.offsets().as_slice(), &[5, 5, 11]); -} - -#[test] -fn empty() { - let array = BinaryArray::::new_empty(DataType::Binary); - assert_eq!(array.values().as_slice(), b""); - assert_eq!(array.offsets().as_slice(), &[0]); - assert_eq!(array.validity(), None); -} - -#[test] -fn from() { - let array = BinaryArray::::from([Some(b"hello".as_ref()), Some(b" ".as_ref()), None]); - - let a = array.validity().unwrap(); - assert_eq!(a, &Bitmap::from([true, true, false])); -} - -#[test] -fn from_trusted_len_iter() { - let iter = std::iter::repeat(b"hello").take(2).map(Some); - let a = BinaryArray::::from_trusted_len_iter(iter); - assert_eq!(a.len(), 2); -} - -#[test] -fn try_from_trusted_len_iter() { - let iter = std::iter::repeat(b"hello".as_ref()) - .take(2) - .map(Some) - .map(Result::Ok); - let a = BinaryArray::::try_from_trusted_len_iter(iter).unwrap(); - assert_eq!(a.len(), 2); -} - -#[test] -fn from_iter() { - let iter = std::iter::repeat(b"hello").take(2).map(Some); - let a: BinaryArray = iter.collect(); - assert_eq!(a.len(), 2); -} - -#[test] -fn with_validity() { - let array = BinaryArray::::from([Some(b"hello".as_ref()), Some(b" ".as_ref()), None]); - - let array = array.with_validity(None); - - let a = array.validity(); - assert_eq!(a, None); -} - -#[test] -#[should_panic] -fn wrong_offsets() { - let offsets = vec![0, 5, 4].try_into().unwrap(); // invalid offsets - let values = Buffer::from(b"abbbbb".to_vec()); - BinaryArray::::new(DataType::Binary, offsets, values, None); -} - -#[test] -#[should_panic] -fn wrong_data_type() { - let offsets = vec![0, 4].try_into().unwrap(); - let values = Buffer::from(b"abbb".to_vec()); - BinaryArray::::new(DataType::Int8, offsets, values, None); -} - -#[test] -#[should_panic] -fn value_with_wrong_offsets_panics() { - let offsets = vec![0, 10, 11, 4].try_into().unwrap(); - let values = Buffer::from(b"abbb".to_vec()); - // the 10-11 is not checked - let array = BinaryArray::::new(DataType::Binary, offsets, values, None); - - // but access is still checked (and panics) - // without checks, this would result in reading beyond bounds - array.value(0); -} - -#[test] -#[should_panic] -fn index_out_of_bounds_panics() { - let offsets = vec![0, 1, 2, 4].try_into().unwrap(); - let values = Buffer::from(b"abbb".to_vec()); - let array = BinaryArray::::new(DataType::Utf8, offsets, values, None); - - array.value(3); -} - -#[test] -#[should_panic] -fn value_unchecked_with_wrong_offsets_panics() { - let offsets = vec![0, 10, 11, 4].try_into().unwrap(); - let values = Buffer::from(b"abbb".to_vec()); - // the 10-11 is not checked - let array = BinaryArray::::new(DataType::Binary, offsets, values, None); - - // but access is still checked (and panics) - // without checks, this would result in reading beyond bounds, - // even if `0` is in bounds - unsafe { array.value_unchecked(0) }; -} - -#[test] -fn debug() { - let array = BinaryArray::::from([Some([1, 2].as_ref()), Some(&[]), None]); - - assert_eq!(format!("{array:?}"), "BinaryArray[[1, 2], [], None]"); -} - -#[test] -fn into_mut_1() { - let offsets = vec![0, 1].try_into().unwrap(); - let values = Buffer::from(b"a".to_vec()); - let a = values.clone(); // cloned values - assert_eq!(a, values); - let array = BinaryArray::::new(DataType::Binary, offsets, values, None); - assert!(array.into_mut().is_left()); -} - -#[test] -fn into_mut_2() { - let offsets: OffsetsBuffer = vec![0, 1].try_into().unwrap(); - let values = Buffer::from(b"a".to_vec()); - let a = offsets.clone(); // cloned offsets - assert_eq!(a, offsets); - let array = BinaryArray::::new(DataType::Binary, offsets, values, None); - assert!(array.into_mut().is_left()); -} - -#[test] -fn into_mut_3() { - let offsets = vec![0, 1].try_into().unwrap(); - let values = Buffer::from(b"a".to_vec()); - let validity = Some([true].into()); - let a = validity.clone(); // cloned validity - assert_eq!(a, validity); - let array = BinaryArray::::new(DataType::Binary, offsets, values, validity); - assert!(array.into_mut().is_left()); -} - -#[test] -fn into_mut_4() { - let offsets = vec![0, 1].try_into().unwrap(); - let values = Buffer::from(b"a".to_vec()); - let validity = Some([true].into()); - let array = BinaryArray::::new(DataType::Binary, offsets, values, validity); - assert!(array.into_mut().is_right()); -} - -#[test] -fn rev_iter() { - let array = BinaryArray::::from([Some("hello".as_bytes()), Some(" ".as_bytes()), None]); - - assert_eq!(array.into_iter().rev().collect::>(), vec![ - None, - Some(" ".as_bytes()), - Some("hello".as_bytes()) - ]); -} - -#[test] -fn iter_nth() { - let array = BinaryArray::::from([Some("hello"), Some(" "), None]); - - assert_eq!(array.iter().nth(1), Some(Some(" ".as_bytes()))); - assert_eq!(array.iter().nth(10), None); -} diff --git a/src/common/arrow/tests/it/arrow/array/binary/mutable.rs b/src/common/arrow/tests/it/arrow/array/binary/mutable.rs deleted file mode 100644 index d20b73dccf28..000000000000 --- a/src/common/arrow/tests/it/arrow/array/binary/mutable.rs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::ops::Deref; - -use databend_common_arrow::arrow::array::BinaryArray; -use databend_common_arrow::arrow::array::MutableArray; -use databend_common_arrow::arrow::array::MutableBinaryArray; -use databend_common_arrow::arrow::array::TryExtendFromSelf; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::error::Error; - -#[test] -fn new() { - assert_eq!(MutableBinaryArray::::new().len(), 0); - - let a = MutableBinaryArray::::with_capacity(2); - assert_eq!(a.len(), 0); - assert!(a.offsets().capacity() >= 2); - assert_eq!(a.values().capacity(), 0); - - let a = MutableBinaryArray::::with_capacities(2, 60); - assert_eq!(a.len(), 0); - assert!(a.offsets().capacity() >= 2); - assert!(a.values().capacity() >= 60); -} - -#[test] -fn from_iter() { - let iter = (0..3u8).map(|x| Some(vec![x; x as usize])); - let a: MutableBinaryArray = iter.clone().collect(); - assert_eq!(a.values().deref(), &[1u8, 2, 2]); - assert_eq!(a.offsets().as_slice(), &[0, 0, 1, 3]); - assert_eq!(a.validity(), None); - - let a = unsafe { MutableBinaryArray::::from_trusted_len_iter_unchecked(iter) }; - assert_eq!(a.values().deref(), &[1u8, 2, 2]); - assert_eq!(a.offsets().as_slice(), &[0, 0, 1, 3]); - assert_eq!(a.validity(), None); -} - -#[test] -fn from_trusted_len_iter() { - let data = [vec![0; 0], vec![1; 1], vec![2; 2]]; - let a: MutableBinaryArray = data.iter().cloned().map(Some).collect(); - assert_eq!(a.values().deref(), &[1u8, 2, 2]); - assert_eq!(a.offsets().as_slice(), &[0, 0, 1, 3]); - assert_eq!(a.validity(), None); - - let a = MutableBinaryArray::::from_trusted_len_iter(data.iter().cloned().map(Some)); - assert_eq!(a.values().deref(), &[1u8, 2, 2]); - assert_eq!(a.offsets().as_slice(), &[0, 0, 1, 3]); - assert_eq!(a.validity(), None); - - let a = MutableBinaryArray::::try_from_trusted_len_iter::( - data.iter().cloned().map(Some).map(Ok), - ) - .unwrap(); - assert_eq!(a.values().deref(), &[1u8, 2, 2]); - assert_eq!(a.offsets().as_slice(), &[0, 0, 1, 3]); - assert_eq!(a.validity(), None); - - let a = MutableBinaryArray::::from_trusted_len_values_iter(data.iter().cloned()); - assert_eq!(a.values().deref(), &[1u8, 2, 2]); - assert_eq!(a.offsets().as_slice(), &[0, 0, 1, 3]); - assert_eq!(a.validity(), None); -} - -#[test] -fn push_null() { - let mut array = MutableBinaryArray::::new(); - array.push::<&str>(None); - - let array: BinaryArray = array.into(); - assert_eq!(array.validity(), Some(&Bitmap::from([false]))); -} - -#[test] -fn pop() { - let mut a = MutableBinaryArray::::new(); - a.push(Some(b"first")); - a.push(Some(b"second")); - a.push::>(None); - a.push_null(); - - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 3); - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 2); - assert_eq!(a.pop(), Some(b"second".to_vec())); - assert_eq!(a.len(), 1); - assert_eq!(a.pop(), Some(b"first".to_vec())); - assert_eq!(a.len(), 0); - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 0); -} - -#[test] -fn pop_all_some() { - let mut a = MutableBinaryArray::::new(); - a.push(Some(b"first")); - a.push(Some(b"second")); - a.push(Some(b"third")); - a.push(Some(b"fourth")); - - for _ in 0..4 { - a.push(Some(b"aaaa")); - } - - a.push(Some(b"bbbb")); - - assert_eq!(a.pop(), Some(b"bbbb".to_vec())); - assert_eq!(a.pop(), Some(b"aaaa".to_vec())); - assert_eq!(a.pop(), Some(b"aaaa".to_vec())); - assert_eq!(a.pop(), Some(b"aaaa".to_vec())); - assert_eq!(a.len(), 5); - assert_eq!(a.pop(), Some(b"aaaa".to_vec())); - assert_eq!(a.pop(), Some(b"fourth".to_vec())); - assert_eq!(a.pop(), Some(b"third".to_vec())); - assert_eq!(a.pop(), Some(b"second".to_vec())); - assert_eq!(a.pop(), Some(b"first".to_vec())); - assert!(a.is_empty()); - assert_eq!(a.pop(), None); -} - -#[test] -fn extend_trusted_len_values() { - let mut array = MutableBinaryArray::::new(); - - array.extend_trusted_len_values(vec![b"first".to_vec(), b"second".to_vec()].into_iter()); - array.extend_trusted_len_values(vec![b"third".to_vec()].into_iter()); - array.extend_trusted_len(vec![None, Some(b"fourth".to_vec())].into_iter()); - - let array: BinaryArray = array.into(); - - assert_eq!(array.values().as_slice(), b"firstsecondthirdfourth"); - assert_eq!(array.offsets().as_slice(), &[0, 5, 11, 16, 16, 22]); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00010111], 5)) - ); -} - -#[test] -fn extend_trusted_len() { - let mut array = MutableBinaryArray::::new(); - - array.extend_trusted_len(vec![Some(b"first".to_vec()), Some(b"second".to_vec())].into_iter()); - array.extend_trusted_len(vec![None, Some(b"third".to_vec())].into_iter()); - - let array: BinaryArray = array.into(); - - assert_eq!(array.values().as_slice(), b"firstsecondthird"); - assert_eq!(array.offsets().as_slice(), &[0, 5, 11, 11, 16]); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00001011], 4)) - ); -} - -#[test] -fn extend_from_self() { - let mut a = MutableBinaryArray::::from([Some(b"aa"), None]); - - a.try_extend_from_self(&a.clone()).unwrap(); - - assert_eq!( - a, - MutableBinaryArray::::from([Some(b"aa"), None, Some(b"aa"), None]) - ); -} - -#[test] -fn test_set_validity() { - let mut array = MutableBinaryArray::::new(); - array.push(Some(b"first")); - array.push(Some(b"second")); - array.push(Some(b"third")); - array.set_validity(Some([false, false, true].into())); - - assert!(!array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); -} - -#[test] -fn test_apply_validity() { - let mut array = MutableBinaryArray::::new(); - array.push(Some(b"first")); - array.push(Some(b"second")); - array.push(Some(b"third")); - array.set_validity(Some([true, true, true].into())); - - array.apply_validity(|mut mut_bitmap| { - mut_bitmap.set(1, false); - mut_bitmap.set(2, false); - mut_bitmap - }); - - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(!array.is_valid(2)); -} - -#[test] -fn test_apply_validity_with_no_validity_inited() { - let mut array = MutableBinaryArray::::new(); - array.push(Some(b"first")); - array.push(Some(b"second")); - array.push(Some(b"third")); - - array.apply_validity(|mut mut_bitmap| { - mut_bitmap.set(1, false); - mut_bitmap.set(2, false); - mut_bitmap - }); - - assert!(array.is_valid(0)); - assert!(array.is_valid(1)); - assert!(array.is_valid(2)); -} diff --git a/src/common/arrow/tests/it/arrow/array/binary/mutable_values.rs b/src/common/arrow/tests/it/arrow/array/binary/mutable_values.rs deleted file mode 100644 index f004394d54c6..000000000000 --- a/src/common/arrow/tests/it/arrow/array/binary/mutable_values.rs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::MutableArray; -use databend_common_arrow::arrow::array::MutableBinaryValuesArray; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn capacity() { - let mut b = MutableBinaryValuesArray::::with_capacity(100); - - assert_eq!(b.values().capacity(), 0); - assert!(b.offsets().capacity() >= 100); - b.shrink_to_fit(); - assert!(b.offsets().capacity() < 100); -} - -#[test] -fn offsets_must_be_in_bounds() { - let offsets = vec![0, 10].try_into().unwrap(); - let values = b"abbbbb".to_vec(); - assert!(MutableBinaryValuesArray::::try_new(DataType::Binary, offsets, values).is_err()); -} - -#[test] -fn data_type_must_be_consistent() { - let offsets = vec![0, 4].try_into().unwrap(); - let values = b"abbb".to_vec(); - assert!(MutableBinaryValuesArray::::try_new(DataType::Int32, offsets, values).is_err()); -} - -#[test] -fn as_box() { - let offsets = vec![0, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - let mut b = - MutableBinaryValuesArray::::try_new(DataType::Binary, offsets, values).unwrap(); - let _ = b.as_box(); -} - -#[test] -fn as_arc() { - let offsets = vec![0, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - let mut b = - MutableBinaryValuesArray::::try_new(DataType::Binary, offsets, values).unwrap(); - let _ = b.as_arc(); -} - -#[test] -fn extend_trusted_len() { - let offsets = vec![0, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - let mut b = - MutableBinaryValuesArray::::try_new(DataType::Binary, offsets, values).unwrap(); - b.extend_trusted_len(vec!["a", "b"].into_iter()); - - let offsets = vec![0, 2, 3, 4].try_into().unwrap(); - let values = b"abab".to_vec(); - assert_eq!( - b.as_box(), - MutableBinaryValuesArray::::try_new(DataType::Binary, offsets, values) - .unwrap() - .as_box() - ) -} - -#[test] -fn from_trusted_len() { - let mut b = MutableBinaryValuesArray::::from_trusted_len_iter(vec!["a", "b"].into_iter()); - - let offsets = vec![0, 1, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - assert_eq!( - b.as_box(), - MutableBinaryValuesArray::::try_new(DataType::Binary, offsets, values) - .unwrap() - .as_box() - ) -} - -#[test] -fn extend_from_iter() { - let offsets = vec![0, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - let mut b = - MutableBinaryValuesArray::::try_new(DataType::Binary, offsets, values).unwrap(); - b.extend_trusted_len(vec!["a", "b"].into_iter()); - - let a = b.clone(); - b.extend_trusted_len(a.iter()); - - let offsets = vec![0, 2, 3, 4, 6, 7, 8].try_into().unwrap(); - let values = b"abababab".to_vec(); - assert_eq!( - b.as_box(), - MutableBinaryValuesArray::::try_new(DataType::Binary, offsets, values) - .unwrap() - .as_box() - ) -} diff --git a/src/common/arrow/tests/it/arrow/array/binary/to_mutable.rs b/src/common/arrow/tests/it/arrow/array/binary/to_mutable.rs deleted file mode 100644 index 069498d42192..000000000000 --- a/src/common/arrow/tests/it/arrow/array/binary/to_mutable.rs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::BinaryArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn not_shared() { - let array = BinaryArray::::from([Some("hello"), Some(" "), None]); - assert!(array.into_mut().is_right()); -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_validity() { - let validity = Bitmap::from([true]); - let array = BinaryArray::::new( - DataType::Binary, - vec![0, 1].try_into().unwrap(), - b"a".to_vec().into(), - Some(validity.clone()), - ); - assert!(array.into_mut().is_left()) -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_values() { - let values: Buffer = b"a".to_vec().into(); - let array = BinaryArray::::new( - DataType::Binary, - vec![0, 1].try_into().unwrap(), - values.clone(), - Some(Bitmap::from([true])), - ); - assert!(array.into_mut().is_left()) -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_offsets_values() { - let offsets: Buffer = vec![0, 1].into(); - let values: Buffer = b"a".to_vec().into(); - let array = BinaryArray::::new( - DataType::Binary, - offsets.clone().try_into().unwrap(), - values.clone(), - Some(Bitmap::from([true])), - ); - assert!(array.into_mut().is_left()) -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_offsets() { - let offsets: Buffer = vec![0, 1].into(); - let array = BinaryArray::::new( - DataType::Binary, - offsets.clone().try_into().unwrap(), - b"a".to_vec().into(), - Some(Bitmap::from([true])), - ); - assert!(array.into_mut().is_left()) -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_all() { - let array = BinaryArray::::from([Some("hello"), Some(" "), None]); - assert!(array.clone().into_mut().is_left()) -} diff --git a/src/common/arrow/tests/it/arrow/array/binview/mod.rs b/src/common/arrow/tests/it/arrow/array/binview/mod.rs deleted file mode 100644 index ff1183375a02..000000000000 --- a/src/common/arrow/tests/it/arrow/array/binview/mod.rs +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod mutable; -mod mutable_values; -mod to_mutable; - -use std::sync::Arc; - -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::BinaryViewArray; -use databend_common_arrow::arrow::array::Utf8ViewArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn basics_string_view() { - let data = vec![ - Some("hello"), - None, - // larger than 12 bytes. - Some("Databend Cloud is a Cost-Effective alternative to Snowflake."), - ]; - - let array: Utf8ViewArray = data.into_iter().collect(); - - assert_eq!(array.value(0), "hello"); - assert_eq!(array.value(1), ""); - assert_eq!( - array.value(2), - "Databend Cloud is a Cost-Effective alternative to Snowflake." - ); - assert_eq!( - unsafe { array.value_unchecked(2) }, - "Databend Cloud is a Cost-Effective alternative to Snowflake." - ); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = Utf8ViewArray::new_unchecked( - DataType::Utf8View, - array.views().clone(), - array.data_buffers().clone(), - array.validity().cloned(), - array.total_bytes_len(), - array.total_buffer_len(), - ); - - assert_eq!(array, array2); - - let array = array.sliced(1, 2); - - assert_eq!(array.value(0), ""); - assert_eq!( - array.value(1), - "Databend Cloud is a Cost-Effective alternative to Snowflake." - ); -} - -#[test] -fn basics_binary_view() { - let data = vec![ - Some(b"hello".to_vec()), - None, - // larger than 12 bytes. - Some(b"Databend Cloud is a Cost-Effective alternative to Snowflake.".to_vec()), - ]; - - let array: BinaryViewArray = data.into_iter().collect(); - - assert_eq!(array.value(0), b"hello"); - assert_eq!(array.value(1), b""); - assert_eq!( - array.value(2), - b"Databend Cloud is a Cost-Effective alternative to Snowflake." - ); - assert_eq!( - unsafe { array.value_unchecked(2) }, - b"Databend Cloud is a Cost-Effective alternative to Snowflake." - ); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = BinaryViewArray::new_unchecked( - DataType::BinaryView, - array.views().clone(), - array.data_buffers().clone(), - array.validity().cloned(), - array.total_bytes_len(), - array.total_buffer_len(), - ); - - assert_eq!(array, array2); - - let array = array.sliced(1, 2); - - assert_eq!(array.value(0), b""); - assert_eq!( - array.value(1), - b"Databend Cloud is a Cost-Effective alternative to Snowflake." - ); -} - -#[test] -fn from() { - let array = Utf8ViewArray::from([Some("hello"), Some(" "), None]); - - let a = array.validity().unwrap(); - assert_eq!(a, &Bitmap::from([true, true, false])); - - let array = BinaryViewArray::from([Some(b"hello".to_vec()), Some(b" ".to_vec()), None]); - - let a = array.validity().unwrap(); - assert_eq!(a, &Bitmap::from([true, true, false])); -} - -#[test] -fn from_iter() { - let iter = std::iter::repeat(b"hello").take(2).map(Some); - let a: BinaryViewArray = iter.collect(); - assert_eq!(a.len(), 2); -} - -#[test] -fn with_validity() { - let array = BinaryViewArray::from([Some(b"hello".as_ref()), Some(b" ".as_ref()), None]); - - let array = array.with_validity(None); - - let a = array.validity(); - assert_eq!(a, None); -} - -#[test] -#[should_panic] -fn wrong_data_type() { - let validity = Some(Bitmap::new_zeroed(3)); - BinaryViewArray::try_new(DataType::Int8, Buffer::zeroed(3), Arc::from([]), validity).unwrap(); -} - -#[test] -fn debug() { - let data = vec![Some([1_u8, 2_u8].to_vec()), Some(vec![]), None]; - - let array: BinaryViewArray = data.into_iter().collect(); - - assert_eq!(format!("{array:?}"), "BinaryViewArray[[1, 2], [], None]"); -} - -#[test] -fn rev_iter() { - let array = BinaryViewArray::from([Some("hello".as_bytes()), Some(" ".as_bytes()), None]); - - assert_eq!(array.into_iter().rev().collect::>(), vec![ - None, - Some(" ".as_bytes()), - Some("hello".as_bytes()) - ]); -} - -#[test] -fn iter_nth() { - let array = BinaryViewArray::from([Some("hello"), Some(" "), None]); - - assert_eq!(array.iter().nth(1), Some(Some(" ".as_bytes()))); - assert_eq!(array.iter().nth(10), None); -} - -#[test] -fn test_slice() { - let data = vec![ - Some("hello"), - Some("world"), - Some("databend"), - None, - Some("y"), - Some("z"), - Some("abc"), - ]; - - let array: Utf8ViewArray = data.into_iter().collect(); - - let a3 = array.sliced(2, 3); - assert_eq!(a3.into_iter().collect::>(), vec![ - Some("databend"), - None, - Some("y"), - ]); -} diff --git a/src/common/arrow/tests/it/arrow/array/binview/mutable.rs b/src/common/arrow/tests/it/arrow/array/binview/mutable.rs deleted file mode 100644 index f2b70037cf7c..000000000000 --- a/src/common/arrow/tests/it/arrow/array/binview/mutable.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::MutableBinaryViewArray; -use databend_common_arrow::arrow::array::Utf8ViewArray; -use databend_common_arrow::arrow::bitmap::Bitmap; - -#[test] -fn new() { - assert_eq!(MutableBinaryViewArray::<[u8]>::new().len(), 0); - - let a = MutableBinaryViewArray::<[u8]>::with_capacity(2); - assert_eq!(a.len(), 0); - assert_eq!(a.capacity(), 2); -} - -#[test] -fn from_iter() { - let iter = (0..3u8).map(|x| Some(vec![x; x as usize])); - let a: MutableBinaryViewArray<[u8]> = iter.clone().collect(); - let mut v_iter = a.values_iter(); - assert_eq!(v_iter.next(), Some(&[] as &[u8])); - assert_eq!(v_iter.next(), Some(&[1u8] as &[u8])); - assert_eq!(v_iter.next(), Some(&[2u8, 2] as &[u8])); - assert_eq!(a.validity(), None); - - let a = MutableBinaryViewArray::<[u8]>::from_iter(iter); - assert_eq!(a.validity(), None); -} - -#[test] -fn push_null() { - let mut array = MutableBinaryViewArray::new(); - array.push::<&str>(None); - - let array: Utf8ViewArray = array.into(); - assert_eq!(array.validity(), Some(&Bitmap::from([false]))); -} diff --git a/src/common/arrow/tests/it/arrow/array/binview/mutable_values.rs b/src/common/arrow/tests/it/arrow/array/binview/mutable_values.rs deleted file mode 100644 index e0384ad7bd39..000000000000 --- a/src/common/arrow/tests/it/arrow/array/binview/mutable_values.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::MutableArray; -use databend_common_arrow::arrow::array::MutableBinaryViewArray; - -#[test] -fn extend_from_iter() { - let mut b = MutableBinaryViewArray::::new(); - b.extend_trusted_len_values(vec!["a", "b"].into_iter()); - - let a = b.clone(); - b.extend_trusted_len_values(a.values_iter()); - - assert_eq!( - b.as_box(), - MutableBinaryViewArray::::from_values_iter(vec!["a", "b", "a", "b"].into_iter()) - .as_box() - ) -} - -#[test] -fn extend_from_repeats() { - let mut b = MutableBinaryViewArray::::new(); - b.extend_constant(4, Some("databend")); - - let a = b.clone(); - b.extend_trusted_len_values(a.values_iter()); - - assert_eq!( - b.as_box(), - MutableBinaryViewArray::::from_values_iter(vec!["databend"; 8].into_iter()).as_box() - ) -} diff --git a/src/common/arrow/tests/it/arrow/array/binview/to_mutable.rs b/src/common/arrow/tests/it/arrow/array/binview/to_mutable.rs deleted file mode 100644 index 7ee7856ba01d..000000000000 --- a/src/common/arrow/tests/it/arrow/array/binview/to_mutable.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::BinaryViewArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn not_shared() { - let array = BinaryViewArray::from([Some("hello"), Some(" "), None]); - assert!(array.into_mut().is_right()); -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared() { - let validity = Bitmap::from([true]); - let data = vec![ - Some(b"hello".to_vec()), - None, - // larger than 12 bytes. - Some(b"Databend Cloud is a Cost-Effective alternative to Snowflake.".to_vec()), - ]; - - let array: BinaryViewArray = data.into_iter().collect(); - let array2 = BinaryViewArray::new_unchecked( - DataType::BinaryView, - array.views().clone(), - array.data_buffers().clone(), - Some(validity.clone()), - array.total_bytes_len(), - array.total_buffer_len(), - ); - assert!(array2.into_mut().is_left()) -} diff --git a/src/common/arrow/tests/it/arrow/array/boolean/mod.rs b/src/common/arrow/tests/it/arrow/array/boolean/mod.rs deleted file mode 100644 index 493d0a372147..000000000000 --- a/src/common/arrow/tests/it/arrow/array/boolean/mod.rs +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::BooleanArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::error::Result; - -mod mutable; - -#[test] -fn basics() { - let data = vec![Some(true), None, Some(false)]; - - let array: BooleanArray = data.into_iter().collect(); - - assert_eq!(array.data_type(), &DataType::Boolean); - - assert!(array.value(0)); - assert!(!array.value(1)); - assert!(!array.value(2)); - assert!(!unsafe { array.value_unchecked(2) }); - assert_eq!(array.values(), &Bitmap::from_u8_slice([0b00000001], 3)); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = BooleanArray::new( - DataType::Boolean, - array.values().clone(), - array.validity().cloned(), - ); - assert_eq!(array, array2); - - let array = array.sliced(1, 2); - assert!(!array.value(0)); - assert!(!array.value(1)); -} - -#[test] -fn try_new_invalid() { - assert!(BooleanArray::try_new(DataType::Int32, [true].into(), None).is_err()); - assert!( - BooleanArray::try_new(DataType::Boolean, [true].into(), Some([false, true].into())) - .is_err() - ); -} - -#[test] -fn with_validity() { - let bitmap = Bitmap::from([true, false, true]); - let a = BooleanArray::new(DataType::Boolean, bitmap, None); - let a = a.with_validity(Some(Bitmap::from([true, false, true]))); - assert!(a.validity().is_some()); -} - -#[test] -fn debug() { - let array = BooleanArray::from([Some(true), None, Some(false)]); - assert_eq!(format!("{array:?}"), "BooleanArray[true, None, false]"); -} - -#[test] -fn into_mut_valid() { - let bitmap = Bitmap::from([true, false, true]); - let a = BooleanArray::new(DataType::Boolean, bitmap, None); - let _ = a.into_mut().right().unwrap(); - - let bitmap = Bitmap::from([true, false, true]); - let validity = Bitmap::from([true, false, true]); - let a = BooleanArray::new(DataType::Boolean, bitmap, Some(validity)); - let _ = a.into_mut().right().unwrap(); -} - -#[test] -fn into_mut_invalid() { - let bitmap = Bitmap::from([true, false, true]); - let _other = bitmap.clone(); // values is shared - let a = BooleanArray::new(DataType::Boolean, bitmap, None); - let _ = a.into_mut().left().unwrap(); - - let bitmap = Bitmap::from([true, false, true]); - let validity = Bitmap::from([true, false, true]); - let _other = validity.clone(); // validity is shared - let a = BooleanArray::new(DataType::Boolean, bitmap, Some(validity)); - let _ = a.into_mut().left().unwrap(); -} - -#[test] -fn empty() { - let array = BooleanArray::new_empty(DataType::Boolean); - assert_eq!(array.values().len(), 0); - assert_eq!(array.validity(), None); -} - -#[test] -fn from_trusted_len_iter() { - let iter = std::iter::repeat(true).take(2).map(Some); - let a = BooleanArray::from_trusted_len_iter(iter.clone()); - assert_eq!(a.len(), 2); - let a = unsafe { BooleanArray::from_trusted_len_iter_unchecked(iter) }; - assert_eq!(a.len(), 2); -} - -#[test] -fn try_from_trusted_len_iter() { - let iter = std::iter::repeat(true).take(2).map(Some).map(Result::Ok); - let a = BooleanArray::try_from_trusted_len_iter(iter.clone()).unwrap(); - assert_eq!(a.len(), 2); - let a = unsafe { BooleanArray::try_from_trusted_len_iter_unchecked(iter).unwrap() }; - assert_eq!(a.len(), 2); -} - -#[test] -fn from_trusted_len_values_iter() { - let iter = std::iter::repeat(true).take(2); - let a = BooleanArray::from_trusted_len_values_iter(iter.clone()); - assert_eq!(a.len(), 2); - let a = unsafe { BooleanArray::from_trusted_len_values_iter_unchecked(iter) }; - assert_eq!(a.len(), 2); -} - -#[test] -fn from_iter() { - let iter = std::iter::repeat(true).take(2).map(Some); - let a: BooleanArray = iter.collect(); - assert_eq!(a.len(), 2); -} - -#[test] -fn into_iter() { - let data = vec![Some(true), None, Some(false)]; - let rev = data.clone().into_iter().rev(); - - let array: BooleanArray = data.clone().into_iter().collect(); - - assert_eq!(array.clone().into_iter().collect::>(), data); - - assert!(array.into_iter().rev().eq(rev)) -} diff --git a/src/common/arrow/tests/it/arrow/array/boolean/mutable.rs b/src/common/arrow/tests/it/arrow/array/boolean/mutable.rs deleted file mode 100644 index 08a02ccc855c..000000000000 --- a/src/common/arrow/tests/it/arrow/array/boolean/mutable.rs +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::MutableArray; -use databend_common_arrow::arrow::array::MutableBooleanArray; -use databend_common_arrow::arrow::array::TryExtendFromSelf; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::error::Result; - -#[test] -fn set() { - let mut a = MutableBooleanArray::from(&[Some(false), Some(true), Some(false)]); - - a.set(1, None); - a.set(0, Some(true)); - assert_eq!( - a, - MutableBooleanArray::from([Some(true), None, Some(false)]) - ); - assert_eq!(a.values(), &MutableBitmap::from([true, false, false])); -} - -#[test] -fn push() { - let mut a = MutableBooleanArray::new(); - a.push(Some(true)); - a.push(Some(false)); - a.push(None); - a.push_null(); - assert_eq!( - a, - MutableBooleanArray::from([Some(true), Some(false), None, None]) - ); -} - -#[test] -fn pop() { - let mut a = MutableBooleanArray::new(); - a.push(Some(true)); - a.push(Some(false)); - a.push(None); - a.push_null(); - - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 3); - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 2); - assert_eq!(a.pop(), Some(false)); - assert_eq!(a.len(), 1); - assert_eq!(a.pop(), Some(true)); - assert_eq!(a.len(), 0); - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 0); -} - -#[test] -fn pop_all_some() { - let mut a = MutableBooleanArray::new(); - for _ in 0..4 { - a.push(Some(true)); - } - - for _ in 0..4 { - a.push(Some(false)); - } - - a.push(Some(true)); - - assert_eq!(a.pop(), Some(true)); - assert_eq!(a.pop(), Some(false)); - assert_eq!(a.pop(), Some(false)); - assert_eq!(a.pop(), Some(false)); - assert_eq!(a.len(), 5); - - assert_eq!( - a, - MutableBooleanArray::from([Some(true), Some(true), Some(true), Some(true), Some(false)]) - ); -} - -#[test] -fn from_trusted_len_iter() { - let iter = std::iter::repeat(true).take(2).map(Some); - let a = MutableBooleanArray::from_trusted_len_iter(iter); - assert_eq!(a, MutableBooleanArray::from([Some(true), Some(true)])); -} - -#[test] -fn from_iter() { - let iter = std::iter::repeat(true).take(2).map(Some); - let a: MutableBooleanArray = iter.collect(); - assert_eq!(a, MutableBooleanArray::from([Some(true), Some(true)])); -} - -#[test] -fn try_from_trusted_len_iter() { - let iter = vec![Some(true), Some(true), None] - .into_iter() - .map(Result::Ok); - let a = MutableBooleanArray::try_from_trusted_len_iter(iter).unwrap(); - assert_eq!(a, MutableBooleanArray::from([Some(true), Some(true), None])); -} - -#[test] -fn reserve() { - let mut a = MutableBooleanArray::try_new( - DataType::Boolean, - MutableBitmap::new(), - Some(MutableBitmap::new()), - ) - .unwrap(); - - a.reserve(10); - assert!(a.validity().unwrap().capacity() > 0); - assert!(a.values().capacity() > 0) -} - -#[test] -fn extend_trusted_len() { - let mut a = MutableBooleanArray::new(); - - a.extend_trusted_len(vec![Some(true), Some(false)].into_iter()); - assert_eq!(a.validity(), None); - - a.extend_trusted_len(vec![None, Some(true)].into_iter()); - assert_eq!( - a.validity(), - Some(&MutableBitmap::from([true, true, false, true])) - ); - assert_eq!(a.values(), &MutableBitmap::from([true, false, false, true])); -} - -#[test] -fn extend_trusted_len_values() { - let mut a = MutableBooleanArray::new(); - - a.extend_trusted_len_values(vec![true, true, false].into_iter()); - assert_eq!(a.validity(), None); - assert_eq!(a.values(), &MutableBitmap::from([true, true, false])); - - let mut a = MutableBooleanArray::new(); - a.push(None); - a.extend_trusted_len_values(vec![true, false].into_iter()); - assert_eq!( - a.validity(), - Some(&MutableBitmap::from([false, true, true])) - ); - assert_eq!(a.values(), &MutableBitmap::from([false, true, false])); -} - -#[test] -fn into_iter() { - let ve = MutableBitmap::from([true, false]) - .into_iter() - .collect::>(); - assert_eq!(ve, vec![true, false]); - let ve = MutableBitmap::from([true, false]) - .iter() - .collect::>(); - assert_eq!(ve, vec![true, false]); -} - -#[test] -fn shrink_to_fit() { - let mut a = MutableBitmap::with_capacity(100); - a.push(true); - a.shrink_to_fit(); - assert_eq!(a.capacity(), 8); -} - -#[test] -fn extend_from_self() { - let mut a = MutableBooleanArray::from([Some(true), None]); - - a.try_extend_from_self(&a.clone()).unwrap(); - - assert_eq!( - a, - MutableBooleanArray::from([Some(true), None, Some(true), None]) - ); -} diff --git a/src/common/arrow/tests/it/arrow/array/dictionary/mod.rs b/src/common/arrow/tests/it/arrow/array/dictionary/mod.rs deleted file mode 100644 index 9bfbde85af92..000000000000 --- a/src/common/arrow/tests/it/arrow/array/dictionary/mod.rs +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod mutable; - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn try_new_ok() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let data_type = - DataType::Dictionary(i32::KEY_TYPE, Box::new(values.data_type().clone()), false); - let array = DictionaryArray::try_new( - data_type, - PrimitiveArray::from_vec(vec![1, 0]), - values.boxed(), - ) - .unwrap(); - - assert_eq!(array.keys(), &PrimitiveArray::from_vec(vec![1i32, 0])); - assert_eq!( - &Utf8Array::::from_slice(["a", "aa"]) as &dyn Array, - array.values().as_ref(), - ); - assert!(!array.is_ordered()); - - assert_eq!(format!("{array:?}"), "DictionaryArray[aa, a]"); -} - -#[test] -fn try_new_incorrect_key() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let data_type = - DataType::Dictionary(i16::KEY_TYPE, Box::new(values.data_type().clone()), false); - - let r = DictionaryArray::try_new( - data_type, - PrimitiveArray::from_vec(vec![1, 0]), - values.boxed(), - ) - .is_err(); - - assert!(r); -} - -#[test] -fn try_new_nulls() { - let key: Option = None; - let keys = PrimitiveArray::from_iter([key]); - let value: &[&str] = &[]; - let values = Utf8Array::::from_slice(value); - - let data_type = - DataType::Dictionary(u32::KEY_TYPE, Box::new(values.data_type().clone()), false); - let r = DictionaryArray::try_new(data_type, keys, values.boxed()).is_ok(); - - assert!(r); -} - -#[test] -fn try_new_incorrect_dt() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let data_type = DataType::Int32; - - let r = DictionaryArray::try_new( - data_type, - PrimitiveArray::from_vec(vec![1, 0]), - values.boxed(), - ) - .is_err(); - - assert!(r); -} - -#[test] -fn try_new_incorrect_values_dt() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let data_type = DataType::Dictionary(i32::KEY_TYPE, Box::new(DataType::LargeUtf8), false); - - let r = DictionaryArray::try_new( - data_type, - PrimitiveArray::from_vec(vec![1, 0]), - values.boxed(), - ) - .is_err(); - - assert!(r); -} - -#[test] -fn try_new_out_of_bounds() { - let values = Utf8Array::::from_slice(["a", "aa"]); - - let r = DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![2, 0]), values.boxed()) - .is_err(); - - assert!(r); -} - -#[test] -fn try_new_out_of_bounds_neg() { - let values = Utf8Array::::from_slice(["a", "aa"]); - - let r = DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![-1, 0]), values.boxed()) - .is_err(); - - assert!(r); -} - -#[test] -fn new_null() { - let dt = DataType::Dictionary(i16::KEY_TYPE, Box::new(DataType::Int32), false); - let array = DictionaryArray::::new_null(dt, 2); - - assert_eq!(format!("{array:?}"), "DictionaryArray[None, None]"); -} - -#[test] -fn new_empty() { - let dt = DataType::Dictionary(i16::KEY_TYPE, Box::new(DataType::Int32), false); - let array = DictionaryArray::::new_empty(dt); - - assert_eq!(format!("{array:?}"), "DictionaryArray[]"); -} - -#[test] -fn with_validity() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let array = - DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![1, 0]), values.boxed()) - .unwrap(); - - let array = array.with_validity(Some([true, false].into())); - - assert_eq!(format!("{array:?}"), "DictionaryArray[aa, None]"); -} - -#[test] -fn rev_iter() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let array = - DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![1, 0]), values.boxed()) - .unwrap(); - - let mut iter = array.into_iter(); - assert_eq!(iter.by_ref().rev().count(), 2); - assert_eq!(iter.size_hint(), (0, Some(0))); -} - -#[test] -fn iter_values() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let array = - DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![1, 0]), values.boxed()) - .unwrap(); - - let mut iter = array.values_iter(); - assert_eq!(iter.by_ref().count(), 2); - assert_eq!(iter.size_hint(), (0, Some(0))); -} - -#[test] -fn keys_values_iter() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let array = - DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![1, 0]), values.boxed()) - .unwrap(); - - assert_eq!(array.keys_values_iter().collect::>(), vec![1, 0]); -} - -#[test] -fn iter_values_typed() { - let values = Utf8Array::::from_slice(["a", "aa"]); - let array = - DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![1, 0, 0]), values.boxed()) - .unwrap(); - - let iter = array.values_iter_typed::>().unwrap(); - assert_eq!(iter.size_hint(), (3, Some(3))); - assert_eq!(iter.collect::>(), vec!["aa", "a", "a"]); - - let iter = array.iter_typed::>().unwrap(); - assert_eq!(iter.size_hint(), (3, Some(3))); - assert_eq!(iter.collect::>(), vec![ - Some("aa"), - Some("a"), - Some("a") - ]); -} - -#[test] -#[should_panic] -fn iter_values_typed_panic() { - let values = Utf8Array::::from_iter([Some("a"), Some("aa"), None]); - let array = - DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![1, 0, 0]), values.boxed()) - .unwrap(); - - // should not be iterating values - let iter = array.values_iter_typed::>().unwrap(); - let _ = iter.collect::>(); -} - -#[test] -#[should_panic] -fn iter_values_typed_panic_2() { - let values = Utf8Array::::from_iter([Some("a"), Some("aa"), None]); - let array = - DictionaryArray::try_from_keys(PrimitiveArray::from_vec(vec![1, 0, 0]), values.boxed()) - .unwrap(); - - // should not be iterating values - let iter = array.iter_typed::>().unwrap(); - let _ = iter.collect::>(); -} diff --git a/src/common/arrow/tests/it/arrow/array/dictionary/mutable.rs b/src/common/arrow/tests/it/arrow/array/dictionary/mutable.rs deleted file mode 100644 index b1fd72a5cc85..000000000000 --- a/src/common/arrow/tests/it/arrow/array/dictionary/mutable.rs +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::borrow::Borrow; -use std::collections::HashSet; -use std::fmt::Debug; -use std::hash::Hash; - -use databend_common_arrow::arrow::array::indexable::AsIndexed; -use databend_common_arrow::arrow::array::indexable::Indexable; -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::error::Result; - -#[test] -fn primitive() -> Result<()> { - let data = vec![Some(1), Some(2), Some(1)]; - - let mut a = MutableDictionaryArray::>::new(); - a.try_extend(data)?; - assert_eq!(a.len(), 3); - assert_eq!(a.values().len(), 2); - Ok(()) -} - -#[test] -fn utf8_natural() -> Result<()> { - let data = vec![Some("a"), Some("b"), Some("a")]; - - let mut a = MutableDictionaryArray::>::new(); - a.try_extend(data)?; - - assert_eq!(a.len(), 3); - assert_eq!(a.values().len(), 2); - Ok(()) -} - -#[test] -fn binary_natural() -> Result<()> { - let data = vec![ - Some("a".as_bytes()), - Some("b".as_bytes()), - Some("a".as_bytes()), - ]; - - let mut a = MutableDictionaryArray::>::new(); - a.try_extend(data)?; - assert_eq!(a.len(), 3); - assert_eq!(a.values().len(), 2); - Ok(()) -} - -#[test] -fn push_utf8() { - let mut new: MutableDictionaryArray> = MutableDictionaryArray::new(); - - for value in [Some("A"), Some("B"), None, Some("C"), Some("A"), Some("B")] { - new.try_push(value).unwrap(); - } - - assert_eq!( - new.values().values(), - MutableUtf8Array::::from_iter_values(["A", "B", "C"].into_iter()).values() - ); - - let mut expected_keys = MutablePrimitiveArray::::from_slice([0, 1]); - expected_keys.push(None); - expected_keys.push(Some(2)); - expected_keys.push(Some(0)); - expected_keys.push(Some(1)); - assert_eq!(*new.keys(), expected_keys); -} - -#[test] -fn into_empty() { - let mut new: MutableDictionaryArray> = MutableDictionaryArray::new(); - for value in [Some("A"), Some("B"), None, Some("C"), Some("A"), Some("B")] { - new.try_push(value).unwrap(); - } - let values = new.values().clone(); - let empty = new.into_empty(); - assert_eq!(empty.values(), &values); - assert!(empty.is_empty()); -} - -#[test] -fn from_values() { - let mut new: MutableDictionaryArray> = MutableDictionaryArray::new(); - for value in [Some("A"), Some("B"), None, Some("C"), Some("A"), Some("B")] { - new.try_push(value).unwrap(); - } - let mut values = new.values().clone(); - let empty = MutableDictionaryArray::::from_values(values.clone()).unwrap(); - assert_eq!(empty.values(), &values); - assert!(empty.is_empty()); - values.push(Some("A")); - assert!(MutableDictionaryArray::::from_values(values).is_err()); -} - -#[test] -fn try_empty() { - let mut values = MutableUtf8Array::::new(); - MutableDictionaryArray::::try_empty(values.clone()).unwrap(); - values.push(Some("A")); - assert!(MutableDictionaryArray::::try_empty(values.clone()).is_err()); -} - -fn test_push_ex(values: Vec, gen: impl Fn(usize) -> T) -where - M: MutableArray + Indexable + TryPush> + TryExtend> + Default + 'static, - M::Type: Eq + Hash + Debug, - T: AsIndexed + Default + Clone + Eq + Hash, -{ - for is_extend in [false, true] { - let mut set = HashSet::new(); - let mut arr = MutableDictionaryArray::::new(); - macro_rules! push { - ($v:expr) => { - if is_extend { - arr.try_extend(std::iter::once($v)) - } else { - arr.try_push($v) - } - }; - } - arr.push_null(); - push!(None).unwrap(); - assert_eq!(arr.len(), 2); - assert_eq!(arr.values().len(), 0); - for (i, v) in values.iter().cloned().enumerate() { - push!(Some(v.clone())).unwrap(); - let is_dup = !set.insert(v.clone()); - if !is_dup { - assert_eq!(arr.values().value_at(i).borrow(), v.as_indexed()); - assert_eq!(arr.keys().value_at(arr.keys().len() - 1), i as u8); - } - assert_eq!(arr.values().len(), set.len()); - assert_eq!(arr.len(), 3 + i); - } - for i in 0..256 - set.len() { - push!(Some(gen(i))).unwrap(); - } - assert!(push!(Some(gen(256))).is_err()); - } -} - -#[test] -fn test_push_utf8_ex() { - test_push_ex::, _>(vec!["a".into(), "b".into(), "a".into()], |i| { - i.to_string() - }) -} - -#[test] -fn test_push_i64_ex() { - test_push_ex::, _>(vec![10, 20, 30, 20], |i| 1000 + i as i64); -} - -#[test] -fn test_big_dict() { - let n = 10; - let strings = (0..10).map(|i| i.to_string()).collect::>(); - let mut arr = MutableDictionaryArray::>::new(); - for s in &strings { - arr.try_push(Some(s)).unwrap(); - } - assert_eq!(arr.values().len(), n); - for _ in 0..10_000 { - for s in &strings { - arr.try_push(Some(s)).unwrap(); - } - } - assert_eq!(arr.values().len(), n); -} diff --git a/src/common/arrow/tests/it/arrow/array/equal/boolean.rs b/src/common/arrow/tests/it/arrow/array/equal/boolean.rs deleted file mode 100644 index 772db1b556c2..000000000000 --- a/src/common/arrow/tests/it/arrow/array/equal/boolean.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; - -use super::test_equal; - -#[test] -fn test_boolean_equal() { - let a = BooleanArray::from_slice([false, false, true]); - let b = BooleanArray::from_slice([false, false, true]); - test_equal(&a, &b, true); - - let b = BooleanArray::from_slice([false, false, false]); - test_equal(&a, &b, false); -} - -#[test] -fn test_boolean_equal_null() { - let a = BooleanArray::from(vec![Some(false), None, None, Some(true)]); - let b = BooleanArray::from(vec![Some(false), None, None, Some(true)]); - test_equal(&a, &b, true); - - let b = BooleanArray::from(vec![None, None, None, Some(true)]); - test_equal(&a, &b, false); - - let b = BooleanArray::from(vec![Some(true), None, None, Some(true)]); - test_equal(&a, &b, false); -} - -#[test] -fn test_boolean_equal_offset() { - let a = BooleanArray::from_slice(vec![false, true, false, true, false, false, true]); - let b = BooleanArray::from_slice(vec![true, false, false, false, true, false, true, true]); - test_equal(&a, &b, false); - - let a_slice = a.sliced(2, 3); - let b_slice = b.sliced(3, 3); - test_equal(&a_slice, &b_slice, true); - - let a_slice = a.sliced(3, 4); - let b_slice = b.sliced(4, 4); - test_equal(&a_slice, &b_slice, false); - - // Elements fill in `u8`'s exactly. - let mut vector = vec![false, false, true, true, true, true, true, true]; - let a = BooleanArray::from_slice(vector.clone()); - let b = BooleanArray::from_slice(vector.clone()); - test_equal(&a, &b, true); - - // Elements fill in `u8`s + suffix bits. - vector.push(true); - let a = BooleanArray::from_slice(vector.clone()); - let b = BooleanArray::from_slice(vector); - test_equal(&a, &b, true); -} diff --git a/src/common/arrow/tests/it/arrow/array/equal/dictionary.rs b/src/common/arrow/tests/it/arrow/array/equal/dictionary.rs deleted file mode 100644 index 9906b1ebfaf7..000000000000 --- a/src/common/arrow/tests/it/arrow/array/equal/dictionary.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; - -use super::test_equal; - -fn create_dictionary_array(values: &[Option<&str>], keys: &[Option]) -> DictionaryArray { - let keys = Int16Array::from(keys); - let values = Utf8Array::::from(values); - - DictionaryArray::try_from_keys(keys, values.boxed()).unwrap() -} - -#[test] -fn dictionary_equal() { - // (a, b, c), (0, 1, 0, 2) => (a, b, a, c) - let a = create_dictionary_array(&[Some("a"), Some("b"), Some("c")], &[ - Some(0), - Some(1), - Some(0), - Some(2), - ]); - // different representation (values and keys are swapped), same result - let b = create_dictionary_array(&[Some("a"), Some("c"), Some("b")], &[ - Some(0), - Some(2), - Some(0), - Some(1), - ]); - test_equal(&a, &b, true); - - // different len - let b = create_dictionary_array(&[Some("a"), Some("c"), Some("b")], &[ - Some(0), - Some(2), - Some(1), - ]); - test_equal(&a, &b, false); - - // different key - let b = create_dictionary_array(&[Some("a"), Some("c"), Some("b")], &[ - Some(0), - Some(2), - Some(0), - Some(0), - ]); - test_equal(&a, &b, false); - - // different values, same keys - let b = create_dictionary_array(&[Some("a"), Some("b"), Some("d")], &[ - Some(0), - Some(1), - Some(0), - Some(2), - ]); - test_equal(&a, &b, false); -} - -#[test] -fn dictionary_equal_null() { - // (a, b, c), (1, 2, 1, 3) => (a, b, a, c) - let a = create_dictionary_array(&[Some("a"), Some("b"), Some("c")], &[ - Some(0), - None, - Some(0), - Some(2), - ]); - - // equal to self - test_equal(&a, &a, true); - - // different representation (values and keys are swapped), same result - let b = create_dictionary_array(&[Some("a"), Some("c"), Some("b")], &[ - Some(0), - None, - Some(0), - Some(1), - ]); - test_equal(&a, &b, true); - - // different null position - let b = create_dictionary_array(&[Some("a"), Some("c"), Some("b")], &[ - Some(0), - Some(2), - Some(0), - None, - ]); - test_equal(&a, &b, false); - - // different key - let b = create_dictionary_array(&[Some("a"), Some("c"), Some("b")], &[ - Some(0), - None, - Some(0), - Some(0), - ]); - test_equal(&a, &b, false); - - // different values, same keys - let b = create_dictionary_array(&[Some("a"), Some("b"), Some("d")], &[ - Some(0), - None, - Some(0), - Some(2), - ]); - test_equal(&a, &b, false); - - // different nulls in keys and values - let a = create_dictionary_array(&[Some("a"), Some("b"), None], &[ - Some(0), - None, - Some(0), - Some(2), - ]); - let b = create_dictionary_array(&[Some("a"), Some("b"), Some("c")], &[ - Some(0), - None, - Some(0), - None, - ]); - test_equal(&a, &b, true); -} diff --git a/src/common/arrow/tests/it/arrow/array/equal/fixed_size_list.rs b/src/common/arrow/tests/it/arrow/array/equal/fixed_size_list.rs deleted file mode 100644 index 9d79c712daf9..000000000000 --- a/src/common/arrow/tests/it/arrow/array/equal/fixed_size_list.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::FixedSizeListArray; -use databend_common_arrow::arrow::array::MutableFixedSizeListArray; -use databend_common_arrow::arrow::array::MutablePrimitiveArray; -use databend_common_arrow::arrow::array::TryExtend; - -use super::test_equal; - -/// Create a fixed size list of 2 value lengths -fn create_fixed_size_list_array, T: AsRef<[Option]>>( - data: T, -) -> FixedSizeListArray { - let data = data.as_ref().iter().map(|x| { - Some(match x { - Some(x) => x.as_ref().iter().map(|x| Some(*x)).collect::>(), - None => std::iter::repeat(None).take(3).collect::>(), - }) - }); - - let mut list = MutableFixedSizeListArray::new(MutablePrimitiveArray::::new(), 3); - list.try_extend(data).unwrap(); - list.into() -} - -#[test] -fn test_fixed_size_list_equal() { - let a = create_fixed_size_list_array([Some(&[1, 2, 3]), Some(&[4, 5, 6])]); - let b = create_fixed_size_list_array([Some(&[1, 2, 3]), Some(&[4, 5, 6])]); - test_equal(&a, &b, true); - - let b = create_fixed_size_list_array([Some(&[1, 2, 3]), Some(&[4, 5, 7])]); - test_equal(&a, &b, false); -} - -// Test the case where null_count > 0 -#[test] -fn test_fixed_list_null() { - let a = - create_fixed_size_list_array([Some(&[1, 2, 3]), None, None, Some(&[4, 5, 6]), None, None]); - // let b = create_fixed_size_list_array(&[ - // Some(&[1, 2, 3]), - // None, - // None, - // Some(&[4, 5, 6]), - // None, - // None, - // ]); - // test_equal(&a, &b, true); - // - // let b = create_fixed_size_list_array(&[ - // Some(&[1, 2, 3]), - // None, - // Some(&[7, 8, 9]), - // Some(&[4, 5, 6]), - // None, - // None, - // ]); - // test_equal(&a, &b, false); - - let b = - create_fixed_size_list_array([Some(&[1, 2, 3]), None, None, Some(&[3, 6, 9]), None, None]); - test_equal(&a, &b, false); -} - -#[test] -fn test_fixed_list_offsets() { - // Test the case where offset != 0 - let a = - create_fixed_size_list_array([Some(&[1, 2, 3]), None, None, Some(&[4, 5, 6]), None, None]); - let b = - create_fixed_size_list_array([Some(&[1, 2, 3]), None, None, Some(&[3, 6, 9]), None, None]); - - let a_slice = a.clone().sliced(0, 3); - let b_slice = b.clone().sliced(0, 3); - test_equal(&a_slice, &b_slice, true); - - let a_slice = a.clone().sliced(0, 5); - let b_slice = b.clone().sliced(0, 5); - test_equal(&a_slice, &b_slice, false); - - let a_slice = a.sliced(4, 1); - let b_slice = b.sliced(4, 1); - test_equal(&a_slice, &b_slice, true); -} diff --git a/src/common/arrow/tests/it/arrow/array/equal/list.rs b/src/common/arrow/tests/it/arrow/array/equal/list.rs deleted file mode 100644 index 7815faadf5d6..000000000000 --- a/src/common/arrow/tests/it/arrow/array/equal/list.rs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::Int32Array; -use databend_common_arrow::arrow::array::ListArray; -use databend_common_arrow::arrow::array::MutableListArray; -use databend_common_arrow::arrow::array::MutablePrimitiveArray; -use databend_common_arrow::arrow::array::TryExtend; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; - -use super::test_equal; - -fn create_list_array, T: AsRef<[Option]>>(data: T) -> ListArray { - let iter = data.as_ref().iter().map(|x| { - x.as_ref() - .map(|x| x.as_ref().iter().map(|x| Some(*x)).collect::>()) - }); - let mut array = MutableListArray::>::new(); - array.try_extend(iter).unwrap(); - array.into() -} - -#[test] -fn test_list_equal() { - let a = create_list_array([Some(&[1, 2, 3]), Some(&[4, 5, 6])]); - let b = create_list_array([Some(&[1, 2, 3]), Some(&[4, 5, 6])]); - test_equal(&a, &b, true); - - let b = create_list_array([Some(&[1, 2, 3]), Some(&[4, 5, 7])]); - test_equal(&a, &b, false); -} - -// Test the case where null_count > 0 -#[test] -fn test_list_null() { - let a = create_list_array([Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); - let b = create_list_array([Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); - test_equal(&a, &b, true); - - let b = create_list_array([ - Some(&[1, 2]), - None, - Some(&[5, 6]), - Some(&[3, 4]), - None, - None, - ]); - test_equal(&a, &b, false); - - let b = create_list_array([Some(&[1, 2]), None, None, Some(&[3, 5]), None, None]); - test_equal(&a, &b, false); -} - -// Test the case where offset != 0 -#[test] -fn test_list_offsets() { - let a = create_list_array([Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); - let b = create_list_array([Some(&[1, 2]), None, None, Some(&[3, 5]), None, None]); - - let a_slice = a.clone().sliced(0, 3); - let b_slice = b.clone().sliced(0, 3); - test_equal(&a_slice, &b_slice, true); - - let a_slice = a.clone().sliced(0, 5); - let b_slice = b.clone().sliced(0, 5); - test_equal(&a_slice, &b_slice, false); - - let a_slice = a.sliced(4, 1); - let b_slice = b.sliced(4, 1); - test_equal(&a_slice, &b_slice, true); -} - -#[test] -fn test_bla() { - let offsets = vec![0, 3, 3, 6].try_into().unwrap(); - let data_type = ListArray::::default_datatype(DataType::Int32); - let values = Box::new(Int32Array::from([ - Some(1), - Some(2), - Some(3), - Some(4), - None, - Some(6), - ])); - let validity = Bitmap::from([true, false, true]); - let lhs = ListArray::::new(data_type, offsets, values, Some(validity)); - let lhs = lhs.sliced(1, 2); - - let offsets = vec![0, 0, 3].try_into().unwrap(); - let data_type = ListArray::::default_datatype(DataType::Int32); - let values = Box::new(Int32Array::from([Some(4), None, Some(6)])); - let validity = Bitmap::from([false, true]); - let rhs = ListArray::::new(data_type, offsets, values, Some(validity)); - - assert_eq!(lhs, rhs); -} diff --git a/src/common/arrow/tests/it/arrow/array/equal/mod.rs b/src/common/arrow/tests/it/arrow/array/equal/mod.rs deleted file mode 100644 index c7f68551543c..000000000000 --- a/src/common/arrow/tests/it/arrow/array/equal/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; - -mod dictionary; -mod fixed_size_list; -mod list; -mod primitive; -mod utf8; - -pub fn test_equal(lhs: &dyn Array, rhs: &dyn Array, expected: bool) { - // equality is symmetric - assert!(equal(lhs, lhs), "\n{lhs:?}\n{lhs:?}"); - assert!(equal(rhs, rhs), "\n{rhs:?}\n{rhs:?}"); - - assert_eq!(equal(lhs, rhs), expected, "\n{lhs:?}\n{rhs:?}"); - assert_eq!(equal(rhs, lhs), expected, "\n{rhs:?}\n{lhs:?}"); -} - -#[allow(clippy::type_complexity)] -fn binary_cases() -> Vec<(Vec>, Vec>, bool)> { - let base = vec![ - Some("hello".to_owned()), - None, - None, - Some("world".to_owned()), - None, - None, - ]; - let not_base = vec![ - Some("hello".to_owned()), - Some("foo".to_owned()), - None, - Some("world".to_owned()), - None, - None, - ]; - vec![ - ( - vec![Some("hello".to_owned()), Some("world".to_owned())], - vec![Some("hello".to_owned()), Some("world".to_owned())], - true, - ), - ( - vec![Some("hello".to_owned()), Some("world".to_owned())], - vec![Some("hello".to_owned()), Some("arrow".to_owned())], - false, - ), - (base.clone(), base.clone(), true), - (base, not_base, false), - ] -} diff --git a/src/common/arrow/tests/it/arrow/array/equal/primitive.rs b/src/common/arrow/tests/it/arrow/array/equal/primitive.rs deleted file mode 100644 index 9aa16dd61491..000000000000 --- a/src/common/arrow/tests/it/arrow/array/equal/primitive.rs +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; - -use super::test_equal; - -#[test] -fn test_primitive() { - let cases = vec![ - ( - vec![Some(1), Some(2), Some(3)], - vec![Some(1), Some(2), Some(3)], - true, - ), - ( - vec![Some(1), Some(2), Some(3)], - vec![Some(1), Some(2), Some(4)], - false, - ), - ( - vec![Some(1), Some(2), None], - vec![Some(1), Some(2), None], - true, - ), - ( - vec![Some(1), None, Some(3)], - vec![Some(1), Some(2), None], - false, - ), - ( - vec![Some(1), None, None], - vec![Some(1), Some(2), None], - false, - ), - ]; - - for (lhs, rhs, expected) in cases { - let lhs = Int32Array::from(&lhs); - let rhs = Int32Array::from(&rhs); - test_equal(&lhs, &rhs, expected); - } -} - -#[test] -fn test_primitive_slice() { - let cases = vec![ - ( - vec![Some(1), Some(2), Some(3)], - (0, 1), - vec![Some(1), Some(2), Some(3)], - (0, 1), - true, - ), - ( - vec![Some(1), Some(2), Some(3)], - (1, 1), - vec![Some(1), Some(2), Some(3)], - (2, 1), - false, - ), - ( - vec![Some(1), Some(2), None], - (1, 1), - vec![Some(1), None, Some(2)], - (2, 1), - true, - ), - ( - vec![None, Some(2), None], - (1, 1), - vec![None, None, Some(2)], - (2, 1), - true, - ), - ( - vec![Some(1), None, Some(2), None, Some(3)], - (2, 2), - vec![None, Some(2), None, Some(3)], - (1, 2), - true, - ), - ]; - - for (lhs, slice_lhs, rhs, slice_rhs, expected) in cases { - let lhs = Int32Array::from(&lhs); - let lhs = lhs.sliced(slice_lhs.0, slice_lhs.1); - let rhs = Int32Array::from(&rhs); - let rhs = rhs.sliced(slice_rhs.0, slice_rhs.1); - - test_equal(&lhs, &rhs, expected); - } -} diff --git a/src/common/arrow/tests/it/arrow/array/fixed_size_binary/mod.rs b/src/common/arrow/tests/it/arrow/array/fixed_size_binary/mod.rs deleted file mode 100644 index ee1b09aaade4..000000000000 --- a/src/common/arrow/tests/it/arrow/array/fixed_size_binary/mod.rs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::FixedSizeBinaryArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType; - -mod mutable; - -#[test] -fn basics() { - let array = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(2), - Buffer::from(vec![1, 2, 3, 4, 5, 6]), - Some(Bitmap::from([true, false, true])), - ); - assert_eq!(array.size(), 2); - assert_eq!(array.len(), 3); - assert_eq!(array.validity(), Some(&Bitmap::from([true, false, true]))); - - assert_eq!(array.value(0), [1, 2]); - assert_eq!(array.value(2), [5, 6]); - - let array = array.sliced(1, 2); - - assert_eq!(array.value(1), [5, 6]); -} - -#[test] -fn with_validity() { - let a = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(2), - vec![1, 2, 3, 4, 5, 6].into(), - None, - ); - let a = a.with_validity(Some(Bitmap::from([true, false, true]))); - assert!(a.validity().is_some()); -} - -#[test] -fn debug() { - let a = FixedSizeBinaryArray::new( - DataType::FixedSizeBinary(2), - vec![1, 2, 3, 4, 5, 6].into(), - Some(Bitmap::from([true, false, true])), - ); - assert_eq!(format!("{a:?}"), "FixedSizeBinary(2)[[1, 2], None, [5, 6]]"); -} - -#[test] -fn empty() { - let array = FixedSizeBinaryArray::new_empty(DataType::FixedSizeBinary(2)); - assert_eq!(array.values().len(), 0); - assert_eq!(array.validity(), None); -} - -#[test] -fn null() { - let array = FixedSizeBinaryArray::new_null(DataType::FixedSizeBinary(2), 2); - assert_eq!(array.values().len(), 4); - assert_eq!(array.validity().cloned(), Some([false, false].into())); -} - -#[test] -fn from_iter() { - let iter = std::iter::repeat(vec![1u8, 2]).take(2).map(Some); - let a = FixedSizeBinaryArray::from_iter(iter, 2); - assert_eq!(a.len(), 2); -} - -#[test] -fn wrong_size() { - let values = Buffer::from(b"abb".to_vec()); - assert!(FixedSizeBinaryArray::try_new(DataType::FixedSizeBinary(2), values, None).is_err()); -} - -#[test] -fn wrong_len() { - let values = Buffer::from(b"abba".to_vec()); - let validity = Some([true, false, false].into()); // it should be 2 - assert!(FixedSizeBinaryArray::try_new(DataType::FixedSizeBinary(2), values, validity).is_err()); -} - -#[test] -fn wrong_data_type() { - let values = Buffer::from(b"abba".to_vec()); - assert!(FixedSizeBinaryArray::try_new(DataType::Binary, values, None).is_err()); -} - -#[test] -fn to() { - let values = Buffer::from(b"abba".to_vec()); - let a = FixedSizeBinaryArray::new(DataType::FixedSizeBinary(2), values, None); - - let extension = DataType::Extension( - "a".to_string(), - Box::new(DataType::FixedSizeBinary(2)), - None, - ); - let _ = a.to(extension); -} diff --git a/src/common/arrow/tests/it/arrow/array/fixed_size_binary/mutable.rs b/src/common/arrow/tests/it/arrow/array/fixed_size_binary/mutable.rs deleted file mode 100644 index 77ff31774fe2..000000000000 --- a/src/common/arrow/tests/it/arrow/array/fixed_size_binary/mutable.rs +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn basic() { - let a = MutableFixedSizeBinaryArray::try_new( - DataType::FixedSizeBinary(2), - Vec::from([1, 2, 3, 4]), - None, - ) - .unwrap(); - assert_eq!(a.len(), 2); - assert_eq!(a.data_type(), &DataType::FixedSizeBinary(2)); - assert_eq!(a.values(), &Vec::from([1, 2, 3, 4])); - assert_eq!(a.validity(), None); - assert_eq!(a.value(1), &[3, 4]); - assert_eq!(unsafe { a.value_unchecked(1) }, &[3, 4]); -} - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let a = MutableFixedSizeBinaryArray::try_new( - DataType::FixedSizeBinary(2), - Vec::from([1, 2, 3, 4]), - None, - ) - .unwrap(); - assert_eq!(a, a); - let b = - MutableFixedSizeBinaryArray::try_new(DataType::FixedSizeBinary(2), Vec::from([1, 2]), None) - .unwrap(); - assert_eq!(b, b); - assert!(a != b); - let a = MutableFixedSizeBinaryArray::try_new( - DataType::FixedSizeBinary(2), - Vec::from([1, 2, 3, 4]), - Some(MutableBitmap::from([true, false])), - ) - .unwrap(); - let b = MutableFixedSizeBinaryArray::try_new( - DataType::FixedSizeBinary(2), - Vec::from([1, 2, 3, 4]), - Some(MutableBitmap::from([false, true])), - ) - .unwrap(); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); -} - -#[test] -fn try_from_iter() { - let array = MutableFixedSizeBinaryArray::try_from_iter( - vec![Some(b"ab"), Some(b"bc"), None, Some(b"fh")], - 2, - ) - .unwrap(); - assert_eq!(array.len(), 4); -} - -#[test] -fn push_null() { - let mut array = MutableFixedSizeBinaryArray::new(2); - array.push::<&[u8]>(None); - - let array: FixedSizeBinaryArray = array.into(); - assert_eq!(array.validity(), Some(&Bitmap::from([false]))); -} - -#[test] -fn pop() { - let mut a = MutableFixedSizeBinaryArray::new(2); - a.push(Some(b"aa")); - a.push::<&[u8]>(None); - a.push(Some(b"bb")); - a.push::<&[u8]>(None); - - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 3); - assert_eq!(a.pop(), Some(b"bb".to_vec())); - assert_eq!(a.len(), 2); - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 1); - assert_eq!(a.pop(), Some(b"aa".to_vec())); - assert!(a.is_empty()); - assert_eq!(a.pop(), None); - assert!(a.is_empty()); -} - -#[test] -fn pop_all_some() { - let mut a = MutableFixedSizeBinaryArray::new(2); - a.push(Some(b"aa")); - a.push(Some(b"bb")); - a.push(Some(b"cc")); - a.push(Some(b"dd")); - - for _ in 0..4 { - a.push(Some(b"11")); - } - - a.push(Some(b"22")); - - assert_eq!(a.pop(), Some(b"22".to_vec())); - assert_eq!(a.pop(), Some(b"11".to_vec())); - assert_eq!(a.pop(), Some(b"11".to_vec())); - assert_eq!(a.pop(), Some(b"11".to_vec())); - assert_eq!(a.len(), 5); - - assert_eq!( - a, - MutableFixedSizeBinaryArray::try_from_iter( - vec![ - Some(b"aa"), - Some(b"bb"), - Some(b"cc"), - Some(b"dd"), - Some(b"11"), - ], - 2, - ) - .unwrap() - ); -} - -#[test] -fn as_arc() { - let mut array = MutableFixedSizeBinaryArray::try_from_iter( - vec![Some(b"ab"), Some(b"bc"), None, Some(b"fh")], - 2, - ) - .unwrap(); - - let array = array.as_arc(); - assert_eq!(array.len(), 4); -} - -#[test] -fn as_box() { - let mut array = MutableFixedSizeBinaryArray::try_from_iter( - vec![Some(b"ab"), Some(b"bc"), None, Some(b"fh")], - 2, - ) - .unwrap(); - - let array = array.as_box(); - assert_eq!(array.len(), 4); -} - -#[test] -fn shrink_to_fit_and_capacity() { - let mut array = MutableFixedSizeBinaryArray::with_capacity(2, 100); - array.push(Some([1, 2])); - array.shrink_to_fit(); - assert_eq!(array.capacity(), 1); -} - -#[test] -fn extend_from_self() { - let mut a = MutableFixedSizeBinaryArray::from([Some([1u8, 2u8]), None]); - - a.try_extend_from_self(&a.clone()).unwrap(); - - assert_eq!( - a, - MutableFixedSizeBinaryArray::from([Some([1u8, 2u8]), None, Some([1u8, 2u8]), None]) - ); -} diff --git a/src/common/arrow/tests/it/arrow/array/fixed_size_list/mod.rs b/src/common/arrow/tests/it/arrow/array/fixed_size_list/mod.rs deleted file mode 100644 index d1cef62dcdf6..000000000000 --- a/src/common/arrow/tests/it/arrow/array/fixed_size_list/mod.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod mutable; - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; - -fn data() -> FixedSizeListArray { - let values = Int32Array::from_slice([10, 20, 0, 0]); - - FixedSizeListArray::try_new( - DataType::FixedSizeList( - Box::new(Field::new("a", values.data_type().clone(), true)), - 2, - ), - values.boxed(), - Some([true, false].into()), - ) - .unwrap() -} - -#[test] -fn basics() { - let array = data(); - assert_eq!(array.size(), 2); - assert_eq!(array.len(), 2); - assert_eq!(array.validity(), Some(&Bitmap::from([true, false]))); - - assert_eq!(array.value(0).as_ref(), Int32Array::from_slice([10, 20])); - assert_eq!(array.value(1).as_ref(), Int32Array::from_slice([0, 0])); - - let array = array.sliced(1, 1); - - assert_eq!(array.value(0).as_ref(), Int32Array::from_slice([0, 0])); -} - -#[test] -fn with_validity() { - let array = data(); - - let a = array.with_validity(None); - assert!(a.validity().is_none()); -} - -#[test] -fn debug() { - let array = data(); - - assert_eq!(format!("{array:?}"), "FixedSizeListArray[[10, 20], None]"); -} - -#[test] -fn empty() { - let array = FixedSizeListArray::new_empty(DataType::FixedSizeList( - Box::new(Field::new("a", DataType::Int32, true)), - 2, - )); - assert_eq!(array.values().len(), 0); - assert_eq!(array.validity(), None); -} - -#[test] -fn null() { - let array = FixedSizeListArray::new_null( - DataType::FixedSizeList(Box::new(Field::new("a", DataType::Int32, true)), 2), - 2, - ); - assert_eq!(array.values().len(), 4); - assert_eq!(array.validity().cloned(), Some([false, false].into())); -} - -#[test] -fn wrong_size() { - let values = Int32Array::from_slice([10, 20, 0]); - assert!( - FixedSizeListArray::try_new( - DataType::FixedSizeList(Box::new(Field::new("a", DataType::Int32, true)), 2), - values.boxed(), - None - ) - .is_err() - ); -} - -#[test] -fn wrong_len() { - let values = Int32Array::from_slice([10, 20, 0]); - assert!( - FixedSizeListArray::try_new( - DataType::FixedSizeList(Box::new(Field::new("a", DataType::Int32, true)), 2), - values.boxed(), - Some([true, false, false].into()), // it should be 2 - ) - .is_err() - ); -} - -#[test] -fn wrong_data_type() { - let values = Int32Array::from_slice([10, 20, 0]); - assert!( - FixedSizeListArray::try_new( - DataType::Binary, - values.boxed(), - Some([true, false, false].into()), // it should be 2 - ) - .is_err() - ); -} diff --git a/src/common/arrow/tests/it/arrow/array/fixed_size_list/mutable.rs b/src/common/arrow/tests/it/arrow/array/fixed_size_list/mutable.rs deleted file mode 100644 index d192392b3641..000000000000 --- a/src/common/arrow/tests/it/arrow/array/fixed_size_list/mutable.rs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; - -#[test] -fn primitive() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - Some(vec![None, None, None]), - Some(vec![Some(4), None, Some(6)]), - ]; - - let mut list = MutableFixedSizeListArray::new(MutablePrimitiveArray::::new(), 3); - list.try_extend(data).unwrap(); - let list: FixedSizeListArray = list.into(); - - let a = list.value(0); - let a = a.as_any().downcast_ref::().unwrap(); - - let expected = Int32Array::from(vec![Some(1i32), Some(2), Some(3)]); - assert_eq!(a, &expected); - - let a = list.value(1); - let a = a.as_any().downcast_ref::().unwrap(); - - let expected = Int32Array::from(vec![None, None, None]); - assert_eq!(a, &expected) -} - -#[test] -fn new_with_field() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - Some(vec![None, None, None]), - Some(vec![Some(4), None, Some(6)]), - ]; - - let mut list = MutableFixedSizeListArray::new_with_field( - MutablePrimitiveArray::::new(), - "custom_items", - false, - 3, - ); - list.try_extend(data).unwrap(); - let list: FixedSizeListArray = list.into(); - - assert_eq!( - list.data_type(), - &DataType::FixedSizeList( - Box::new(Field::new("custom_items", DataType::Int32, false)), - 3 - ) - ); - - let a = list.value(0); - let a = a.as_any().downcast_ref::().unwrap(); - - let expected = Int32Array::from(vec![Some(1i32), Some(2), Some(3)]); - assert_eq!(a, &expected); - - let a = list.value(1); - let a = a.as_any().downcast_ref::().unwrap(); - - let expected = Int32Array::from(vec![None, None, None]); - assert_eq!(a, &expected) -} - -#[test] -fn extend_from_self() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(4), None, Some(6)]), - ]; - let mut a = MutableFixedSizeListArray::new(MutablePrimitiveArray::::new(), 3); - a.try_extend(data.clone()).unwrap(); - - a.try_extend_from_self(&a.clone()).unwrap(); - let a: FixedSizeListArray = a.into(); - - let mut expected = data.clone(); - expected.extend(data); - - let mut b = MutableFixedSizeListArray::new(MutablePrimitiveArray::::new(), 3); - b.try_extend(expected).unwrap(); - let b: FixedSizeListArray = b.into(); - - assert_eq!(a, b); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/binary.rs b/src/common/arrow/tests/it/arrow/array/growable/binary.rs deleted file mode 100644 index f69e6c9334ce..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/binary.rs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableBinary; -use databend_common_arrow::arrow::array::BinaryArray; - -#[test] -fn no_offsets() { - let array = BinaryArray::::from([Some("a"), Some("bc"), None, Some("defh")]); - - let mut a = GrowableBinary::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from([Some("bc"), None]); - assert_eq!(result, expected); -} - -/// tests extending from a variable-sized (strings and binary) array -/// with an offset and nulls -#[test] -fn with_offsets() { - let array = BinaryArray::::from([Some("a"), Some("bc"), None, Some("defh")]); - let array = array.sliced(1, 3); - - let mut a = GrowableBinary::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - assert_eq!(a.len(), 3); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from([Some("bc"), None, Some("defh")]); - assert_eq!(result, expected); -} - -#[test] -fn test_string_offsets() { - let array = BinaryArray::::from([Some("a"), Some("bc"), None, Some("defh")]); - let array = array.sliced(1, 3); - - let mut a = GrowableBinary::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - assert_eq!(a.len(), 3); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from([Some("bc"), None, Some("defh")]); - assert_eq!(result, expected); -} - -#[test] -fn test_multiple_with_validity() { - let array1 = BinaryArray::::from_slice([b"hello", b"world"]); - let array2 = BinaryArray::::from([Some("1"), None]); - - let mut a = GrowableBinary::new(vec![&array1, &array2], false, 5); - - a.extend(0, 0, 2); - a.extend(1, 0, 2); - assert_eq!(a.len(), 4); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from([Some("hello"), Some("world"), Some("1"), None]); - assert_eq!(result, expected); -} - -#[test] -fn test_string_null_offset_validity() { - let array = BinaryArray::::from([Some("a"), Some("bc"), None, Some("defh")]); - let array = array.sliced(1, 3); - - let mut a = GrowableBinary::new(vec![&array], true, 0); - - a.extend(0, 1, 2); - a.extend_validity(1); - assert_eq!(a.len(), 3); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from([None, Some("defh"), None]); - assert_eq!(result, expected); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/boolean.rs b/src/common/arrow/tests/it/arrow/array/growable/boolean.rs deleted file mode 100644 index faca21710c98..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/boolean.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableBoolean; -use databend_common_arrow::arrow::array::BooleanArray; - -#[test] -fn test_bool() { - let array = BooleanArray::from(vec![Some(false), Some(true), None, Some(false)]); - - let mut a = GrowableBoolean::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - - let result: BooleanArray = a.into(); - - let expected = BooleanArray::from(vec![Some(true), None]); - assert_eq!(result, expected); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/dictionary.rs b/src/common/arrow/tests/it/arrow/array/growable/dictionary.rs deleted file mode 100644 index 20f94a6e7089..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/dictionary.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableDictionary; -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::error::Result; - -#[test] -fn test_single() -> Result<()> { - let original_data = vec![Some("a"), Some("b"), Some("a")]; - - let data = original_data.clone(); - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data)?; - let array = array.into(); - - // same values, less keys - let expected = DictionaryArray::try_from_keys( - PrimitiveArray::from_vec(vec![1, 0]), - Box::new(Utf8Array::::from(&original_data)), - ) - .unwrap(); - - let mut growable = GrowableDictionary::new(&[&array], false, 0); - - growable.extend(0, 1, 2); - assert_eq!(growable.len(), 2); - - let result: DictionaryArray = growable.into(); - - assert_eq!(result, expected); - Ok(()) -} - -#[test] -fn test_multi() -> Result<()> { - let mut original_data1 = vec![Some("a"), Some("b"), None, Some("a")]; - let original_data2 = vec![Some("c"), Some("b"), None, Some("a")]; - - let data1 = original_data1.clone(); - let data2 = original_data2.clone(); - - let mut array1 = MutableDictionaryArray::>::new(); - array1.try_extend(data1)?; - let array1: DictionaryArray = array1.into(); - - let mut array2 = MutableDictionaryArray::>::new(); - array2.try_extend(data2)?; - let array2: DictionaryArray = array2.into(); - - // same values, less keys - original_data1.extend(original_data2.iter().cloned()); - let expected = DictionaryArray::try_from_keys( - PrimitiveArray::from(&[Some(1), None, Some(3), None]), - Utf8Array::::from_slice(["a", "b", "c", "b", "a"]).boxed(), - ) - .unwrap(); - - let mut growable = GrowableDictionary::new(&[&array1, &array2], false, 0); - - growable.extend(0, 1, 2); - growable.extend(1, 1, 2); - assert_eq!(growable.len(), 4); - - let result: DictionaryArray = growable.into(); - - assert_eq!(result, expected); - Ok(()) -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/fixed_binary.rs b/src/common/arrow/tests/it/arrow/array/growable/fixed_binary.rs deleted file mode 100644 index 97394fbd71e7..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/fixed_binary.rs +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableFixedSizeBinary; -use databend_common_arrow::arrow::array::FixedSizeBinaryArray; - -/// tests extending from a variable-sized (strings and binary) array w/ offset with nulls -#[test] -fn basic() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some(b"ab"), Some(b"bc"), None, Some(b"de")], 2); - - let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter(vec![Some("bc"), None], 2); - assert_eq!(result, expected); -} - -/// tests extending from a variable-sized (strings and binary) array -/// with an offset and nulls -#[test] -fn offsets() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some(b"ab"), Some(b"bc"), None, Some(b"fh")], 2); - let array = array.sliced(1, 3); - - let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - assert_eq!(a.len(), 3); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter(vec![Some(b"bc"), None, Some(b"fh")], 2); - assert_eq!(result, expected); -} - -#[test] -fn multiple_with_validity() { - let array1 = FixedSizeBinaryArray::from_iter(vec![Some("hello"), Some("world")], 5); - let array2 = FixedSizeBinaryArray::from_iter(vec![Some("12345"), None], 5); - - let mut a = GrowableFixedSizeBinary::new(vec![&array1, &array2], false, 5); - - a.extend(0, 0, 2); - a.extend(1, 0, 2); - assert_eq!(a.len(), 4); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = - FixedSizeBinaryArray::from_iter(vec![Some("hello"), Some("world"), Some("12345"), None], 5); - assert_eq!(result, expected); -} - -#[test] -fn null_offset_validity() { - let array = FixedSizeBinaryArray::from_iter(vec![Some("aa"), Some("bc"), None, Some("fh")], 2); - let array = array.sliced(1, 3); - - let mut a = GrowableFixedSizeBinary::new(vec![&array], true, 0); - - a.extend(0, 1, 2); - a.extend_validity(1); - assert_eq!(a.len(), 3); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter(vec![None, Some("fh"), None], 2); - assert_eq!(result, expected); -} - -#[test] -fn sized_offsets() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some(&[0, 0]), Some(&[0, 1]), Some(&[0, 2])], 2); - let array = array.sliced(1, 2); - // = [[0, 1], [0, 2]] due to the offset = 1 - - let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); - - a.extend(0, 1, 1); - a.extend(0, 0, 1); - assert_eq!(a.len(), 2); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter(vec![Some(&[0, 2]), Some(&[0, 1])], 2); - assert_eq!(result, expected); -} - -/// to, as_box, as_arc -#[test] -fn as_box() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some(b"ab"), Some(b"bc"), None, Some(b"de")], 2); - let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); - a.extend(0, 1, 2); - - let result = a.as_box(); - let result = result - .as_any() - .downcast_ref::() - .unwrap(); - - let expected = FixedSizeBinaryArray::from_iter(vec![Some("bc"), None], 2); - assert_eq!(&expected, result); -} - -/// as_arc -#[test] -fn as_arc() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some(b"ab"), Some(b"bc"), None, Some(b"de")], 2); - let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); - a.extend(0, 1, 2); - - let result = a.as_arc(); - let result = result - .as_any() - .downcast_ref::() - .unwrap(); - - let expected = FixedSizeBinaryArray::from_iter(vec![Some("bc"), None], 2); - assert_eq!(&expected, result); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/fixed_size_list.rs b/src/common/arrow/tests/it/arrow/array/growable/fixed_size_list.rs deleted file mode 100644 index aa12f3bda234..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/fixed_size_list.rs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableFixedSizeList; -use databend_common_arrow::arrow::array::FixedSizeListArray; -use databend_common_arrow::arrow::array::MutableFixedSizeListArray; -use databend_common_arrow::arrow::array::MutablePrimitiveArray; -use databend_common_arrow::arrow::array::TryExtend; - -fn create_list_array(data: Vec>>>) -> FixedSizeListArray { - let mut array = MutableFixedSizeListArray::new(MutablePrimitiveArray::::new(), 3); - array.try_extend(data).unwrap(); - array.into() -} - -#[test] -fn basic() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - Some(vec![Some(4), Some(5), Some(6)]), - Some(vec![Some(7i32), Some(8), Some(9)]), - ]; - - let array = create_list_array(data); - - let mut a = GrowableFixedSizeList::new(vec![&array], false, 0); - a.extend(0, 0, 1); - assert_eq!(a.len(), 1); - - let result: FixedSizeListArray = a.into(); - - let expected = vec![Some(vec![Some(1i32), Some(2), Some(3)])]; - let expected = create_list_array(expected); - - assert_eq!(result, expected) -} - -#[test] -fn null_offset() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(6i32), Some(7), Some(8)]), - ]; - let array = create_list_array(data); - let array = array.sliced(1, 2); - - let mut a = GrowableFixedSizeList::new(vec![&array], false, 0); - a.extend(0, 1, 1); - assert_eq!(a.len(), 1); - - let result: FixedSizeListArray = a.into(); - - let expected = vec![Some(vec![Some(6i32), Some(7), Some(8)])]; - let expected = create_list_array(expected); - - assert_eq!(result, expected) -} - -#[test] -fn test_from_two_lists() { - let data_1 = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(6i32), None, Some(8)]), - ]; - let array_1 = create_list_array(data_1); - - let data_2 = vec![ - Some(vec![Some(8i32), Some(7), Some(6)]), - Some(vec![Some(5i32), None, Some(4)]), - Some(vec![Some(2i32), Some(1), Some(0)]), - ]; - let array_2 = create_list_array(data_2); - - let mut a = GrowableFixedSizeList::new(vec![&array_1, &array_2], false, 6); - a.extend(0, 0, 2); - a.extend(1, 1, 1); - assert_eq!(a.len(), 3); - - let result: FixedSizeListArray = a.into(); - - let expected = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(5i32), None, Some(4)]), - ]; - let expected = create_list_array(expected); - - assert_eq!(result, expected); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/list.rs b/src/common/arrow/tests/it/arrow/array/growable/list.rs deleted file mode 100644 index 094956a68a87..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/list.rs +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableList; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::ListArray; -use databend_common_arrow::arrow::array::MutableListArray; -use databend_common_arrow::arrow::array::MutablePrimitiveArray; -use databend_common_arrow::arrow::array::TryExtend; -use databend_common_arrow::arrow::datatypes::DataType; - -fn create_list_array(data: Vec>>>) -> ListArray { - let mut array = MutableListArray::>::new(); - array.try_extend(data).unwrap(); - array.into() -} - -#[test] -fn extension() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - Some(vec![Some(4), Some(5)]), - Some(vec![Some(6i32), Some(7), Some(8)]), - ]; - - let array = create_list_array(data); - - let data_type = - DataType::Extension("ext".to_owned(), Box::new(array.data_type().clone()), None); - let array_ext = ListArray::new( - data_type, - array.offsets().clone(), - array.values().clone(), - array.validity().cloned(), - ); - - let mut a = GrowableList::new(vec![&array_ext], false, 0); - a.extend(0, 0, 1); - assert_eq!(a.len(), 1); - - let result: ListArray = a.into(); - assert_eq!(array_ext.data_type(), result.data_type()); - dbg!(result); -} - -#[test] -fn basic() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - Some(vec![Some(4), Some(5)]), - Some(vec![Some(6i32), Some(7), Some(8)]), - ]; - - let array = create_list_array(data); - - let mut a = GrowableList::new(vec![&array], false, 0); - a.extend(0, 0, 1); - assert_eq!(a.len(), 1); - - let result: ListArray = a.into(); - - let expected = vec![Some(vec![Some(1i32), Some(2), Some(3)])]; - let expected = create_list_array(expected); - - assert_eq!(result, expected) -} - -#[test] -fn null_offset() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(6i32), Some(7), Some(8)]), - ]; - let array = create_list_array(data); - let array = array.sliced(1, 2); - - let mut a = GrowableList::new(vec![&array], false, 0); - a.extend(0, 1, 1); - assert_eq!(a.len(), 1); - - let result: ListArray = a.into(); - - let expected = vec![Some(vec![Some(6i32), Some(7), Some(8)])]; - let expected = create_list_array(expected); - - assert_eq!(result, expected) -} - -#[test] -fn null_offsets() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(6i32), None, Some(8)]), - ]; - let array = create_list_array(data); - let array = array.sliced(1, 2); - - let mut a = GrowableList::new(vec![&array], false, 0); - a.extend(0, 1, 1); - assert_eq!(a.len(), 1); - - let result: ListArray = a.into(); - - let expected = vec![Some(vec![Some(6i32), None, Some(8)])]; - let expected = create_list_array(expected); - - assert_eq!(result, expected) -} - -#[test] -fn test_from_two_lists() { - let data_1 = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(6i32), None, Some(8)]), - ]; - let array_1 = create_list_array(data_1); - - let data_2 = vec![ - Some(vec![Some(8i32), Some(7), Some(6)]), - Some(vec![Some(5i32), None, Some(4)]), - Some(vec![Some(2i32), Some(1), Some(0)]), - ]; - let array_2 = create_list_array(data_2); - - let mut a = GrowableList::new(vec![&array_1, &array_2], false, 6); - a.extend(0, 0, 2); - a.extend(1, 1, 1); - assert_eq!(a.len(), 3); - - let result: ListArray = a.into(); - - let expected = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(5i32), None, Some(4)]), - ]; - let expected = create_list_array(expected); - - assert_eq!(result, expected); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/map.rs b/src/common/arrow/tests/it/arrow/array/growable/map.rs deleted file mode 100644 index 9b08dabfacf3..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/map.rs +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableMap; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::MapArray; -use databend_common_arrow::arrow::array::PrimitiveArray; -use databend_common_arrow::arrow::array::StructArray; -use databend_common_arrow::arrow::array::Utf8Array; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::offset::OffsetsBuffer; - -fn some_values() -> (DataType, Vec>) { - let strings: Box = Box::new(Utf8Array::::from([ - Some("a"), - Some("aa"), - Some("bc"), - Some("mark"), - Some("doe"), - Some("xyz"), - ])); - let ints: Box = Box::new(PrimitiveArray::::from(&[ - Some(1), - Some(2), - Some(3), - Some(4), - Some(5), - Some(6), - ])); - let fields = vec![ - Field::new("key", DataType::Utf8, true), - Field::new("val", DataType::Int32, true), - ]; - (DataType::Struct(fields), vec![strings, ints]) -} - -#[test] -fn basic() { - let (fields, values) = some_values(); - - let kv_array = StructArray::new(fields.clone(), values, None).boxed(); - let kv_field = Field::new("kv", fields, false); - let data_type = DataType::Map(Box::new(kv_field), false); - let offsets = OffsetsBuffer::try_from(vec![0, 1, 2, 4, 6]).unwrap(); - - let array = MapArray::new(data_type.clone(), offsets, kv_array.clone(), None); - - let mut a = GrowableMap::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - let result: MapArray = a.into(); - - let kv_array = kv_array.sliced(1, 4); - let offsets = OffsetsBuffer::try_from(vec![0, 1, 3]).unwrap(); - let expected = MapArray::new(data_type, offsets, kv_array, None); - - assert_eq!(result, expected) -} - -#[test] -fn offset() { - let (fields, values) = some_values(); - - let kv_array = StructArray::new(fields.clone(), values, None).boxed(); - let kv_field = Field::new("kv", fields, false); - let data_type = DataType::Map(Box::new(kv_field), false); - let offsets = OffsetsBuffer::try_from(vec![0, 1, 2, 4, 6]).unwrap(); - - let array = MapArray::new(data_type.clone(), offsets, kv_array.clone(), None).sliced(1, 3); - - let mut a = GrowableMap::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - let result: MapArray = a.into(); - - let kv_array = kv_array.sliced(2, 4); - let offsets = OffsetsBuffer::try_from(vec![0, 2, 4]).unwrap(); - let expected = MapArray::new(data_type, offsets, kv_array, None); - - assert_eq!(result, expected) -} - -#[test] -fn nulls() { - let (fields, values) = some_values(); - - let kv_array = StructArray::new(fields.clone(), values, None).boxed(); - let kv_field = Field::new("kv", fields, false); - let data_type = DataType::Map(Box::new(kv_field), false); - let offsets = OffsetsBuffer::try_from(vec![0, 1, 2, 4, 6]).unwrap(); - - let array = MapArray::new( - data_type.clone(), - offsets, - kv_array.clone(), - Some(Bitmap::from_u8_slice([0b00000010], 4)), - ); - - let mut a = GrowableMap::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - let result: MapArray = a.into(); - - let kv_array = kv_array.sliced(1, 4); - let offsets = OffsetsBuffer::try_from(vec![0, 1, 3]).unwrap(); - let expected = MapArray::new( - data_type, - offsets, - kv_array, - Some(Bitmap::from_u8_slice([0b00000010], 4).sliced(1, 2)), - ); - - assert_eq!(result, expected) -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/mod.rs b/src/common/arrow/tests/it/arrow/array/growable/mod.rs deleted file mode 100644 index dda375c45954..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/mod.rs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod binary; -mod boolean; -mod dictionary; -mod fixed_binary; -mod fixed_size_list; -mod list; -mod map; -mod null; -mod primitive; -mod struct_; -mod union; -mod utf8; - -use databend_common_arrow::arrow::array::growable::make_growable; -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; - -#[test] -fn test_make_growable() { - let array = Int32Array::from_slice([1, 2]); - make_growable(&[&array], false, 2); - - let array = Utf8Array::::from_slice(["a", "aa"]); - make_growable(&[&array], false, 2); - - let array = Utf8Array::::from_slice(["a", "aa"]); - make_growable(&[&array], false, 2); - - let array = BinaryArray::::from_slice([b"a".as_ref(), b"aa".as_ref()]); - make_growable(&[&array], false, 2); - - let array = BinaryArray::::from_slice([b"a".as_ref(), b"aa".as_ref()]); - make_growable(&[&array], false, 2); - - let array = BinaryArray::::from_slice([b"a".as_ref(), b"aa".as_ref()]); - make_growable(&[&array], false, 2); - - let array = - FixedSizeBinaryArray::new(DataType::FixedSizeBinary(2), b"abcd".to_vec().into(), None); - make_growable(&[&array], false, 2); -} - -#[test] -fn test_make_growable_extension() { - let array = DictionaryArray::try_from_keys( - Int32Array::from_slice([1, 0]), - Int32Array::from_slice([1, 2]).boxed(), - ) - .unwrap(); - make_growable(&[&array], false, 2); - - let data_type = DataType::Extension("ext".to_owned(), Box::new(DataType::Int32), None); - let array = Int32Array::from_slice([1, 2]).to(data_type.clone()); - let array_grown = make_growable(&[&array], false, 2).as_box(); - assert_eq!(array_grown.data_type(), &data_type); - - let data_type = DataType::Extension( - "ext".to_owned(), - Box::new(DataType::Struct(vec![Field::new( - "a", - DataType::Int32, - false, - )])), - None, - ); - let array = StructArray::new( - data_type.clone(), - vec![Int32Array::from_slice([1, 2]).boxed()], - None, - ); - let array_grown = make_growable(&[&array], false, 2).as_box(); - assert_eq!(array_grown.data_type(), &data_type); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/null.rs b/src/common/arrow/tests/it/arrow/array/growable/null.rs deleted file mode 100644 index 3abd2efab423..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/null.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableNull; -use databend_common_arrow::arrow::array::NullArray; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn null() { - let mut mutable = GrowableNull::default(); - - mutable.extend(0, 1, 2); - mutable.extend(1, 0, 1); - assert_eq!(mutable.len(), 3); - - let result: NullArray = mutable.into(); - - let expected = NullArray::new(DataType::Null, 3); - assert_eq!(result, expected); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/primitive.rs b/src/common/arrow/tests/it/arrow/array/growable/primitive.rs deleted file mode 100644 index 6af36a8e1306..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/primitive.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowablePrimitive; -use databend_common_arrow::arrow::array::PrimitiveArray; - -/// tests extending from a primitive array w/ offset nor nulls -#[test] -fn basics() { - let b = PrimitiveArray::::from(vec![Some(1), Some(2), Some(3)]); - let mut a = GrowablePrimitive::new(vec![&b], false, 3); - a.extend(0, 0, 2); - assert_eq!(a.len(), 2); - let result: PrimitiveArray = a.into(); - let expected = PrimitiveArray::::from(vec![Some(1), Some(2)]); - assert_eq!(result, expected); -} - -/// tests extending from a primitive array with offset w/ nulls -#[test] -fn offset() { - let b = PrimitiveArray::::from(vec![Some(1), Some(2), Some(3)]); - let b = b.sliced(1, 2); - let mut a = GrowablePrimitive::new(vec![&b], false, 2); - a.extend(0, 0, 2); - assert_eq!(a.len(), 2); - let result: PrimitiveArray = a.into(); - let expected = PrimitiveArray::::from(vec![Some(2), Some(3)]); - assert_eq!(result, expected); -} - -/// tests extending from a primitive array with offset and nulls -#[test] -fn null_offset() { - let b = PrimitiveArray::::from(vec![Some(1), None, Some(3)]); - let b = b.sliced(1, 2); - let mut a = GrowablePrimitive::new(vec![&b], false, 2); - a.extend(0, 0, 2); - assert_eq!(a.len(), 2); - let result: PrimitiveArray = a.into(); - let expected = PrimitiveArray::::from(vec![None, Some(3)]); - assert_eq!(result, expected); -} - -#[test] -fn null_offset_validity() { - let b = PrimitiveArray::::from(&[Some(1), Some(2), Some(3)]); - let b = b.sliced(1, 2); - let mut a = GrowablePrimitive::new(vec![&b], true, 2); - a.extend(0, 0, 2); - a.extend_validity(3); - a.extend(0, 1, 1); - assert_eq!(a.len(), 6); - let result: PrimitiveArray = a.into(); - let expected = PrimitiveArray::::from(&[Some(2), Some(3), None, None, None, Some(3)]); - assert_eq!(result, expected); -} - -#[test] -fn joining_arrays() { - let b = PrimitiveArray::::from(&[Some(1), Some(2), Some(3)]); - let c = PrimitiveArray::::from(&[Some(4), Some(5), Some(6)]); - let mut a = GrowablePrimitive::new(vec![&b, &c], false, 4); - a.extend(0, 0, 2); - a.extend(1, 1, 2); - assert_eq!(a.len(), 4); - let result: PrimitiveArray = a.into(); - - let expected = PrimitiveArray::::from(&[Some(1), Some(2), Some(5), Some(6)]); - assert_eq!(result, expected); -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/struct_.rs b/src/common/arrow/tests/it/arrow/array/growable/struct_.rs deleted file mode 100644 index fa911ae940c1..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/struct_.rs +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableStruct; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::PrimitiveArray; -use databend_common_arrow::arrow::array::StructArray; -use databend_common_arrow::arrow::array::Utf8Array; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; - -fn some_values() -> (DataType, Vec>) { - let strings: Box = Box::new(Utf8Array::::from([ - Some("a"), - Some("aa"), - None, - Some("mark"), - Some("doe"), - ])); - let ints: Box = Box::new(PrimitiveArray::::from(&[ - Some(1), - Some(2), - Some(3), - Some(4), - Some(5), - ])); - let fields = vec![ - Field::new("f1", DataType::Utf8, true), - Field::new("f2", DataType::Int32, true), - ]; - (DataType::Struct(fields), vec![strings, ints]) -} - -#[test] -fn basic() { - let (fields, values) = some_values(); - - let array = StructArray::new(fields.clone(), values.clone(), None); - - let mut a = GrowableStruct::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - let result: StructArray = a.into(); - - let expected = StructArray::new( - fields, - vec![values[0].sliced(1, 2), values[1].sliced(1, 2)], - None, - ); - assert_eq!(result, expected) -} - -#[test] -fn offset() { - let (fields, values) = some_values(); - - let array = StructArray::new(fields.clone(), values.clone(), None).sliced(1, 3); - - let mut a = GrowableStruct::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - let result: StructArray = a.into(); - - let expected = StructArray::new( - fields, - vec![values[0].sliced(2, 2), values[1].sliced(2, 2)], - None, - ); - - assert_eq!(result, expected); -} - -#[test] -fn nulls() { - let (fields, values) = some_values(); - - let array = StructArray::new( - fields.clone(), - values.clone(), - Some(Bitmap::from_u8_slice([0b00000010], 5)), - ); - - let mut a = GrowableStruct::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - assert_eq!(a.len(), 2); - let result: StructArray = a.into(); - - let expected = StructArray::new( - fields, - vec![values[0].sliced(1, 2), values[1].sliced(1, 2)], - Some(Bitmap::from_u8_slice([0b00000010], 5).sliced(1, 2)), - ); - - assert_eq!(result, expected) -} - -#[test] -fn many() { - let (fields, values) = some_values(); - - let array = StructArray::new(fields.clone(), values.clone(), None); - - let mut mutable = GrowableStruct::new(vec![&array, &array], true, 0); - - mutable.extend(0, 1, 2); - mutable.extend(1, 0, 2); - mutable.extend_validity(1); - assert_eq!(mutable.len(), 5); - let result = mutable.as_box(); - - let expected_string: Box = Box::new(Utf8Array::::from([ - Some("aa"), - None, - Some("a"), - Some("aa"), - None, - ])); - let expected_int: Box = Box::new(PrimitiveArray::::from(vec![ - Some(2), - Some(3), - Some(1), - Some(2), - None, - ])); - - let expected = StructArray::new( - fields, - vec![expected_string, expected_int], - Some(Bitmap::from([true, true, true, true, false])), - ); - assert_eq!(expected, result.as_ref()) -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/union.rs b/src/common/arrow/tests/it/arrow/array/growable/union.rs deleted file mode 100644 index 342fe5fd1a14..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/union.rs +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableUnion; -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::*; -use databend_common_arrow::arrow::error::Result; - -#[test] -fn sparse() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let types = vec![0, 0, 1].into(); - let fields = vec![ - Int32Array::from(&[Some(1), None, Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - let array = UnionArray::new(data_type, types, fields, None); - - for length in 1..2 { - for index in 0..(array.len() - length + 1) { - let mut a = GrowableUnion::new(vec![&array], 10); - - a.extend(0, index, length); - assert_eq!(a.len(), length); - - let expected = array.clone().sliced(index, length); - - let result: UnionArray = a.into(); - - assert_eq!(expected, result); - } - } - - Ok(()) -} - -#[test] -fn dense() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); - let types = vec![0, 0, 1].into(); - let fields = vec![ - Int32Array::from(&[Some(1), None, Some(2)]).boxed(), - Utf8Array::::from([Some("c")]).boxed(), - ]; - let offsets = Some(vec![0, 1, 0].into()); - - let array = UnionArray::new(data_type, types, fields, offsets); - - for length in 1..2 { - for index in 0..(array.len() - length + 1) { - let mut a = GrowableUnion::new(vec![&array], 10); - - a.extend(0, index, length); - assert_eq!(a.len(), length); - let expected = array.clone().sliced(index, length); - - let result: UnionArray = a.into(); - - assert_eq!(expected, result); - } - } - - Ok(()) -} - -#[test] -fn complex_dense() -> Result<()> { - let fixed_size_type = - DataType::FixedSizeList(Box::new(Field::new("i", DataType::UInt16, true)), 3); - - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - Field::new("c", fixed_size_type.clone(), true), - ]; - - let data_type = DataType::Union(fields, None, UnionMode::Dense); - - // UnionArray[1, [11, 12, 13], abcd, [21, 22, 23], 2] - let types = vec![0, 2, 1, 2, 0].into(); - let fields = vec![ - Int32Array::from(&[Some(1), Some(2)]).boxed(), - Utf8Array::::from([Some("abcd")]).boxed(), - FixedSizeListArray::try_new( - fixed_size_type.clone(), - UInt16Array::from_iter([11, 12, 13, 21, 22, 23].into_iter().map(Some)).boxed(), - None, - ) - .unwrap() - .boxed(), - ]; - let offsets = Some(vec![0, 0, 0, 1, 1].into()); - - let array1 = UnionArray::new(data_type.clone(), types, fields, offsets); - - // UnionArray[[31, 32, 33], [41, 42, 43], ef, ghijk, 3] - let types = vec![2, 2, 1, 1, 0].into(); - let fields = vec![ - Int32Array::from(&[Some(3)]).boxed(), - Utf8Array::::from([Some("ef"), Some("ghijk")]).boxed(), - FixedSizeListArray::try_new( - fixed_size_type.clone(), - UInt16Array::from_iter([31, 32, 33, 41, 42, 43].into_iter().map(Some)).boxed(), - None, - ) - .unwrap() - .boxed(), - ]; - let offsets = Some(vec![0, 1, 0, 1, 0].into()); - - let array2 = UnionArray::new(data_type.clone(), types, fields, offsets); - - let mut a = GrowableUnion::new(vec![&array1, &array2], 10); - - // Take the whole first array - a.extend(0, 0, 5); - // Skip the first value from the second array: [31, 32, 33] - a.extend(1, 1, 4); - assert_eq!(a.len(), 9); - - let result: UnionArray = a.into(); - - // UnionArray[1, [11, 12, 13], abcd, [21, 22, 23], 2, [41, 42, 43], ef, ghijk, 3] - let types = vec![0, 2, 1, 2, 0, 2, 1, 1, 0].into(); - let fields = vec![ - Int32Array::from(&[Some(1), Some(2), Some(3)]).boxed(), - Utf8Array::::from([Some("abcd"), Some("ef"), Some("ghijk")]).boxed(), - FixedSizeListArray::try_new( - fixed_size_type, - UInt16Array::from_iter([11, 12, 13, 21, 22, 23, 41, 42, 43].into_iter().map(Some)) - .boxed(), - None, - ) - .unwrap() - .boxed(), - ]; - let offsets = Some(vec![0, 0, 0, 1, 1, 2, 1, 2, 2].into()); - - let expected = UnionArray::new(data_type, types, fields, offsets); - - assert_eq!(expected, result); - - Ok(()) -} diff --git a/src/common/arrow/tests/it/arrow/array/growable/utf8.rs b/src/common/arrow/tests/it/arrow/array/growable/utf8.rs deleted file mode 100644 index 582504c36d6b..000000000000 --- a/src/common/arrow/tests/it/arrow/array/growable/utf8.rs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::growable::Growable; -use databend_common_arrow::arrow::array::growable::GrowableUtf8; -use databend_common_arrow::arrow::array::Utf8Array; - -/// tests extending from a variable-sized (strings and binary) array w/ offset with nulls -#[test] -fn validity() { - let array = Utf8Array::::from([Some("a"), Some("bc"), None, Some("defh")]); - - let mut a = GrowableUtf8::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from([Some("bc"), None]); - assert_eq!(result, expected); -} - -/// tests extending from a variable-sized (strings and binary) array -/// with an offset and nulls -#[test] -fn offsets() { - let array = Utf8Array::::from([Some("a"), Some("bc"), None, Some("defh")]); - let array = array.sliced(1, 3); - - let mut a = GrowableUtf8::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - assert_eq!(a.len(), 3); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from([Some("bc"), None, Some("defh")]); - assert_eq!(result, expected); -} - -#[test] -fn offsets2() { - let array = Utf8Array::::from([Some("a"), Some("bc"), None, Some("defh")]); - let array = array.sliced(1, 3); - - let mut a = GrowableUtf8::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - assert_eq!(a.len(), 3); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from([Some("bc"), None, Some("defh")]); - assert_eq!(result, expected); -} - -#[test] -fn multiple_with_validity() { - let array1 = Utf8Array::::from_slice(["hello", "world"]); - let array2 = Utf8Array::::from([Some("1"), None]); - - let mut a = GrowableUtf8::new(vec![&array1, &array2], false, 5); - - a.extend(0, 0, 2); - a.extend(1, 0, 2); - assert_eq!(a.len(), 4); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from([Some("hello"), Some("world"), Some("1"), None]); - assert_eq!(result, expected); -} - -#[test] -fn null_offset_validity() { - let array = Utf8Array::::from([Some("a"), Some("bc"), None, Some("defh")]); - let array = array.sliced(1, 3); - - let mut a = GrowableUtf8::new(vec![&array], true, 0); - - a.extend(0, 1, 2); - a.extend_validity(1); - assert_eq!(a.len(), 3); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from([None, Some("defh"), None]); - assert_eq!(result, expected); -} diff --git a/src/common/arrow/tests/it/arrow/array/list/mod.rs b/src/common/arrow/tests/it/arrow/array/list/mod.rs deleted file mode 100644 index 1deec7a392fe..000000000000 --- a/src/common/arrow/tests/it/arrow/array/list/mod.rs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType; - -mod mutable; - -#[test] -fn debug() { - let values = Buffer::from(vec![1, 2, 3, 4, 5]); - let values = PrimitiveArray::::new(DataType::Int32, values, None); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::new( - data_type, - vec![0, 2, 2, 3, 5].try_into().unwrap(), - Box::new(values), - None, - ); - - assert_eq!(format!("{array:?}"), "ListArray[[1, 2], [], [3], [4, 5]]"); -} - -#[test] -#[should_panic] -fn test_nested_panic() { - let values = Buffer::from(vec![1, 2, 3, 4, 5]); - let values = PrimitiveArray::::new(DataType::Int32, values, None); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::new( - data_type.clone(), - vec![0, 2, 2, 3, 5].try_into().unwrap(), - Box::new(values), - None, - ); - - // The datatype for the nested array has to be created considering - // the nested structure of the child data - let _ = ListArray::::new( - data_type, - vec![0, 2, 4].try_into().unwrap(), - Box::new(array), - None, - ); -} - -#[test] -fn test_nested_display() { - let values = Buffer::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); - let values = PrimitiveArray::::new(DataType::Int32, values, None); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::new( - data_type, - vec![0, 2, 4, 7, 7, 8, 10].try_into().unwrap(), - Box::new(values), - None, - ); - - let data_type = ListArray::::default_datatype(array.data_type().clone()); - let nested = ListArray::::new( - data_type, - vec![0, 2, 5, 6].try_into().unwrap(), - Box::new(array), - None, - ); - - let expected = "ListArray[[[1, 2], [3, 4]], [[5, 6, 7], [], [8]], [[9, 10]]]"; - assert_eq!(format!("{nested:?}"), expected); -} diff --git a/src/common/arrow/tests/it/arrow/array/list/mutable.rs b/src/common/arrow/tests/it/arrow/array/list/mutable.rs deleted file mode 100644 index a18d2e41a1a2..000000000000 --- a/src/common/arrow/tests/it/arrow/array/list/mutable.rs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn basics() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(4), None, Some(6)]), - ]; - - let mut array = MutableListArray::>::new(); - array.try_extend(data).unwrap(); - let array: ListArray = array.into(); - - let values = PrimitiveArray::::new( - DataType::Int32, - Buffer::from(vec![1, 2, 3, 4, 0, 6]), - Some(Bitmap::from([true, true, true, true, false, true])), - ); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let expected = ListArray::::new( - data_type, - vec![0, 3, 3, 6].try_into().unwrap(), - Box::new(values), - Some(Bitmap::from([true, false, true])), - ); - assert_eq!(expected, array); -} - -#[test] -fn with_capacity() { - let array = MutableListArray::>::with_capacity(10); - assert!(array.offsets().capacity() >= 10); - assert_eq!(array.offsets().len_proxy(), 0); - assert_eq!(array.values().values().capacity(), 0); - assert_eq!(array.validity(), None); -} - -#[test] -fn push() { - let mut array = MutableListArray::>::new(); - array - .try_push(Some(vec![Some(1i32), Some(2), Some(3)])) - .unwrap(); - assert_eq!(array.len(), 1); - assert_eq!(array.values().values().as_ref(), [1, 2, 3]); - assert_eq!(array.offsets().as_slice(), [0, 3]); - assert_eq!(array.validity(), None); -} - -#[test] -fn extend_from_self() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(4), None, Some(6)]), - ]; - let mut a = MutableListArray::>::new(); - a.try_extend(data.clone()).unwrap(); - - a.try_extend_from_self(&a.clone()).unwrap(); - let a: ListArray = a.into(); - - let mut expected = data.clone(); - expected.extend(data); - - let mut b = MutableListArray::>::new(); - b.try_extend(expected).unwrap(); - let b: ListArray = b.into(); - - assert_eq!(a, b); -} diff --git a/src/common/arrow/tests/it/arrow/array/map/mod.rs b/src/common/arrow/tests/it/arrow/array/map/mod.rs deleted file mode 100644 index c5f2322fef87..000000000000 --- a/src/common/arrow/tests/it/arrow/array/map/mod.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; - -#[test] -fn basics() { - let dt = DataType::Struct(vec![ - Field::new("a", DataType::Utf8, true), - Field::new("b", DataType::Utf8, true), - ]); - let data_type = DataType::Map(Box::new(Field::new("a", dt.clone(), true)), false); - - let field = StructArray::new( - dt.clone(), - vec![ - Box::new(Utf8Array::::from_slice(["a", "aa", "aaa"])) as _, - Box::new(Utf8Array::::from_slice(["b", "bb", "bbb"])), - ], - None, - ); - - let array = MapArray::new( - data_type, - vec![0, 1, 2].try_into().unwrap(), - Box::new(field), - None, - ); - - assert_eq!( - array.value(0), - Box::new(StructArray::new( - dt.clone(), - vec![ - Box::new(Utf8Array::::from_slice(["a"])) as _, - Box::new(Utf8Array::::from_slice(["b"])), - ], - None, - )) as Box - ); - - let sliced = array.sliced(1, 1); - assert_eq!( - sliced.value(0), - Box::new(StructArray::new( - dt, - vec![ - Box::new(Utf8Array::::from_slice(["aa"])) as _, - Box::new(Utf8Array::::from_slice(["bb"])), - ], - None, - )) as Box - ); -} diff --git a/src/common/arrow/tests/it/arrow/array/mod.rs b/src/common/arrow/tests/it/arrow/array/mod.rs deleted file mode 100644 index 96fc64824da2..000000000000 --- a/src/common/arrow/tests/it/arrow/array/mod.rs +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod binary; -mod binview; -mod boolean; -mod dictionary; -mod equal; -mod fixed_size_binary; -mod fixed_size_list; -mod growable; -mod list; -mod map; -mod ord; -mod primitive; -mod struct_; -mod union; -mod utf8; - -use databend_common_arrow::arrow::array::clone; -use databend_common_arrow::arrow::array::new_empty_array; -use databend_common_arrow::arrow::array::new_null_array; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::PrimitiveArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::datatypes::UnionMode; - -#[test] -fn nulls() { - let datatypes = vec![ - DataType::Int32, - DataType::Float64, - DataType::Utf8, - DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), - ]; - let a = datatypes - .into_iter() - .all(|x| new_null_array(x, 10).null_count() == 10); - assert!(a); - - // unions' null count is always 0 - let datatypes = vec![ - DataType::Union( - vec![Field::new("a", DataType::Binary, true)], - None, - UnionMode::Dense, - ), - DataType::Union( - vec![Field::new("a", DataType::Binary, true)], - None, - UnionMode::Sparse, - ), - ]; - let a = datatypes - .into_iter() - .all(|x| new_null_array(x, 10).null_count() == 0); - assert!(a); -} - -#[test] -fn empty() { - let datatypes = vec![ - DataType::Int32, - DataType::Float64, - DataType::Utf8, - DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), - DataType::List(Box::new(Field::new( - "a", - DataType::Extension("ext".to_owned(), Box::new(DataType::Int32), None), - true, - ))), - DataType::Union( - vec![Field::new("a", DataType::Binary, true)], - None, - UnionMode::Sparse, - ), - DataType::Union( - vec![Field::new("a", DataType::Binary, true)], - None, - UnionMode::Dense, - ), - DataType::Struct(vec![Field::new("a", DataType::Int32, true)]), - ]; - let a = datatypes.into_iter().all(|x| new_empty_array(x).len() == 0); - assert!(a); -} - -#[test] -fn empty_extension() { - let datatypes = vec![ - DataType::Int32, - DataType::Float64, - DataType::Utf8, - DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), - DataType::Union( - vec![Field::new("a", DataType::Binary, true)], - None, - UnionMode::Sparse, - ), - DataType::Union( - vec![Field::new("a", DataType::Binary, true)], - None, - UnionMode::Dense, - ), - DataType::Struct(vec![Field::new("a", DataType::Int32, true)]), - ]; - let a = datatypes - .into_iter() - .map(|dt| DataType::Extension("ext".to_owned(), Box::new(dt), None)) - .all(|x| { - let a = new_empty_array(x); - a.len() == 0 && matches!(a.data_type(), DataType::Extension(_, _, _)) - }); - assert!(a); -} - -#[test] -fn test_clone() { - let datatypes = vec![ - DataType::Int32, - DataType::Float64, - DataType::Utf8, - DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), - ]; - let a = datatypes - .into_iter() - .all(|x| clone(new_null_array(x.clone(), 10).as_ref()) == new_null_array(x, 10)); - assert!(a); -} - -#[test] -fn test_with_validity() { - let arr = PrimitiveArray::from_slice([1i32, 2, 3]); - let validity = Bitmap::from(&[true, false, true]); - let arr = arr.with_validity(Some(validity)); - let arr_ref = arr.as_any().downcast_ref::>().unwrap(); - - let expected = PrimitiveArray::from(&[Some(1i32), None, Some(3)]); - assert_eq!(arr_ref, &expected); -} - -// check that we ca derive stuff -#[allow(dead_code)] -#[derive(PartialEq, Clone, Debug)] -struct A { - array: Box, -} diff --git a/src/common/arrow/tests/it/arrow/array/ord.rs b/src/common/arrow/tests/it/arrow/array/ord.rs deleted file mode 100644 index a5d297a2aec2..000000000000 --- a/src/common/arrow/tests/it/arrow/array/ord.rs +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::cmp::Ordering; - -use databend_common_arrow::arrow::array::ord::build_compare; -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::error::Result; - -#[test] -fn i32() -> Result<()> { - let array = Int32Array::from_slice([1, 2]); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - Ok(()) -} - -#[test] -fn i32_i32() -> Result<()> { - let array1 = Int32Array::from_slice([1]); - let array2 = Int32Array::from_slice([2]); - - let cmp = build_compare(&array1, &array2)?; - - assert_eq!(Ordering::Less, (cmp)(0, 0)); - Ok(()) -} - -#[test] -fn f32() -> Result<()> { - let array = &Float32Array::from_slice([1.0, 2.0]); - - let cmp = build_compare(array, array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - Ok(()) -} - -#[test] -fn f64() -> Result<()> { - let array = Float64Array::from_slice([1.0, 2.0]); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - Ok(()) -} - -#[test] -fn f64_nan() -> Result<()> { - let array = Float64Array::from_slice([1.0, f64::NAN]); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - Ok(()) -} - -#[test] -fn f64_zeros() -> Result<()> { - let array = Float64Array::from_slice([-0.0, 0.0]); - - let cmp = build_compare(&array, &array)?; - - // official IEEE 754 (2008 revision) - assert_eq!(Ordering::Less, (cmp)(0, 1)); - assert_eq!(Ordering::Greater, (cmp)(1, 0)); - Ok(()) -} - -#[test] -fn decimal() -> Result<()> { - let array = Int128Array::from_slice([1, 2]).to(DataType::Decimal(38, 0)); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - assert_eq!(Ordering::Equal, (cmp)(1, 1)); - assert_eq!(Ordering::Greater, (cmp)(1, 0)); - - Ok(()) -} - -#[test] -fn dict_utf8() -> Result<()> { - let data = vec!["a", "b", "c", "a", "a", "c", "c"]; - - let data = data.into_iter().map(Some); - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data)?; - let array: DictionaryArray = array.into(); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - assert_eq!(Ordering::Equal, (cmp)(3, 4)); - assert_eq!(Ordering::Greater, (cmp)(2, 3)); - Ok(()) -} - -#[test] -fn dict_i32() -> Result<()> { - let data = vec![1, 2, 3, 1, 1, 3, 3]; - - let data = data.into_iter().map(Some); - - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data)?; - let array = array.into_arc(); - - let cmp = build_compare(array.as_ref(), array.as_ref())?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - assert_eq!(Ordering::Equal, (cmp)(3, 4)); - assert_eq!(Ordering::Greater, (cmp)(2, 3)); - Ok(()) -} diff --git a/src/common/arrow/tests/it/arrow/array/primitive/fmt.rs b/src/common/arrow/tests/it/arrow/array/primitive/fmt.rs deleted file mode 100644 index 96515ca91eeb..000000000000 --- a/src/common/arrow/tests/it/arrow/array/primitive/fmt.rs +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::*; -use databend_common_arrow::arrow::types::days_ms; -use databend_common_arrow::arrow::types::months_days_ns; - -#[test] -fn debug_int32() { - let array = Int32Array::from(&[Some(1), None, Some(2)]); - assert_eq!(format!("{array:?}"), "Int32[1, None, 2]"); -} - -#[test] -fn debug_date32() { - let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Date32); - assert_eq!(format!("{array:?}"), "Date32[1970-01-02, None, 1970-01-03]"); -} - -#[test] -fn debug_time32s() { - let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Time32(TimeUnit::Second)); - assert_eq!( - format!("{array:?}"), - "Time32(Second)[00:00:01, None, 00:00:02]" - ); -} - -#[test] -fn debug_time32ms() { - let array = - Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Time32(TimeUnit::Millisecond)); - assert_eq!( - format!("{array:?}"), - "Time32(Millisecond)[00:00:00.001, None, 00:00:00.002]" - ); -} - -#[test] -fn debug_interval_d() { - let array = - Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Interval(IntervalUnit::YearMonth)); - assert_eq!(format!("{array:?}"), "Interval(YearMonth)[1m, None, 2m]"); -} - -#[test] -fn debug_int64() { - let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Int64); - assert_eq!(format!("{array:?}"), "Int64[1, None, 2]"); -} - -#[test] -fn debug_date64() { - let array = Int64Array::from(&[Some(1), None, Some(86400000)]).to(DataType::Date64); - assert_eq!(format!("{array:?}"), "Date64[1970-01-01, None, 1970-01-02]"); -} - -#[test] -fn debug_time64us() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Time64(TimeUnit::Microsecond)); - assert_eq!( - format!("{array:?}"), - "Time64(Microsecond)[00:00:00.000001, None, 00:00:00.000002]" - ); -} - -#[test] -fn debug_time64ns() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Time64(TimeUnit::Nanosecond)); - assert_eq!( - format!("{array:?}"), - "Time64(Nanosecond)[00:00:00.000000001, None, 00:00:00.000000002]" - ); -} - -#[test] -fn debug_timestamp_s() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp(TimeUnit::Second, None)); - assert_eq!( - format!("{array:?}"), - "Timestamp(Second, None)[1970-01-01 00:00:01, None, 1970-01-01 00:00:02]" - ); -} - -#[test] -fn debug_timestamp_ms() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Timestamp(TimeUnit::Millisecond, None)); - assert_eq!( - format!("{array:?}"), - "Timestamp(Millisecond, None)[1970-01-01 00:00:00.001, None, 1970-01-01 00:00:00.002]" - ); -} - -#[test] -fn debug_timestamp_us() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Timestamp(TimeUnit::Microsecond, None)); - assert_eq!( - format!("{array:?}"), - "Timestamp(Microsecond, None)[1970-01-01 00:00:00.000001, None, 1970-01-01 00:00:00.000002]" - ); -} - -#[test] -fn debug_timestamp_ns() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Timestamp(TimeUnit::Nanosecond, None)); - assert_eq!( - format!("{array:?}"), - "Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000001, None, 1970-01-01 00:00:00.000000002]" - ); -} - -#[test] -fn debug_timestamp_tz_ns() { - let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp( - TimeUnit::Nanosecond, - Some("+02:00".to_string()), - )); - assert_eq!( - format!("{array:?}"), - "Timestamp(Nanosecond, Some(\"+02:00\"))[1970-01-01 02:00:00.000000001 +02:00, None, 1970-01-01 02:00:00.000000002 +02:00]" - ); -} - -#[test] -fn debug_timestamp_tz_not_parsable() { - let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp( - TimeUnit::Nanosecond, - Some("aa".to_string()), - )); - assert_eq!( - format!("{array:?}"), - "Timestamp(Nanosecond, Some(\"aa\"))[1 (aa), None, 2 (aa)]" - ); -} - -#[cfg(feature = "chrono-tz")] -#[test] -fn debug_timestamp_tz1_ns() { - let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp( - TimeUnit::Nanosecond, - Some("Europe/Lisbon".to_string()), - )); - assert_eq!( - format!("{array:?}"), - "Timestamp(Nanosecond, Some(\"Europe/Lisbon\"))[1970-01-01 01:00:00.000000001 CET, None, 1970-01-01 01:00:00.000000002 CET]" - ); -} - -#[test] -fn debug_duration_ms() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Millisecond)); - assert_eq!( - format!("{array:?}"), - "Duration(Millisecond)[1ms, None, 2ms]" - ); -} - -#[test] -fn debug_duration_s() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Second)); - assert_eq!(format!("{array:?}"), "Duration(Second)[1s, None, 2s]"); -} - -#[test] -fn debug_duration_us() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Microsecond)); - assert_eq!( - format!("{array:?}"), - "Duration(Microsecond)[1us, None, 2us]" - ); -} - -#[test] -fn debug_duration_ns() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Nanosecond)); - assert_eq!(format!("{array:?}"), "Duration(Nanosecond)[1ns, None, 2ns]"); -} - -#[test] -fn debug_decimal() { - let array = Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal(5, 2)); - assert_eq!(format!("{array:?}"), "Decimal(5, 2)[123.45, None, 234.56]"); -} - -#[test] -fn debug_decimal1() { - let array = Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal(5, 1)); - assert_eq!(format!("{array:?}"), "Decimal(5, 1)[1234.5, None, 2345.6]"); -} - -#[test] -fn debug_interval_days_ms() { - let array = DaysMsArray::from(&[Some(days_ms::new(1, 1)), None, Some(days_ms::new(2, 2))]); - assert_eq!( - format!("{array:?}"), - "Interval(DayTime)[1d1ms, None, 2d2ms]" - ); -} - -#[test] -fn debug_months_days_ns() { - let data = &[ - Some(months_days_ns::new(1, 1, 2)), - None, - Some(months_days_ns::new(2, 3, 3)), - ]; - - let array = MonthsDaysNsArray::from(&data); - - assert_eq!( - format!("{array:?}"), - "Interval(MonthDayNano)[1m1d2ns, None, 2m3d3ns]" - ); -} diff --git a/src/common/arrow/tests/it/arrow/array/primitive/mod.rs b/src/common/arrow/tests/it/arrow/array/primitive/mod.rs deleted file mode 100644 index ad9c1ce27872..000000000000 --- a/src/common/arrow/tests/it/arrow/array/primitive/mod.rs +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::*; -use databend_common_arrow::arrow::types::months_days_ns; - -mod fmt; -mod mutable; -mod to_mutable; - -#[test] -fn basics() { - let data = vec![Some(1), None, Some(10)]; - - let array = Int32Array::from_iter(data); - - assert_eq!(array.value(0), 1); - assert_eq!(array.value(1), 0); - assert_eq!(array.value(2), 10); - assert_eq!(array.values().as_slice(), &[1, 0, 10]); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = Int32Array::new( - DataType::Int32, - array.values().clone(), - array.validity().cloned(), - ); - assert_eq!(array, array2); - - let array = array.sliced(1, 2); - assert_eq!(array.value(0), 0); - assert_eq!(array.value(1), 10); - assert_eq!(array.values().as_slice(), &[0, 10]); - - unsafe { - assert_eq!(array.value_unchecked(0), 0); - assert_eq!(array.value_unchecked(1), 10); - } -} - -#[test] -fn empty() { - let array = Int32Array::new_empty(DataType::Int32); - assert_eq!(array.values().len(), 0); - assert_eq!(array.validity(), None); -} - -#[test] -fn from() { - let data = vec![Some(1), None, Some(10)]; - - let array = PrimitiveArray::from(data.clone()); - assert_eq!(array.len(), 3); - - let array = PrimitiveArray::from_iter(data.clone()); - assert_eq!(array.len(), 3); - - let array = PrimitiveArray::from_trusted_len_iter(data.into_iter()); - assert_eq!(array.len(), 3); - - let data = vec![1i32, 2, 3]; - - let array = PrimitiveArray::from_values(data.clone()); - assert_eq!(array.len(), 3); - - let array = PrimitiveArray::from_trusted_len_values_iter(data.into_iter()); - assert_eq!(array.len(), 3); -} - -#[test] -fn months_days_ns_from_slice() { - let data = &[ - months_days_ns::new(1, 1, 2), - months_days_ns::new(1, 1, 3), - months_days_ns::new(2, 3, 3), - ]; - - let array = MonthsDaysNsArray::from_slice(data); - - let a = array.values().as_slice(); - assert_eq!(a, data.as_ref()); -} - -#[test] -fn wrong_data_type() { - let values = Buffer::from(b"abbb".to_vec()); - assert!(PrimitiveArray::try_new(DataType::Utf8, values, None).is_err()); -} - -#[test] -fn wrong_len() { - let values = Buffer::from(b"abbb".to_vec()); - let validity = Some([true, false].into()); - assert!(PrimitiveArray::try_new(DataType::Utf8, values, validity).is_err()); -} - -#[test] -fn into_mut_1() { - let values = Buffer::::from(vec![0, 1]); - let a = values.clone(); // cloned values - assert_eq!(a, values); - let array = PrimitiveArray::new(DataType::Int32, values, None); - assert!(array.into_mut().is_left()); -} - -#[test] -fn into_mut_2() { - let values = Buffer::::from(vec![0, 1]); - let validity = Some([true, false].into()); - let a = validity.clone(); // cloned values - assert_eq!(a, validity); - let array = PrimitiveArray::new(DataType::Int32, values, validity); - assert!(array.into_mut().is_left()); -} - -#[test] -fn into_mut_3() { - let values = Buffer::::from(vec![0, 1]); - let validity = Some([true, false].into()); - let array = PrimitiveArray::new(DataType::Int32, values, validity); - assert!(array.into_mut().is_right()); -} - -#[test] -fn into_iter() { - let data = vec![Some(1), None, Some(10)]; - let rev = data.clone().into_iter().rev(); - - let array: Int32Array = data.clone().into_iter().collect(); - - assert_eq!(array.clone().into_iter().collect::>(), data); - - assert!(array.into_iter().rev().eq(rev)) -} diff --git a/src/common/arrow/tests/it/arrow/array/primitive/mutable.rs b/src/common/arrow/tests/it/arrow/array/primitive/mutable.rs deleted file mode 100644 index 57f173dd6cc5..000000000000 --- a/src/common/arrow/tests/it/arrow/array/primitive/mutable.rs +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter::FromIterator; - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::error::Result; - -#[test] -fn from_and_into_data() { - let a = MutablePrimitiveArray::try_new( - DataType::Int32, - vec![1i32, 0], - Some(MutableBitmap::from([true, false])), - ) - .unwrap(); - assert_eq!(a.len(), 2); - let (a, b, c) = a.into_inner(); - assert_eq!(a, DataType::Int32); - assert_eq!(b, Vec::from([1i32, 0])); - assert_eq!(c, Some(MutableBitmap::from([true, false]))); -} - -#[test] -fn from_vec() { - let a = MutablePrimitiveArray::from_vec(Vec::from([1i32, 0])); - assert_eq!(a.len(), 2); -} - -#[test] -fn to() { - let a = MutablePrimitiveArray::try_new( - DataType::Int32, - vec![1i32, 0], - Some(MutableBitmap::from([true, false])), - ) - .unwrap(); - let a = a.to(DataType::Date32); - assert_eq!(a.data_type(), &DataType::Date32); -} - -#[test] -fn values_mut_slice() { - let mut a = MutablePrimitiveArray::try_new( - DataType::Int32, - vec![1i32, 0], - Some(MutableBitmap::from([true, false])), - ) - .unwrap(); - let values = a.values_mut_slice(); - - values[0] = 10; - assert_eq!(a.values()[0], 10); -} - -#[test] -fn push() { - let mut a = MutablePrimitiveArray::::new(); - a.push(Some(1)); - a.push(None); - a.push_null(); - assert_eq!(a.len(), 3); - assert!(a.is_valid(0)); - assert!(!a.is_valid(1)); - assert!(!a.is_valid(2)); - - assert_eq!(a.values(), &Vec::from([1, 0, 0])); -} - -#[test] -fn pop() { - let mut a = MutablePrimitiveArray::::new(); - a.push(Some(1)); - a.push(None); - a.push(Some(2)); - a.push_null(); - assert_eq!(a.pop(), None); - assert_eq!(a.pop(), Some(2)); - assert_eq!(a.pop(), None); - assert!(a.is_valid(0)); - assert_eq!(a.values(), &Vec::from([1])); - assert_eq!(a.pop(), Some(1)); - assert_eq!(a.len(), 0); - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 0); -} - -#[test] -fn pop_all_some() { - let mut a = MutablePrimitiveArray::::new(); - for v in 0..8 { - a.push(Some(v)); - } - - a.push(Some(8)); - assert_eq!(a.pop(), Some(8)); - assert_eq!(a.pop(), Some(7)); - assert_eq!(a.pop(), Some(6)); - assert_eq!(a.pop(), Some(5)); - assert_eq!(a.pop(), Some(4)); - assert_eq!(a.len(), 4); - assert!(a.is_valid(0)); - assert!(a.is_valid(1)); - assert!(a.is_valid(2)); - assert!(a.is_valid(3)); - assert_eq!(a.values(), &Vec::from([0, 1, 2, 3])); -} - -#[test] -fn set() { - let mut a = MutablePrimitiveArray::::from([Some(1), None]); - - a.set(0, Some(2)); - a.set(1, Some(1)); - - assert_eq!(a.len(), 2); - assert!(a.is_valid(0)); - assert!(a.is_valid(1)); - - assert_eq!(a.values(), &Vec::from([2, 1])); - - let mut a = MutablePrimitiveArray::::from_slice([1, 2]); - - a.set(0, Some(2)); - a.set(1, None); - - assert_eq!(a.len(), 2); - assert!(a.is_valid(0)); - assert!(!a.is_valid(1)); - - assert_eq!(a.values(), &Vec::from([2, 0])); -} - -#[test] -fn from_iter() { - let a = MutablePrimitiveArray::::from_iter((0..2).map(Some)); - assert_eq!(a.len(), 2); - let validity = a.validity().unwrap(); - assert_eq!(validity.unset_bits(), 0); -} - -#[test] -fn natural_arc() { - let a = MutablePrimitiveArray::::from_slice([0, 1]).into_arc(); - assert_eq!(a.len(), 2); -} - -#[test] -fn as_arc() { - let a = MutablePrimitiveArray::::from_slice([0, 1]).as_arc(); - assert_eq!(a.len(), 2); -} - -#[test] -fn as_box() { - let a = MutablePrimitiveArray::::from_slice([0, 1]).as_box(); - assert_eq!(a.len(), 2); -} - -#[test] -fn shrink_to_fit_and_capacity() { - let mut a = MutablePrimitiveArray::::with_capacity(100); - a.push(Some(1)); - a.try_push(None).unwrap(); - assert!(a.capacity() >= 100); - (&mut a as &mut dyn MutableArray).shrink_to_fit(); - assert_eq!(a.capacity(), 2); -} - -#[test] -fn only_nulls() { - let mut a = MutablePrimitiveArray::::new(); - a.push(None); - a.push(None); - let a: PrimitiveArray = a.into(); - assert_eq!(a.validity(), Some(&Bitmap::from([false, false]))); -} - -#[test] -fn from_trusted_len() { - let a = - MutablePrimitiveArray::::from_trusted_len_iter(vec![Some(1), None, None].into_iter()); - let a: PrimitiveArray = a.into(); - assert_eq!(a.validity(), Some(&Bitmap::from([true, false, false]))); - - let a = unsafe { - MutablePrimitiveArray::::from_trusted_len_iter_unchecked( - vec![Some(1), None].into_iter(), - ) - }; - let a: PrimitiveArray = a.into(); - assert_eq!(a.validity(), Some(&Bitmap::from([true, false]))); -} - -#[test] -fn extend_trusted_len() { - let mut a = MutablePrimitiveArray::::new(); - a.extend_trusted_len(vec![Some(1), Some(2)].into_iter()); - let validity = a.validity().unwrap(); - assert_eq!(validity.unset_bits(), 0); - a.extend_trusted_len(vec![None, Some(4)].into_iter()); - assert_eq!( - a.validity(), - Some(&MutableBitmap::from([true, true, false, true])) - ); - assert_eq!(a.values(), &Vec::::from([1, 2, 0, 4])); -} - -#[test] -fn extend_constant_no_validity() { - let mut a = MutablePrimitiveArray::::new(); - a.push(Some(1)); - a.extend_constant(2, Some(3)); - assert_eq!(a.validity(), None); - assert_eq!(a.values(), &Vec::::from([1, 3, 3])); -} - -#[test] -fn extend_constant_validity() { - let mut a = MutablePrimitiveArray::::new(); - a.push(Some(1)); - a.extend_constant(2, None); - assert_eq!( - a.validity(), - Some(&MutableBitmap::from([true, false, false])) - ); - assert_eq!(a.values(), &Vec::::from([1, 0, 0])); -} - -#[test] -fn extend_constant_validity_inverse() { - let mut a = MutablePrimitiveArray::::new(); - a.push(None); - a.extend_constant(2, Some(1)); - assert_eq!( - a.validity(), - Some(&MutableBitmap::from([false, true, true])) - ); - assert_eq!(a.values(), &Vec::::from([0, 1, 1])); -} - -#[test] -fn extend_constant_validity_none() { - let mut a = MutablePrimitiveArray::::new(); - a.push(None); - a.extend_constant(2, None); - assert_eq!( - a.validity(), - Some(&MutableBitmap::from([false, false, false])) - ); - assert_eq!(a.values(), &Vec::::from([0, 0, 0])); -} - -#[test] -fn extend_trusted_len_values() { - let mut a = MutablePrimitiveArray::::new(); - a.extend_trusted_len_values(vec![1, 2, 3].into_iter()); - assert_eq!(a.validity(), None); - assert_eq!(a.values(), &Vec::::from([1, 2, 3])); - - let mut a = MutablePrimitiveArray::::new(); - a.push(None); - a.extend_trusted_len_values(vec![1, 2].into_iter()); - assert_eq!( - a.validity(), - Some(&MutableBitmap::from([false, true, true])) - ); -} - -#[test] -fn extend_from_slice() { - let mut a = MutablePrimitiveArray::::new(); - a.extend_from_slice(&[1, 2, 3]); - assert_eq!(a.validity(), None); - assert_eq!(a.values(), &Vec::::from([1, 2, 3])); - - let mut a = MutablePrimitiveArray::::new(); - a.push(None); - a.extend_from_slice(&[1, 2]); - assert_eq!( - a.validity(), - Some(&MutableBitmap::from([false, true, true])) - ); -} - -#[test] -fn set_validity() { - let mut a = MutablePrimitiveArray::::new(); - a.extend_trusted_len(vec![Some(1), Some(2)].into_iter()); - let validity = a.validity().unwrap(); - assert_eq!(validity.unset_bits(), 0); - - // test that upon conversion to array the bitmap is set to None - let arr: PrimitiveArray<_> = a.clone().into(); - assert_eq!(arr.validity(), None); - - // test set_validity - a.set_validity(Some(MutableBitmap::from([false, true]))); - assert_eq!(a.validity(), Some(&MutableBitmap::from([false, true]))); -} - -#[test] -fn set_values() { - let mut a = MutablePrimitiveArray::::from_slice([1, 2]); - a.set_values(Vec::from([1, 3])); - assert_eq!(a.values().as_slice(), [1, 3]); -} - -#[test] -fn try_from_trusted_len_iter() { - let iter = std::iter::repeat(Some(1)).take(2).map(Result::Ok); - let a = MutablePrimitiveArray::try_from_trusted_len_iter(iter).unwrap(); - assert_eq!(a, MutablePrimitiveArray::from([Some(1), Some(1)])); -} - -#[test] -fn wrong_data_type() { - assert!(MutablePrimitiveArray::::try_new(DataType::Utf8, vec![], None).is_err()); -} - -#[test] -fn extend_from_self() { - let mut a = MutablePrimitiveArray::from([Some(1), None]); - - a.try_extend_from_self(&a.clone()).unwrap(); - - assert_eq!( - a, - MutablePrimitiveArray::from([Some(1), None, Some(1), None]) - ); -} diff --git a/src/common/arrow/tests/it/arrow/array/primitive/to_mutable.rs b/src/common/arrow/tests/it/arrow/array/primitive/to_mutable.rs deleted file mode 100644 index b8ed0689ab9c..000000000000 --- a/src/common/arrow/tests/it/arrow/array/primitive/to_mutable.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::PrimitiveArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use either::Either; - -#[test] -fn array_to_mutable() { - let data = vec![1, 2, 3]; - let arr = PrimitiveArray::new(DataType::Int32, data.into(), None); - - // to mutable push and freeze again - let mut mut_arr = arr.into_mut().unwrap_right(); - mut_arr.push(Some(5)); - let immut: PrimitiveArray = mut_arr.into(); - assert_eq!(immut.values().as_slice(), [1, 2, 3, 5]); - - // let's cause a realloc and see if miri is ok - let mut mut_arr = immut.into_mut().unwrap_right(); - mut_arr.extend_constant(256, Some(9)); - let immut: PrimitiveArray = mut_arr.into(); - assert_eq!(immut.values().len(), 256 + 4); -} - -#[test] -fn array_to_mutable_not_owned() { - let data = vec![1, 2, 3]; - let arr = PrimitiveArray::new(DataType::Int32, data.into(), None); - let arr2 = arr.clone(); - - // to the `to_mutable` should fail and we should get back the original array - match arr2.into_mut() { - Either::Left(arr2) => { - assert_eq!(arr, arr2); - } - _ => panic!(), - } -} - -#[test] -#[allow(clippy::redundant_clone)] -fn array_to_mutable_validity() { - let data = vec![1, 2, 3]; - - // both have a single reference should be ok - let bitmap = Bitmap::from_iter([true, false, true]); - let arr = PrimitiveArray::new(DataType::Int32, data.clone().into(), Some(bitmap)); - assert!(matches!(arr.into_mut(), Either::Right(_))); - - // now we clone the bitmap increasing the ref count - let bitmap = Bitmap::from_iter([true, false, true]); - let arr = PrimitiveArray::new(DataType::Int32, data.into(), Some(bitmap.clone())); - assert!(matches!(arr.into_mut(), Either::Left(_))); -} diff --git a/src/common/arrow/tests/it/arrow/array/struct_/iterator.rs b/src/common/arrow/tests/it/arrow/array/struct_/iterator.rs deleted file mode 100644 index b28c4ff3fd1a..000000000000 --- a/src/common/arrow/tests/it/arrow/array/struct_/iterator.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::*; -use databend_common_arrow::arrow::scalar::new_scalar; - -#[test] -fn test_simple_iter() { - let boolean = BooleanArray::from_slice([false, false, true, true]).boxed(); - let int = Int32Array::from_slice([42, 28, 19, 31]).boxed(); - - let fields = vec![ - Field::new("b", DataType::Boolean, false), - Field::new("c", DataType::Int32, false), - ]; - - let array = StructArray::new( - DataType::Struct(fields), - vec![boolean.clone(), int.clone()], - None, - ); - - for (i, item) in array.iter().enumerate() { - let expected = Some(vec![ - new_scalar(boolean.as_ref(), i), - new_scalar(int.as_ref(), i), - ]); - assert_eq!(expected, item); - } -} diff --git a/src/common/arrow/tests/it/arrow/array/struct_/mod.rs b/src/common/arrow/tests/it/arrow/array/struct_/mod.rs deleted file mode 100644 index fe9618fdb702..000000000000 --- a/src/common/arrow/tests/it/arrow/array/struct_/mod.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod iterator; -mod mutable; - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::*; - -#[test] -fn debug() { - let boolean = BooleanArray::from_slice([false, false, true, true]).boxed(); - let int = Int32Array::from_slice([42, 28, 19, 31]).boxed(); - - let fields = vec![ - Field::new("b", DataType::Boolean, false), - Field::new("c", DataType::Int32, false), - ]; - - let array = StructArray::new( - DataType::Struct(fields), - vec![boolean.clone(), int.clone()], - Some(Bitmap::from([true, true, false, true])), - ); - assert_eq!( - format!("{array:?}"), - "StructArray[{b: false, c: 42}, {b: false, c: 28}, None, {b: true, c: 31}]" - ); -} diff --git a/src/common/arrow/tests/it/arrow/array/struct_/mutable.rs b/src/common/arrow/tests/it/arrow/array/struct_/mutable.rs deleted file mode 100644 index 21c0fdc7eca3..000000000000 --- a/src/common/arrow/tests/it/arrow/array/struct_/mutable.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; - -#[test] -fn push() { - let c1 = Box::new(MutablePrimitiveArray::::new()) as Box; - let values = vec![c1]; - let data_type = DataType::Struct(vec![Field::new("f1", DataType::Int32, true)]); - let mut a = MutableStructArray::new(data_type, values); - - a.value::>(0) - .unwrap() - .push(Some(1)); - a.push(true); - a.value::>(0).unwrap().push(None); - a.push(false); - a.value::>(0) - .unwrap() - .push(Some(2)); - a.push(true); - - assert_eq!(a.len(), 3); - assert!(a.is_valid(0)); - assert!(!a.is_valid(1)); - assert!(a.is_valid(2)); - - assert_eq!( - a.value::>(0).unwrap().values(), - &Vec::from([1, 0, 2]) - ); -} diff --git a/src/common/arrow/tests/it/arrow/array/union.rs b/src/common/arrow/tests/it/arrow/array/union.rs deleted file mode 100644 index 8b2b88b70aff..000000000000 --- a/src/common/arrow/tests/it/arrow/array/union.rs +++ /dev/null @@ -1,390 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::*; -use databend_common_arrow::arrow::error::Result; -use databend_common_arrow::arrow::scalar::new_scalar; -use databend_common_arrow::arrow::scalar::PrimitiveScalar; -use databend_common_arrow::arrow::scalar::Scalar; -use databend_common_arrow::arrow::scalar::UnionScalar; -use databend_common_arrow::arrow::scalar::Utf8Scalar; - -fn next_unwrap(iter: &mut I) -> T -where - I: Iterator>, - T: Clone + 'static, -{ - iter.next() - .unwrap() - .as_any() - .downcast_ref::() - .unwrap() - .clone() -} - -#[test] -fn sparse_debug() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let types = vec![0, 0, 1].into(); - let fields = vec![ - Int32Array::from(&[Some(1), None, Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - let array = UnionArray::new(data_type, types, fields, None); - - assert_eq!(format!("{array:?}"), "UnionArray[1, None, c]"); - - Ok(()) -} - -#[test] -fn dense_debug() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); - let types = vec![0, 0, 1].into(); - let fields = vec![ - Int32Array::from(&[Some(1), None, Some(2)]).boxed(), - Utf8Array::::from([Some("c")]).boxed(), - ]; - let offsets = Some(vec![0, 1, 0].into()); - - let array = UnionArray::new(data_type, types, fields, offsets); - - assert_eq!(format!("{array:?}"), "UnionArray[1, None, c]"); - - Ok(()) -} - -#[test] -fn slice() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let types = Buffer::from(vec![0, 0, 1]); - let fields = vec![ - Int32Array::from(&[Some(1), None, Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - let array = UnionArray::new(data_type.clone(), types, fields.clone(), None); - - let result = array.sliced(1, 2); - - let sliced_types = Buffer::from(vec![0, 1]); - let sliced_fields = vec![ - Int32Array::from(&[None, Some(2)]).boxed(), - Utf8Array::::from([Some("b"), Some("c")]).boxed(), - ]; - let expected = UnionArray::new(data_type, sliced_types, sliced_fields, None); - - assert_eq!(expected, result); - Ok(()) -} - -#[test] -fn iter_sparse() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let types = Buffer::from(vec![0, 0, 1]); - let fields = vec![ - Int32Array::from(&[Some(1), None, Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - let array = UnionArray::new(data_type, types, fields.clone(), None); - let mut iter = array.iter(); - - assert_eq!( - next_unwrap::, _>(&mut iter).value(), - &Some(1) - ); - assert_eq!( - next_unwrap::, _>(&mut iter).value(), - &None - ); - assert_eq!( - next_unwrap::, _>(&mut iter).value(), - Some("c") - ); - assert_eq!(iter.next(), None); - - Ok(()) -} - -#[test] -fn iter_dense() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); - let types = Buffer::from(vec![0, 0, 1]); - let offsets = Buffer::::from(vec![0, 1, 0]); - let fields = vec![ - Int32Array::from(&[Some(1), None]).boxed(), - Utf8Array::::from([Some("c")]).boxed(), - ]; - - let array = UnionArray::new(data_type, types, fields.clone(), Some(offsets)); - let mut iter = array.iter(); - - assert_eq!( - next_unwrap::, _>(&mut iter).value(), - &Some(1) - ); - assert_eq!( - next_unwrap::, _>(&mut iter).value(), - &None - ); - assert_eq!( - next_unwrap::, _>(&mut iter).value(), - Some("c") - ); - assert_eq!(iter.next(), None); - - Ok(()) -} - -#[test] -fn iter_sparse_slice() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let types = Buffer::from(vec![0, 0, 1]); - let fields = vec![ - Int32Array::from(&[Some(1), Some(3), Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - let array = UnionArray::new(data_type, types, fields.clone(), None); - let array_slice = array.sliced(1, 1); - let mut iter = array_slice.iter(); - - assert_eq!( - next_unwrap::, _>(&mut iter).value(), - &Some(3) - ); - assert_eq!(iter.next(), None); - - Ok(()) -} - -#[test] -fn iter_dense_slice() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); - let types = Buffer::from(vec![0, 0, 1]); - let offsets = Buffer::::from(vec![0, 1, 0]); - let fields = vec![ - Int32Array::from(&[Some(1), Some(3)]).boxed(), - Utf8Array::::from([Some("c")]).boxed(), - ]; - - let array = UnionArray::new(data_type, types, fields.clone(), Some(offsets)); - let array_slice = array.sliced(1, 1); - let mut iter = array_slice.iter(); - - assert_eq!( - next_unwrap::, _>(&mut iter).value(), - &Some(3) - ); - assert_eq!(iter.next(), None); - - Ok(()) -} - -#[test] -fn scalar() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); - let types = Buffer::from(vec![0, 0, 1]); - let offsets = Buffer::::from(vec![0, 1, 0]); - let fields = vec![ - Int32Array::from(&[Some(1), None]).boxed(), - Utf8Array::::from([Some("c")]).boxed(), - ]; - - let array = UnionArray::new(data_type, types, fields.clone(), Some(offsets)); - - let scalar = new_scalar(&array, 0); - let union_scalar = scalar.as_any().downcast_ref::().unwrap(); - assert_eq!( - union_scalar - .value() - .as_any() - .downcast_ref::>() - .unwrap() - .value(), - &Some(1) - ); - assert_eq!(union_scalar.type_(), 0); - let scalar = new_scalar(&array, 1); - let union_scalar = scalar.as_any().downcast_ref::().unwrap(); - assert_eq!( - union_scalar - .value() - .as_any() - .downcast_ref::>() - .unwrap() - .value(), - &None - ); - assert_eq!(union_scalar.type_(), 0); - - let scalar = new_scalar(&array, 2); - let union_scalar = scalar.as_any().downcast_ref::().unwrap(); - assert_eq!( - union_scalar - .value() - .as_any() - .downcast_ref::>() - .unwrap() - .value(), - Some("c") - ); - assert_eq!(union_scalar.type_(), 1); - - Ok(()) -} - -#[test] -fn dense_without_offsets_is_error() { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); - let types = vec![0, 0, 1].into(); - let fields = vec![ - Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - assert!(UnionArray::try_new(data_type, types, fields.clone(), None).is_err()); -} - -#[test] -fn fields_must_match() { - let fields = vec![ - Field::new("a", DataType::Int64, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let types = vec![0, 0, 1].into(); - let fields = vec![ - Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - assert!(UnionArray::try_new(data_type, types, fields.clone(), None).is_err()); -} - -#[test] -fn sparse_with_offsets_is_error() { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let fields = vec![ - Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - let types = vec![0, 0, 1].into(); - let offsets = vec![0, 1, 0].into(); - - assert!(UnionArray::try_new(data_type, types, fields.clone(), Some(offsets)).is_err()); -} - -#[test] -fn offsets_must_be_in_bounds() { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let fields = vec![ - Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - let types = vec![0, 0, 1].into(); - // it must be equal to length og types - let offsets = vec![0, 1].into(); - - assert!(UnionArray::try_new(data_type, types, fields.clone(), Some(offsets)).is_err()); -} - -#[test] -fn sparse_with_wrong_offsets1_is_error() { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let fields = vec![ - Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - let types = vec![0, 0, 1].into(); - // it must be equal to length of types - let offsets = vec![0, 1, 10].into(); - - assert!(UnionArray::try_new(data_type, types, fields.clone(), Some(offsets)).is_err()); -} - -#[test] -fn types_must_be_in_bounds() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); - let fields = vec![ - Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), - Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), - ]; - - // 10 > num fields - let types = vec![0, 10].into(); - - assert!(UnionArray::try_new(data_type, types, fields.clone(), None).is_err()); - Ok(()) -} diff --git a/src/common/arrow/tests/it/arrow/array/utf8/mod.rs b/src/common/arrow/tests/it/arrow/array/utf8/mod.rs deleted file mode 100644 index 7cbe4116336f..000000000000 --- a/src/common/arrow/tests/it/arrow/array/utf8/mod.rs +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::error::Result; -use databend_common_arrow::arrow::offset::OffsetsBuffer; - -mod mutable; -mod mutable_values; -mod to_mutable; - -#[test] -fn basics() { - let data = vec![Some("hello"), None, Some("hello2")]; - - let array: Utf8Array = data.into_iter().collect(); - - assert_eq!(array.value(0), "hello"); - assert_eq!(array.value(1), ""); - assert_eq!(array.value(2), "hello2"); - assert_eq!(unsafe { array.value_unchecked(2) }, "hello2"); - assert_eq!(array.values().as_slice(), b"hellohello2"); - assert_eq!(array.offsets().as_slice(), &[0, 5, 5, 11]); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = Utf8Array::::new( - DataType::Utf8, - array.offsets().clone(), - array.values().clone(), - array.validity().cloned(), - ); - assert_eq!(array, array2); - - let array = array.sliced(1, 2); - assert_eq!(array.value(0), ""); - assert_eq!(array.value(1), "hello2"); - // note how this keeps everything: the offsets were sliced - assert_eq!(array.values().as_slice(), b"hellohello2"); - assert_eq!(array.offsets().as_slice(), &[5, 5, 11]); -} - -#[test] -fn empty() { - let array = Utf8Array::::new_empty(DataType::Utf8); - assert_eq!(array.values().as_slice(), b""); - assert_eq!(array.offsets().as_slice(), &[0]); - assert_eq!(array.validity(), None); -} - -#[test] -fn from() { - let array = Utf8Array::::from([Some("hello"), Some(" "), None]); - - let a = array.validity().unwrap(); - assert_eq!(a, &Bitmap::from([true, true, false])); -} - -#[test] -fn from_slice() { - let b = Utf8Array::::from_slice(["a", "b", "cc"]); - - let offsets = vec![0, 1, 2, 4].try_into().unwrap(); - let values = b"abcc".to_vec().into(); - assert_eq!( - b, - Utf8Array::::new(DataType::Utf8, offsets, values, None) - ); -} - -#[test] -fn from_iter_values() { - let b = Utf8Array::::from_iter_values(["a", "b", "cc"].iter()); - - let offsets = vec![0, 1, 2, 4].try_into().unwrap(); - let values = b"abcc".to_vec().into(); - assert_eq!( - b, - Utf8Array::::new(DataType::Utf8, offsets, values, None) - ); -} - -#[test] -fn from_trusted_len_iter() { - let b = - Utf8Array::::from_trusted_len_iter(vec![Some("a"), Some("b"), Some("cc")].into_iter()); - - let offsets = vec![0, 1, 2, 4].try_into().unwrap(); - let values = b"abcc".to_vec().into(); - assert_eq!( - b, - Utf8Array::::new(DataType::Utf8, offsets, values, None) - ); -} - -#[test] -fn try_from_trusted_len_iter() { - let b = Utf8Array::::try_from_trusted_len_iter( - vec![Some("a"), Some("b"), Some("cc")] - .into_iter() - .map(Result::Ok), - ) - .unwrap(); - - let offsets = vec![0, 1, 2, 4].try_into().unwrap(); - let values = b"abcc".to_vec().into(); - assert_eq!( - b, - Utf8Array::::new(DataType::Utf8, offsets, values, None) - ); -} - -#[test] -fn not_utf8() { - let offsets = vec![0, 4].try_into().unwrap(); - let values = vec![0, 159, 146, 150].into(); // invalid utf8 - assert!(Utf8Array::::try_new(DataType::Utf8, offsets, values, None).is_err()); -} - -#[test] -fn not_utf8_individually() { - let offsets = vec![0, 1, 2].try_into().unwrap(); - let values = vec![207, 128].into(); // each is invalid utf8, but together is valid - assert!(Utf8Array::::try_new(DataType::Utf8, offsets, values, None).is_err()); -} - -#[test] -fn wrong_data_type() { - let offsets = vec![0, 4].try_into().unwrap(); - let values = b"abbb".to_vec().into(); - assert!(Utf8Array::::try_new(DataType::Int32, offsets, values, None).is_err()); -} - -#[test] -fn out_of_bounds_offsets_panics() { - // the 10 is out of bounds - let offsets = vec![0, 10, 11].try_into().unwrap(); - let values = b"abbb".to_vec().into(); - assert!(Utf8Array::::try_new(DataType::Utf8, offsets, values, None).is_err()); -} - -#[test] -#[should_panic] -fn index_out_of_bounds_panics() { - let offsets = vec![0, 1, 2, 4].try_into().unwrap(); - let values = b"abbb".to_vec().into(); - let array = Utf8Array::::new(DataType::Utf8, offsets, values, None); - - array.value(3); -} - -#[test] -fn debug() { - let array = Utf8Array::::from([Some("aa"), Some(""), None]); - - assert_eq!(format!("{array:?}"), "Utf8Array[aa, , None]"); -} - -#[test] -fn into_mut_1() { - let offsets = vec![0, 1].try_into().unwrap(); - let values = Buffer::from(b"a".to_vec()); - let a = values.clone(); // cloned values - assert_eq!(a, values); - let array = Utf8Array::::new(DataType::Utf8, offsets, values, None); - assert!(array.into_mut().is_left()); -} - -#[test] -fn into_mut_2() { - let offsets: OffsetsBuffer = vec![0, 1].try_into().unwrap(); - let values = b"a".to_vec().into(); - let a = offsets.clone(); // cloned offsets - assert_eq!(a, offsets); - let array = Utf8Array::::new(DataType::Utf8, offsets, values, None); - assert!(array.into_mut().is_left()); -} - -#[test] -fn into_mut_3() { - let offsets = vec![0, 1].try_into().unwrap(); - let values = b"a".to_vec().into(); - let validity = Some([true].into()); - let a = validity.clone(); // cloned validity - assert_eq!(a, validity); - let array = Utf8Array::::new(DataType::Utf8, offsets, values, validity); - assert!(array.into_mut().is_left()); -} - -#[test] -fn into_mut_4() { - let offsets = vec![0, 1].try_into().unwrap(); - let values = b"a".to_vec().into(); - let validity = Some([true].into()); - let array = Utf8Array::::new(DataType::Utf8, offsets, values, validity); - assert!(array.into_mut().is_right()); -} - -#[test] -fn rev_iter() { - let array = Utf8Array::::from([Some("hello"), Some(" "), None]); - - assert_eq!(array.into_iter().rev().collect::>(), vec![ - None, - Some(" "), - Some("hello") - ]); -} - -#[test] -fn iter_nth() { - let array = Utf8Array::::from([Some("hello"), Some(" "), None]); - - assert_eq!(array.iter().nth(1), Some(Some(" "))); - assert_eq!(array.iter().nth(10), None); -} - -#[test] -fn test_apply_validity() { - let mut array = Utf8Array::::from([Some("Red"), Some("Green"), Some("Blue")]); - array.set_validity(Some([true, true, true].into())); - - array.apply_validity(|bitmap| { - let mut mut_bitmap = bitmap.into_mut().right().unwrap(); - mut_bitmap.set(1, false); - mut_bitmap.set(2, false); - mut_bitmap.into() - }); - - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(!array.is_valid(2)); -} diff --git a/src/common/arrow/tests/it/arrow/array/utf8/mutable.rs b/src/common/arrow/tests/it/arrow/array/utf8/mutable.rs deleted file mode 100644 index 4f03eb2ef00a..000000000000 --- a/src/common/arrow/tests/it/arrow/array/utf8/mutable.rs +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::MutableArray; -use databend_common_arrow::arrow::array::MutableUtf8Array; -use databend_common_arrow::arrow::array::TryExtendFromSelf; -use databend_common_arrow::arrow::array::Utf8Array; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn capacities() { - let b = MutableUtf8Array::::with_capacities(1, 10); - - assert!(b.values().capacity() >= 10); - assert!(b.offsets().capacity() >= 1); -} - -#[test] -fn push_null() { - let mut array = MutableUtf8Array::::new(); - array.push::<&str>(None); - - let array: Utf8Array = array.into(); - assert_eq!(array.validity(), Some(&Bitmap::from([false]))); -} - -#[test] -fn pop() { - let mut a = MutableUtf8Array::::new(); - a.push(Some("first")); - a.push(Some("second")); - a.push(Some("third")); - a.push::<&str>(None); - - assert_eq!(a.pop(), None); - assert_eq!(a.len(), 3); - assert_eq!(a.pop(), Some("third".to_owned())); - assert_eq!(a.len(), 2); - assert_eq!(a.pop(), Some("second".to_string())); - assert_eq!(a.len(), 1); - assert_eq!(a.pop(), Some("first".to_string())); - assert!(a.is_empty()); - assert_eq!(a.pop(), None); - assert!(a.is_empty()); -} - -#[test] -fn pop_all_some() { - let mut a = MutableUtf8Array::::new(); - a.push(Some("first")); - a.push(Some("second")); - a.push(Some("third")); - a.push(Some("fourth")); - for _ in 0..4 { - a.push(Some("aaaa")); - } - a.push(Some("こんにちは")); - - assert_eq!(a.pop(), Some("こんにちは".to_string())); - assert_eq!(a.pop(), Some("aaaa".to_string())); - assert_eq!(a.pop(), Some("aaaa".to_string())); - assert_eq!(a.pop(), Some("aaaa".to_string())); - assert_eq!(a.len(), 5); - assert_eq!(a.pop(), Some("aaaa".to_string())); - assert_eq!(a.pop(), Some("fourth".to_string())); - assert_eq!(a.pop(), Some("third".to_string())); - assert_eq!(a.pop(), Some("second".to_string())); - assert_eq!(a.pop(), Some("first".to_string())); - assert!(a.is_empty()); - assert_eq!(a.pop(), None); -} - -/// Safety guarantee -#[test] -fn not_utf8() { - let offsets = vec![0, 4].try_into().unwrap(); - let values = vec![0, 159, 146, 150]; // invalid utf8 - assert!(MutableUtf8Array::::try_new(DataType::Utf8, offsets, values, None).is_err()); -} - -#[test] -fn wrong_data_type() { - let offsets = vec![0, 4].try_into().unwrap(); - let values = vec![1, 2, 3, 4]; - assert!(MutableUtf8Array::::try_new(DataType::Int8, offsets, values, None).is_err()); -} - -#[test] -fn test_extend_trusted_len_values() { - let mut array = MutableUtf8Array::::new(); - - array.extend_trusted_len_values(["hi", "there"].iter()); - array.extend_trusted_len_values(["hello"].iter()); - array.extend_trusted_len(vec![Some("again"), None].into_iter()); - - let array: Utf8Array = array.into(); - - assert_eq!(array.values().as_slice(), b"hitherehelloagain"); - assert_eq!(array.offsets().as_slice(), &[0, 2, 7, 12, 17, 17]); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00001111], 5)) - ); -} - -#[test] -fn test_extend_trusted_len() { - let mut array = MutableUtf8Array::::new(); - - array.extend_trusted_len(vec![Some("hi"), Some("there")].into_iter()); - array.extend_trusted_len(vec![None, Some("hello")].into_iter()); - array.extend_trusted_len_values(["again"].iter()); - - let array: Utf8Array = array.into(); - - assert_eq!(array.values().as_slice(), b"hitherehelloagain"); - assert_eq!(array.offsets().as_slice(), &[0, 2, 7, 7, 12, 17]); - assert_eq!( - array.validity(), - Some(&Bitmap::from_u8_slice([0b00011011], 5)) - ); -} - -#[test] -fn test_extend_values() { - let mut array = MutableUtf8Array::::new(); - - array.extend_values([Some("hi"), None, Some("there"), None].iter().flatten()); - array.extend_values([Some("hello"), None].iter().flatten()); - array.extend_values(vec![Some("again"), None].into_iter().flatten()); - - let array: Utf8Array = array.into(); - - assert_eq!(array.values().as_slice(), b"hitherehelloagain"); - assert_eq!(array.offsets().as_slice(), &[0, 2, 7, 12, 17]); - assert_eq!(array.validity(), None,); -} - -#[test] -fn test_extend() { - let mut array = MutableUtf8Array::::new(); - - array.extend([Some("hi"), None, Some("there"), None]); - - let array: Utf8Array = array.into(); - - assert_eq!( - array, - Utf8Array::::from([Some("hi"), None, Some("there"), None]) - ); -} - -#[test] -fn as_arc() { - let mut array = MutableUtf8Array::::new(); - - array.extend([Some("hi"), None, Some("there"), None]); - - assert_eq!( - Utf8Array::::from([Some("hi"), None, Some("there"), None]), - array.as_arc().as_ref() - ); -} - -#[test] -fn test_iter() { - let mut array = MutableUtf8Array::::new(); - - array.extend_trusted_len(vec![Some("hi"), Some("there")].into_iter()); - array.extend_trusted_len(vec![None, Some("hello")].into_iter()); - array.extend_trusted_len_values(["again"].iter()); - - let result = array.iter().collect::>(); - assert_eq!(result, vec![ - Some("hi"), - Some("there"), - None, - Some("hello"), - Some("again"), - ]); -} - -#[test] -fn as_box_twice() { - let mut a = MutableUtf8Array::::new(); - let _ = a.as_box(); - let _ = a.as_box(); - let mut a = MutableUtf8Array::::new(); - let _ = a.as_arc(); - let _ = a.as_arc(); -} - -#[test] -fn extend_from_self() { - let mut a = MutableUtf8Array::::from([Some("aa"), None]); - - a.try_extend_from_self(&a.clone()).unwrap(); - - assert_eq!( - a, - MutableUtf8Array::::from([Some("aa"), None, Some("aa"), None]) - ); -} - -#[test] -fn test_set_validity() { - let mut array = MutableUtf8Array::::from([Some("Red"), Some("Green"), Some("Blue")]); - array.set_validity(Some([false, false, true].into())); - - assert!(!array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); -} - -#[test] -fn test_apply_validity() { - let mut array = MutableUtf8Array::::from([Some("Red"), Some("Green"), Some("Blue")]); - array.set_validity(Some([true, true, true].into())); - - array.apply_validity(|mut mut_bitmap| { - mut_bitmap.set(1, false); - mut_bitmap.set(2, false); - mut_bitmap - }); - - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(!array.is_valid(2)); -} - -#[test] -fn test_apply_validity_with_no_validity_inited() { - let mut array = MutableUtf8Array::::from([Some("Red"), Some("Green"), Some("Blue")]); - - array.apply_validity(|mut mut_bitmap| { - mut_bitmap.set(1, false); - mut_bitmap.set(2, false); - mut_bitmap - }); - - assert!(array.is_valid(0)); - assert!(array.is_valid(1)); - assert!(array.is_valid(2)); -} diff --git a/src/common/arrow/tests/it/arrow/array/utf8/mutable_values.rs b/src/common/arrow/tests/it/arrow/array/utf8/mutable_values.rs deleted file mode 100644 index edf61ceac642..000000000000 --- a/src/common/arrow/tests/it/arrow/array/utf8/mutable_values.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::MutableArray; -use databend_common_arrow::arrow::array::MutableUtf8ValuesArray; -use databend_common_arrow::arrow::datatypes::DataType; - -#[test] -fn capacity() { - let mut b = MutableUtf8ValuesArray::::with_capacity(100); - - assert_eq!(b.values().capacity(), 0); - assert!(b.offsets().capacity() >= 100); - b.shrink_to_fit(); - assert!(b.offsets().capacity() < 100); -} - -#[test] -fn offsets_must_be_in_bounds() { - let offsets = vec![0, 10].try_into().unwrap(); - let values = b"abbbbb".to_vec(); - assert!(MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values).is_err()); -} - -#[test] -fn data_type_must_be_consistent() { - let offsets = vec![0, 4].try_into().unwrap(); - let values = b"abbb".to_vec(); - assert!(MutableUtf8ValuesArray::::try_new(DataType::Int32, offsets, values).is_err()); -} - -#[test] -fn must_be_utf8() { - let offsets = vec![0, 4].try_into().unwrap(); - let values = vec![0, 159, 146, 150]; - assert!(std::str::from_utf8(&values).is_err()); - assert!(MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values).is_err()); -} - -#[test] -fn as_box() { - let offsets = vec![0, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - let mut b = MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values).unwrap(); - let _ = b.as_box(); -} - -#[test] -fn as_arc() { - let offsets = vec![0, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - let mut b = MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values).unwrap(); - let _ = b.as_arc(); -} - -#[test] -fn extend_trusted_len() { - let offsets = vec![0, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - let mut b = MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values).unwrap(); - b.extend_trusted_len(vec!["a", "b"].into_iter()); - - let offsets = vec![0, 2, 3, 4].try_into().unwrap(); - let values = b"abab".to_vec(); - assert_eq!( - b.as_box(), - MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values) - .unwrap() - .as_box() - ) -} - -#[test] -fn from_trusted_len() { - let mut b = MutableUtf8ValuesArray::::from_trusted_len_iter(vec!["a", "b"].into_iter()); - - let offsets = vec![0, 1, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - assert_eq!( - b.as_box(), - MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values) - .unwrap() - .as_box() - ) -} - -#[test] -fn extend_from_iter() { - let offsets = vec![0, 2].try_into().unwrap(); - let values = b"ab".to_vec(); - let mut b = MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values).unwrap(); - b.extend_trusted_len(vec!["a", "b"].into_iter()); - - let a = b.clone(); - b.extend_trusted_len(a.iter()); - - let offsets = vec![0, 2, 3, 4, 6, 7, 8].try_into().unwrap(); - let values = b"abababab".to_vec(); - assert_eq!( - b.as_box(), - MutableUtf8ValuesArray::::try_new(DataType::Utf8, offsets, values) - .unwrap() - .as_box() - ) -} diff --git a/src/common/arrow/tests/it/arrow/array/utf8/to_mutable.rs b/src/common/arrow/tests/it/arrow/array/utf8/to_mutable.rs deleted file mode 100644 index 5dd214f6bb17..000000000000 --- a/src/common/arrow/tests/it/arrow/array/utf8/to_mutable.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::Utf8Array; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::offset::OffsetsBuffer; - -#[test] -fn not_shared() { - let array = Utf8Array::::from([Some("hello"), Some(" "), None]); - assert!(array.into_mut().is_right()); -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_validity() { - let validity = Bitmap::from([true]); - let array = Utf8Array::::new( - DataType::Utf8, - vec![0, 1].try_into().unwrap(), - b"a".to_vec().into(), - Some(validity.clone()), - ); - assert!(array.into_mut().is_left()) -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_values() { - let values: Buffer = b"a".to_vec().into(); - let array = Utf8Array::::new( - DataType::Utf8, - vec![0, 1].try_into().unwrap(), - values.clone(), - Some(Bitmap::from([true])), - ); - assert!(array.into_mut().is_left()) -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_offsets_values() { - let offsets: OffsetsBuffer = vec![0, 1].try_into().unwrap(); - let values: Buffer = b"a".to_vec().into(); - let array = Utf8Array::::new( - DataType::Utf8, - offsets.clone(), - values.clone(), - Some(Bitmap::from([true])), - ); - assert!(array.into_mut().is_left()) -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_offsets() { - let offsets: OffsetsBuffer = vec![0, 1].try_into().unwrap(); - let array = Utf8Array::::new( - DataType::Utf8, - offsets.clone(), - b"a".to_vec().into(), - Some(Bitmap::from([true])), - ); - assert!(array.into_mut().is_left()) -} - -#[test] -#[allow(clippy::redundant_clone)] -fn shared_all() { - let array = Utf8Array::::from([Some("hello"), Some(" "), None]); - assert!(array.clone().into_mut().is_left()) -} diff --git a/src/common/arrow/tests/it/arrow/arrow_data.rs b/src/common/arrow/tests/it/arrow/arrow_data.rs deleted file mode 100644 index 61831769e862..000000000000 --- a/src/common/arrow/tests/it/arrow/arrow_data.rs +++ /dev/null @@ -1,385 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use arrow_array::ArrayRef; -use arrow_data::ArrayDataBuilder; -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::datatypes::IntegerType; -use databend_common_arrow::arrow::datatypes::TimeUnit; -use databend_common_arrow::arrow::datatypes::UnionMode; -use databend_common_arrow::arrow::offset::Offsets; -use proptest::num::i32; - -fn test_arrow2_roundtrip(array: &dyn arrow_array::Array) { - let arrow2 = Box::::from(array); - assert_eq!(arrow2.len(), array.len()); - - let back = ArrayRef::from(arrow2); - assert_eq!(back.len(), array.len()); - - match array.data_type() { - d @ arrow_schema::DataType::Union(_, arrow_schema::UnionMode::Sparse) => { - // Temporary workaround https://github.com/apache/arrow-rs/issues/4044 - let data = array.to_data(); - let type_ids = data.buffers()[0].slice_with_length(data.offset(), data.len()); - let child_data = data - .child_data() - .iter() - .map(|x| x.slice(data.offset(), data.len())) - .collect(); - - let data = ArrayDataBuilder::new(d.clone()) - .len(data.len()) - .buffers(vec![type_ids]) - .child_data(child_data) - .build() - .unwrap(); - - assert_eq!(back.to_data(), data); - } - _ => assert_eq!(array, back.as_ref()), - } - assert_eq!(array.data_type(), back.data_type()); -} - -fn test_arrow_roundtrip(array: &dyn Array) { - let arrow = ArrayRef::from(array); - assert_eq!(arrow.len(), array.len()); - - let back = Box::::from(arrow); - assert_eq!(back.len(), array.len()); - - assert_eq!(array, back.as_ref()); - assert_eq!(array.data_type(), back.data_type()); -} - -fn test_conversion(array: &dyn Array) { - test_arrow_roundtrip(array); - let to_arrow = ArrayRef::from(array); - test_arrow2_roundtrip(to_arrow.as_ref()); - - if !array.is_empty() { - let sliced = array.sliced(1, array.len() - 1); - test_arrow_roundtrip(sliced.as_ref()); - - let sliced = to_arrow.slice(1, array.len() - 1); - test_arrow2_roundtrip(sliced.as_ref()); - - let sliced = array.sliced(0, array.len() - 1); - test_arrow_roundtrip(sliced.as_ref()); - - let sliced = to_arrow.slice(0, array.len() - 1); - test_arrow2_roundtrip(sliced.as_ref()); - } - - if array.len() > 2 { - let sliced = array.sliced(1, array.len() - 2); - test_arrow_roundtrip(sliced.as_ref()); - - let sliced = to_arrow.slice(1, array.len() - 2); - test_arrow2_roundtrip(sliced.as_ref()); - } -} - -#[test] -fn test_null() { - let data_type = DataType::Null; - let array = NullArray::new(data_type, 7); - test_conversion(&array); -} - -#[test] -fn test_primitive() { - let data_type = DataType::Int32; - let array = PrimitiveArray::new(data_type, vec![1, 2, 3].into(), None); - test_conversion(&array); - - let data_type = DataType::Timestamp(TimeUnit::Second, Some("UTC".into())); - let nulls = Bitmap::from_iter([true, true, false]); - let array = PrimitiveArray::new(data_type, vec![1_i64, 24, 0].into(), Some(nulls)); - test_conversion(&array); -} - -#[test] -fn test_boolean() { - let data_type = DataType::Boolean; - let values = [false, false, true, true, true].into_iter().collect(); - let validity = [false, true, true, false, false].into_iter().collect(); - let array = BooleanArray::new(data_type, values, Some(validity)); - test_conversion(&array); -} - -#[test] -fn test_utf8() { - let array = Utf8Array::::from_iter([Some("asd\0"), None, Some("45\0848"), Some("")]); - test_conversion(&array); - - let array = Utf8Array::::from_iter([Some("asd"), None, Some("45\n848"), Some("")]); - test_conversion(&array); - - let array = Utf8Array::::new_empty(DataType::Utf8); - test_conversion(&array); -} - -#[test] -fn test_binary() { - let array = BinaryArray::::from_iter([Some("s".as_bytes()), Some(b"sd\xFFfk\x23"), None]); - test_conversion(&array); - - let array = BinaryArray::::from_iter([Some("45848".as_bytes()), Some(b"\x03\xFF"), None]); - test_conversion(&array); - - let array = BinaryArray::::new_empty(DataType::Binary); - test_conversion(&array); -} - -/// Returns a 3 element struct array -fn make_struct() -> StructArray { - let a1 = BinaryArray::::from_iter([Some("s".as_bytes()), Some(b"sd\xFFfk\x23"), None]); - let a2 = BinaryArray::::from_iter([Some("45848".as_bytes()), Some(b"\x03\xFF"), None]); - - let data_type = DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())); - let nulls = Bitmap::from_iter([true, true, false]); - let a3 = PrimitiveArray::new(data_type, vec![1_i64, 24, 0].into(), Some(nulls)); - - let nulls = [true, true, false].into_iter().collect(); - StructArray::new( - DataType::Struct(vec![ - Field::new("a1", a1.data_type().clone(), true), - Field::new("a2", a2.data_type().clone(), true), - Field::new("a3", a3.data_type().clone(), true), - ]), - vec![Box::new(a1), Box::new(a2), Box::new(a3)], - Some(nulls), - ) -} - -#[test] -fn test_struct() { - let array = make_struct(); - test_conversion(&array); -} - -#[test] -fn test_list() { - let values = Utf8Array::::from_iter([ - Some("asd\0"), - None, - Some("45\0848"), - Some(""), - Some("335"), - Some("test"), - ]); - - let validity = [true, true, false, false, true].into_iter().collect(); - let offsets = Offsets::try_from_iter(vec![0, 2, 2, 2, 0]).unwrap(); - let data_type = DataType::List(Box::new(Field::new("element", DataType::Utf8, true))); - let list = ListArray::::new( - data_type.clone(), - offsets.into(), - Box::new(values.clone()), - Some(validity), - ); - - test_conversion(&list); - - let list = ListArray::::new_empty(data_type); - test_conversion(&list); - - let validity = [true, true, false, false, true].into_iter().collect(); - let offsets = Offsets::try_from_iter(vec![0, 2, 2, 2, 0]).unwrap(); - let data_type = DataType::LargeList(Box::new(Field::new("element", DataType::Utf8, true))); - let list = ListArray::::new( - data_type.clone(), - offsets.into(), - Box::new(values), - Some(validity), - ); - - test_conversion(&list); - - let list = ListArray::::new_empty(data_type); - test_conversion(&list); -} - -#[test] -fn test_list_struct() { - let values = make_struct(); - let validity = [true, true, false, true].into_iter().collect(); - let offsets = Offsets::try_from_iter(vec![0, 1, 0, 2]).unwrap(); - let list = ListArray::::new( - DataType::List(Box::new(Field::new( - "element", - values.data_type().clone(), - true, - ))), - offsets.into(), - Box::new(values), - Some(validity), - ); - - test_conversion(&list); -} - -#[test] -fn test_dictionary() { - let nulls = [true, false, true, true, true].into_iter().collect(); - let keys = PrimitiveArray::new(DataType::Int16, vec![1_i16, 1, 0, 2, 2].into(), Some(nulls)); - let values = make_struct(); - let dictionary = DictionaryArray::try_new( - DataType::Dictionary( - IntegerType::Int16, - Box::new(values.data_type().clone()), - false, - ), - keys, - Box::new(values), - ) - .unwrap(); - - test_conversion(&dictionary); -} - -#[test] -fn test_fixed_size_binary() { - let data = (0_u8..16).collect::>(); - let nulls = [false, false, true, true, true, false, false, true] - .into_iter() - .collect(); - - let array = FixedSizeBinaryArray::new(DataType::FixedSizeBinary(2), data.into(), Some(nulls)); - test_conversion(&array); -} - -#[test] -fn test_fixed_size_list() { - let values = vec![1_i64, 2, 3, 4, 5, 6, 7, 8]; - let nulls = [false, false, true, true, true, true, false, false] - .into_iter() - .collect(); - let values = PrimitiveArray::new(DataType::Int64, values.into(), Some(nulls)); - - let nulls = [true, true, false, true].into_iter().collect(); - let array = FixedSizeListArray::new( - DataType::FixedSizeList(Box::new(Field::new("element", DataType::Int64, true)), 2), - Box::new(values), - Some(nulls), - ); - - test_conversion(&array); -} - -#[test] -fn test_map() { - let keys = Utf8Array::::from_iter( - ["key1", "key2", "key3", "key1", "key2"] - .into_iter() - .map(Some), - ); - let values = PrimitiveArray::::from_iter([Some(1), None, Some(3), Some(1), None]); - let fields = StructArray::new( - DataType::Struct(vec![ - Field::new("keys", DataType::Utf8, false), // Cannot be nullable - Field::new("values", DataType::Int32, true), - ]), - vec![Box::new(keys), Box::new(values)], - None, // Cannot be nullable - ); - - let validity = [true, true, false, false].into_iter().collect(); - let offsets = Offsets::try_from_iter(vec![0, 2, 0, 2]).unwrap(); - let data_type = DataType::Map( - Box::new(Field::new("entries", fields.data_type().clone(), true)), - false, - ); - let map = MapArray::new( - data_type.clone(), - offsets.into(), - Box::new(fields), - Some(validity), - ); - - test_conversion(&map); - - let map = MapArray::new_empty(data_type); - test_conversion(&map); -} - -#[test] -fn test_dense_union() { - let fields = vec![ - Field::new("a1", DataType::Int32, true), - Field::new("a2", DataType::Int64, true), - ]; - - let a1 = PrimitiveArray::from_iter([Some(2), None]); - let a2 = PrimitiveArray::from_iter([Some(2_i64), None, Some(3)]); - - let types = vec![1, 0, 0, 1, 1]; - let offsets = vec![0, 0, 1, 1, 2]; - let union = UnionArray::new( - DataType::Union(fields.clone(), Some(vec![0, 1]), UnionMode::Dense), - types.into(), - vec![Box::new(a1.clone()), Box::new(a2.clone())], - Some(offsets.into()), - ); - - test_conversion(&union); - - let types = vec![1, 4, 4, 1, 1]; - let offsets = vec![0, 0, 1, 1, 2]; - let union = UnionArray::new( - DataType::Union(fields, Some(vec![4, 1]), UnionMode::Dense), - types.into(), - vec![Box::new(a1), Box::new(a2)], - Some(offsets.into()), - ); - - test_conversion(&union); -} - -#[test] -fn test_sparse_union() { - let fields = vec![ - Field::new("a1", DataType::Int32, true), - Field::new("a2", DataType::Int64, true), - ]; - - let a1 = PrimitiveArray::from_iter([None, Some(2), None, None, None]); - let a2 = PrimitiveArray::from_iter([Some(2_i64), None, None, None, Some(3)]); - - let types = vec![1, 0, 0, 1, 1]; - let union = UnionArray::new( - DataType::Union(fields.clone(), Some(vec![0, 1]), UnionMode::Sparse), - types.into(), - vec![Box::new(a1.clone()), Box::new(a2.clone())], - None, - ); - - test_conversion(&union); - - let types = vec![1, 4, 4, 1, 1]; - let union = UnionArray::new( - DataType::Union(fields, Some(vec![4, 1]), UnionMode::Sparse), - types.into(), - vec![Box::new(a1), Box::new(a2)], - None, - ); - - test_conversion(&union); -} diff --git a/src/common/arrow/tests/it/arrow/buffer/mod.rs b/src/common/arrow/tests/it/arrow/buffer/mod.rs deleted file mode 100644 index 37f54df9889e..000000000000 --- a/src/common/arrow/tests/it/arrow/buffer/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod immutable; diff --git a/src/common/arrow/tests/it/arrow/compute/concatenate.rs b/src/common/arrow/tests/it/arrow/compute/concatenate.rs deleted file mode 100644 index 4efb72284424..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/concatenate.rs +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::compute::concatenate::concatenate; -use databend_common_arrow::arrow::error::Result; - -#[test] -fn empty_vec() { - let re = concatenate(&[]); - assert!(re.is_err()); -} - -#[test] -fn incompatible_datatypes() { - let re = concatenate(&[ - &Int64Array::from([Some(-1), Some(2), None]), - &Utf8Array::::from([Some("hello"), Some("bar"), Some("world")]), - ]); - assert!(re.is_err()); -} - -#[test] -fn string_arrays() -> Result<()> { - let arr = concatenate(&[ - &Utf8Array::::from_slice(["hello", "world"]), - &Utf8Array::::from_slice(["2", "3", "4"]), - &Utf8Array::::from([Some("foo"), Some("bar"), None, Some("baz")]), - ])?; - - let expected_output = Utf8Array::::from([ - Some("hello"), - Some("world"), - Some("2"), - Some("3"), - Some("4"), - Some("foo"), - Some("bar"), - None, - Some("baz"), - ]); - - assert_eq!(expected_output, arr.as_ref()); - - Ok(()) -} - -#[test] -fn primitive_arrays() -> Result<()> { - let arr = concatenate(&[ - &Int64Array::from(&[Some(-1), Some(-1), Some(2), None, None]), - &Int64Array::from(&[Some(101), Some(102), Some(103), None]), - &Int64Array::from(&[Some(256), Some(512), Some(1024)]), - ])?; - - let expected_output = Int64Array::from(vec![ - Some(-1), - Some(-1), - Some(2), - None, - None, - Some(101), - Some(102), - Some(103), - None, - Some(256), - Some(512), - Some(1024), - ]); - - assert_eq!(expected_output, arr.as_ref()); - - Ok(()) -} - -#[test] -fn primitive_array_slices() -> Result<()> { - let input_1 = Int64Array::from(&[Some(-1), Some(-1), Some(2), None, None]).sliced(1, 3); - - let input_2 = Int64Array::from(&[Some(101), Some(102), Some(103), None]).sliced(1, 3); - let arr = concatenate(&[&input_1, &input_2])?; - - let expected_output = Int64Array::from(&[Some(-1), Some(2), None, Some(102), Some(103), None]); - - assert_eq!(expected_output, arr.as_ref()); - - Ok(()) -} - -#[test] -fn boolean_primitive_arrays() -> Result<()> { - let arr = concatenate(&[ - &BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - None, - None, - Some(false), - ]), - &BooleanArray::from(vec![None, Some(false), Some(true), Some(false)]), - ])?; - - let expected_output = BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - None, - None, - Some(false), - None, - Some(false), - Some(true), - Some(false), - ]); - - assert_eq!(expected_output, arr.as_ref()); - - Ok(()) -} diff --git a/src/common/arrow/tests/it/arrow/compute/mod.rs b/src/common/arrow/tests/it/arrow/compute/mod.rs deleted file mode 100644 index 9f3816f2a629..000000000000 --- a/src/common/arrow/tests/it/arrow/compute/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(feature = "compute_concatenate")] -mod concatenate; diff --git a/src/common/arrow/tests/it/arrow/scalar/binary.rs b/src/common/arrow/tests/it/arrow/scalar/binary.rs deleted file mode 100644 index 3f1d4c6e0baa..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/binary.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::scalar::BinaryScalar; -use databend_common_arrow::arrow::scalar::Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let a = BinaryScalar::::from(Some("a")); - let b = BinaryScalar::::from(None::<&str>); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - let b = BinaryScalar::::from(Some("b")); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let a = BinaryScalar::::from(Some("a")); - - assert_eq!(a.value(), Some(b"a".as_ref())); - assert_eq!(a.data_type(), &DataType::Binary); - assert!(a.is_valid()); - - let a = BinaryScalar::::from(None::<&str>); - - assert_eq!(a.data_type(), &DataType::LargeBinary); - assert!(!a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/boolean.rs b/src/common/arrow/tests/it/arrow/scalar/boolean.rs deleted file mode 100644 index 55f7989809c1..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/boolean.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::scalar::BooleanScalar; -use databend_common_arrow::arrow::scalar::Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let a = BooleanScalar::from(Some(true)); - let b = BooleanScalar::from(None); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - let b = BooleanScalar::from(Some(false)); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let a = BooleanScalar::new(Some(true)); - - assert_eq!(a.value(), Some(true)); - assert_eq!(a.data_type(), &DataType::Boolean); - assert!(a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/fixed_size_binary.rs b/src/common/arrow/tests/it/arrow/scalar/fixed_size_binary.rs deleted file mode 100644 index f49b4f5db726..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/fixed_size_binary.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::scalar::FixedSizeBinaryScalar; -use databend_common_arrow::arrow::scalar::Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let a = FixedSizeBinaryScalar::new(DataType::FixedSizeBinary(1), Some("a")); - let b = FixedSizeBinaryScalar::new(DataType::FixedSizeBinary(1), None::<&str>); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - let b = FixedSizeBinaryScalar::new(DataType::FixedSizeBinary(1), Some("b")); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let a = FixedSizeBinaryScalar::new(DataType::FixedSizeBinary(1), Some("a")); - - assert_eq!(a.value(), Some(b"a".as_ref())); - assert_eq!(a.data_type(), &DataType::FixedSizeBinary(1)); - assert!(a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/fixed_size_list.rs b/src/common/arrow/tests/it/arrow/scalar/fixed_size_list.rs deleted file mode 100644 index 88f893dc9ae0..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/fixed_size_list.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::BooleanArray; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::scalar::FixedSizeListScalar; -use databend_common_arrow::arrow::scalar::Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let dt = DataType::FixedSizeList(Box::new(Field::new("a", DataType::Boolean, true)), 2); - let a = FixedSizeListScalar::new( - dt.clone(), - Some(BooleanArray::from_slice([true, false]).boxed()), - ); - - let b = FixedSizeListScalar::new(dt.clone(), None); - - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - - let b = FixedSizeListScalar::new(dt, Some(BooleanArray::from_slice([true, true]).boxed())); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let dt = DataType::FixedSizeList(Box::new(Field::new("a", DataType::Boolean, true)), 2); - let a = FixedSizeListScalar::new( - dt.clone(), - Some(BooleanArray::from_slice([true, false]).boxed()), - ); - - assert_eq!( - BooleanArray::from_slice([true, false]), - a.values().unwrap().as_ref() - ); - assert_eq!(a.data_type(), &dt); - assert!(a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/list.rs b/src/common/arrow/tests/it/arrow/scalar/list.rs deleted file mode 100644 index 310ddc2da943..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/list.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::BooleanArray; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::scalar::ListScalar; -use databend_common_arrow::arrow::scalar::Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let dt = DataType::List(Box::new(Field::new("a", DataType::Boolean, true))); - let a = ListScalar::::new( - dt.clone(), - Some(BooleanArray::from_slice([true, false]).boxed()), - ); - let b = ListScalar::::new(dt.clone(), None); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - let b = ListScalar::::new(dt, Some(BooleanArray::from_slice([true, true]).boxed())); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let dt = DataType::List(Box::new(Field::new("a", DataType::Boolean, true))); - let a = ListScalar::::new( - dt.clone(), - Some(BooleanArray::from_slice([true, false]).boxed()), - ); - - assert_eq!(BooleanArray::from_slice([true, false]), a.values().as_ref()); - assert_eq!(a.data_type(), &dt); - assert!(a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/map.rs b/src/common/arrow/tests/it/arrow/scalar/map.rs deleted file mode 100644 index db2a4a44052f..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/map.rs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::BooleanArray; -use databend_common_arrow::arrow::array::StructArray; -use databend_common_arrow::arrow::array::Utf8Array; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::scalar::MapScalar; -use databend_common_arrow::arrow::scalar::Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let kv_dt = DataType::Struct(vec![ - Field::new("key", DataType::Utf8, false), - Field::new("value", DataType::Boolean, true), - ]); - let kv_array1 = StructArray::try_new( - kv_dt.clone(), - vec![ - Utf8Array::::from([Some("k1"), Some("k2")]).boxed(), - BooleanArray::from_slice([true, false]).boxed(), - ], - None, - ) - .unwrap(); - let kv_array2 = StructArray::try_new( - kv_dt.clone(), - vec![ - Utf8Array::::from([Some("k1"), Some("k3")]).boxed(), - BooleanArray::from_slice([true, true]).boxed(), - ], - None, - ) - .unwrap(); - - let dt = DataType::Map(Box::new(Field::new("entries", kv_dt, true)), false); - let a = MapScalar::new(dt.clone(), Some(Box::new(kv_array1))); - let b = MapScalar::new(dt.clone(), None); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - let b = MapScalar::new(dt, Some(Box::new(kv_array2))); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let kv_dt = DataType::Struct(vec![ - Field::new("key", DataType::Utf8, false), - Field::new("value", DataType::Boolean, true), - ]); - let kv_array = StructArray::try_new( - kv_dt.clone(), - vec![ - Utf8Array::::from([Some("k1"), Some("k2")]).boxed(), - BooleanArray::from_slice([true, false]).boxed(), - ], - None, - ) - .unwrap(); - - let dt = DataType::Map(Box::new(Field::new("entries", kv_dt, true)), false); - let a = MapScalar::new(dt.clone(), Some(Box::new(kv_array.clone()))); - - assert_eq!(kv_array, a.values().as_ref()); - assert_eq!(a.data_type(), &dt); - assert!(a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/mod.rs b/src/common/arrow/tests/it/arrow/scalar/mod.rs deleted file mode 100644 index cab89def259e..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod binary; -mod boolean; -mod fixed_size_binary; -mod fixed_size_list; -mod list; -mod map; -mod null; -mod primitive; -mod struct_; -mod utf8; - -use databend_common_arrow::arrow::scalar::Scalar; - -// check that `PartialEq` can be derived -#[allow(dead_code)] -#[derive(PartialEq)] -struct A { - array: Box, -} diff --git a/src/common/arrow/tests/it/arrow/scalar/null.rs b/src/common/arrow/tests/it/arrow/scalar/null.rs deleted file mode 100644 index 36ea8f893e02..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/null.rs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::scalar::NullScalar; -use databend_common_arrow::arrow::scalar::Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let a = NullScalar::new(); - assert_eq!(a, a); -} - -#[test] -fn basics() { - let a = NullScalar::default(); - - assert_eq!(a.data_type(), &DataType::Null); - assert!(!a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/primitive.rs b/src/common/arrow/tests/it/arrow/scalar/primitive.rs deleted file mode 100644 index a5b1b36b2d9f..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/primitive.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::scalar::PrimitiveScalar; -use databend_common_arrow::arrow::scalar::Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let a = PrimitiveScalar::from(Some(2i32)); - let b = PrimitiveScalar::::from(None); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - let b = PrimitiveScalar::::from(Some(1i32)); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let a = PrimitiveScalar::from(Some(2i32)); - - assert_eq!(a.value(), &Some(2i32)); - assert_eq!(a.data_type(), &DataType::Int32); - - let a = a.to(DataType::Date32); - assert_eq!(a.data_type(), &DataType::Date32); - - let a = PrimitiveScalar::::from(None); - - assert_eq!(a.data_type(), &DataType::Int32); - assert!(!a.is_valid()); - - let a = a.to(DataType::Date32); - assert_eq!(a.data_type(), &DataType::Date32); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/struct_.rs b/src/common/arrow/tests/it/arrow/scalar/struct_.rs deleted file mode 100644 index c12a523aec31..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/struct_.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::scalar::BooleanScalar; -use databend_common_arrow::arrow::scalar::Scalar; -use databend_common_arrow::arrow::scalar::StructScalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let dt = DataType::Struct(vec![Field::new("a", DataType::Boolean, true)]); - let a = StructScalar::new( - dt.clone(), - Some(vec![ - Box::new(BooleanScalar::from(Some(true))) as Box - ]), - ); - let b = StructScalar::new(dt.clone(), None); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - let b = StructScalar::new( - dt, - Some(vec![ - Box::new(BooleanScalar::from(Some(false))) as Box - ]), - ); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let dt = DataType::Struct(vec![Field::new("a", DataType::Boolean, true)]); - - let values = vec![Box::new(BooleanScalar::from(Some(true))) as Box]; - - let a = StructScalar::new(dt.clone(), Some(values.clone())); - - assert_eq!(a.values(), &values); - assert_eq!(a.data_type(), &dt); - assert!(a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/scalar/utf8.rs b/src/common/arrow/tests/it/arrow/scalar/utf8.rs deleted file mode 100644 index fda7458f8cd3..000000000000 --- a/src/common/arrow/tests/it/arrow/scalar/utf8.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::scalar::Scalar; -use databend_common_arrow::arrow::scalar::Utf8Scalar; - -#[allow(clippy::eq_op)] -#[test] -fn equal() { - let a = Utf8Scalar::::from(Some("a")); - let b = Utf8Scalar::::from(None::<&str>); - assert_eq!(a, a); - assert_eq!(b, b); - assert!(a != b); - let b = Utf8Scalar::::from(Some("b")); - assert!(a != b); - assert_eq!(b, b); -} - -#[test] -fn basics() { - let a = Utf8Scalar::::from(Some("a")); - - assert_eq!(a.value(), Some("a")); - assert_eq!(a.data_type(), &DataType::Utf8); - assert!(a.is_valid()); - - let a = Utf8Scalar::::from(None::<&str>); - - assert_eq!(a.data_type(), &DataType::LargeUtf8); - assert!(!a.is_valid()); - - let _: &dyn std::any::Any = a.as_any(); -} diff --git a/src/common/arrow/tests/it/arrow/temporal_conversions.rs b/src/common/arrow/tests/it/arrow/temporal_conversions.rs deleted file mode 100644 index e913f994c57b..000000000000 --- a/src/common/arrow/tests/it/arrow/temporal_conversions.rs +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use chrono::NaiveDateTime; -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::datatypes::TimeUnit; -use databend_common_arrow::arrow::temporal_conversions; -use databend_common_arrow::arrow::types::months_days_ns; - -#[test] -fn naive() { - let expected = "Timestamp(Nanosecond, None)[1996-12-19 16:39:57, 1996-12-19 13:39:57, None]"; - let fmt = "%Y-%m-%dT%H:%M:%S:z"; - let array = Utf8Array::::from_slice([ - "1996-12-19T16:39:57-02:00", - "1996-12-19T13:39:57-03:00", - "1996-12-19 13:39:57-03:00", // missing T - ]); - let r = temporal_conversions::utf8_to_naive_timestamp_ns(&array, fmt); - assert_eq!(format!("{r:?}"), expected); - - let fmt = "%Y-%m-%dT%H:%M:%S"; // no tz info - let array = Utf8Array::::from_slice([ - "1996-12-19T16:39:57-02:00", - "1996-12-19T13:39:57-03:00", - "1996-12-19 13:39:57-03:00", // missing T - ]); - let r = temporal_conversions::utf8_to_naive_timestamp_ns(&array, fmt); - assert_eq!(format!("{r:?}"), expected); -} - -#[test] -fn naive_scalar() { - let fmt = "%Y-%m-%dT%H:%M:%S.%9f%z"; - let str = "2023-04-07T12:23:34.123456789Z"; - - let nanos_expected = 1680870214123456789; - - // seconds - let r = temporal_conversions::utf8_to_naive_timestamp_scalar(str, fmt, &TimeUnit::Second); - assert_eq!(r, Some(nanos_expected / 1_000_000_000)); - // milliseconds - let r = temporal_conversions::utf8_to_naive_timestamp_scalar(str, fmt, &TimeUnit::Millisecond); - assert_eq!(r, Some(nanos_expected / 1_000_000)); - // microseconds - let r = temporal_conversions::utf8_to_naive_timestamp_scalar(str, fmt, &TimeUnit::Microsecond); - assert_eq!(r, Some(nanos_expected / 1_000)); - // nanoseconds - let r = temporal_conversions::utf8_to_naive_timestamp_scalar(str, fmt, &TimeUnit::Nanosecond); - assert_eq!(r, Some(nanos_expected)); -} - -#[test] -fn naive_scalar_no_tz() { - let fmt = "%Y-%m-%dT%H:%M:%S.%9f"; - - let str = "2023-04-07T12:23:34.123456789"; - let nanos_expected = 1680870214123456789; - - // seconds - let r = temporal_conversions::utf8_to_naive_timestamp_scalar(str, fmt, &TimeUnit::Second); - assert_eq!(r, Some(nanos_expected / 1_000_000_000)); - // milliseconds - let r = temporal_conversions::utf8_to_naive_timestamp_scalar(str, fmt, &TimeUnit::Millisecond); - assert_eq!(r, Some(nanos_expected / 1_000_000)); - // microseconds - let r = temporal_conversions::utf8_to_naive_timestamp_scalar(str, fmt, &TimeUnit::Microsecond); - assert_eq!(r, Some(nanos_expected / 1_000)); - // nanoseconds - let r = temporal_conversions::utf8_to_naive_timestamp_scalar(str, fmt, &TimeUnit::Nanosecond); - assert_eq!(r, Some(nanos_expected)); -} - -#[test] -fn scalar_tz_aware() { - let fmt = "%Y-%m-%dT%H:%M:%S%.f%:z"; - - let tz = temporal_conversions::parse_offset("-02:00").unwrap(); - let str = "2023-04-07T10:23:34.000000000-02:00"; - let nanos_expected = 1680870214000000000; - - // seconds - let r = temporal_conversions::utf8_to_timestamp_scalar(str, fmt, &tz, &TimeUnit::Second); - assert_eq!(r, Some(nanos_expected / 1_000_000_000)); - // milliseconds - let r = temporal_conversions::utf8_to_timestamp_scalar(str, fmt, &tz, &TimeUnit::Millisecond); - assert_eq!(r, Some(nanos_expected / 1_000_000)); - // microseconds - let r = temporal_conversions::utf8_to_timestamp_scalar(str, fmt, &tz, &TimeUnit::Microsecond); - assert_eq!(r, Some(nanos_expected / 1_000)); - // nanoseconds - let r = temporal_conversions::utf8_to_timestamp_scalar(str, fmt, &tz, &TimeUnit::Nanosecond); - assert_eq!(r, Some(nanos_expected)); -} -#[test] -fn scalar_tz_aware_no_timezone() { - let fmt = "%Y-%m-%dT%H:%M:%S%.f"; - - let tz = temporal_conversions::parse_offset("-02:00").unwrap(); - let str = "2023-04-07T10:23:34.000000000-02:00"; - - // seconds - let r = temporal_conversions::utf8_to_timestamp_scalar(str, fmt, &tz, &TimeUnit::Second); - assert_eq!(r, None); - // milliseconds - let r = temporal_conversions::utf8_to_timestamp_scalar(str, fmt, &tz, &TimeUnit::Millisecond); - assert_eq!(r, None); - // microseconds - let r = temporal_conversions::utf8_to_timestamp_scalar(str, fmt, &tz, &TimeUnit::Microsecond); - assert_eq!(r, None); - // nanoseconds - let r = temporal_conversions::utf8_to_timestamp_scalar(str, fmt, &tz, &TimeUnit::Nanosecond); - assert_eq!(r, None); -} - -#[test] -fn naive_no_tz() { - let expected = "Timestamp(Nanosecond, None)[1996-12-19 16:39:57, 1996-12-19 13:39:57, None]"; - let fmt = "%Y-%m-%dT%H:%M:%S"; // no tz info - let array = Utf8Array::::from_slice([ - "1996-12-19T16:39:57", - "1996-12-19T13:39:57", - "1996-12-19 13:39:57", // missing T - ]); - let r = temporal_conversions::utf8_to_naive_timestamp_ns(&array, fmt); - assert_eq!(format!("{r:?}"), expected); -} - -#[test] -fn timestamp_to_datetime() { - let fmt = "%Y-%m-%dT%H:%M:%S.%9f"; - let ts = 1680870214123456789; - - // positive milliseconds - assert_eq!( - temporal_conversions::timestamp_ms_to_datetime(ts / 1_000_000), - NaiveDateTime::parse_from_str("2023-04-07T12:23:34.123000000", fmt).unwrap() - ); - // positive microseconds - assert_eq!( - temporal_conversions::timestamp_us_to_datetime(ts / 1_000), - NaiveDateTime::parse_from_str("2023-04-07T12:23:34.123456000", fmt).unwrap() - ); - // positive nanoseconds - assert_eq!( - temporal_conversions::timestamp_ns_to_datetime(ts), - NaiveDateTime::parse_from_str("2023-04-07T12:23:34.123456789", fmt).unwrap() - ); - - let ts = -15548276987654321; - - // negative milliseconds - assert_eq!( - temporal_conversions::timestamp_ms_to_datetime(ts / 1_000_000), - NaiveDateTime::parse_from_str("1969-07-05T01:02:03.013000000", fmt).unwrap() - ); - // negative microseconds - assert_eq!( - temporal_conversions::timestamp_us_to_datetime(ts / 1_000), - NaiveDateTime::parse_from_str("1969-07-05T01:02:03.012346000", fmt).unwrap() - ); - // negative nanoseconds - assert_eq!( - temporal_conversions::timestamp_ns_to_datetime(ts), - NaiveDateTime::parse_from_str("1969-07-05T01:02:03.012345679", fmt).unwrap() - ); - - let fmt = "%Y-%m-%dT%H:%M:%S"; - let ts = -2209075200000000000; - let expected = NaiveDateTime::parse_from_str("1899-12-31T00:00:00", fmt).unwrap(); - - assert_eq!( - temporal_conversions::timestamp_ms_to_datetime(ts / 1_000_000), - expected - ); - assert_eq!( - temporal_conversions::timestamp_us_to_datetime(ts / 1_000), - expected - ); - assert_eq!(temporal_conversions::timestamp_ns_to_datetime(ts), expected); -} - -#[test] -fn timestamp_to_negative_datetime() { - let fmt = "%Y-%m-%d %H:%M:%S"; - let ts = -63135596800000000; - let expected = NaiveDateTime::parse_from_str("-0031-04-24 22:13:20", fmt).unwrap(); - - assert_eq!( - temporal_conversions::timestamp_ms_to_datetime(ts / 1_000), - expected - ); - assert_eq!(temporal_conversions::timestamp_us_to_datetime(ts), expected); -} - -#[test] -fn tz_aware() { - let tz = "-02:00".to_string(); - let expected = "Timestamp(Nanosecond, Some(\"-02:00\"))[1996-12-19 16:39:57 -02:00, 1996-12-19 17:39:57 -02:00, None]"; - let fmt = "%Y-%m-%dT%H:%M:%S%.f%:z"; - let array = Utf8Array::::from_slice([ - "1996-12-19T16:39:57.0-02:00", - "1996-12-19T16:39:57.0-03:00", // same time at a different TZ - "1996-12-19 13:39:57.0-03:00", - ]); - let r = temporal_conversions::utf8_to_timestamp_ns(&array, fmt, tz).unwrap(); - assert_eq!(format!("{r:?}"), expected); -} - -#[test] -fn tz_aware_no_timezone() { - let tz = "-02:00".to_string(); - let expected = "Timestamp(Nanosecond, Some(\"-02:00\"))[None, None, None]"; - let fmt = "%Y-%m-%dT%H:%M:%S%.f"; - let array = Utf8Array::::from_slice([ - "1996-12-19T16:39:57.0", - "1996-12-19T17:39:57.0", - "1996-12-19 13:39:57.0", - ]); - let r = temporal_conversions::utf8_to_timestamp_ns(&array, fmt, tz).unwrap(); - assert_eq!(format!("{r:?}"), expected); -} - -#[test] -fn add_interval_fixed_offset() { - // 1972 has a leap year on the 29th. - let timestamp = 68086800; // Mon Feb 28 1972 01:00:00 GMT+0000 - let timeunit = TimeUnit::Second; - let timezone = temporal_conversions::parse_offset("+01:00").unwrap(); - - let r = temporal_conversions::add_interval( - timestamp, - timeunit, - months_days_ns::new(0, 1, 60_000_000_000), - &timezone, - ); - let r = temporal_conversions::timestamp_to_datetime(r, timeunit, &timezone); - assert_eq!("1972-02-29 02:01:00 +01:00", format!("{r}")); - - let r = temporal_conversions::add_interval( - timestamp, - timeunit, - months_days_ns::new(1, 1, 60_000_000_000), - &timezone, - ); - let r = temporal_conversions::timestamp_to_datetime(r, timeunit, &timezone); - assert_eq!("1972-03-29 02:01:00 +01:00", format!("{r}")); - - let r = temporal_conversions::add_interval( - timestamp, - timeunit, - months_days_ns::new(24, 1, 60_000_000_000), - &timezone, - ); - let r = temporal_conversions::timestamp_to_datetime(r, timeunit, &timezone); - assert_eq!("1974-03-01 02:01:00 +01:00", format!("{r}")); - - let r = temporal_conversions::add_interval( - timestamp, - timeunit, - months_days_ns::new(-1, 1, 60_000_000_000), - &timezone, - ); - let r = temporal_conversions::timestamp_to_datetime(r, timeunit, &timezone); - assert_eq!("1972-01-29 02:01:00 +01:00", format!("{r}")); -} - -#[cfg(feature = "chrono-tz")] -#[test] -fn add_interval_timezone() { - // current time is Sun Mar 29 2020 00:00:00 GMT+0000 (Western European Standard Time) - // 1 hour later is Sun Mar 29 2020 02:00:00 GMT+0100 (Western European Summer Time) - let timestamp = 1585440000; - let timeunit = TimeUnit::Second; - let timezone = temporal_conversions::parse_offset_tz("Europe/Lisbon").unwrap(); - - let r = temporal_conversions::add_interval( - timestamp, - timeunit, - months_days_ns::new(0, 0, 60 * 60 * 1_000_000_000), - &timezone, - ); - let r = temporal_conversions::timestamp_to_datetime(r, timeunit, &timezone); - assert_eq!("2020-03-29 02:00:00 WEST", format!("{r}")); - - // crosses two summer time changes and thus adds only 1 hour - let r = temporal_conversions::add_interval( - timestamp, - timeunit, - months_days_ns::new(7, 0, 60 * 60 * 1_000_000_000), - &timezone, - ); - let r = temporal_conversions::timestamp_to_datetime(r, timeunit, &timezone); - assert_eq!("2020-10-29 01:00:00 WET", format!("{r}")); -} diff --git a/src/common/arrow/tests/it/arrow/types.rs b/src/common/arrow/tests/it/arrow/types.rs deleted file mode 100644 index e84e6ad81c46..000000000000 --- a/src/common/arrow/tests/it/arrow/types.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::types::days_ms; -use databend_common_arrow::arrow::types::months_days_ns; -use databend_common_arrow::arrow::types::BitChunkIter; -use databend_common_arrow::arrow::types::BitChunkOnes; -use databend_common_arrow::arrow::types::NativeType; - -#[test] -fn test_basic1() { - let a = [0b00000001, 0b00010000]; // 0th and 13th entry - let a = u16::from_ne_bytes(a); - let iter = BitChunkIter::new(a, 16); - let r = iter.collect::>(); - assert_eq!(r, (0..16).map(|x| x == 0 || x == 12).collect::>(),); -} - -#[test] -fn test_ones() { - let a = [0b00000001, 0b00010000]; // 0th and 13th entry - let a = u16::from_ne_bytes(a); - let mut iter = BitChunkOnes::new(a); - assert_eq!(iter.size_hint(), (2, Some(2))); - assert_eq!(iter.next(), Some(0)); - assert_eq!(iter.next(), Some(12)); -} - -#[test] -fn months_days_ns_roundtrip() { - let a = months_days_ns(1, 2, 3); - let bytes = a.to_le_bytes(); - assert_eq!(bytes, [1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0]); - - let a = months_days_ns(1, 1, 1); - assert_eq!(a, months_days_ns::from_be_bytes(a.to_be_bytes())); -} - -#[test] -fn days_ms_roundtrip() { - let a = days_ms(1, 2); - let bytes = a.to_le_bytes(); - assert_eq!(bytes, [1, 0, 0, 0, 2, 0, 0, 0]); - - let a = days_ms(1, 2); - assert_eq!(a, days_ms::from_be_bytes(a.to_be_bytes())); -} diff --git a/src/common/arrow/tests/it/native/io.rs b/src/common/arrow/tests/it/native/io.rs deleted file mode 100644 index 7dbb4f27a2aa..000000000000 --- a/src/common/arrow/tests/it/native/io.rs +++ /dev/null @@ -1,557 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::io::BufRead; -use std::io::BufReader; - -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::BinaryArray; -use databend_common_arrow::arrow::array::BinaryViewArray; -use databend_common_arrow::arrow::array::BooleanArray; -use databend_common_arrow::arrow::array::Float32Array; -use databend_common_arrow::arrow::array::Float64Array; -use databend_common_arrow::arrow::array::Int16Array; -use databend_common_arrow::arrow::array::Int32Array; -use databend_common_arrow::arrow::array::Int64Array; -use databend_common_arrow::arrow::array::Int8Array; -use databend_common_arrow::arrow::array::ListArray; -use databend_common_arrow::arrow::array::MapArray; -use databend_common_arrow::arrow::array::PrimitiveArray; -use databend_common_arrow::arrow::array::StructArray; -use databend_common_arrow::arrow::array::UInt16Array; -use databend_common_arrow::arrow::array::UInt32Array; -use databend_common_arrow::arrow::array::UInt64Array; -use databend_common_arrow::arrow::array::UInt8Array; -use databend_common_arrow::arrow::array::Utf8Array; -use databend_common_arrow::arrow::array::Utf8ViewArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::chunk::Chunk; -use databend_common_arrow::arrow::compute; -use databend_common_arrow::arrow::datatypes::DataType; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::datatypes::Schema; -use databend_common_arrow::arrow::offset::OffsetsBuffer; -use databend_common_arrow::native::n_columns; -use databend_common_arrow::native::read::batch_read::batch_read_array; -use databend_common_arrow::native::read::deserialize::column_iter_to_arrays; -use databend_common_arrow::native::read::reader::NativeReader; -use databend_common_arrow::native::write::NativeWriter; -use databend_common_arrow::native::write::WriteOptions; -use databend_common_arrow::native::ColumnMeta; -use databend_common_arrow::native::CommonCompression; -use databend_common_arrow::native::PageMeta; -use rand::rngs::StdRng; -use rand::Rng; -use rand::SeedableRng; - -pub const WRITE_PAGE: usize = 2048; -pub const SMALL_WRITE_PAGE: usize = 2; - -pub fn new_test_chunk() -> Chunk> { - Chunk::new(vec![ - Box::new(BooleanArray::from_slice([ - true, true, true, false, false, false, - ])) as _, - Box::new(UInt8Array::from_vec(vec![1, 2, 3, 4, 5, 6])) as _, - Box::new(UInt16Array::from_vec(vec![1, 2, 3, 4, 5, 6])) as _, - Box::new(UInt32Array::from_vec(vec![1, 2, 3, 4, 5, 6])) as _, - Box::new(UInt64Array::from_vec(vec![1, 2, 3, 4, 5, 6])) as _, - Box::new(Int8Array::from_vec(vec![1, 2, 3, 4, 5, 6])) as _, - Box::new(Int16Array::from_vec(vec![1, 2, 3, 4, 5, 6])) as _, - Box::new(Int32Array::from_vec(vec![1, 2, 3, 4, 5, 6])) as _, - Box::new(Int64Array::from_vec(vec![1, 2, 3, 4, 5, 6])) as _, - Box::new(Float32Array::from_vec(vec![1.1, 2.2, 3.3, 4.4, 5.5, 6.6])) as _, - Box::new(Float64Array::from_vec(vec![1.1, 2.2, 3.3, 4.4, 5.5, 6.6])) as _, - Box::new(Utf8Array::::from_iter_values( - ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), - )) as _, - Box::new(BinaryArray::::from_iter_values( - ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), - )) as _, - Box::new(Utf8ViewArray::from_slice_values( - ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), - )) as _, - Box::new(BinaryViewArray::from_slice_values( - ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), - )) as _, - ]) -} - -#[test] -fn test_basic() { - test_write_read(new_test_chunk()); -} - -#[test] -fn test_random_nonull() { - let size: usize = 10000; - let chunk = Chunk::new(vec![ - Box::new(create_random_bool(size, 0.0)) as _, - Box::new(create_random_index(size, 0.0, size)) as _, - Box::new(create_random_double(size, 0.0, size)) as _, - Box::new(create_random_string(size, 0.0, size)) as _, - Box::new(create_random_view(size, 0.0, size)) as _, - ]); - test_write_read(chunk); -} - -#[test] -fn test_random() { - let size = 10000; - let chunk = Chunk::new(vec![ - Box::new(create_random_bool(size, 0.1)) as _, - Box::new(create_random_index(size, 0.1, size)) as _, - Box::new(create_random_index(size, 0.2, size)) as _, - Box::new(create_random_index(size, 0.3, size)) as _, - Box::new(create_random_index(size, 0.4, size)) as _, - Box::new(create_random_double(size, 0.5, size)) as _, - Box::new(create_random_string(size, 0.4, size)) as _, - Box::new(create_random_view(size, 0.4, size)) as _, - ]); - test_write_read(chunk); -} - -#[test] -fn test_dict() { - let size = 10000; - let chunk = Chunk::new(vec![ - Box::new(create_random_bool(size, 0.1)) as _, - Box::new(create_random_index(size, 0.1, 8)) as _, - Box::new(create_random_index(size, 0.2, 8)) as _, - Box::new(create_random_index(size, 0.3, 8)) as _, - Box::new(create_random_index(size, 0.4, 8)) as _, - Box::new(create_random_double(size, 0.5, 8)) as _, - Box::new(create_random_string(size, 0.4, 8)) as _, - Box::new(create_random_view(size, 0.4, size)) as _, - ]); - test_write_read(chunk); -} - -#[test] -fn test_freq() { - let size = WRITE_PAGE * 5; - let mut values: Vec = Vec::with_capacity(size); - for _ in 0..5 { - values.extend_from_slice(&vec![20; WRITE_PAGE - 3]); - values.push(10000); - values.push(10000); - values.push(10000); - } - - let chunk = Chunk::new(vec![Box::new(UInt32Array::from_vec(values)) as _]); - test_write_read(chunk); -} - -#[test] -fn test_bitpacking() { - let size = WRITE_PAGE * 5; - let chunk = Chunk::new(vec![ - Box::new(create_random_index(size, 0.1, 8)) as _, - Box::new(create_random_index(size, 0.5, 8)) as _, - ]); - test_write_read(chunk); -} - -#[test] -fn test_deleta_bitpacking() { - let size = WRITE_PAGE * 5; - let chunk = Chunk::new(vec![ - Box::new(UInt32Array::from_vec((0..size as u32).collect())) as _, - Box::new(Int32Array::from_vec((0..size as i32).collect())) as _, - ]); - test_write_read(chunk); -} - -#[test] -fn test_onevalue() { - let size = 10000; - let chunk = Chunk::new(vec![ - Box::new(BooleanArray::from_iter((0..size).map(|_| Some(true)))) as _, - Box::new(BooleanArray::from_iter((0..size).map(|_| Some(false)))) as _, - Box::new(UInt32Array::from_vec(vec![3; size])) as _, - Box::new(create_random_index(size, 0.3, 1)) as _, - Box::new(create_random_string(size, 0.4, 1)) as _, - ]); - test_write_read(chunk); -} - -#[test] -fn test_struct() { - let struct_array = create_struct(1000, 0.2, 1000); - let chunk = Chunk::new(vec![Box::new(struct_array) as _]); - test_write_read(chunk); -} - -#[test] -fn test_float() { - let size = 1000; - let chunk = Chunk::new(vec![Box::new(create_random_double(size, 0.5, size)) as _]); - test_write_read(chunk); -} - -#[test] -fn test_list() { - let list_array = create_list(1000, 0.2); - let chunk = Chunk::new(vec![Box::new(list_array) as _]); - test_write_read(chunk); -} - -#[test] -fn test_map() { - let map_array = create_map(1000, 0.2); - let chunk = Chunk::new(vec![Box::new(map_array) as _]); - test_write_read(chunk); -} - -#[test] -fn test_list_list() { - let l1 = create_list(2000, 0.2); - - let mut offsets = vec![]; - for i in (0..=1000).step_by(2) { - offsets.push(i); - } - let list_array = ListArray::try_new( - DataType::List(Box::new(Field::new("item", l1.data_type().clone(), true))), - OffsetsBuffer::try_from(offsets).unwrap(), - l1.boxed(), - None, - ) - .unwrap(); - - let chunk = Chunk::new(vec![Box::new(list_array) as _]); - test_write_read(chunk); -} - -#[test] -fn test_list_struct() { - let s1 = create_struct(2000, 0.2, 2000); - - let mut offsets = vec![]; - for i in (0..=1000).step_by(2) { - offsets.push(i); - } - let list_array = ListArray::try_new( - DataType::List(Box::new(Field::new("item", s1.data_type().clone(), true))), - OffsetsBuffer::try_from(offsets).unwrap(), - s1.boxed(), - None, - ) - .unwrap(); - - let chunk = Chunk::new(vec![Box::new(list_array) as _]); - test_write_read(chunk); -} - -#[test] -fn test_list_map() { - let m1 = create_map(2000, 0.2); - - let mut offsets = vec![]; - for i in (0..=1000).step_by(2) { - offsets.push(i); - } - let list_array = ListArray::try_new( - DataType::List(Box::new(Field::new("item", m1.data_type().clone(), true))), - OffsetsBuffer::try_from(offsets).unwrap(), - m1.boxed(), - None, - ) - .unwrap(); - - let chunk = Chunk::new(vec![Box::new(list_array) as _]); - test_write_read(chunk); -} - -#[test] -fn test_struct_list() { - let size = 10000; - let null_density = 0.2; - let dt = DataType::Struct(vec![ - Field::new("name", DataType::LargeBinary, true), - Field::new( - "age", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), - true, - ), - ]); - let struct_array = StructArray::try_new( - dt, - vec![ - Box::new(create_random_string(size, null_density, size)) as _, - Box::new(create_list(size, null_density)) as _, - ], - None, - ) - .unwrap(); - let chunk = Chunk::new(vec![Box::new(struct_array) as _]); - test_write_read(chunk); -} - -fn create_list(size: usize, null_density: f32) -> ListArray { - let (offsets, bitmap) = create_random_offsets(size, 0.1); - let length = *offsets.last().unwrap() as usize; - let l1 = create_random_index(length, null_density, length); - - ListArray::try_new( - DataType::List(Box::new(Field::new("item", l1.data_type().clone(), true))), - OffsetsBuffer::try_from(offsets).unwrap(), - l1.boxed(), - bitmap, - ) - .unwrap() -} - -fn create_map(size: usize, null_density: f32) -> MapArray { - let (offsets, bitmap) = create_random_offsets(size, 0.1); - let length = *offsets.last().unwrap() as usize; - let dt = DataType::Struct(vec![ - Field::new("key", DataType::Int32, false), - Field::new("value", DataType::LargeBinary, true), - ]); - let struct_array = StructArray::try_new( - dt, - vec![ - Box::new(create_random_index(length, 0.0, length)) as _, - Box::new(create_random_string(length, null_density, length)) as _, - ], - None, - ) - .unwrap(); - - MapArray::try_new( - DataType::Map( - Box::new(Field::new( - "entries", - struct_array.data_type().clone(), - false, - )), - false, - ), - OffsetsBuffer::try_from(offsets).unwrap(), - struct_array.boxed(), - bitmap, - ) - .unwrap() -} - -fn create_struct(size: usize, null_density: f32, uniq: usize) -> StructArray { - let dt = DataType::Struct(vec![ - Field::new("age", DataType::Int32, true), - Field::new("name", DataType::Utf8View, true), - Field::new("name2", DataType::LargeBinary, true), - ]); - StructArray::try_new( - dt, - vec![ - Box::new(create_random_index(size, null_density, uniq)) as _, - Box::new(create_random_view(size, null_density, uniq)) as _, - Box::new(create_random_string(size, null_density, uniq)) as _, - ], - None, - ) - .unwrap() -} - -fn create_random_bool(size: usize, null_density: f32) -> BooleanArray { - let mut rng = StdRng::seed_from_u64(42); - (0..size) - .map(|_| { - if rng.gen::() > null_density { - let value = rng.gen::(); - Some(value) - } else { - None - } - }) - .collect::() -} - -fn create_random_index(size: usize, null_density: f32, uniq: usize) -> PrimitiveArray { - let mut rng = StdRng::seed_from_u64(42); - (0..size) - .map(|_| { - if rng.gen::() > null_density { - let value = rng.gen_range::(0i32..uniq as i32); - Some(value) - } else { - None - } - }) - .collect::>() -} - -fn create_random_double(size: usize, null_density: f32, uniq: usize) -> PrimitiveArray { - let mut rng = StdRng::seed_from_u64(42); - (0..size) - .map(|_| { - if rng.gen::() > null_density { - let value = rng.gen_range::(0i32..uniq as i32); - Some(value as f64) - } else { - None - } - }) - .collect::>() -} - -fn create_random_string(size: usize, null_density: f32, uniq: usize) -> BinaryArray { - let mut rng = StdRng::seed_from_u64(42); - (0..size) - .map(|_| { - if rng.gen::() > null_density { - let value = rng.gen_range::(0i32..uniq as i32); - Some(format!("{value}")) - } else { - None - } - }) - .collect::>() -} - -fn create_random_view(size: usize, null_density: f32, uniq: usize) -> Utf8ViewArray { - let mut rng = StdRng::seed_from_u64(42); - (0..size) - .map(|_| { - if rng.gen::() > null_density { - let value = rng.gen_range::(0i32..uniq as i32); - Some(format!("{value}")) - } else { - None - } - }) - .collect::() -} - -fn create_random_offsets(size: usize, null_density: f32) -> (Vec, Option) { - let mut offsets = Vec::with_capacity(size + 1); - offsets.push(0i32); - let mut builder = MutableBitmap::with_capacity(size); - let mut rng = StdRng::seed_from_u64(42); - for _ in 0..size { - if rng.gen::() > null_density { - let offset = rng.gen_range::(0i32..3i32); - offsets.push(*offsets.last().unwrap() + offset); - builder.push(true); - } else { - offsets.push(*offsets.last().unwrap()); - builder.push(false); - } - } - (offsets, builder.into()) -} - -fn test_write_read(chunk: Chunk>) { - let _ = env_logger::try_init(); - - let compressions = vec![ - CommonCompression::Lz4, - CommonCompression::Zstd, - CommonCompression::Snappy, - CommonCompression::None, - ]; - let page_sizes = vec![WRITE_PAGE, SMALL_WRITE_PAGE]; - - for compression in compressions { - for page_size in &page_sizes { - test_write_read_with_options(chunk.clone(), WriteOptions { - default_compression: compression, - max_page_size: Some(*page_size), - default_compress_ratio: Some(2.0f64), - forbidden_compressions: vec![], - }); - } - } -} - -fn test_write_read_with_options(chunk: Chunk>, options: WriteOptions) { - let mut bytes = Vec::new(); - let fields: Vec = chunk - .iter() - .map(|array| { - Field::new( - "name", - array.data_type().clone(), - array.validity().is_some(), - ) - }) - .collect(); - - let schema = Schema::from(fields); - let mut writer = NativeWriter::new(&mut bytes, schema.clone(), options).unwrap(); - - writer.start().unwrap(); - writer.write(&chunk).unwrap(); - writer.finish().unwrap(); - - log::info!("write finished, start to read"); - - let mut batch_metas = writer.metas.clone(); - let mut metas = writer.metas.clone(); - let mut results = Vec::with_capacity(schema.fields.len()); - for field in schema.fields.iter() { - let n = n_columns(&field.data_type); - - let curr_metas: Vec = metas.drain(..n).collect(); - - let mut native_readers = Vec::with_capacity(n); - for curr_meta in curr_metas.iter() { - let mut range_bytes = std::io::Cursor::new(bytes.clone()); - range_bytes.consume(curr_meta.offset as usize); - - let native_reader = NativeReader::new(range_bytes, curr_meta.pages.clone(), vec![]); - native_readers.push(native_reader); - } - - let mut array_iter = column_iter_to_arrays(native_readers, field.clone(), vec![]).unwrap(); - - let mut arrays = vec![]; - for array in array_iter.by_ref() { - arrays.push(array.unwrap().to_boxed()); - } - let arrays: Vec<&dyn Array> = arrays.iter().map(|v| v.as_ref()).collect(); - let result = compute::concatenate::concatenate(&arrays).unwrap(); - results.push(result); - } - let result_chunk = Chunk::new(results); - - assert_eq!(chunk, result_chunk); - - // test batch read - let mut batch_results = Vec::with_capacity(schema.fields.len()); - for field in schema.fields.iter() { - let n = n_columns(&field.data_type); - - let curr_metas: Vec = batch_metas.drain(..n).collect(); - - let mut pages: Vec> = Vec::with_capacity(n); - let mut readers = Vec::with_capacity(n); - for curr_meta in curr_metas.iter() { - pages.push(curr_meta.pages.clone()); - let mut reader = std::io::Cursor::new(bytes.clone()); - reader.consume(curr_meta.offset as usize); - - let buffer_size = curr_meta.total_len().min(8192) as usize; - let reader = BufReader::with_capacity(buffer_size, reader); - - readers.push(reader); - } - let batch_result = batch_read_array(readers, field.clone(), pages).unwrap(); - batch_results.push(batch_result); - } - let batch_result_chunk = Chunk::new(batch_results); - - assert_eq!(chunk, batch_result_chunk); -} diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 0560b19d3a25..c1dddaafb6ca 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -26,6 +26,7 @@ databend-common-exception = { workspace = true } async-backtrace = { workspace = true } async-trait = { workspace = true } borsh = { workspace = true } +bytemuck = { workspace = true } bytes = { workspace = true } bytesize = { workspace = true } chrono = { workspace = true } diff --git a/src/common/base/src/base/ordered_float.rs b/src/common/base/src/base/ordered_float.rs index 23fd5946c592..aaa103192431 100644 --- a/src/common/base/src/base/ordered_float.rs +++ b/src/common/base/src/base/ordered_float.rs @@ -42,6 +42,8 @@ use core::ops::SubAssign; use core::str::FromStr; use std::error::Error; +use bytemuck::Pod; +use bytemuck::Zeroable; use micromarshal::Unmarshal; use num_traits::float::FloatCore; use num_traits::AsPrimitive; @@ -583,6 +585,11 @@ impl Bounded for OrderedFloat { } } +unsafe impl Zeroable for OrderedFloat {} +unsafe impl Pod for OrderedFloat {} +unsafe impl Zeroable for OrderedFloat {} +unsafe impl Pod for OrderedFloat {} + impl FromStr for OrderedFloat { type Err = T::Err; diff --git a/src/common/column/Cargo.toml b/src/common/column/Cargo.toml new file mode 100644 index 000000000000..5c5ecf86d325 --- /dev/null +++ b/src/common/column/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "databend-common-column" +version = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +edition = { workspace = true } + +[lib] +doctest = false +test = true + +[features] +default = ["column-default"] +serde_types = ["serde", "serde_derive"] +simd = [] + +column-default = [ + "serde_types", + "simd", +] + +[dependencies] + +databend-common-base = { workspace = true } +databend-common-exception = { workspace = true } + +arrow-buffer = { workspace = true } +arrow-data = { workspace = true } +arrow-schema = { workspace = true } +bytemuck = { workspace = true } +either = { workspace = true } +ethnum = { workspace = true } +foreign_vec = { workspace = true } +hex = { workspace = true } +match-template = { workspace = true } +num-traits = { workspace = true } +serde = { workspace = true, features = ["rc"], optional = true } +serde_derive = { workspace = true, optional = true } +serde_json = { workspace = true } +simdutf8 = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } + +[lints] +workspace = true diff --git a/src/common/column/src/binary/builder.rs b/src/common/column/src/binary/builder.rs new file mode 100644 index 000000000000..d32d4a00946a --- /dev/null +++ b/src/common/column/src/binary/builder.rs @@ -0,0 +1,247 @@ +// Copyright (c) 2020 Ritchie Vink +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Debug; +use std::iter::once; + +use serde::Deserialize; +use serde::Serialize; + +use super::BinaryColumn; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BinaryColumnBuilder { + // if the BinaryColumnBuilder is created with `data_capacity`, need_estimated is false + pub need_estimated: bool, + pub data: Vec, + pub offsets: Vec, +} + +impl BinaryColumnBuilder { + pub fn with_capacity(len: usize, data_capacity: usize) -> Self { + let mut offsets = Vec::with_capacity(len + 1); + offsets.push(0); + BinaryColumnBuilder { + need_estimated: data_capacity == 0 && len > 0, + data: Vec::with_capacity(data_capacity), + offsets, + } + } + + pub fn from_column(col: BinaryColumn) -> Self { + BinaryColumnBuilder { + need_estimated: col.data.is_empty(), + data: col.data.make_mut(), + offsets: col.offsets.to_vec(), + } + } + + pub fn from_data(data: Vec, offsets: Vec) -> Self { + debug_assert!({ offsets.windows(2).all(|w| w[0] <= w[1]) }); + + BinaryColumnBuilder { + need_estimated: false, + data, + offsets, + } + } + + pub fn repeat(scalar: &[u8], n: usize) -> Self { + let len = scalar.len(); + let data = scalar.repeat(n); + let offsets = once(0) + .chain((0..n).map(|i| (len * (i + 1)) as u64)) + .collect(); + BinaryColumnBuilder { + data, + offsets, + need_estimated: false, + } + } + + pub fn repeat_default(n: usize) -> Self { + BinaryColumnBuilder { + data: vec![], + offsets: vec![0; n + 1], + need_estimated: false, + } + } + + pub fn len(&self) -> usize { + self.offsets.len() - 1 + } + + pub fn is_empty(&self) -> bool { + self.offsets.len() <= 1 + } + + pub fn memory_size(&self) -> usize { + self.offsets.len() * 8 + self.data.len() + } + + pub fn put_u8(&mut self, item: u8) { + self.data.push(item); + } + + pub fn put_char(&mut self, item: char) { + self.data + .extend_from_slice(item.encode_utf8(&mut [0; 4]).as_bytes()); + } + + #[inline] + pub fn put_str(&mut self, item: &str) { + self.data.extend_from_slice(item.as_bytes()); + } + + #[inline] + pub fn put_slice(&mut self, item: &[u8]) { + self.data.extend_from_slice(item); + } + + pub fn put_char_iter(&mut self, iter: impl Iterator) { + for c in iter { + let mut buf = [0; 4]; + let result = c.encode_utf8(&mut buf); + self.data.extend_from_slice(result.as_bytes()); + } + } + + pub fn put(&mut self, item: &[u8]) { + self.data.extend_from_slice(item); + } + + #[inline] + pub fn commit_row(&mut self) { + self.offsets.push(self.data.len() as u64); + + if self.need_estimated + && self.offsets.len() - 1 == 64 + && self.offsets.len() < self.offsets.capacity() + { + let bytes_per_row = self.data.len() / 64 + 1; + let bytes_estimate = bytes_per_row * self.offsets.capacity(); + + const MAX_HINT_SIZE: usize = 1_000_000_000; + // if we are more than 10% over the capacity, we reserve more + if bytes_estimate < MAX_HINT_SIZE + && bytes_estimate as f64 > self.data.capacity() as f64 * 1.10f64 + { + self.data.reserve(bytes_estimate - self.data.capacity()); + } + } + } + + pub fn append_column(&mut self, other: &BinaryColumn) { + // the first offset of other column may not be zero + let other_start = *other.offsets.first().unwrap(); + let other_last = *other.offsets.last().unwrap(); + let start = self.offsets.last().cloned().unwrap(); + self.data + .extend_from_slice(&other.data[(other_start as usize)..(other_last as usize)]); + self.offsets.extend( + other + .offsets + .iter() + .skip(1) + .map(|offset| start + offset - other_start), + ); + } + + pub fn build(self) -> BinaryColumn { + BinaryColumn::new(self.data.into(), self.offsets.into()) + } + + pub fn build_scalar(self) -> Vec { + assert_eq!(self.offsets.len(), 2); + + self.data[(self.offsets[0] as usize)..(self.offsets[1] as usize)].to_vec() + } + + #[inline] + pub fn may_resize(&self, add_size: usize) -> bool { + self.data.len() + add_size > self.data.capacity() + } + + /// # Safety + /// + /// Calling this method with an out-of-bounds index is *[undefined behavior]* + pub unsafe fn index_unchecked(&self, row: usize) -> &[u8] { + debug_assert!(row + 1 < self.offsets.len()); + + let start = *self.offsets.get_unchecked(row) as usize; + let end = *self.offsets.get_unchecked(row + 1) as usize; + self.data.get_unchecked(start..end) + } + + pub fn push_repeat(&mut self, item: &[u8], n: usize) { + self.data.reserve(item.len() * n); + if self.need_estimated && self.offsets.len() - 1 < 64 { + for _ in 0..n { + self.data.extend_from_slice(item); + self.commit_row(); + } + } else { + let start = self.data.len(); + let len = item.len(); + for _ in 0..n { + self.data.extend_from_slice(item) + } + self.offsets + .extend((1..=n).map(|i| (start + len * i) as u64)); + } + } + + pub fn pop(&mut self) -> Option> { + if !self.is_empty() { + let index = self.len() - 1; + let start = unsafe { *self.offsets.get_unchecked(index) as usize }; + self.offsets.pop(); + let val = self.data.split_off(start); + Some(val) + } else { + None + } + } + + /// Extends the [`MutableBinaryArray`] from an iterator of values. + /// This differs from `extended_trusted_len` which accepts iterator of optional values. + #[inline] + pub fn extend_values(&mut self, iterator: I) + where + P: AsRef<[u8]>, + I: Iterator, + { + for item in iterator { + self.put_slice(item.as_ref()); + self.commit_row(); + } + } +} + +impl> FromIterator

for BinaryColumnBuilder { + fn from_iter>(iter: I) -> Self { + let iter = iter.into_iter(); + let (lower, _) = iter.size_hint(); + let mut builder = BinaryColumnBuilder::with_capacity(lower, 0); + builder.extend_values(iter); + builder + } +} + +impl From for BinaryColumn { + fn from(value: BinaryColumnBuilder) -> Self { + value.build() + } +} diff --git a/src/common/arrow/src/arrow/array/binary/fmt.rs b/src/common/column/src/binary/fmt.rs similarity index 52% rename from src/common/arrow/src/arrow/array/binary/fmt.rs rename to src/common/column/src/binary/fmt.rs index 44149a298b95..dbbf95623063 100644 --- a/src/common/arrow/src/arrow/array/binary/fmt.rs +++ b/src/common/column/src/binary/fmt.rs @@ -1,4 +1,4 @@ -// Copyright 2020-2022 Jorge C. Leitão +// Copyright (c) 2020 Ritchie Vink // Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,27 +18,32 @@ use std::fmt::Formatter; use std::fmt::Result; use std::fmt::Write; -use super::super::fmt::write_vec; -use super::BinaryArray; -use crate::arrow::offset::Offset; +use super::BinaryColumn; +use crate::fmt::write_vec; -pub fn write_value(array: &BinaryArray, index: usize, f: &mut W) -> Result { +pub fn write_value(array: &BinaryColumn, index: usize, f: &mut W) -> Result { let bytes = array.value(index); let writer = |f: &mut W, index| write!(f, "{}", bytes[index]); write_vec(f, writer, None, bytes.len(), "None", false) } -impl Debug for BinaryArray { - fn fmt(&self, f: &mut Formatter) -> Result { - let writer = |f: &mut Formatter, index| write_value(self, index, f); - - let head = if O::IS_LARGE { - "LargeBinaryArray" - } else { - "BinaryArray" - }; - write!(f, "{head}")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) +impl Debug for BinaryColumn { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("BinaryColumn") + .field( + "data", + &format_args!("0x{}", &hex::encode(self.data().as_slice())), + ) + .field("offsets", &self.offsets()) + .finish() } } + +// impl Debug for BinaryColumn { +// fn fmt(&self, f: &mut Formatter) -> Result { +// let writer = |f: &mut Formatter, index| write_value(self, index, f); +// write!(f, "BinaryColumn")?; +// write_vec(f, writer, None, self.len(), "None", false) +// } +// } diff --git a/src/common/column/src/binary/iterator.rs b/src/common/column/src/binary/iterator.rs new file mode 100644 index 000000000000..74a35f2ce6f7 --- /dev/null +++ b/src/common/column/src/binary/iterator.rs @@ -0,0 +1,61 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::builder::BinaryColumnBuilder; +use super::BinaryColumn; +use crate::iterator::ColumnAccessor; +use crate::iterator::ColumnValuesIter; + +unsafe impl<'a> ColumnAccessor<'a> for BinaryColumn { + type Item = &'a [u8]; + + #[inline] + unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { + self.index_unchecked(index) + } + + #[inline] + fn len(&self) -> usize { + self.offsets().len() - 1 + } +} + +/// Iterator of values of an [`BinaryArray`]. +pub type BinaryColumnIter<'a> = ColumnValuesIter<'a, BinaryColumn>; + +impl<'a> IntoIterator for &'a BinaryColumn { + type Item = &'a [u8]; + type IntoIter = BinaryColumnIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +unsafe impl<'a> ColumnAccessor<'a> for BinaryColumnBuilder { + type Item = &'a [u8]; + + #[inline] + unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { + self.index_unchecked(index) + } + + #[inline] + fn len(&self) -> usize { + self.offsets.len() - 1 + } +} + +/// Iterator of values of an [`BinaryColumnBuilder`]. +pub type BinaryColumnBuilderIter<'a> = ColumnValuesIter<'a, BinaryColumnBuilder>; diff --git a/src/common/column/src/binary/mod.rs b/src/common/column/src/binary/mod.rs new file mode 100644 index 000000000000..d6b4774b5cac --- /dev/null +++ b/src/common/column/src/binary/mod.rs @@ -0,0 +1,170 @@ +// Copyright (c) 2020 Ritchie Vink +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod builder; +pub(crate) mod fmt; +mod iterator; + +use std::ops::Range; + +use arrow_data::ArrayData; +use arrow_data::ArrayDataBuilder; +use arrow_schema::DataType; +pub use builder::BinaryColumnBuilder; +pub use iterator::BinaryColumnBuilderIter; +pub use iterator::BinaryColumnIter; + +use crate::bitmap::utils::BitmapIter; +use crate::bitmap::utils::ZipValidity; +use crate::bitmap::Bitmap; +use crate::buffer::Buffer; +use crate::error::Error; +use crate::error::Result; + +#[derive(Clone, PartialEq)] +pub struct BinaryColumn { + pub(crate) data: Buffer, + pub(crate) offsets: Buffer, +} + +impl BinaryColumn { + pub fn new(data: Buffer, offsets: Buffer) -> Self { + debug_assert!({ offsets.windows(2).all(|w| w[0] <= w[1]) }); + + BinaryColumn { data, offsets } + } + + pub fn len(&self) -> usize { + self.offsets.len() - 1 + } + + pub fn is_empty(&self) -> bool { + self.offsets.len() <= 1 + } + + pub fn total_bytes_len(&self) -> usize { + (*self.offsets().last().unwrap() - *self.offsets().first().unwrap()) as _ + } + + pub fn data(&self) -> &Buffer { + &self.data + } + + pub fn offsets(&self) -> &Buffer { + &self.offsets + } + + pub fn memory_size(&self) -> usize { + let offsets = self.offsets.as_slice(); + let len = offsets.len(); + len * 8 + (offsets[len - 1] - offsets[0]) as usize + } + + pub fn index(&self, index: usize) -> Option<&[u8]> { + if index + 1 < self.offsets.len() { + Some(&self.data[(self.offsets[index] as usize)..(self.offsets[index + 1] as usize)]) + } else { + None + } + } + + pub fn value(&self, index: usize) -> &[u8] { + assert!(index + 1 < self.offsets.len()); + &self.data[(self.offsets[index] as usize)..(self.offsets[index + 1] as usize)] + } + + /// # Safety + /// + /// Calling this method with an out-of-bounds index is *[undefined behavior]* + #[inline] + pub unsafe fn index_unchecked(&self, index: usize) -> &[u8] { + let start = *self.offsets.get_unchecked(index) as usize; + let end = *self.offsets.get_unchecked(index + 1) as usize; + self.data.get_unchecked(start..end) + } + + pub fn slice(&self, range: Range) -> Self { + let offsets = self + .offsets + .clone() + .sliced(range.start, range.end - range.start + 1); + BinaryColumn { + data: self.data.clone(), + offsets, + } + } + + pub fn iter(&self) -> BinaryColumnIter { + BinaryColumnIter::new(self) + } + + pub fn option_iter<'a>( + &'a self, + validity: Option<&'a Bitmap>, + ) -> ZipValidity<&'a [u8], BinaryColumnIter, BitmapIter<'a>> { + let bitmap_iter = validity.as_ref().map(|v| v.iter()); + ZipValidity::new(self.iter(), bitmap_iter) + } + + pub fn into_buffer(self) -> (Buffer, Buffer) { + (self.data, self.offsets) + } + + pub fn check_valid(&self) -> Result<()> { + let offsets = self.offsets.as_slice(); + let len = offsets.len(); + if len < 1 { + return Err(Error::OutOfSpec(format!( + "BinaryColumn offsets length must be equal or greater than 1, but got {}", + len + ))); + } + + for i in 1..len { + if offsets[i] < offsets[i - 1] { + return Err(Error::OutOfSpec(format!( + "BinaryColumn offsets value must be equal or greater than previous value, but got {}", + offsets[i] + ))); + } + } + Ok(()) + } +} + +impl From for ArrayData { + fn from(column: BinaryColumn) -> Self { + let builder = ArrayDataBuilder::new(DataType::LargeBinary) + .len(column.len()) + .buffers(vec![column.offsets.into(), column.data.into()]); + + unsafe { builder.build_unchecked() } + } +} + +impl From for BinaryColumn { + fn from(data: ArrayData) -> Self { + let offsets = data.buffers()[0].clone(); + let values = data.buffers()[1].clone(); + + BinaryColumn::new(values.into(), offsets.into()) + } +} + +impl> FromIterator

for BinaryColumn { + fn from_iter>(iter: I) -> Self { + BinaryColumnBuilder::from_iter(iter).into() + } +} diff --git a/src/common/arrow/src/arrow/array/binview/mutable.rs b/src/common/column/src/binview/builder.rs similarity index 62% rename from src/common/arrow/src/arrow/array/binview/mutable.rs rename to src/common/column/src/binview/builder.rs index c4a4f7a3c866..7ea031694a1c 100644 --- a/src/common/arrow/src/arrow/array/binview/mutable.rs +++ b/src/common/column/src/binview/builder.rs @@ -13,32 +13,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::fmt::Debug; use std::fmt::Formatter; +use std::iter::TrustedLen; use std::sync::Arc; -use crate::arrow::array::binview::iterator::MutableBinaryViewValueIter; -use crate::arrow::array::binview::view::validate_utf8_only; -use crate::arrow::array::binview::BinaryViewArrayGeneric; -use crate::arrow::array::binview::View; -use crate::arrow::array::binview::ViewType; -use crate::arrow::array::Array; -use crate::arrow::array::MutableArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::arrow::trusted_len::TrustedLen; -use crate::arrow::types::NativeType; +use super::view::CheckUTF8; +use crate::binary::BinaryColumn; +use crate::binview::iterator::BinaryViewBuilderIter; +use crate::binview::view::validate_utf8_only; +use crate::binview::BinaryViewColumnGeneric; +use crate::binview::View; +use crate::binview::ViewType; +use crate::buffer::Buffer; +use crate::error::Result; +use crate::types::NativeType; const DEFAULT_BLOCK_SIZE: usize = 8 * 1024; -pub struct MutableBinaryViewArray { +pub struct BinaryViewColumnBuilder { pub(super) views: Vec, pub(super) completed_buffers: Vec>, pub(super) in_progress_buffer: Vec, - pub(super) validity: Option, pub(super) phantom: std::marker::PhantomData, /// Total bytes length if we would concatenate them all. pub total_bytes_len: usize, @@ -46,13 +42,12 @@ pub struct MutableBinaryViewArray { pub total_buffer_len: usize, } -impl Clone for MutableBinaryViewArray { +impl Clone for BinaryViewColumnBuilder { fn clone(&self) -> Self { Self { views: self.views.clone(), completed_buffers: self.completed_buffers.clone(), in_progress_buffer: self.in_progress_buffer.clone(), - validity: self.validity.clone(), phantom: Default::default(), total_bytes_len: self.total_bytes_len, total_buffer_len: self.total_buffer_len, @@ -60,33 +55,31 @@ impl Clone for MutableBinaryViewArray { } } -impl Debug for MutableBinaryViewArray { +impl Debug for BinaryViewColumnBuilder { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "mutable-binview{:?}", T::DATA_TYPE) + write!(f, "BinaryViewColumnBuilder{:?}", T::name()) } } -impl Default for MutableBinaryViewArray { +impl Default for BinaryViewColumnBuilder { fn default() -> Self { Self::with_capacity(0) } } -impl From> for BinaryViewArrayGeneric { - fn from(mut value: MutableBinaryViewArray) -> Self { +impl From> for BinaryViewColumnGeneric { + fn from(mut value: BinaryViewColumnBuilder) -> Self { value.finish_in_progress(); Self::new_unchecked( - T::DATA_TYPE, value.views.into(), Arc::from(value.completed_buffers), - value.validity.map(|b| b.into()), value.total_bytes_len, value.total_buffer_len, ) } } -impl MutableBinaryViewArray { +impl BinaryViewColumnBuilder { pub fn new() -> Self { Self::default() } @@ -96,7 +89,6 @@ impl MutableBinaryViewArray { views: Vec::with_capacity(capacity), completed_buffers: vec![], in_progress_buffer: vec![], - validity: None, phantom: Default::default(), total_buffer_len: 0, total_bytes_len: 0, @@ -113,14 +105,6 @@ impl MutableBinaryViewArray { &self.views } - pub fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - pub fn validity_mut(&mut self) -> Option<&mut MutableBitmap> { - self.validity.as_mut() - } - /// Reserves `additional` elements and `additional_buffer` on the buffer. pub fn reserve(&mut self, additional: usize) { self.views.reserve(additional); @@ -140,15 +124,6 @@ impl MutableBinaryViewArray { self.views.capacity() } - fn init_validity(&mut self, unset_last: bool) { - let mut validity = MutableBitmap::with_capacity(self.views.capacity()); - validity.extend_constant(self.len(), true); - if unset_last { - validity.set(self.len() - 1, false); - } - self.validity = Some(validity); - } - /// # Safety /// - caller must allocate enough capacity /// - caller must ensure the view and buffers match. @@ -164,11 +139,11 @@ impl MutableBinaryViewArray { let offset = v.offset as usize; let bytes = data.get_unchecked(offset..offset + len as usize); let t = T::from_bytes_unchecked(bytes); - self.push_value_ignore_validity(t) + self.push_value(t) } } - pub fn push_value_ignore_validity>(&mut self, value: V) { + pub fn push_value>(&mut self, value: V) { let value = value.as_ref(); let bytes = value.to_bytes(); self.total_bytes_len += bytes.len(); @@ -216,71 +191,17 @@ impl MutableBinaryViewArray { self.views.push(value); } - pub fn push_value>(&mut self, value: V) { - if let Some(validity) = &mut self.validity { - validity.push(true) - } - self.push_value_ignore_validity(value) - } - - pub fn push>(&mut self, value: Option) { - if let Some(value) = value { - self.push_value(value) - } else { - self.push_null() - } - } - - pub fn push_null(&mut self) { - self.views.push(View::default()); - match &mut self.validity { - Some(validity) => validity.push(false), - None => self.init_validity(true), - } - } - - pub fn extend_null(&mut self, additional: usize) { - if self.validity.is_none() && additional > 0 { - self.init_validity(false); - } - self.views - .extend(std::iter::repeat(View::default()).take(additional)); - if let Some(validity) = &mut self.validity { - validity.extend_constant(additional, false); - } - } - - pub fn extend_constant>(&mut self, additional: usize, value: Option) { - if value.is_none() && self.validity.is_none() { - self.init_validity(false); - } - - if let Some(validity) = &mut self.validity { - validity.extend_constant(additional, value.is_some()) - } - - // Push and pop to get the properly encoded value. - // For long string this leads to a dictionary encoding, - // as we push the string only once in the buffers - + pub fn extend_constant>(&mut self, additional: usize, value: V) { let old_bytes_len = self.total_bytes_len; - let view_value = value - .map(|v| { - self.push_value_ignore_validity(v); - self.views.pop().unwrap() - }) - .unwrap_or_default(); + self.push_value(value); + let value = self.views.pop().unwrap(); self.total_bytes_len += (self.total_bytes_len - old_bytes_len) * additional.saturating_sub(1); - - self.views - .extend(std::iter::repeat(view_value).take(additional)); + self.views.extend(std::iter::repeat(value).take(additional)); } - impl_mutable_array_mut_validity!(); - #[inline] pub fn extend_values(&mut self, iterator: I) where @@ -305,19 +226,19 @@ impl MutableBinaryViewArray { #[inline] pub fn extend(&mut self, iterator: I) where - I: Iterator>, + I: Iterator, P: AsRef, { self.reserve(iterator.size_hint().0); for p in iterator { - self.push(p) + self.push_value(p) } } #[inline] pub fn extend_trusted_len(&mut self, iterator: I) where - I: TrustedLen>, + I: TrustedLen, P: AsRef, { self.extend(iterator) @@ -326,12 +247,12 @@ impl MutableBinaryViewArray { #[inline] pub fn from_iterator(iterator: I) -> Self where - I: Iterator>, + I: Iterator, P: AsRef, { - let mut mutable = Self::with_capacity(iterator.size_hint().0); - mutable.extend(iterator); - mutable + let mut builder = Self::with_capacity(iterator.size_hint().0); + builder.extend(iterator); + builder } pub fn from_values_iter(iterator: I) -> Self @@ -339,12 +260,12 @@ impl MutableBinaryViewArray { I: Iterator, P: AsRef, { - let mut mutable = Self::with_capacity(iterator.size_hint().0); - mutable.extend_values(iterator); - mutable + let mut builder = Self::with_capacity(iterator.size_hint().0); + builder.extend_values(iterator); + builder } - pub fn from, P: AsRef<[Option]>>(slice: P) -> Self { + pub fn from, P: AsRef<[S]>>(slice: P) -> Self { Self::from_iterator(slice.as_ref().iter().map(|opt_v| opt_v.as_ref())) } @@ -356,7 +277,7 @@ impl MutableBinaryViewArray { } #[inline] - pub fn freeze(self) -> BinaryViewArrayGeneric { + pub fn freeze(self) -> BinaryViewColumnGeneric { self.into() } @@ -397,17 +318,17 @@ impl MutableBinaryViewArray { T::from_bytes_unchecked(bytes) } - /// Returns an iterator of `&[u8]` over every element of this array, ignoring the validity - pub fn values_iter(&self) -> MutableBinaryViewValueIter { - MutableBinaryViewValueIter::new(self) + /// Returns an iterator of `&[u8]` over every element of this array + pub fn iter(&self) -> BinaryViewBuilderIter { + BinaryViewBuilderIter::new(self) } pub fn values(&self) -> Vec<&T> { - self.values_iter().collect() + self.iter().collect() } } -impl MutableBinaryViewArray<[u8]> { +impl BinaryViewColumnBuilder<[u8]> { pub fn validate_utf8(&mut self) -> Result<()> { self.finish_in_progress(); // views are correct @@ -415,70 +336,40 @@ impl MutableBinaryViewArray<[u8]> { } } -impl MutableBinaryViewArray { +impl BinaryViewColumnBuilder { + pub fn try_from_bin_column(col: BinaryColumn) -> Result { + let mut data = Self::with_capacity(col.len()); + col.data.as_slice().check_utf8()?; + + for v in col.iter() { + data.push_value(unsafe { std::str::from_utf8_unchecked(v) }); + } + + Ok(data) + } + pub fn pop(&mut self) -> Option { if self.is_empty() { return None; } let value = unsafe { self.value_unchecked(self.len() - 1).to_string() }; - - self.views.pop(); self.total_bytes_len -= value.len(); + self.views.pop(); Some(value) } } -impl> Extend> for MutableBinaryViewArray { +impl> Extend

for BinaryViewColumnBuilder { #[inline] - fn extend>>(&mut self, iter: I) { + fn extend>(&mut self, iter: I) { Self::extend(self, iter.into_iter()) } } -impl> FromIterator> for MutableBinaryViewArray { +impl> FromIterator

for BinaryViewColumnBuilder { #[inline] - fn from_iter>>(iter: I) -> Self { + fn from_iter>(iter: I) -> Self { Self::from_iterator(iter.into_iter()) } } - -impl MutableArray for MutableBinaryViewArray { - fn data_type(&self) -> &DataType { - T::data_type() - } - - fn len(&self) -> usize { - MutableBinaryViewArray::len(self) - } - - fn validity(&self) -> Option<&MutableBitmap> { - self.validity.as_ref() - } - - fn as_box(&mut self) -> Box { - let mutable = std::mem::take(self); - let arr: BinaryViewArrayGeneric = mutable.into(); - arr.boxed() - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn Any { - self - } - - fn push_null(&mut self) { - MutableBinaryViewArray::push_null(self) - } - - fn reserve(&mut self, additional: usize) { - MutableBinaryViewArray::reserve(self, additional) - } - - fn shrink_to_fit(&mut self) { - self.views.shrink_to_fit() - } -} diff --git a/src/common/arrow/src/arrow/array/binview/fmt.rs b/src/common/column/src/binview/fmt.rs similarity index 68% rename from src/common/arrow/src/arrow/array/binview/fmt.rs rename to src/common/column/src/binview/fmt.rs index 463f4ab3641b..6c7a0300e5aa 100644 --- a/src/common/arrow/src/arrow/array/binview/fmt.rs +++ b/src/common/column/src/binview/fmt.rs @@ -18,15 +18,14 @@ use std::fmt::Formatter; use std::fmt::Result; use std::fmt::Write; -use crate::arrow::array::binview::BinaryViewArray; -use crate::arrow::array::binview::BinaryViewArrayGeneric; -use crate::arrow::array::binview::Utf8ViewArray; -use crate::arrow::array::binview::ViewType; -use crate::arrow::array::fmt::write_vec; -use crate::arrow::array::Array; +use crate::binview::BinaryViewColumn; +use crate::binview::BinaryViewColumnGeneric; +use crate::binview::Utf8ViewColumn; +use crate::binview::ViewType; +use crate::fmt::write_vec; pub fn write_value<'a, T: ViewType + ?Sized, W: Write>( - array: &'a BinaryViewArrayGeneric, + array: &'a BinaryViewColumnGeneric, index: usize, f: &mut W, ) -> Result @@ -39,18 +38,18 @@ where write_vec(f, writer, None, bytes.len(), "None", false) } -impl Debug for BinaryViewArray { +impl Debug for BinaryViewColumn { fn fmt(&self, f: &mut Formatter) -> Result { let writer = |f: &mut Formatter, index| write_value(self, index, f); - write!(f, "BinaryViewArray")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) + write!(f, "BinaryViewColumn")?; + write_vec(f, writer, None, self.len(), "None", false) } } -impl Debug for Utf8ViewArray { +impl Debug for Utf8ViewColumn { fn fmt(&self, f: &mut Formatter) -> Result { let writer = |f: &mut Formatter, index| write!(f, "{}", self.value(index)); - write!(f, "Utf8ViewArray")?; - write_vec(f, writer, self.validity(), self.len(), "None", false) + write!(f, "StringColumn")?; + write_vec(f, writer, None, self.len(), "None", false) } } diff --git a/src/common/arrow/src/arrow/array/binview/iterator.rs b/src/common/column/src/binview/iterator.rs similarity index 55% rename from src/common/arrow/src/arrow/array/binview/iterator.rs rename to src/common/column/src/binview/iterator.rs index 26511537efdf..874d0e15ce07 100644 --- a/src/common/arrow/src/arrow/array/binview/iterator.rs +++ b/src/common/column/src/binview/iterator.rs @@ -12,15 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::arrow::array::binview::mutable::MutableBinaryViewArray; -use crate::arrow::array::binview::BinaryViewArrayGeneric; -use crate::arrow::array::binview::ViewType; -use crate::arrow::array::ArrayAccessor; -use crate::arrow::array::ArrayValuesIter; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; +use super::BinaryViewColumnGeneric; +use crate::binview::builder::BinaryViewColumnBuilder; +use crate::binview::ViewType; +use crate::iterator::ColumnAccessor; +use crate::iterator::ColumnValuesIter; -unsafe impl<'a, T: ViewType + ?Sized> ArrayAccessor<'a> for BinaryViewArrayGeneric { +unsafe impl<'a, T: ViewType + ?Sized> ColumnAccessor<'a> for BinaryViewColumnGeneric { type Item = &'a T; #[inline] @@ -35,18 +33,18 @@ unsafe impl<'a, T: ViewType + ?Sized> ArrayAccessor<'a> for BinaryViewArrayGener } /// Iterator of values of an [`BinaryArray`]. -pub type BinaryViewValueIter<'a, T> = ArrayValuesIter<'a, BinaryViewArrayGeneric>; +pub type BinaryViewColumnIter<'a, T> = ColumnValuesIter<'a, BinaryViewColumnGeneric>; -impl<'a, T: ViewType + ?Sized> IntoIterator for &'a BinaryViewArrayGeneric { - type Item = Option<&'a T>; - type IntoIter = ZipValidity<&'a T, BinaryViewValueIter<'a, T>, BitmapIter<'a>>; +impl<'a, T: ViewType + ?Sized> IntoIterator for &'a BinaryViewColumnGeneric { + type Item = &'a T; + type IntoIter = BinaryViewColumnIter<'a, T>; fn into_iter(self) -> Self::IntoIter { self.iter() } } -unsafe impl<'a, T: ViewType + ?Sized> ArrayAccessor<'a> for MutableBinaryViewArray { +unsafe impl<'a, T: ViewType + ?Sized> ColumnAccessor<'a> for BinaryViewColumnBuilder { type Item = &'a T; #[inline] @@ -60,5 +58,5 @@ unsafe impl<'a, T: ViewType + ?Sized> ArrayAccessor<'a> for MutableBinaryViewArr } } -/// Iterator of values of an [`MutableBinaryViewArray`]. -pub type MutableBinaryViewValueIter<'a, T> = ArrayValuesIter<'a, MutableBinaryViewArray>; +/// Iterator of values of an [`BinaryViewColumnBuilder`]. +pub type BinaryViewBuilderIter<'a, T> = ColumnValuesIter<'a, BinaryViewColumnBuilder>; diff --git a/src/common/arrow/src/arrow/array/binview/mod.rs b/src/common/column/src/binview/mod.rs similarity index 51% rename from src/common/arrow/src/arrow/array/binview/mod.rs rename to src/common/column/src/binview/mod.rs index 2d0ebc49e5b9..58e52b234f33 100644 --- a/src/common/arrow/src/arrow/array/binview/mod.rs +++ b/src/common/column/src/binview/mod.rs @@ -13,52 +13,49 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod builder; pub(crate) mod fmt; -mod from; mod iterator; -mod mutable; mod view; -mod private { - pub trait Sealed: Send + Sync {} - - impl Sealed for str {} - - impl Sealed for [u8] {} -} - -use std::any::Any; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::sync::Arc; +use arrow_data::ArrayData; +use arrow_data::ArrayDataBuilder; +use arrow_schema::DataType; +pub use builder::BinaryViewColumnBuilder; use either::Either; -pub use iterator::BinaryViewValueIter; -pub use mutable::MutableBinaryViewArray; +pub use iterator::BinaryViewColumnIter; use private::Sealed; +use view::validate_utf8_only; +pub use view::CheckUTF8; pub use view::View; -use crate::arrow::array::binview::view::validate_utf8_only; -use crate::arrow::array::iterator::NonNullValuesIter; -use crate::arrow::array::Array; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::utils::ZipValidity; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; +use crate::binary::BinaryColumn; +use crate::binary::BinaryColumnBuilder; +use crate::bitmap::utils::BitmapIter; +use crate::bitmap::utils::ZipValidity; +use crate::bitmap::Bitmap; +use crate::buffer::Buffer; +use crate::error::Result; +use crate::impl_sliced; -static BIN_VIEW_TYPE: DataType = DataType::BinaryView; -static UTF8_VIEW_TYPE: DataType = DataType::Utf8View; +mod private { + pub trait Sealed: Send + Sync {} + + impl Sealed for str {} + + impl Sealed for [u8] {} +} const UNKNOWN_LEN: u64 = u64::MAX; pub trait ViewType: Sealed + 'static + PartialEq + AsRef { const IS_UTF8: bool; - const DATA_TYPE: DataType; type Owned: Debug + Clone + Sync + Send + AsRef; /// # Safety @@ -70,12 +67,17 @@ pub trait ViewType: Sealed + 'static + PartialEq + AsRef { #[allow(clippy::wrong_self_convention)] fn into_owned(&self) -> Self::Owned; - fn data_type() -> &'static DataType; + fn name() -> &'static str { + if Self::IS_UTF8 { + "StringView" + } else { + "BinaryView" + } + } } impl ViewType for str { const IS_UTF8: bool = true; - const DATA_TYPE: DataType = DataType::Utf8View; type Owned = String; #[inline(always)] @@ -91,15 +93,10 @@ impl ViewType for str { fn into_owned(&self) -> Self::Owned { self.to_string() } - - fn data_type() -> &'static DataType { - &UTF8_VIEW_TYPE - } } impl ViewType for [u8] { const IS_UTF8: bool = false; - const DATA_TYPE: DataType = DataType::BinaryView; type Owned = Vec; #[inline(always)] @@ -115,17 +112,11 @@ impl ViewType for [u8] { fn into_owned(&self) -> Self::Owned { self.to_vec() } - - fn data_type() -> &'static DataType { - &BIN_VIEW_TYPE - } } -pub struct BinaryViewArrayGeneric { - data_type: DataType, +pub struct BinaryViewColumnGeneric { views: Buffer, buffers: Arc<[Buffer]>, - validity: Option, phantom: PhantomData, /// Total bytes length if we would concat them all total_bytes_len: AtomicU64, @@ -133,19 +124,12 @@ pub struct BinaryViewArrayGeneric { total_buffer_len: usize, } -impl PartialEq for BinaryViewArrayGeneric { - fn eq(&self, other: &Self) -> bool { - self.into_iter().zip(other).all(|(l, r)| l == r) - } -} - -impl Clone for BinaryViewArrayGeneric { +impl Clone for BinaryViewColumnGeneric { fn clone(&self) -> Self { Self { - data_type: self.data_type.clone(), views: self.views.clone(), buffers: self.buffers.clone(), - validity: self.validity.clone(), + phantom: Default::default(), total_bytes_len: AtomicU64::new(self.total_bytes_len.load(Ordering::Relaxed)), total_buffer_len: self.total_buffer_len, @@ -153,16 +137,15 @@ impl Clone for BinaryViewArrayGeneric { } } -unsafe impl Send for BinaryViewArrayGeneric {} +unsafe impl Send for BinaryViewColumnGeneric {} -unsafe impl Sync for BinaryViewArrayGeneric {} +unsafe impl Sync for BinaryViewColumnGeneric {} -impl BinaryViewArrayGeneric { +impl BinaryViewColumnGeneric { pub fn new_unchecked( - data_type: DataType, views: Buffer, buffers: Arc<[Buffer]>, - validity: Option, + total_bytes_len: usize, total_buffer_len: usize, ) -> Self { @@ -183,37 +166,28 @@ impl BinaryViewArrayGeneric { // - the data is valid utf8 (if required) // - the offsets match the buffers. Self { - data_type, views, buffers, - validity, + phantom: Default::default(), total_bytes_len: AtomicU64::new(total_bytes_len as u64), total_buffer_len, } } - /// Create a new BinaryViewArray but initialize a statistics compute. + /// Create a new BinaryViewColumn but initialize a statistics compute. /// # Safety /// The caller must ensure the invariants pub unsafe fn new_unchecked_unknown_md( - data_type: DataType, views: Buffer, buffers: Arc<[Buffer]>, - validity: Option, + total_buffer_len: Option, ) -> Self { let total_bytes_len = UNKNOWN_LEN as usize; let total_buffer_len = total_buffer_len.unwrap_or_else(|| buffers.iter().map(|b| b.len()).sum()); - Self::new_unchecked( - data_type, - views, - buffers, - validity, - total_bytes_len, - total_buffer_len, - ) + Self::new_unchecked(views, buffers, total_bytes_len, total_buffer_len) } pub fn data_buffers(&self) -> &Arc<[Buffer]> { @@ -228,72 +202,29 @@ impl BinaryViewArrayGeneric { &self.views } - pub fn try_new( - data_type: DataType, - views: Buffer, - buffers: Arc<[Buffer]>, - validity: Option, - ) -> Result { - if data_type.to_physical_type() != Self::default_data_type().to_physical_type() { - return Err(Error::oos( - "BinaryViewArray can only be initialized with DataType::BinaryView or DataType::Utf8View", - )); - } - + pub fn try_new(views: Buffer, buffers: Arc<[Buffer]>) -> Result { #[cfg(debug_assertions)] { if T::IS_UTF8 { - crate::arrow::array::binview::view::validate_utf8_view( - views.as_ref(), - buffers.as_ref(), - )?; + crate::binview::view::validate_utf8_view(views.as_ref(), buffers.as_ref())?; } else { - crate::arrow::array::binview::view::validate_binary_view( - views.as_ref(), - buffers.as_ref(), - )?; + crate::binview::view::validate_binary_view(views.as_ref(), buffers.as_ref())?; } } - if let Some(validity) = &validity { - if validity.len() != views.len() { - return Err(Error::oos( - "validity mask length must match the number of values", - )); - } - } - - unsafe { - Ok(Self::new_unchecked_unknown_md( - data_type, views, buffers, validity, None, - )) - } + unsafe { Ok(Self::new_unchecked_unknown_md(views, buffers, None)) } } - /// Returns a new [`BinaryViewArrayGeneric`] from a slice of `&T`. + /// Returns a new [`BinaryViewColumnGeneric`] from a slice of `&T`. // Note: this can't be `impl From` because Rust does not allow double `AsRef` on it. - pub fn from, P: AsRef<[Option]>>(slice: P) -> Self { - MutableBinaryViewArray::::from(slice).into() + pub fn from, P: AsRef<[V]>>(slice: P) -> Self { + BinaryViewColumnBuilder::::from(slice).into() } - /// Creates an empty [`BinaryViewArrayGeneric`], i.e. whose `.len` is zero. + /// Creates an empty [`BinaryViewColumnGeneric`], i.e. whose `.len` is zero. #[inline] - pub fn new_empty(data_type: DataType) -> Self { - Self::new_unchecked(data_type, Buffer::new(), Arc::from([]), None, 0, 0) - } - - /// Returns a new null [`BinaryViewArrayGeneric`] of `length`. - #[inline] - pub fn new_null(data_type: DataType, length: usize) -> Self { - let validity = Some(Bitmap::new_zeroed(length)); - Self::new_unchecked( - data_type, - Buffer::zeroed(length), - Arc::from([]), - validity, - 0, - 0, - ) + pub fn new_empty() -> Self { + Self::new_unchecked(Buffer::new(), Arc::from([]), 0, 0) } /// Returns the element at index `i` @@ -305,6 +236,16 @@ impl BinaryViewArrayGeneric { unsafe { self.value_unchecked(i) } } + /// Returns the element at index `i` + #[inline] + pub fn index(&self, i: usize) -> Option<&T> { + if i < self.len() { + Some(unsafe { self.value_unchecked(i) }) + } else { + None + } + } + /// Returns the element at index `i` /// # Safety /// Assumes that the `i < self.len`. @@ -314,36 +255,43 @@ impl BinaryViewArrayGeneric { T::from_bytes_unchecked(v.get_slice_unchecked(&self.buffers)) } - /// Returns an iterator of `Option<&T>` over every element of this array. - pub fn iter(&self) -> ZipValidity<&T, BinaryViewValueIter, BitmapIter> { - ZipValidity::new_with_validity(self.values_iter(), self.validity.as_ref()) + /// same as value_unchecked + /// # Safety + /// Assumes that the `i < self.len`. + #[inline] + pub unsafe fn index_unchecked(&self, i: usize) -> &T { + let v = self.views.get_unchecked(i); + T::from_bytes_unchecked(v.get_slice_unchecked(&self.buffers)) } - /// Returns an iterator of `&[u8]` over every element of this array, ignoring the validity - pub fn values_iter(&self) -> BinaryViewValueIter { - BinaryViewValueIter::new(self) + /// same as value_unchecked, yet it will return bytes + /// # Safety + /// Assumes that the `i < self.len`. + #[inline] + pub unsafe fn index_unchecked_bytes(&self, i: usize) -> &[u8] { + let v = self.views.get_unchecked(i); + v.get_slice_unchecked(&self.buffers) } - pub fn len_iter(&self) -> impl Iterator + '_ { - self.views.iter().map(|v| v.length) + /// Returns an iterator of `&[u8]` over every element of this array, ignoring the validity + pub fn iter(&self) -> BinaryViewColumnIter { + BinaryViewColumnIter::new(self) } - /// Returns an iterator of the non-null values. - pub fn non_null_values_iter(&self) -> NonNullValuesIter<'_, BinaryViewArrayGeneric> { - NonNullValuesIter::new(self, self.validity()) + pub fn option_iter<'a>( + &'a self, + validity: Option<&'a Bitmap>, + ) -> ZipValidity<&'a T, BinaryViewColumnIter, BitmapIter<'a>> { + let bitmap_iter = validity.as_ref().map(|v| v.iter()); + ZipValidity::new(self.iter(), bitmap_iter) } - /// Returns an iterator of the non-null values. - pub fn non_null_views_iter(&self) -> NonNullValuesIter<'_, Buffer> { - NonNullValuesIter::new(self.views(), self.validity()) + pub fn len_iter(&self) -> impl Iterator + '_ { + self.views.iter().map(|v| v.length) } - impl_sliced!(); - impl_mut_validity!(); - impl_into_array!(); - - pub fn from_slice, P: AsRef<[Option]>>(slice: P) -> Self { - let mutable = MutableBinaryViewArray::from_iterator( + pub fn from_slice, P: AsRef<[S]>>(slice: P) -> Self { + let mutable = BinaryViewColumnBuilder::from_iterator( slice.as_ref().iter().map(|opt_v| opt_v.as_ref()), ); mutable.into() @@ -351,7 +299,7 @@ impl BinaryViewArrayGeneric { pub fn from_slice_values, P: AsRef<[S]>>(slice: P) -> Self { let mutable = - MutableBinaryViewArray::from_values_iter(slice.as_ref().iter().map(|v| v.as_ref())); + BinaryViewColumnBuilder::from_values_iter(slice.as_ref().iter().map(|v| v.as_ref())); mutable.into() } @@ -367,6 +315,10 @@ impl BinaryViewArrayGeneric { } } + pub fn memory_size(&self) -> usize { + self.total_buffer_len() + self.len() * 12 + } + fn total_unshared_buffer_len(&self) -> usize { // Given this function is only called in `maybe_gc()`, // it may not be worthy to add an extra field for this. @@ -402,19 +354,35 @@ impl BinaryViewArrayGeneric { if self.buffers.is_empty() { return self; } - let mut mutable = MutableBinaryViewArray::with_capacity(self.len()); + let mut mutable = BinaryViewColumnBuilder::with_capacity(self.len()); let buffers = self.buffers.as_ref(); for view in self.views.as_ref() { unsafe { mutable.push_view_unchecked(*view, buffers) } } - mutable.freeze().with_validity(self.validity) + mutable.freeze() } pub fn is_sliced(&self) -> bool { self.views.as_ptr() != self.views.data_ptr() } + fn slice(&mut self, offset: usize, length: usize) { + assert!( + offset + length <= self.len(), + "the offset of the new Buffer cannot exceed the existing length" + ); + unsafe { self.slice_unchecked(offset, length) } + } + + unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { + debug_assert!(offset + length <= self.len()); + self.views.slice_unchecked(offset, length); + self.total_bytes_len.store(UNKNOWN_LEN, Ordering::Relaxed) + } + + impl_sliced!(); + pub fn maybe_gc(self) -> Self { const GC_MINIMUM_SAVINGS: usize = 16 * 1024; // At least 16 KiB. @@ -447,15 +415,14 @@ impl BinaryViewArrayGeneric { } } - pub fn make_mut(self) -> MutableBinaryViewArray { + pub fn make_mut(self) -> BinaryViewColumnBuilder { let views = self.views.make_mut(); let completed_buffers = self.buffers.to_vec(); - let validity = self.validity.map(|bitmap| bitmap.make_mut()); - MutableBinaryViewArray { + BinaryViewColumnBuilder { views, completed_buffers, in_progress_buffer: vec![], - validity, + phantom: Default::default(), total_bytes_len: self.total_bytes_len.load(Ordering::Relaxed) as usize, total_buffer_len: self.total_buffer_len, @@ -463,185 +430,213 @@ impl BinaryViewArrayGeneric { } #[must_use] - pub fn into_mut(self) -> Either> { + pub fn into_mut(self) -> Either> { use Either::*; let is_unique = (Arc::strong_count(&self.buffers) + Arc::weak_count(&self.buffers)) == 1; - if let Some(bitmap) = self.validity { - match bitmap.into_mut() { - Left(bitmap) => Left(Self::new_unchecked( - self.data_type, - self.views, - self.buffers, - Some(bitmap), - self.total_bytes_len.load(Ordering::Relaxed) as usize, - self.total_buffer_len, - )), - Right(mutable_bitmap) => match (self.views.into_mut(), is_unique) { - (Right(views), true) => Right(MutableBinaryViewArray { - views, - completed_buffers: self.buffers.to_vec(), - in_progress_buffer: vec![], - validity: Some(mutable_bitmap), - phantom: Default::default(), - total_bytes_len: self.total_bytes_len.load(Ordering::Relaxed) as usize, - total_buffer_len: self.total_buffer_len, - }), - (Right(views), false) => Left(Self::new_unchecked( - self.data_type, - views.into(), - self.buffers, - Some(mutable_bitmap.into()), - self.total_bytes_len.load(Ordering::Relaxed) as usize, - self.total_buffer_len, - )), - (Left(views), _) => Left(Self::new_unchecked( - self.data_type, - views, - self.buffers, - Some(mutable_bitmap.into()), - self.total_bytes_len.load(Ordering::Relaxed) as usize, - self.total_buffer_len, - )), - }, - } - } else { - match (self.views.into_mut(), is_unique) { - (Right(views), true) => Right(MutableBinaryViewArray { - views, - completed_buffers: self.buffers.to_vec(), - in_progress_buffer: vec![], - validity: None, - phantom: Default::default(), - total_bytes_len: self.total_bytes_len.load(Ordering::Relaxed) as usize, - total_buffer_len: self.total_buffer_len, - }), - (Right(views), false) => Left(Self::new_unchecked( - self.data_type, - views.into(), - self.buffers, - None, - self.total_bytes_len.load(Ordering::Relaxed) as usize, - self.total_buffer_len, - )), - (Left(views), _) => Left(Self::new_unchecked( - self.data_type, - views, - self.buffers, - None, - self.total_bytes_len.load(Ordering::Relaxed) as usize, - self.total_buffer_len, - )), - } + match (self.views.into_mut(), is_unique) { + (Right(views), true) => Right(BinaryViewColumnBuilder { + views, + completed_buffers: self.buffers.to_vec(), + in_progress_buffer: vec![], + phantom: Default::default(), + total_bytes_len: self.total_bytes_len.load(Ordering::Relaxed) as usize, + total_buffer_len: self.total_buffer_len, + }), + (Right(views), false) => Left(Self::new_unchecked( + views.into(), + self.buffers, + self.total_bytes_len.load(Ordering::Relaxed) as usize, + self.total_buffer_len, + )), + (Left(views), _) => Left(Self::new_unchecked( + views, + self.buffers, + self.total_bytes_len.load(Ordering::Relaxed) as usize, + self.total_buffer_len, + )), } } - pub fn default_data_type() -> &'static DataType { - T::data_type() + pub fn compare(col_i: &Self, i: usize, col_j: &Self, j: usize) -> std::cmp::Ordering { + let view_i = unsafe { col_i.views().as_slice().get_unchecked(i) }; + let view_j = unsafe { col_j.views().as_slice().get_unchecked(j) }; + + if view_i.prefix == view_j.prefix { + unsafe { + let value_i = col_i + .views + .get_unchecked(i) + .get_slice_unchecked(&col_i.buffers); + let value_j = col_j + .views + .get_unchecked(j) + .get_slice_unchecked(&col_j.buffers); + value_i.cmp(value_j) + } + } else { + view_i + .prefix + .to_le_bytes() + .cmp(&view_j.prefix.to_le_bytes()) + } } +} - pub fn with_data_type(mut self, data_type: DataType) -> Self { - self.data_type = data_type; - self +impl> FromIterator

for BinaryViewColumnGeneric { + #[inline] + fn from_iter>(iter: I) -> Self { + BinaryViewColumnBuilder::::from_iter(iter).into() } } -pub type BinaryViewArray = BinaryViewArrayGeneric<[u8]>; -pub type Utf8ViewArray = BinaryViewArrayGeneric; +pub type BinaryViewColumn = BinaryViewColumnGeneric<[u8]>; +pub type Utf8ViewColumn = BinaryViewColumnGeneric; +pub type StringColumn = BinaryViewColumnGeneric; -pub type MutableUtf8ViewArray = MutableBinaryViewArray; +pub type Utf8ViewColumnBuilder = BinaryViewColumnBuilder; +pub type StringColumnBuilder = BinaryViewColumnBuilder; -impl BinaryViewArray { +impl BinaryViewColumn { /// Validate the underlying bytes on UTF-8. pub fn validate_utf8(&self) -> Result<()> { // SAFETY: views are correct unsafe { validate_utf8_only(&self.views, &self.buffers) } } - /// Convert [`BinaryViewArray`] to [`Utf8ViewArray`]. - pub fn to_utf8view(&self) -> Result { + /// Convert [`BinaryViewColumn`] to [`Utf8ViewColumn`]. + pub fn to_utf8view(&self) -> Result { self.validate_utf8()?; unsafe { Ok(self.to_utf8view_unchecked()) } } - /// Convert [`BinaryViewArray`] to [`Utf8ViewArray`] without checking UTF-8. + /// Convert [`BinaryViewColumn`] to [`Utf8ViewColumn`] without checking UTF-8. /// /// # Safety /// The caller must ensure the underlying data is valid UTF-8. - pub unsafe fn to_utf8view_unchecked(&self) -> Utf8ViewArray { - Utf8ViewArray::new_unchecked( - DataType::Utf8View, + pub unsafe fn to_utf8view_unchecked(&self) -> Utf8ViewColumn { + Utf8ViewColumn::new_unchecked( self.views.clone(), self.buffers.clone(), - self.validity.clone(), self.total_bytes_len.load(Ordering::Relaxed) as usize, self.total_buffer_len, ) } } -impl Utf8ViewArray { - pub fn to_binview(&self) -> BinaryViewArray { - BinaryViewArray::new_unchecked( - DataType::BinaryView, +impl Utf8ViewColumn { + pub fn to_binview(&self) -> BinaryViewColumn { + BinaryViewColumn::new_unchecked( self.views.clone(), self.buffers.clone(), - self.validity.clone(), self.total_bytes_len.load(Ordering::Relaxed) as usize, self.total_buffer_len, ) } -} -impl Array for BinaryViewArrayGeneric { - fn as_any(&self) -> &dyn Any { - self - } + pub fn compare_str(col: &Self, i: usize, value: &str) -> std::cmp::Ordering { + let view = unsafe { col.views().as_slice().get_unchecked(i) }; + let prefix = load_prefix(value.as_bytes()); - fn as_any_mut(&mut self) -> &mut dyn Any { - self + if view.prefix == prefix { + let value_i = unsafe { col.value_unchecked(i) }; + value_i.cmp(value) + } else { + view.prefix.to_le_bytes().as_slice().cmp(value.as_bytes()) + } } +} - #[inline(always)] - fn len(&self) -> usize { - BinaryViewArrayGeneric::len(self) +impl PartialEq for BinaryViewColumnGeneric { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == std::cmp::Ordering::Equal } +} +impl Eq for BinaryViewColumnGeneric {} - fn data_type(&self) -> &DataType { - &self.data_type +impl PartialOrd for BinaryViewColumnGeneric { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) } +} - fn validity(&self) -> Option<&Bitmap> { - self.validity.as_ref() +impl Ord for BinaryViewColumnGeneric { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + for i in 0..self.len().max(other.len()) { + match Self::compare(self, i, other, i) { + std::cmp::Ordering::Equal => continue, + other => return other, + } + } + + std::cmp::Ordering::Equal } +} - fn slice(&mut self, offset: usize, length: usize) { - assert!( - offset + length <= self.len(), - "the offset of the new Buffer cannot exceed the existing length" - ); - unsafe { self.slice_unchecked(offset, length) } +impl TryFrom for Utf8ViewColumn { + type Error = crate::error::Error; + + fn try_from(col: BinaryColumn) -> Result { + let builder = Utf8ViewColumnBuilder::try_from_bin_column(col)?; + Ok(builder.into()) } +} - unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - debug_assert!(offset + length <= self.len()); - self.validity = self - .validity - .take() - .map(|bitmap| bitmap.sliced_unchecked(offset, length)) - .filter(|bitmap| bitmap.unset_bits() > 0); - self.views.slice_unchecked(offset, length); +impl From for BinaryColumn { + fn from(col: Utf8ViewColumn) -> BinaryColumn { + BinaryColumnBuilder::from_iter(col.iter()).into() + } +} - self.total_bytes_len.store(UNKNOWN_LEN, Ordering::Relaxed) +impl From for ArrayData { + fn from(column: Utf8ViewColumn) -> Self { + let builder = ArrayDataBuilder::new(DataType::Utf8View) + .len(column.len()) + .add_buffer(column.views.into()) + .add_buffers( + column + .buffers + .iter() + .map(|x| x.clone().into()) + .collect::>(), + ); + unsafe { builder.build_unchecked() } } +} - fn with_validity(&self, validity: Option) -> Box { - let mut new = self.clone(); - new.validity = validity; - Box::new(new) +impl From for ArrayData { + fn from(column: BinaryViewColumn) -> Self { + let builder = ArrayDataBuilder::new(DataType::BinaryView) + .len(column.len()) + .add_buffer(column.views.into()) + .add_buffers( + column + .buffers + .iter() + .map(|x| x.clone().into()) + .collect::>(), + ); + unsafe { builder.build_unchecked() } } +} - fn to_boxed(&self) -> Box { - Box::new(self.clone()) +impl From for Utf8ViewColumn { + fn from(data: ArrayData) -> Self { + let views = data.buffers()[0].clone(); + let buffers = data.buffers()[1..] + .iter() + .map(|x| x.clone().into()) + .collect(); + + unsafe { Utf8ViewColumn::new_unchecked_unknown_md(views.into(), buffers, None) } } } + +// Loads (up to) the first 4 bytes of s as little-endian, padded with zeros. +#[inline] +fn load_prefix(s: &[u8]) -> u32 { + let start = &s[..s.len().min(4)]; + let mut tmp = [0u8; 4]; + tmp[..start.len()].copy_from_slice(start); + u32::from_le_bytes(tmp) +} diff --git a/src/common/arrow/src/arrow/array/binview/view.rs b/src/common/column/src/binview/view.rs similarity index 76% rename from src/common/arrow/src/arrow/array/binview/view.rs rename to src/common/column/src/binview/view.rs index 1707b6a58a9c..f0ad3ddece68 100644 --- a/src/common/arrow/src/arrow/array/binview/view.rs +++ b/src/common/column/src/binview/view.rs @@ -19,11 +19,13 @@ use std::ops::Add; use bytemuck::Pod; use bytemuck::Zeroable; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::PrimitiveType; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; +use crate::binary::BinaryColumn; +use crate::binary::BinaryColumnBuilder; +use crate::buffer::Buffer; +use crate::error::Error; +use crate::error::Result; +use crate::types::NativeType; +use crate::types::PrimitiveType; #[derive(Debug, Copy, Clone, Default)] #[repr(C)] @@ -284,3 +286,84 @@ pub(super) unsafe fn validate_utf8_only(views: &[View], buffers: &[Buffer]) Ok(()) } + +pub trait CheckUTF8 { + fn check_utf8(&self) -> Result<()>; +} + +impl CheckUTF8 for &[u8] { + fn check_utf8(&self) -> Result<()> { + validate_utf8(self) + } +} + +impl CheckUTF8 for Vec { + fn check_utf8(&self) -> Result<()> { + self.as_slice().check_utf8() + } +} + +impl CheckUTF8 for BinaryColumn { + fn check_utf8(&self) -> Result<()> { + for bytes in self.iter() { + bytes.check_utf8()?; + } + Ok(()) + } +} + +impl CheckUTF8 for BinaryColumnBuilder { + fn check_utf8(&self) -> Result<()> { + check_utf8_column(&self.offsets, &self.data) + } +} + +/// # Check if any slice of `values` between two consecutive pairs from `offsets` is invalid `utf8` +fn check_utf8_column(offsets: &[u64], data: &[u8]) -> Result<()> { + let res: Option<()> = try { + if offsets.len() == 1 { + return Ok(()); + } + + if data.is_ascii() { + return Ok(()); + } + + simdutf8::basic::from_utf8(data).ok()?; + + let last = if let Some(last) = offsets.last() { + if *last as usize == data.len() { + return Ok(()); + } else { + *last as usize + } + } else { + // given `l = data.len()`, this branch is hit iff either: + // * `offsets = [0, l, l, ...]`, which was covered by `from_utf8(data)` above + // * `offsets = [0]`, which never happens because offsets.len() == 1 is short-circuited above + return Ok(()); + }; + + // truncate to relevant offsets. Note: `=last` because last was computed skipping the first item + // following the example: starts = [0, 5] + let starts = unsafe { offsets.get_unchecked(..=last) }; + + let mut any_invalid = false; + for start in starts { + let start = *start as usize; + + // Safety: `try_check_offsets_bounds` just checked for bounds + let b = *unsafe { data.get_unchecked(start) }; + + // A valid code-point iff it does not start with 0b10xxxxxx + // Bit-magic taken from `std::str::is_char_boundary` + if (b as i8) < -0x40 { + any_invalid = true + } + } + if any_invalid { + None?; + } + }; + res.ok_or_else(|| Error::oos("invalid utf8")) +} diff --git a/src/common/arrow/src/arrow/bitmap/assign_ops.rs b/src/common/column/src/bitmap/assign_ops.rs similarity index 96% rename from src/common/arrow/src/arrow/bitmap/assign_ops.rs rename to src/common/column/src/bitmap/assign_ops.rs index bb1ae719ba08..b4ad1f52e89f 100644 --- a/src/common/arrow/src/arrow/bitmap/assign_ops.rs +++ b/src/common/column/src/bitmap/assign_ops.rs @@ -16,8 +16,8 @@ use super::utils::BitChunk; use super::utils::BitChunkIterExact; use super::utils::BitChunksExact; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; +use crate::bitmap::Bitmap; +use crate::bitmap::MutableBitmap; /// Applies a function to every bit of this [`MutableBitmap`] in chunks /// @@ -122,11 +122,11 @@ where F: Fn(T, T) -> T { #[inline] /// Compute bitwise OR operation in-place fn or_assign(lhs: &mut MutableBitmap, rhs: &Bitmap) { - if rhs.unset_bits() == 0 { + if rhs.null_count() == 0 { assert_eq!(lhs.len(), rhs.len()); lhs.clear(); lhs.extend_constant(rhs.len(), true); - } else if rhs.unset_bits() == rhs.len() { + } else if rhs.null_count() == rhs.len() { // bitmap remains } else { binary_assign(lhs, rhs, |x: T, y| x | y) @@ -153,10 +153,10 @@ impl<'a> std::ops::BitOr<&'a Bitmap> for MutableBitmap { #[inline] /// Compute bitwise `&` between `lhs` and `rhs`, assigning it to `lhs` fn and_assign(lhs: &mut MutableBitmap, rhs: &Bitmap) { - if rhs.unset_bits() == 0 { + if rhs.null_count() == 0 { // bitmap remains } - if rhs.unset_bits() == rhs.len() { + if rhs.null_count() == rhs.len() { assert_eq!(lhs.len(), rhs.len()); lhs.clear(); lhs.extend_constant(rhs.len(), false); diff --git a/src/common/arrow/src/arrow/bitmap/bitmap_ops.rs b/src/common/column/src/bitmap/bitmap_ops.rs similarity index 96% rename from src/common/arrow/src/arrow/bitmap/bitmap_ops.rs rename to src/common/column/src/bitmap/bitmap_ops.rs index 09d7177e5d8b..c03a998c7bd5 100644 --- a/src/common/arrow/src/arrow/bitmap/bitmap_ops.rs +++ b/src/common/column/src/bitmap/bitmap_ops.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::iter::TrustedLen; use std::ops::BitAnd; use std::ops::BitOr; use std::ops::BitXor; @@ -22,8 +23,7 @@ use super::utils::BitChunk; use super::utils::BitChunkIterExact; use super::utils::BitChunksExact; use super::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::trusted_len::TrustedLen; +use crate::bitmap::MutableBitmap; /// Creates a [Vec] from an [`Iterator`] of [`BitChunk`]. /// # Safety @@ -175,7 +175,7 @@ pub(crate) fn align(bitmap: &Bitmap, new_offset: usize) -> Bitmap { #[inline] /// Compute bitwise AND operation pub fn and(lhs: &Bitmap, rhs: &Bitmap) -> Bitmap { - if lhs.unset_bits() == lhs.len() || rhs.unset_bits() == rhs.len() { + if lhs.null_count() == lhs.len() || rhs.null_count() == rhs.len() { assert_eq!(lhs.len(), rhs.len()); Bitmap::new_zeroed(lhs.len()) } else { @@ -186,7 +186,7 @@ pub fn and(lhs: &Bitmap, rhs: &Bitmap) -> Bitmap { #[inline] /// Compute bitwise OR operation pub fn or(lhs: &Bitmap, rhs: &Bitmap) -> Bitmap { - if lhs.unset_bits() == 0 || rhs.unset_bits() == 0 { + if lhs.null_count() == 0 || rhs.null_count() == 0 { assert_eq!(lhs.len(), rhs.len()); let mut mutable = MutableBitmap::with_capacity(lhs.len()); mutable.extend_constant(lhs.len(), true); @@ -199,8 +199,8 @@ pub fn or(lhs: &Bitmap, rhs: &Bitmap) -> Bitmap { #[inline] /// Compute bitwise XOR operation pub fn xor(lhs: &Bitmap, rhs: &Bitmap) -> Bitmap { - let lhs_nulls = lhs.unset_bits(); - let rhs_nulls = rhs.unset_bits(); + let lhs_nulls = lhs.null_count(); + let rhs_nulls = rhs.null_count(); // all false or all true if lhs_nulls == rhs_nulls && rhs_nulls == rhs.len() || lhs_nulls == 0 && rhs_nulls == 0 { diff --git a/src/common/arrow/src/arrow/bitmap/bitmask.rs b/src/common/column/src/bitmap/bitmask.rs similarity index 99% rename from src/common/arrow/src/arrow/bitmap/bitmask.rs rename to src/common/column/src/bitmap/bitmask.rs index d0f74f11e480..4776d43f6c22 100644 --- a/src/common/arrow/src/arrow/bitmap/bitmask.rs +++ b/src/common/column/src/bitmap/bitmask.rs @@ -22,7 +22,7 @@ use std::simd::MaskElement; #[cfg(feature = "simd")] use std::simd::SupportedLaneCount; -use crate::arrow::bitmap::Bitmap; +use crate::bitmap::Bitmap; /// Returns the nth set bit in w, if n+1 bits are set. The indexing is /// zero-based, nth_set_bit_u32(w, 0) returns the least significant set bit in w. diff --git a/src/common/arrow/src/arrow/bitmap/immutable.rs b/src/common/column/src/bitmap/immutable.rs similarity index 90% rename from src/common/arrow/src/arrow/bitmap/immutable.rs rename to src/common/column/src/bitmap/immutable.rs index 353efdc5adeb..1ca754fd26c1 100644 --- a/src/common/arrow/src/arrow/bitmap/immutable.rs +++ b/src/common/column/src/bitmap/immutable.rs @@ -14,9 +14,12 @@ // limitations under the License. use std::iter::FromIterator; +use std::iter::TrustedLen; use std::ops::Deref; use std::sync::Arc; +use arrow_data::ArrayData; +use arrow_data::ArrayDataBuilder; use either::Either; use super::chunk_iter_to_vec; @@ -27,19 +30,19 @@ use super::utils::get_bit_unchecked; use super::utils::BitChunk; use super::utils::BitChunks; use super::utils::BitmapIter; +use super::utils::ZipValidity; use super::IntoIter; use super::MutableBitmap; -use crate::arrow::buffer::Bytes; -use crate::arrow::error::Error; -use crate::arrow::trusted_len::TrustedLen; +use crate::buffer::Bytes; +use crate::error::Error; /// An immutable container semantically equivalent to `Arc>` but represented as `Arc>` where /// each boolean is represented as a single bit. /// /// # Examples /// ``` -/// use arrow2::bitmap::Bitmap; -/// use arrow2::bitmap::MutableBitmap; +/// use crate::bitmap::Bitmap; +/// use crate::bitmap::MutableBitmap; /// /// let bitmap = Bitmap::from([true, false, true]); /// assert_eq!(bitmap.iter().collect::>(), vec![true, false, true]); @@ -144,6 +147,14 @@ impl Bitmap { BitmapIter::new(&self.bytes, self.offset, self.length) } + pub fn option_iter<'a>( + &'a self, + validity: Option<&'a Bitmap>, + ) -> ZipValidity, BitmapIter<'a>> { + let bitmap_iter = validity.as_ref().map(|v| v.iter()); + ZipValidity::new(self.iter(), bitmap_iter) + } + /// Returns an iterator over bits in bit chunks [`BitChunk`]. /// /// This iterator is useful to operate over multiple bits via e.g. bitwise. @@ -175,14 +186,7 @@ impl Bitmap { /// # Implementation /// This function is `O(1)` - the number of unset bits is computed when the bitmap is /// created - pub const fn unset_bits(&self) -> usize { - self.unset_bits - } - - /// Returns the number of unset bits on this [`Bitmap`]. - #[inline] - #[deprecated(since = "0.13.0", note = "use `unset_bits` instead")] - pub fn null_count(&self) -> usize { + pub const fn null_count(&self) -> usize { self.unset_bits } @@ -480,7 +484,7 @@ impl Bitmap { /// Create a new [`Bitmap`] from an arrow [`NullBuffer`] /// /// [`NullBuffer`]: arrow_buffer::buffer::NullBuffer - #[cfg(feature = "arrow")] + pub fn from_null_buffer(value: arrow_buffer::buffer::NullBuffer) -> Self { let offset = value.offset(); let length = value.len(); @@ -489,9 +493,13 @@ impl Bitmap { offset, length, unset_bits, - bytes: Arc::new(crate::arrow::buffer::to_bytes(value.buffer().clone())), + bytes: Arc::new(crate::buffer::to_bytes(value.buffer().clone())), } } + + pub fn into_array_data(&self) -> ArrayData { + ArrayData::from(self) + } } impl<'a> IntoIterator for &'a Bitmap { @@ -512,13 +520,43 @@ impl IntoIterator for Bitmap { } } -#[cfg(feature = "arrow")] impl From for arrow_buffer::buffer::NullBuffer { fn from(value: Bitmap) -> Self { let null_count = value.unset_bits; - let buffer = crate::arrow::buffer::to_buffer(value.bytes); + let buffer = crate::buffer::to_buffer(value.bytes); let buffer = arrow_buffer::buffer::BooleanBuffer::new(buffer, value.offset, value.length); // Safety: null count is accurate unsafe { arrow_buffer::buffer::NullBuffer::new_unchecked(buffer, null_count) } } } + +impl From<&Bitmap> for ArrayData { + fn from(value: &Bitmap) -> Self { + let buffer = arrow_buffer::buffer::NullBuffer::from(value.clone()); + let builder = ArrayDataBuilder::new(arrow_schema::DataType::Boolean) + .len(buffer.len()) + .offset(buffer.offset()) + .buffers(vec![buffer.into_inner().into_inner()]); + + // Safety: Array is valid + unsafe { builder.build_unchecked() } + } +} + +impl From for ArrayData { + fn from(value: Bitmap) -> Self { + ArrayData::from(&value) + } +} + +impl Bitmap { + pub fn from_array_data(data: ArrayData) -> Self { + assert_eq!(data.data_type(), &arrow_schema::DataType::Boolean); + + let buffers = data.buffers(); + let buffer = + arrow_buffer::BooleanBuffer::new(buffers[0].clone(), data.offset(), data.len()); + // Use NullBuffer to compute set count + Bitmap::from_null_buffer(arrow_buffer::NullBuffer::new(buffer)) + } +} diff --git a/src/common/arrow/src/arrow/bitmap/iterator.rs b/src/common/column/src/bitmap/iterator.rs similarity index 96% rename from src/common/arrow/src/arrow/bitmap/iterator.rs rename to src/common/column/src/bitmap/iterator.rs index 439ddd1b3198..278b96832b71 100644 --- a/src/common/arrow/src/arrow/bitmap/iterator.rs +++ b/src/common/column/src/bitmap/iterator.rs @@ -13,9 +13,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::iter::TrustedLen; + use super::Bitmap; -use crate::arrow::bitmap::bitmask::BitMask; -use crate::arrow::trusted_len::TrustedLen; +use crate::bitmap::bitmask::BitMask; pub struct TrueIdxIter<'a> { mask: BitMask<'a>, @@ -34,7 +35,7 @@ impl<'a> TrueIdxIter<'a> { mask: BitMask::from_bitmap(bitmap), first_unknown: 0, i: 0, - remaining: bitmap.len() - bitmap.unset_bits(), + remaining: bitmap.len() - bitmap.null_count(), len, } } else { diff --git a/src/common/arrow/src/arrow/bitmap/mod.rs b/src/common/column/src/bitmap/mod.rs similarity index 100% rename from src/common/arrow/src/arrow/bitmap/mod.rs rename to src/common/column/src/bitmap/mod.rs diff --git a/src/common/arrow/src/arrow/bitmap/mutable.rs b/src/common/column/src/bitmap/mutable.rs similarity index 97% rename from src/common/arrow/src/arrow/bitmap/mutable.rs rename to src/common/column/src/bitmap/mutable.rs index c0d0e058113c..1d19016cf494 100644 --- a/src/common/arrow/src/arrow/bitmap/mutable.rs +++ b/src/common/column/src/bitmap/mutable.rs @@ -15,6 +15,7 @@ use std::hint::unreachable_unchecked; use std::iter::FromIterator; +use std::iter::TrustedLen; use std::ops::Range; use std::sync::Arc; @@ -27,10 +28,9 @@ use super::utils::BitChunk; use super::utils::BitChunksExactMut; use super::utils::BitmapIter; use super::Bitmap; -use crate::arrow::bitmap::utils::merge_reversed; -use crate::arrow::bitmap::utils::set_bit_unchecked; -use crate::arrow::error::Error; -use crate::arrow::trusted_len::TrustedLen; +use crate::bitmap::utils::merge_reversed; +use crate::bitmap::utils::set_bit_unchecked; +use crate::error::Error; /// A container of booleans. [`MutableBitmap`] is semantically equivalent /// to [`Vec`]. @@ -43,7 +43,7 @@ use crate::arrow::trusted_len::TrustedLen; /// A [`MutableBitmap`] can be converted to a [`Bitmap`] at `O(1)`. /// # Examples /// ``` -/// use arrow2::bitmap::MutableBitmap; +/// use crate::bitmap::MutableBitmap; /// /// let bitmap = MutableBitmap::from([true, false, true]); /// assert_eq!(bitmap.iter().collect::>(), vec![true, false, true]); @@ -274,18 +274,8 @@ impl MutableBitmap { } /// Returns the number of unset bits on this [`MutableBitmap`]. - /// - /// Guaranteed to be `<= self.len()`. - /// # Implementation - /// This function is `O(N)` - pub fn unset_bits(&self) -> usize { - count_zeros(&self.buffer, 0, self.length) - } - - /// Returns the number of unset bits on this [`MutableBitmap`]. - #[deprecated(since = "0.13.0", note = "use `unset_bits` instead")] pub fn null_count(&self) -> usize { - self.unset_bits() + count_zeros(&self.buffer, 0, self.length) } /// Returns the length of the [`MutableBitmap`]. @@ -390,7 +380,7 @@ impl From for Bitmap { impl From for Option { #[inline] fn from(buffer: MutableBitmap) -> Self { - let unset_bits = buffer.unset_bits(); + let unset_bits = buffer.null_count(); if unset_bits > 0 { // safety: // invariants of the `MutableBitmap` equal that of `Bitmap` diff --git a/src/common/arrow/src/arrow/bitmap/utils/chunk_iterator/chunks_exact.rs b/src/common/column/src/bitmap/utils/chunk_iterator/chunks_exact.rs similarity index 98% rename from src/common/arrow/src/arrow/bitmap/utils/chunk_iterator/chunks_exact.rs rename to src/common/column/src/bitmap/utils/chunk_iterator/chunks_exact.rs index 0801bb729fd8..0e6117273426 100644 --- a/src/common/arrow/src/arrow/bitmap/utils/chunk_iterator/chunks_exact.rs +++ b/src/common/column/src/bitmap/utils/chunk_iterator/chunks_exact.rs @@ -14,12 +14,11 @@ // limitations under the License. use std::convert::TryInto; +use std::iter::TrustedLen; use std::slice::ChunksExact; use super::BitChunk; use super::BitChunkIterExact; -use crate::arrow::trusted_len::TrustedLen; - /// An iterator over a slice of bytes in [`BitChunk`]s. #[derive(Debug)] pub struct BitChunksExact<'a, T: BitChunk> { diff --git a/src/common/arrow/src/arrow/bitmap/utils/chunk_iterator/merge.rs b/src/common/column/src/bitmap/utils/chunk_iterator/merge.rs similarity index 100% rename from src/common/arrow/src/arrow/bitmap/utils/chunk_iterator/merge.rs rename to src/common/column/src/bitmap/utils/chunk_iterator/merge.rs diff --git a/src/common/arrow/src/arrow/bitmap/utils/chunk_iterator/mod.rs b/src/common/column/src/bitmap/utils/chunk_iterator/mod.rs similarity index 98% rename from src/common/arrow/src/arrow/bitmap/utils/chunk_iterator/mod.rs rename to src/common/column/src/bitmap/utils/chunk_iterator/mod.rs index ec0ca9c196f0..e8503d92e0ad 100644 --- a/src/common/arrow/src/arrow/bitmap/utils/chunk_iterator/mod.rs +++ b/src/common/column/src/bitmap/utils/chunk_iterator/mod.rs @@ -14,6 +14,7 @@ // limitations under the License. use std::convert::TryInto; +use std::iter::TrustedLen; mod chunks_exact; mod merge; @@ -21,9 +22,8 @@ mod merge; pub use chunks_exact::BitChunksExact; pub(crate) use merge::merge_reversed; -use crate::arrow::trusted_len::TrustedLen; -pub use crate::arrow::types::BitChunk; -use crate::arrow::types::BitChunkIter; +pub use crate::types::BitChunk; +use crate::types::BitChunkIter; /// Trait representing an exact iterator over bytes in [`BitChunk`]. pub trait BitChunkIterExact: TrustedLen { diff --git a/src/common/arrow/src/arrow/bitmap/utils/chunks_exact_mut.rs b/src/common/column/src/bitmap/utils/chunks_exact_mut.rs similarity index 100% rename from src/common/arrow/src/arrow/bitmap/utils/chunks_exact_mut.rs rename to src/common/column/src/bitmap/utils/chunks_exact_mut.rs diff --git a/src/common/arrow/src/arrow/bitmap/utils/fmt.rs b/src/common/column/src/bitmap/utils/fmt.rs similarity index 100% rename from src/common/arrow/src/arrow/bitmap/utils/fmt.rs rename to src/common/column/src/bitmap/utils/fmt.rs diff --git a/src/common/arrow/src/arrow/bitmap/utils/iterator.rs b/src/common/column/src/bitmap/utils/iterator.rs similarity index 98% rename from src/common/arrow/src/arrow/bitmap/utils/iterator.rs rename to src/common/column/src/bitmap/utils/iterator.rs index eecf13230cfd..bb71c68565dc 100644 --- a/src/common/arrow/src/arrow/bitmap/utils/iterator.rs +++ b/src/common/column/src/bitmap/utils/iterator.rs @@ -13,8 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::iter::TrustedLen; + use super::get_bit_unchecked; -use crate::arrow::trusted_len::TrustedLen; /// An iterator over bits according to the [LSB](https://en.wikipedia.org/wiki/Bit_numbering#Least_significant_bit), /// i.e. the bytes `[4u8, 128u8]` correspond to `[false, false, true, false, ..., true]`. diff --git a/src/common/arrow/src/arrow/bitmap/utils/mod.rs b/src/common/column/src/bitmap/utils/mod.rs similarity index 100% rename from src/common/arrow/src/arrow/bitmap/utils/mod.rs rename to src/common/column/src/bitmap/utils/mod.rs diff --git a/src/common/arrow/src/arrow/bitmap/utils/slice_iterator.rs b/src/common/column/src/bitmap/utils/slice_iterator.rs similarity index 98% rename from src/common/arrow/src/arrow/bitmap/utils/slice_iterator.rs rename to src/common/column/src/bitmap/utils/slice_iterator.rs index 2cb234fb8a3e..f2cad26cd27a 100644 --- a/src/common/arrow/src/arrow/bitmap/utils/slice_iterator.rs +++ b/src/common/column/src/bitmap/utils/slice_iterator.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::arrow::bitmap::Bitmap; +use crate::bitmap::Bitmap; /// Internal state of [`SlicesIterator`] #[derive(Debug, Clone, PartialEq)] @@ -54,7 +54,7 @@ impl<'a> SlicesIterator<'a> { Self { state, - count: values.len() - values.unset_bits(), + count: values.len() - values.null_count(), max_len: values.len(), values: iter, mask: 1u8.rotate_left(offset as u32), diff --git a/src/common/arrow/src/arrow/bitmap/utils/zip_validity.rs b/src/common/column/src/bitmap/utils/zip_validity.rs similarity index 97% rename from src/common/arrow/src/arrow/bitmap/utils/zip_validity.rs rename to src/common/column/src/bitmap/utils/zip_validity.rs index 112d6d89bc19..a2150cc862c2 100644 --- a/src/common/arrow/src/arrow/bitmap/utils/zip_validity.rs +++ b/src/common/column/src/bitmap/utils/zip_validity.rs @@ -13,9 +13,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::trusted_len::TrustedLen; +use std::iter::TrustedLen; + +use crate::bitmap::utils::BitmapIter; +use crate::bitmap::Bitmap; /// An [`Iterator`] over validity and values. #[derive(Debug, Clone)] @@ -142,7 +143,7 @@ where I: Iterator /// are valid. pub fn new_with_validity(values: I, validity: Option<&'a Bitmap>) -> Self { // only if the validity has nulls we take the optional branch. - match validity.and_then(|validity| (validity.unset_bits() > 0).then(|| validity.iter())) { + match validity.and_then(|validity| (validity.null_count() > 0).then(|| validity.iter())) { Some(validity) => Self::Optional(ZipValidityIter::new(values, validity)), _ => Self::Required(values), } diff --git a/src/common/arrow/src/arrow/buffer/immutable.rs b/src/common/column/src/buffer/immutable.rs similarity index 88% rename from src/common/arrow/src/arrow/buffer/immutable.rs rename to src/common/column/src/buffer/immutable.rs index 94aa8ac4d591..94724df55a73 100644 --- a/src/common/arrow/src/arrow/buffer/immutable.rs +++ b/src/common/column/src/buffer/immutable.rs @@ -17,12 +17,17 @@ use std::iter::FromIterator; use std::ops::Deref; use std::sync::Arc; +use arrow_data::ArrayData; +use arrow_data::ArrayDataBuilder; +use arrow_schema::DataType; use either::Either; use num_traits::Zero; use super::Bytes; -use super::IntoIter; -use crate::arrow::array::ArrayAccessor; +use crate::bitmap::utils::BitmapIter; +use crate::bitmap::utils::ZipValidity; +use crate::bitmap::Bitmap; +use crate::types::NativeType; /// [`Buffer`] is a contiguous memory region that can be shared across /// thread boundaries. @@ -36,7 +41,7 @@ use crate::arrow::array::ArrayAccessor; /// /// # Examples /// ``` -/// use arrow2::buffer::Buffer; +/// use crate::buffer::Buffer; /// /// let mut buffer: Buffer = vec![1, 2, 3].into(); /// assert_eq!(buffer.as_ref(), [1, 2, 3].as_ref()); @@ -332,42 +337,54 @@ impl FromIterator for Buffer { } } +impl<'a, T> IntoIterator for &'a Buffer { + type Item = &'a T; + type IntoIter = std::slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.as_ref().iter() + } +} + impl IntoIterator for Buffer { type Item = T; - type IntoIter = IntoIter; + type IntoIter = super::iterator::IntoIter; fn into_iter(self) -> Self::IntoIter { - IntoIter::new(self) + super::iterator::IntoIter::new(self) } } -#[cfg(feature = "arrow")] -impl From for Buffer { +impl Buffer { + pub fn option_iter<'a>( + &self, + validity: Option<&'a Bitmap>, + ) -> ZipValidity, BitmapIter<'a>> { + let iter = IntoIterator::into_iter(self.clone()); + let bitmap_iter = validity.as_ref().map(|v| v.iter()); + ZipValidity::new(iter, bitmap_iter) + } +} + +impl From for Buffer { fn from(value: arrow_buffer::Buffer) -> Self { - Self::from_bytes(crate::arrow::buffer::to_bytes(value)) + Self::from_bytes(crate::buffer::to_bytes(value)) } } -#[cfg(feature = "arrow")] -impl From> for arrow_buffer::Buffer { +impl From> for arrow_buffer::Buffer { fn from(value: Buffer) -> Self { - crate::arrow::buffer::to_buffer(value.data).slice_with_length( + crate::buffer::to_buffer(value.data).slice_with_length( value.offset * std::mem::size_of::(), value.length * std::mem::size_of::(), ) } } -unsafe impl<'a, T: 'a> ArrayAccessor<'a> for Buffer { - type Item = &'a T; - - unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - debug_assert!(index < self.length); - unsafe { self.get_unchecked(self.offset + index) } - } - - fn len(&self) -> usize { - Buffer::len(self) - } +pub fn buffer_to_array_data(value: (Buffer, DataType)) -> ArrayData { + let l = value.0.len(); + let buffer = value.0.into(); + let builder = ArrayDataBuilder::new(value.1).len(l).buffers(vec![buffer]); + unsafe { builder.build_unchecked() } } diff --git a/src/common/arrow/src/arrow/buffer/iterator.rs b/src/common/column/src/buffer/iterator.rs similarity index 98% rename from src/common/arrow/src/arrow/buffer/iterator.rs rename to src/common/column/src/buffer/iterator.rs index cf908272015f..91b4d351b5d5 100644 --- a/src/common/arrow/src/arrow/buffer/iterator.rs +++ b/src/common/column/src/buffer/iterator.rs @@ -13,8 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::iter::TrustedLen; + use super::Buffer; -use crate::arrow::trusted_len::TrustedLen; /// This crates' equivalent of [`std::vec::IntoIter`] for [`Buffer`]. #[derive(Debug, Clone)] diff --git a/src/common/arrow/src/arrow/buffer/mod.rs b/src/common/column/src/buffer/mod.rs similarity index 91% rename from src/common/arrow/src/arrow/buffer/mod.rs rename to src/common/column/src/buffer/mod.rs index 873655a813da..c76d93a02738 100644 --- a/src/common/arrow/src/arrow/buffer/mod.rs +++ b/src/common/column/src/buffer/mod.rs @@ -20,9 +20,11 @@ mod iterator; use std::ops::Deref; +pub use immutable::buffer_to_array_data; +pub use immutable::Buffer; + #[allow(dead_code)] pub(crate) enum BytesAllocator { - #[cfg(feature = "arrow")] Arrow(arrow_buffer::Buffer), } pub(crate) type BytesInner = foreign_vec::ForeignVec; @@ -77,8 +79,7 @@ impl From> for Bytes { } } -#[cfg(feature = "arrow")] -pub(crate) fn to_buffer( +pub(crate) fn to_buffer( value: std::sync::Arc>, ) -> arrow_buffer::Buffer { // This should never panic as ForeignVec pointer must be non-null @@ -88,8 +89,7 @@ pub(crate) fn to_buffer( unsafe { arrow_buffer::Buffer::from_custom_allocation(ptr, len, value) } } -#[cfg(feature = "arrow")] -pub(crate) fn to_bytes( +pub(crate) fn to_bytes( value: arrow_buffer::Buffer, ) -> Bytes { let ptr = value.as_ptr(); @@ -100,11 +100,8 @@ pub(crate) fn to_bytes( // Valid as `NativeType: Pod` and checked alignment above let ptr = value.as_ptr() as *const T; - let owner = crate::arrow::buffer::BytesAllocator::Arrow(value); + let owner = crate::buffer::BytesAllocator::Arrow(value); // Safety: slice is valid for len elements of T unsafe { Bytes::from_foreign(ptr, len, owner) } } - -pub use immutable::Buffer; -pub(super) use iterator::IntoIter; diff --git a/src/common/arrow/src/arrow/error.rs b/src/common/column/src/error.rs similarity index 93% rename from src/common/arrow/src/arrow/error.rs rename to src/common/column/src/error.rs index 8841b8ce40df..191739dd35b7 100644 --- a/src/common/arrow/src/arrow/error.rs +++ b/src/common/column/src/error.rs @@ -20,6 +20,8 @@ use std::fmt::Display; /// Defines [`Error`], representing all errors returned by this crate. use std::fmt::Formatter; +use databend_common_exception::ErrorCode; + /// Enum with all errors in this crate. #[derive(Debug)] #[non_exhaustive] @@ -123,3 +125,12 @@ impl std::error::Error for Error {} /// Typedef for a [`std::result::Result`] of an [`Error`]. pub type Result = std::result::Result; + +impl From for ErrorCode { + fn from(error: Error) -> Self { + match error { + Error::NotYetImplemented(v) => ErrorCode::Unimplemented(format!("arrow: {v}")), + v => ErrorCode::from_std_error(v), + } + } +} diff --git a/src/common/column/src/fmt.rs b/src/common/column/src/fmt.rs new file mode 100644 index 000000000000..829bc881b131 --- /dev/null +++ b/src/common/column/src/fmt.rs @@ -0,0 +1,85 @@ +// Copyright 2020-2022 Jorge C. Leitão +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Result; +use std::fmt::Write; + +use crate::bitmap::Bitmap; + +pub fn write_vec( + f: &mut F, + d: D, + validity: Option<&Bitmap>, + len: usize, + null: &'static str, + new_lines: bool, +) -> Result +where + D: Fn(&mut F, usize) -> Result, + F: Write, +{ + f.write_char('[')?; + write_list(f, d, validity, len, null, new_lines)?; + f.write_char(']')?; + Ok(()) +} + +fn write_list( + f: &mut F, + d: D, + validity: Option<&Bitmap>, + len: usize, + null: &'static str, + new_lines: bool, +) -> Result +where + D: Fn(&mut F, usize) -> Result, + F: Write, +{ + for index in 0..len { + if index != 0 { + f.write_char(',')?; + f.write_char(if new_lines { '\n' } else { ' ' })?; + } + if let Some(val) = validity { + if val.get_bit(index) { + d(f, index) + } else { + write!(f, "{null}") + } + } else { + d(f, index) + }?; + } + Ok(()) +} + +pub fn write_map( + f: &mut F, + d: D, + validity: Option<&Bitmap>, + len: usize, + null: &'static str, + new_lines: bool, +) -> Result +where + D: Fn(&mut F, usize) -> Result, + F: Write, +{ + f.write_char('{')?; + write_list(f, d, validity, len, null, new_lines)?; + f.write_char('}')?; + Ok(()) +} diff --git a/src/common/arrow/src/arrow/array/iterator.rs b/src/common/column/src/iterator.rs similarity index 71% rename from src/common/arrow/src/arrow/array/iterator.rs rename to src/common/column/src/iterator.rs index f4e74b6e37c9..479691d3c6c5 100644 --- a/src/common/arrow/src/arrow/array/iterator.rs +++ b/src/common/column/src/iterator.rs @@ -1,4 +1,3 @@ -// Copyright 2020-2022 Jorge C. Leitão // Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,36 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::TrueIdxIter; -use crate::arrow::trusted_len::TrustedLen; +use std::iter::TrustedLen; -mod private { - pub trait Sealed {} - - impl<'a, T: super::ArrayAccessor<'a>> Sealed for T {} -} +use crate::bitmap::Bitmap; +use crate::bitmap::TrueIdxIter; /// Sealed trait representing assess to a value of an array. /// # Safety /// Implementers of this trait guarantee that /// `value_unchecked` is safe when called up to `len` -pub unsafe trait ArrayAccessor<'a>: private::Sealed { +#[allow(clippy::missing_safety_doc)] +pub unsafe trait ColumnAccessor<'a> { type Item: 'a; unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item; fn len(&self) -> usize; } -/// Iterator of values of an [`ArrayAccessor`]. +/// Iterator of values of an [`ColumnAccessor`]. #[derive(Debug, Clone)] -pub struct ArrayValuesIter<'a, A: ArrayAccessor<'a>> { +pub struct ColumnValuesIter<'a, A: ColumnAccessor<'a>> { array: &'a A, index: usize, end: usize, } -impl<'a, A: ArrayAccessor<'a>> ArrayValuesIter<'a, A> { - /// Creates a new [`ArrayValuesIter`] +impl<'a, A: ColumnAccessor<'a>> ColumnValuesIter<'a, A> { + /// Creates a new [`ColumnValuesIter`] #[inline] pub fn new(array: &'a A) -> Self { Self { @@ -53,7 +48,7 @@ impl<'a, A: ArrayAccessor<'a>> ArrayValuesIter<'a, A> { } } -impl<'a, A: ArrayAccessor<'a>> Iterator for ArrayValuesIter<'a, A> { +impl<'a, A: ColumnAccessor<'a>> Iterator for ColumnValuesIter<'a, A> { type Item = A::Item; #[inline] @@ -84,7 +79,7 @@ impl<'a, A: ArrayAccessor<'a>> Iterator for ArrayValuesIter<'a, A> { } } -impl<'a, A: ArrayAccessor<'a>> DoubleEndedIterator for ArrayValuesIter<'a, A> { +impl<'a, A: ColumnAccessor<'a>> DoubleEndedIterator for ColumnValuesIter<'a, A> { #[inline] fn next_back(&mut self) -> Option { if self.index == self.end { @@ -96,15 +91,15 @@ impl<'a, A: ArrayAccessor<'a>> DoubleEndedIterator for ArrayValuesIter<'a, A> { } } -unsafe impl<'a, A: ArrayAccessor<'a>> TrustedLen for ArrayValuesIter<'a, A> {} -impl<'a, A: ArrayAccessor<'a>> ExactSizeIterator for ArrayValuesIter<'a, A> {} +unsafe impl<'a, A: ColumnAccessor<'a>> TrustedLen for ColumnValuesIter<'a, A> {} +impl<'a, A: ColumnAccessor<'a>> ExactSizeIterator for ColumnValuesIter<'a, A> {} pub struct NonNullValuesIter<'a, A: ?Sized> { accessor: &'a A, idxs: TrueIdxIter<'a>, } -impl<'a, A: ArrayAccessor<'a> + ?Sized> NonNullValuesIter<'a, A> { +impl<'a, A: ColumnAccessor<'a> + ?Sized> NonNullValuesIter<'a, A> { pub fn new(accessor: &'a A, validity: Option<&'a Bitmap>) -> Self { Self { idxs: TrueIdxIter::new(accessor.len(), validity), @@ -113,7 +108,7 @@ impl<'a, A: ArrayAccessor<'a> + ?Sized> NonNullValuesIter<'a, A> { } } -impl<'a, A: ArrayAccessor<'a> + ?Sized> Iterator for NonNullValuesIter<'a, A> { +impl<'a, A: ColumnAccessor<'a> + ?Sized> Iterator for NonNullValuesIter<'a, A> { type Item = A::Item; #[inline] @@ -129,4 +124,4 @@ impl<'a, A: ArrayAccessor<'a> + ?Sized> Iterator for NonNullValuesIter<'a, A> { } } -unsafe impl<'a, A: ArrayAccessor<'a> + ?Sized> TrustedLen for NonNullValuesIter<'a, A> {} +unsafe impl<'a, A: ColumnAccessor<'a> + ?Sized> TrustedLen for NonNullValuesIter<'a, A> {} diff --git a/src/common/arrow/src/lib.rs b/src/common/column/src/lib.rs similarity index 71% rename from src/common/arrow/src/lib.rs rename to src/common/column/src/lib.rs index ab58813bfb7a..7d9748e4f413 100644 --- a/src/common/arrow/src/lib.rs +++ b/src/common/column/src/lib.rs @@ -13,17 +13,23 @@ // limitations under the License. #![feature(iter_advance_by)] +#![feature(portable_simd)] #![allow(clippy::unconditional_recursion)] -#![cfg_attr(feature = "simd", feature(portable_simd))] -#![allow(clippy::redundant_closure_call)] #![allow(clippy::non_canonical_partial_ord_impl)] +#![allow(clippy::len_without_is_empty)] #![allow(dead_code)] +#![feature(trusted_len)] +#![feature(try_blocks)] -//#[macro_use] -// mod errors; +pub mod binary; +pub mod binview; +pub mod bitmap; +pub mod buffer; +pub mod error; +pub mod fmt; +pub mod iterator; +pub mod offset; +pub mod types; -pub mod arrow; -pub mod native; -pub mod schema_projection; - -pub type ArrayRef = Box; +#[macro_use] +pub(crate) mod utils; diff --git a/src/common/arrow/src/arrow/offset.rs b/src/common/column/src/offset.rs similarity index 99% rename from src/common/arrow/src/arrow/offset.rs rename to src/common/column/src/offset.rs index dd24cd0bce21..60ff5d6041f0 100644 --- a/src/common/arrow/src/arrow/offset.rs +++ b/src/common/column/src/offset.rs @@ -16,9 +16,9 @@ //! Contains the declaration of [`Offset`] use std::hint::unreachable_unchecked; -use crate::arrow::buffer::Buffer; -use crate::arrow::error::Error; -pub use crate::arrow::types::Offset; +use crate::buffer::Buffer; +use crate::error::Error; +pub use crate::types::Offset; /// A wrapper type of [`Vec`] representing the invariants of Arrow's offsets. /// It is guaranteed to (sound to assume that): diff --git a/src/common/arrow/src/arrow/types/bit_chunk.rs b/src/common/column/src/types/bit_chunk.rs similarity index 96% rename from src/common/arrow/src/arrow/types/bit_chunk.rs rename to src/common/column/src/types/bit_chunk.rs index 84e90b29410a..ffbd74ca9b15 100644 --- a/src/common/arrow/src/arrow/types/bit_chunk.rs +++ b/src/common/column/src/types/bit_chunk.rs @@ -120,7 +120,7 @@ impl Iterator for BitChunkIter { // # Safety // a mathematical invariant of this iterator -unsafe impl crate::arrow::trusted_len::TrustedLen for BitChunkIter {} +unsafe impl std::iter::TrustedLen for BitChunkIter {} /// An [`Iterator`] over a [`BitChunk`] returning the index of each bit set in the chunk /// See for details @@ -171,4 +171,4 @@ impl Iterator for BitChunkOnes { // # Safety // a mathematical invariant of this iterator -unsafe impl crate::arrow::trusted_len::TrustedLen for BitChunkOnes {} +unsafe impl std::iter::TrustedLen for BitChunkOnes {} diff --git a/src/common/arrow/src/arrow/types/index.rs b/src/common/column/src/types/index.rs similarity index 98% rename from src/common/arrow/src/arrow/types/index.rs rename to src/common/column/src/types/index.rs index 28b262eee4c9..64570ec269ce 100644 --- a/src/common/arrow/src/arrow/types/index.rs +++ b/src/common/column/src/types/index.rs @@ -14,9 +14,9 @@ // limitations under the License. use std::convert::TryFrom; +use std::iter::TrustedLen; use super::NativeType; -use crate::arrow::trusted_len::TrustedLen; /// Sealed trait describing the subset of [`NativeType`] (`i32`, `i64`, `u32` and `u64`) /// that can be used to index a slot of an array. diff --git a/src/common/arrow/src/arrow/types/mod.rs b/src/common/column/src/types/mod.rs similarity index 94% rename from src/common/arrow/src/arrow/types/mod.rs rename to src/common/column/src/types/mod.rs index f669cd0eaf42..e31e249fb660 100644 --- a/src/common/arrow/src/arrow/types/mod.rs +++ b/src/common/column/src/types/mod.rs @@ -40,9 +40,9 @@ pub use bit_chunk::BitChunk; pub use bit_chunk::BitChunkIter; pub use bit_chunk::BitChunkOnes; mod index; -pub mod simd; pub use index::*; mod native; +pub mod simd; pub use native::*; mod offset; pub use offset::*; @@ -90,7 +90,9 @@ pub enum PrimitiveType { } mod private { - use crate::arrow::array::View; + use databend_common_base::base::OrderedFloat; + + use crate::binview::View; pub trait Sealed {} @@ -105,9 +107,12 @@ mod private { impl Sealed for i128 {} impl Sealed for u128 {} impl Sealed for super::i256 {} + impl Sealed for ethnum::i256 {} impl Sealed for super::f16 {} impl Sealed for f32 {} impl Sealed for f64 {} + impl Sealed for OrderedFloat {} + impl Sealed for OrderedFloat {} impl Sealed for super::days_ms {} impl Sealed for super::months_days_ns {} impl Sealed for View {} diff --git a/src/common/arrow/src/arrow/types/native.rs b/src/common/column/src/types/native.rs similarity index 93% rename from src/common/arrow/src/arrow/types/native.rs rename to src/common/column/src/types/native.rs index 7a96101daf29..1ba84c4bb19e 100644 --- a/src/common/arrow/src/arrow/types/native.rs +++ b/src/common/column/src/types/native.rs @@ -19,6 +19,7 @@ use std::panic::RefUnwindSafe; use bytemuck::Pod; use bytemuck::Zeroable; +use databend_common_base::base::OrderedFloat; use super::PrimitiveType; @@ -91,6 +92,9 @@ macro_rules! native_type { }; } +type F32 = OrderedFloat; +type F64 = OrderedFloat; + native_type!(u8, PrimitiveType::UInt8); native_type!(u16, PrimitiveType::UInt16); native_type!(u32, PrimitiveType::UInt32); @@ -103,6 +107,49 @@ native_type!(f32, PrimitiveType::Float32); native_type!(f64, PrimitiveType::Float64); native_type!(i128, PrimitiveType::Int128); +impl NativeType for F32 { + const PRIMITIVE: PrimitiveType = (PrimitiveType::Float32); + type Bytes = [u8; std::mem::size_of::()]; + #[inline] + fn to_le_bytes(&self) -> Self::Bytes { + self.0.to_le_bytes() + } + #[inline] + fn to_be_bytes(&self) -> Self::Bytes { + self.0.to_be_bytes() + } + #[inline] + fn from_le_bytes(bytes: Self::Bytes) -> Self { + Self(f32::from_le_bytes(bytes)) + } + #[inline] + fn from_be_bytes(bytes: Self::Bytes) -> Self { + Self(f32::from_be_bytes(bytes)) + } +} + +impl NativeType for F64 { + const PRIMITIVE: PrimitiveType = (PrimitiveType::Float64); + type Bytes = [u8; std::mem::size_of::()]; + + #[inline] + fn to_le_bytes(&self) -> Self::Bytes { + self.0.to_le_bytes() + } + #[inline] + fn to_be_bytes(&self) -> Self::Bytes { + self.0.to_be_bytes() + } + #[inline] + fn from_le_bytes(bytes: Self::Bytes) -> Self { + Self(f64::from_le_bytes(bytes)) + } + #[inline] + fn from_be_bytes(bytes: Self::Bytes) -> Self { + Self(f64::from_be_bytes(bytes)) + } +} + /// The in-memory representation of the DayMillisecond variant of arrow's "Interval" logical type. #[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, Zeroable, Pod)] #[allow(non_camel_case_types)] diff --git a/src/common/arrow/src/arrow/types/offset.rs b/src/common/column/src/types/offset.rs similarity index 100% rename from src/common/arrow/src/arrow/types/offset.rs rename to src/common/column/src/types/offset.rs diff --git a/src/common/arrow/src/arrow/types/simd/mod.rs b/src/common/column/src/types/simd/mod.rs similarity index 100% rename from src/common/arrow/src/arrow/types/simd/mod.rs rename to src/common/column/src/types/simd/mod.rs diff --git a/src/common/arrow/src/arrow/types/simd/native.rs b/src/common/column/src/types/simd/native.rs similarity index 96% rename from src/common/arrow/src/arrow/types/simd/native.rs rename to src/common/column/src/types/simd/native.rs index 9ff787e33005..a7345be924de 100644 --- a/src/common/arrow/src/arrow/types/simd/native.rs +++ b/src/common/column/src/types/simd/native.rs @@ -16,7 +16,7 @@ use std::convert::TryInto; use super::*; -use crate::arrow::types::BitChunkIter; +use crate::types::BitChunkIter; native_simd!(u8x64, u8, 64, u64); native_simd!(u16x32, u16, 32, u32); diff --git a/src/common/arrow/src/arrow/types/simd/packed.rs b/src/common/column/src/types/simd/packed.rs similarity index 100% rename from src/common/arrow/src/arrow/types/simd/packed.rs rename to src/common/column/src/types/simd/packed.rs diff --git a/src/common/column/src/utils.rs b/src/common/column/src/utils.rs new file mode 100644 index 000000000000..99a771dc02cb --- /dev/null +++ b/src/common/column/src/utils.rs @@ -0,0 +1,56 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// macro implementing `sliced` and `sliced_unchecked` +#[macro_export] +macro_rules! impl_sliced { + () => { + /// Returns this array sliced. + /// # Implementation + /// This function is `O(1)`. + /// # Panics + /// iff `offset + length > self.len()`. + #[inline] + #[must_use] + pub fn sliced(self, offset: usize, length: usize) -> Self { + assert!( + offset + length <= self.len(), + "the offset of the new Buffer cannot exceed the existing length" + ); + unsafe { self.sliced_unchecked(offset, length) } + } + + /// Returns this array sliced. + /// # Implementation + /// This function is `O(1)`. + /// # Safety + /// The caller must ensure that `offset + length <= self.len()`. + #[inline] + #[must_use] + pub unsafe fn sliced_unchecked(mut self, offset: usize, length: usize) -> Self { + self.slice_unchecked(offset, length); + self + } + }; +} + +#[macro_export] +macro_rules! with_number_type { + ( | $t:tt | $($tail:tt)* ) => { + match_template::match_template! { + $t = [UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64], + $($tail)* + } + } +} diff --git a/src/common/column/tests/it/binview/builder.rs b/src/common/column/tests/it/binview/builder.rs new file mode 100644 index 000000000000..d12990191364 --- /dev/null +++ b/src/common/column/tests/it/binview/builder.rs @@ -0,0 +1,65 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_column::binview::BinaryViewColumnBuilder; +use databend_common_column::binview::Utf8ViewColumn; + +#[test] +fn extend_from_iter() { + let mut b = BinaryViewColumnBuilder::::new(); + b.extend_trusted_len_values(vec!["a", "b"].into_iter()); + + let a = b.clone(); + b.extend_trusted_len_values(a.iter()); + + let b: Utf8ViewColumn = b.into(); + let c: Utf8ViewColumn = + BinaryViewColumnBuilder::::from_iter(vec!["a", "b", "a", "b"]).into(); + + assert_eq!(b, c) +} + +#[test] +fn new() { + assert_eq!(BinaryViewColumnBuilder::<[u8]>::new().len(), 0); + + let a = BinaryViewColumnBuilder::<[u8]>::with_capacity(2); + assert_eq!(a.len(), 0); + assert_eq!(a.capacity(), 2); +} + +#[test] +fn from_iter() { + let iter = (0..3u8).map(|x| vec![x; x as usize]); + let a: BinaryViewColumnBuilder<[u8]> = iter.clone().collect(); + let mut v_iter = a.iter(); + assert_eq!(v_iter.next(), Some(&[] as &[u8])); + assert_eq!(v_iter.next(), Some(&[1u8] as &[u8])); + assert_eq!(v_iter.next(), Some(&[2u8, 2] as &[u8])); + + let b = BinaryViewColumnBuilder::<[u8]>::from_iter(iter); + assert_eq!(a.freeze(), b.freeze()) +} + +#[test] +fn test_pop_gc() { + let iter = (0..1024).map(|x| format!("{}", x)); + let mut a: BinaryViewColumnBuilder = iter.clone().collect(); + let item = a.pop(); + assert_eq!(item, Some("1023".to_string())); + + let column = a.freeze(); + let column = column.sliced(10, 10); + column.gc(); +} diff --git a/src/common/column/tests/it/binview/mod.rs b/src/common/column/tests/it/binview/mod.rs new file mode 100644 index 000000000000..0e42c0061ace --- /dev/null +++ b/src/common/column/tests/it/binview/mod.rs @@ -0,0 +1,156 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod builder; + +use databend_common_column::binview::BinaryViewColumn; +use databend_common_column::binview::Utf8ViewColumn; + +#[test] +fn basics_string_view() { + let data = vec![ + "hello", + "", + // larger than 12 bytes. + "Databend Cloud is a Cost-Effective alternative to Snowflake.", + ]; + + let array: Utf8ViewColumn = data.into_iter().collect(); + + assert_eq!(array.value(0), "hello"); + assert_eq!(array.value(1), ""); + assert_eq!( + array.value(2), + "Databend Cloud is a Cost-Effective alternative to Snowflake." + ); + assert_eq!( + unsafe { array.value_unchecked(2) }, + "Databend Cloud is a Cost-Effective alternative to Snowflake." + ); + + let array2 = Utf8ViewColumn::new_unchecked( + array.views().clone(), + array.data_buffers().clone(), + array.total_bytes_len(), + array.total_buffer_len(), + ); + + assert_eq!(array, array2); + + let array = array.sliced(1, 2); + + assert_eq!(array.value(0), ""); + assert_eq!( + array.value(1), + "Databend Cloud is a Cost-Effective alternative to Snowflake." + ); +} + +#[test] +fn basics_binary_view() { + let data = vec![ + b"hello".to_vec(), + b"".to_vec(), + // larger than 12 bytes. + b"Databend Cloud is a Cost-Effective alternative to Snowflake.".to_vec(), + ]; + + let array: BinaryViewColumn = data.into_iter().collect(); + + assert_eq!(array.value(0), b"hello"); + assert_eq!(array.value(1), b""); + assert_eq!( + array.value(2), + b"Databend Cloud is a Cost-Effective alternative to Snowflake." + ); + assert_eq!( + unsafe { array.value_unchecked(2) }, + b"Databend Cloud is a Cost-Effective alternative to Snowflake." + ); + + let array2 = BinaryViewColumn::new_unchecked( + array.views().clone(), + array.data_buffers().clone(), + array.total_bytes_len(), + array.total_buffer_len(), + ); + + assert_eq!(array, array2); + + let array = array.sliced(1, 2); + + assert_eq!(array.value(0), b""); + assert_eq!( + array.value(1), + b"Databend Cloud is a Cost-Effective alternative to Snowflake." + ); +} + +#[test] +fn from() { + let array1 = Utf8ViewColumn::from(["hello", " ", ""]); + let array2 = BinaryViewColumn::from([b"hello".to_vec(), b" ".to_vec(), b"".to_vec()]); + assert_eq!(array1.to_binview(), array2); +} + +#[test] +fn from_iter() { + let iter = std::iter::repeat(b"hello").take(2); + let a: BinaryViewColumn = iter.collect(); + assert_eq!(a.len(), 2); +} + +#[test] +fn test_slice() { + let data = vec!["hello", "world", "databend", "y", "z", "abc"]; + + let array: Utf8ViewColumn = data.into_iter().collect(); + + let a3 = array.sliced(2, 3); + assert_eq!(a3.into_iter().collect::>(), vec![ + "databend", "y", "z" + ]); +} + +#[test] +fn test_compare() { + let data = vec![ + "aaaz", + "aaaaaaaahello", + "bbbbbbbbbbbbbbbbbbbbhello", + "ccccccccccccccchello", + "y", + "z", + "zzzzzz", + "abc", + ]; + + let array: Utf8ViewColumn = data.into_iter().collect(); + + let min = array.iter().min().unwrap(); + let max = array.iter().max().unwrap(); + + let min_expect = (0..array.len()) + .min_by(|i, j| Utf8ViewColumn::compare(&array, *i, &array, *j)) + .unwrap(); + let min_expect = array.value(min_expect); + + let max_expect = (0..array.len()) + .max_by(|i, j| Utf8ViewColumn::compare(&array, *i, &array, *j)) + .unwrap(); + let max_expect = array.value(max_expect); + + assert_eq!(min, min_expect); + assert_eq!(max, max_expect); +} diff --git a/src/common/arrow/tests/it/arrow/bitmap/assign_ops.rs b/src/common/column/tests/it/bitmap/assign_ops.rs similarity index 91% rename from src/common/arrow/tests/it/arrow/bitmap/assign_ops.rs rename to src/common/column/tests/it/bitmap/assign_ops.rs index f4335128bb75..d6d17d873494 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/assign_ops.rs +++ b/src/common/column/tests/it/bitmap/assign_ops.rs @@ -13,13 +13,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::binary_assign; -use databend_common_arrow::arrow::bitmap::unary_assign; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; +use databend_common_column::bitmap::binary_assign; +use databend_common_column::bitmap::unary_assign; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; use proptest::prelude::*; -use crate::arrow::bitmap::bitmap_strategy; +use crate::bitmap::bitmap_strategy; #[test] fn basics() { diff --git a/src/common/arrow/tests/it/arrow/bitmap/bitmap_ops.rs b/src/common/column/tests/it/bitmap/bitmap_ops.rs similarity index 89% rename from src/common/arrow/tests/it/arrow/bitmap/bitmap_ops.rs rename to src/common/column/tests/it/bitmap/bitmap_ops.rs index db4d2c673860..1b20810a7ace 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/bitmap_ops.rs +++ b/src/common/column/tests/it/bitmap/bitmap_ops.rs @@ -13,13 +13,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::and; -use databend_common_arrow::arrow::bitmap::or; -use databend_common_arrow::arrow::bitmap::xor; -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::and; +use databend_common_column::bitmap::or; +use databend_common_column::bitmap::xor; +use databend_common_column::bitmap::Bitmap; use proptest::prelude::*; -use crate::arrow::bitmap::bitmap_strategy; +use crate::bitmap::bitmap_strategy; proptest! { /// Asserts that !bitmap equals all bits flipped diff --git a/src/common/arrow/tests/it/arrow/bitmap/immutable.rs b/src/common/column/tests/it/bitmap/immutable.rs similarity index 91% rename from src/common/arrow/tests/it/arrow/bitmap/immutable.rs rename to src/common/column/tests/it/bitmap/immutable.rs index 6724c571eb26..96bd9f209b2b 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/immutable.rs +++ b/src/common/column/tests/it/bitmap/immutable.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; #[test] fn as_slice() { @@ -55,7 +55,7 @@ fn new_constant() { assert!((slice[1] & 0b00000001) > 0); assert_eq!(offset, 0); assert_eq!(length, 9); - assert_eq!(b.unset_bits(), 0); + assert_eq!(b.null_count(), 0); let b = Bitmap::new_constant(false, 9); let (slice, offset, length) = b.as_slice(); @@ -63,7 +63,7 @@ fn new_constant() { assert!((slice[1] & 0b00000001) == 0); assert_eq!(offset, 0); assert_eq!(length, 9); - assert_eq!(b.unset_bits(), 9); + assert_eq!(b.null_count(), 9); } #[test] @@ -75,7 +75,6 @@ fn debug() { } #[test] -#[cfg(feature = "arrow")] fn from_arrow() { use arrow_buffer::buffer::BooleanBuffer; use arrow_buffer::buffer::NullBuffer; @@ -85,7 +84,7 @@ fn from_arrow() { assert_eq!(nulls.null_count(), 3); let bitmap = Bitmap::from_null_buffer(nulls.clone()); - assert_eq!(nulls.null_count(), bitmap.unset_bits()); + assert_eq!(nulls.null_count(), bitmap.null_count()); assert_eq!(nulls.len(), bitmap.len()); let back = NullBuffer::from(bitmap); assert_eq!(nulls, back); @@ -95,7 +94,7 @@ fn from_arrow() { assert_eq!(nulls.len(), 3); let bitmap = Bitmap::from_null_buffer(nulls.clone()); - assert_eq!(nulls.null_count(), bitmap.unset_bits()); + assert_eq!(nulls.null_count(), bitmap.null_count()); assert_eq!(nulls.len(), bitmap.len()); let back = NullBuffer::from(bitmap); assert_eq!(nulls, back); diff --git a/src/common/arrow/tests/it/arrow/bitmap/mod.rs b/src/common/column/tests/it/bitmap/mod.rs similarity index 95% rename from src/common/arrow/tests/it/arrow/bitmap/mod.rs rename to src/common/column/tests/it/bitmap/mod.rs index 339cfc27d05e..3d4922e11aff 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/mod.rs +++ b/src/common/column/tests/it/bitmap/mod.rs @@ -19,7 +19,7 @@ mod immutable; mod mutable; mod utils; -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use proptest::prelude::*; /// Returns a strategy of an arbitrary sliced [`Bitmap`] of size up to 1000 @@ -127,13 +127,13 @@ fn not() { #[test] fn subslicing_gives_correct_null_count() { let base = Bitmap::from([false, true, true, false, false, true, true, true]); - assert_eq!(base.unset_bits(), 3); + assert_eq!(base.null_count(), 3); let view1 = base.clone().sliced(0, 1); let view2 = base.sliced(1, 7); - assert_eq!(view1.unset_bits(), 1); - assert_eq!(view2.unset_bits(), 2); + assert_eq!(view1.null_count(), 1); + assert_eq!(view2.null_count(), 2); let view3 = view2.sliced(0, 1); - assert_eq!(view3.unset_bits(), 0); + assert_eq!(view3.null_count(), 0); } diff --git a/src/common/arrow/tests/it/arrow/bitmap/mutable.rs b/src/common/column/tests/it/bitmap/mutable.rs similarity index 98% rename from src/common/arrow/tests/it/arrow/bitmap/mutable.rs rename to src/common/column/tests/it/bitmap/mutable.rs index 3d470d8a2dcb..14eea8a04458 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/mutable.rs +++ b/src/common/column/tests/it/bitmap/mutable.rs @@ -13,8 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; #[test] fn from_slice() { @@ -27,14 +27,14 @@ fn from_slice() { fn from_len_zeroed() { let a = MutableBitmap::from_len_zeroed(10); assert_eq!(a.len(), 10); - assert_eq!(a.unset_bits(), 10); + assert_eq!(a.null_count(), 10); } #[test] fn from_len_set() { let a = MutableBitmap::from_len_set(10); assert_eq!(a.len(), 10); - assert_eq!(a.unset_bits(), 0); + assert_eq!(a.null_count(), 0); } #[test] diff --git a/src/common/arrow/tests/it/arrow/bitmap/utils/bit_chunks_exact.rs b/src/common/column/tests/it/bitmap/utils/bit_chunks_exact.rs similarity index 95% rename from src/common/arrow/tests/it/arrow/bitmap/utils/bit_chunks_exact.rs rename to src/common/column/tests/it/bitmap/utils/bit_chunks_exact.rs index 0fa1307d9b68..847dd6881ef8 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/utils/bit_chunks_exact.rs +++ b/src/common/column/tests/it/bitmap/utils/bit_chunks_exact.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::utils::BitChunksExact; +use databend_common_column::bitmap::utils::BitChunksExact; #[test] fn basics() { diff --git a/src/common/arrow/tests/it/arrow/bitmap/utils/chunk_iter.rs b/src/common/column/tests/it/bitmap/utils/chunk_iter.rs similarity index 97% rename from src/common/arrow/tests/it/arrow/bitmap/utils/chunk_iter.rs rename to src/common/column/tests/it/bitmap/utils/chunk_iter.rs index 9e038d6e0887..c35c622bf95c 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/utils/chunk_iter.rs +++ b/src/common/column/tests/it/bitmap/utils/chunk_iter.rs @@ -13,8 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::utils::BitChunks; -use databend_common_arrow::arrow::types::BitChunkIter; +use databend_common_column::bitmap::utils::BitChunks; +use databend_common_column::types::BitChunkIter; #[test] fn basics() { diff --git a/src/common/arrow/tests/it/arrow/bitmap/utils/fmt.rs b/src/common/column/tests/it/bitmap/utils/fmt.rs similarity index 97% rename from src/common/arrow/tests/it/arrow/bitmap/utils/fmt.rs rename to src/common/column/tests/it/bitmap/utils/fmt.rs index bae56be2a513..6978e7f52e56 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/utils/fmt.rs +++ b/src/common/column/tests/it/bitmap/utils/fmt.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::utils::fmt; +use databend_common_column::bitmap::utils::fmt; struct A<'a>(&'a [u8], usize, usize); diff --git a/src/common/arrow/tests/it/arrow/bitmap/utils/iterator.rs b/src/common/column/tests/it/bitmap/utils/iterator.rs similarity index 96% rename from src/common/arrow/tests/it/arrow/bitmap/utils/iterator.rs rename to src/common/column/tests/it/bitmap/utils/iterator.rs index 1c7dd9d8fe1f..3cdc774fee43 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/utils/iterator.rs +++ b/src/common/column/tests/it/bitmap/utils/iterator.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::utils::BitmapIter; +use databend_common_column::bitmap::utils::BitmapIter; #[test] fn basic() { diff --git a/src/common/arrow/tests/it/arrow/bitmap/utils/mod.rs b/src/common/column/tests/it/bitmap/utils/mod.rs similarity index 95% rename from src/common/arrow/tests/it/arrow/bitmap/utils/mod.rs rename to src/common/column/tests/it/bitmap/utils/mod.rs index 561ba99e5387..28f30043b812 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/utils/mod.rs +++ b/src/common/column/tests/it/bitmap/utils/mod.rs @@ -13,10 +13,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::utils::*; +use databend_common_column::bitmap::utils::*; use proptest::prelude::*; -use crate::arrow::bitmap::bitmap_strategy; +use crate::bitmap::bitmap_strategy; mod bit_chunks_exact; mod chunk_iter; @@ -93,6 +93,6 @@ proptest! { #[cfg_attr(miri, ignore)] // miri and proptest do not work well :( fn null_count(bitmap in bitmap_strategy()) { let sum_of_sets: usize = (0..bitmap.len()).map(|x| (!bitmap.get_bit(x)) as usize).sum(); - assert_eq!(bitmap.unset_bits(), sum_of_sets); + assert_eq!(bitmap.null_count(), sum_of_sets); } } diff --git a/src/common/arrow/tests/it/arrow/bitmap/utils/slice_iterator.rs b/src/common/column/tests/it/bitmap/utils/slice_iterator.rs similarity index 92% rename from src/common/arrow/tests/it/arrow/bitmap/utils/slice_iterator.rs rename to src/common/column/tests/it/bitmap/utils/slice_iterator.rs index c96efee79d1c..a971c5bbea74 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/utils/slice_iterator.rs +++ b/src/common/column/tests/it/bitmap/utils/slice_iterator.rs @@ -13,11 +13,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::utils::SlicesIterator; -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::utils::SlicesIterator; +use databend_common_column::bitmap::Bitmap; use proptest::prelude::*; -use crate::arrow::bitmap::bitmap_strategy; +use crate::bitmap::bitmap_strategy; proptest! { /// Asserts that: @@ -31,7 +31,7 @@ proptest! { let slots = iter.slots(); - assert_eq!(bitmap.len() - bitmap.unset_bits(), slots); + assert_eq!(bitmap.len() - bitmap.null_count(), slots); let slices = iter.collect::>(); let mut sum = 0; @@ -124,7 +124,7 @@ fn bla() { .collect::(); let iter = SlicesIterator::new(&values); let count = iter.slots(); - assert_eq!(values.unset_bits() + iter.slots(), values.len()); + assert_eq!(values.null_count() + iter.slots(), values.len()); let total = iter.into_iter().fold(0, |acc, x| acc + x.1); @@ -136,7 +136,7 @@ fn past_end_should_not_be_returned() { let values = Bitmap::from_u8_slice([0b11111010], 3); let iter = SlicesIterator::new(&values); let count = iter.slots(); - assert_eq!(values.unset_bits() + iter.slots(), values.len()); + assert_eq!(values.null_count() + iter.slots(), values.len()); let total = iter.into_iter().fold(0, |acc, x| acc + x.1); diff --git a/src/common/arrow/tests/it/arrow/bitmap/utils/zip_validity.rs b/src/common/column/tests/it/bitmap/utils/zip_validity.rs similarity index 95% rename from src/common/arrow/tests/it/arrow/bitmap/utils/zip_validity.rs rename to src/common/column/tests/it/bitmap/utils/zip_validity.rs index 73ffb7249f81..de7043ea5d5e 100644 --- a/src/common/arrow/tests/it/arrow/bitmap/utils/zip_validity.rs +++ b/src/common/column/tests/it/bitmap/utils/zip_validity.rs @@ -13,9 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::utils::BitmapIter; -use databend_common_arrow::arrow::bitmap::utils::ZipValidity; -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::utils::BitmapIter; +use databend_common_column::bitmap::utils::ZipValidity; +use databend_common_column::bitmap::Bitmap; #[test] fn basic() { diff --git a/src/common/arrow/tests/it/arrow/buffer/immutable.rs b/src/common/column/tests/it/buffer/immutable.rs similarity index 96% rename from src/common/arrow/tests/it/arrow/buffer/immutable.rs rename to src/common/column/tests/it/buffer/immutable.rs index 4252be713ca7..7cf5457260f6 100644 --- a/src/common/arrow/tests/it/arrow/buffer/immutable.rs +++ b/src/common/column/tests/it/buffer/immutable.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::buffer::Buffer; #[test] fn new() { @@ -60,7 +60,7 @@ fn from_vec() { } #[test] -#[cfg(feature = "arrow")] + fn from_arrow() { let buffer = arrow_buffer::Buffer::from_vec(vec![1_i32, 2_i32, 3_i32]); let b = Buffer::::from(buffer.clone()); @@ -92,7 +92,6 @@ fn from_arrow() { } #[test] -#[cfg(feature = "arrow")] fn from_arrow_vec() { // Zero-copy vec conversion in arrow-rs let buffer = arrow_buffer::Buffer::from_vec(vec![1_i32, 2_i32, 3_i32]); @@ -116,7 +115,6 @@ fn from_arrow_vec() { } #[test] -#[cfg(feature = "arrow")] #[should_panic(expected = "not aligned")] fn from_arrow_misaligned() { let buffer = arrow_buffer::Buffer::from_vec(vec![1_i32, 2_i32, 3_i32]).slice(1); @@ -124,7 +122,6 @@ fn from_arrow_misaligned() { } #[test] -#[cfg(feature = "arrow")] fn from_arrow_sliced() { let buffer = arrow_buffer::Buffer::from_vec(vec![1_i32, 2_i32, 3_i32]); let b = Buffer::::from(buffer); diff --git a/src/common/arrow/tests/it/arrow/array/equal/utf8.rs b/src/common/column/tests/it/buffer/mod.rs similarity index 50% rename from src/common/arrow/tests/it/arrow/array/equal/utf8.rs rename to src/common/column/tests/it/buffer/mod.rs index 594de71c708f..a42b42419669 100644 --- a/src/common/arrow/tests/it/arrow/array/equal/utf8.rs +++ b/src/common/column/tests/it/buffer/mod.rs @@ -13,30 +13,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::array::*; -use databend_common_arrow::arrow::offset::Offset; +use databend_common_column::binview::BinaryViewColumnBuilder; +use databend_common_column::buffer::Buffer; -use super::binary_cases; -use super::test_equal; +mod immutable; -fn test_generic_string_equal() { - let cases = binary_cases(); +#[test] +fn new_basic() { + let mut buffer = Buffer::::new(); + assert_eq!(buffer.len(), 0); + assert!(buffer.is_empty()); - for (lhs, rhs, expected) in cases { - let lhs = lhs.iter().map(|x| x.as_deref()); - let rhs = rhs.iter().map(|x| x.as_deref()); - let lhs = Utf8Array::::from_trusted_len_iter(lhs); - let rhs = Utf8Array::::from_trusted_len_iter(rhs); - test_equal(&lhs, &rhs, expected); - } + buffer = Buffer::::from(vec![1, 2, 3]); + assert_eq!(buffer.len(), 3); } #[test] -fn utf8_equal() { - test_generic_string_equal::() -} +fn extend_from_repeats() { + let mut b = BinaryViewColumnBuilder::::new(); + b.extend_constant(4, "databend"); -#[test] -fn large_utf8_equal() { - test_generic_string_equal::() + let a = b.clone(); + b.extend_trusted_len_values(a.iter()); + + assert_eq!( + b.freeze(), + BinaryViewColumnBuilder::::from_values_iter(vec!["databend"; 8].into_iter()).freeze() + ) } diff --git a/src/common/arrow/tests/it/arrow/mod.rs b/src/common/column/tests/it/main.rs similarity index 86% rename from src/common/arrow/tests/it/arrow/mod.rs rename to src/common/column/tests/it/main.rs index a5d66bee6cbe..b4d29c706dec 100644 --- a/src/common/arrow/tests/it/arrow/mod.rs +++ b/src/common/column/tests/it/main.rs @@ -16,14 +16,6 @@ // this landed on 1.60. Let's not force everyone to bump just yet #![allow(clippy::unnecessary_lazy_evaluations)] -mod array; -#[cfg(feature = "arrow")] -mod arrow_data; - +mod binview; mod bitmap; mod buffer; -mod compute; - -mod scalar; -mod temporal_conversions; -mod types; diff --git a/src/common/exception/Cargo.toml b/src/common/exception/Cargo.toml index 74eb2c38e5ae..f7d33a1a0277 100644 --- a/src/common/exception/Cargo.toml +++ b/src/common/exception/Cargo.toml @@ -11,7 +11,6 @@ doctest = false test = true [dependencies] -databend-common-arrow = { workspace = true } databend-common-ast = { workspace = true } anyhow = { workspace = true } diff --git a/src/common/exception/src/exception_into.rs b/src/common/exception/src/exception_into.rs index 72788198bd35..6d0819f1d609 100644 --- a/src/common/exception/src/exception_into.rs +++ b/src/common/exception/src/exception_into.rs @@ -107,16 +107,6 @@ impl From for ErrorCode { } } -impl From for ErrorCode { - fn from(error: databend_common_arrow::arrow::error::Error) -> Self { - use databend_common_arrow::arrow::error::Error; - match error { - Error::NotYetImplemented(v) => ErrorCode::Unimplemented(format!("arrow: {v}")), - v => ErrorCode::from_std_error(v), - } - } -} - impl From for ErrorCode { fn from(error: arrow_schema::ArrowError) -> Self { match error { diff --git a/src/common/hashtable/Cargo.toml b/src/common/hashtable/Cargo.toml index 55af3ee47c48..fdaa4f24eadf 100644 --- a/src/common/hashtable/Cargo.toml +++ b/src/common/hashtable/Cargo.toml @@ -11,8 +11,8 @@ doctest = false test = true [dependencies] -databend-common-arrow = { workspace = true } databend-common-base = { workspace = true } +databend-common-column = { workspace = true } ## Must disable feature "runtime-rng", it will make the hash results unstable in cluster ahash = { workspace = true, features = ["no-rng"] } bumpalo = { workspace = true } diff --git a/src/common/hashtable/src/hashjoin_hashtable.rs b/src/common/hashtable/src/hashjoin_hashtable.rs index 7d7987b493b7..12a4c1ac20c9 100644 --- a/src/common/hashtable/src/hashjoin_hashtable.rs +++ b/src/common/hashtable/src/hashjoin_hashtable.rs @@ -17,8 +17,8 @@ use std::marker::PhantomData; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_base::mem_allocator::MmapAllocator; +use databend_common_column::bitmap::Bitmap; use super::traits::HashJoinHashtableLike; use super::traits::Keyable; @@ -165,12 +165,12 @@ where fn probe(&self, hashes: &mut [u64], bitmap: Option) -> usize { let mut valids = None; if let Some(bitmap) = bitmap { - if bitmap.unset_bits() == bitmap.len() { + if bitmap.null_count() == bitmap.len() { hashes.iter_mut().for_each(|hash| { *hash = 0; }); return 0; - } else if bitmap.unset_bits() > 0 { + } else if bitmap.null_count() > 0 { valids = Some(bitmap); } } @@ -220,7 +220,7 @@ where ) -> (usize, usize) { let mut valids = None; if let Some(bitmap) = bitmap { - if bitmap.unset_bits() == bitmap.len() { + if bitmap.null_count() == bitmap.len() { unmatched_selection .iter_mut() .enumerate() @@ -228,7 +228,7 @@ where *val = idx as u32; }); return (0, hashes.len()); - } else if bitmap.unset_bits() > 0 { + } else if bitmap.null_count() > 0 { valids = Some(bitmap); } } @@ -290,9 +290,9 @@ where ) -> usize { let mut valids = None; if let Some(bitmap) = bitmap { - if bitmap.unset_bits() == bitmap.len() { + if bitmap.null_count() == bitmap.len() { return 0; - } else if bitmap.unset_bits() > 0 { + } else if bitmap.null_count() > 0 { valids = Some(bitmap); } } diff --git a/src/common/hashtable/src/hashjoin_string_hashtable.rs b/src/common/hashtable/src/hashjoin_string_hashtable.rs index 4776d00449ae..8fdc7aa13208 100644 --- a/src/common/hashtable/src/hashjoin_string_hashtable.rs +++ b/src/common/hashtable/src/hashjoin_string_hashtable.rs @@ -16,8 +16,8 @@ use std::alloc::Allocator; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_base::mem_allocator::MmapAllocator; +use databend_common_column::bitmap::Bitmap; use super::traits::HashJoinHashtableLike; use crate::hashjoin_hashtable::combine_header; @@ -97,12 +97,12 @@ where A: Allocator + Clone + 'static fn probe(&self, hashes: &mut [u64], bitmap: Option) -> usize { let mut valids = None; if let Some(bitmap) = bitmap { - if bitmap.unset_bits() == bitmap.len() { + if bitmap.null_count() == bitmap.len() { hashes.iter_mut().for_each(|hash| { *hash = 0; }); return 0; - } else if bitmap.unset_bits() > 0 { + } else if bitmap.null_count() > 0 { valids = Some(bitmap); } } @@ -149,7 +149,7 @@ where A: Allocator + Clone + 'static ) -> (usize, usize) { let mut valids = None; if let Some(bitmap) = bitmap { - if bitmap.unset_bits() == bitmap.len() { + if bitmap.null_count() == bitmap.len() { unmatched_selection .iter_mut() .enumerate() @@ -157,7 +157,7 @@ where A: Allocator + Clone + 'static *val = idx as u32; }); return (0, hashes.len()); - } else if bitmap.unset_bits() > 0 { + } else if bitmap.null_count() > 0 { valids = Some(bitmap); } } @@ -216,9 +216,9 @@ where A: Allocator + Clone + 'static ) -> usize { let mut valids = None; if let Some(bitmap) = bitmap { - if bitmap.unset_bits() == bitmap.len() { + if bitmap.null_count() == bitmap.len() { return 0; - } else if bitmap.unset_bits() > 0 { + } else if bitmap.null_count() > 0 { valids = Some(bitmap); } } diff --git a/src/common/hashtable/src/traits.rs b/src/common/hashtable/src/traits.rs index 05da2c6f769a..fb3cad40bcd7 100644 --- a/src/common/hashtable/src/traits.rs +++ b/src/common/hashtable/src/traits.rs @@ -18,8 +18,8 @@ use std::iter::TrustedLen; use std::mem::MaybeUninit; use std::num::NonZeroU64; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_base::base::OrderedFloat; +use databend_common_column::bitmap::Bitmap; use ethnum::i256; use ethnum::U256; diff --git a/src/common/native/Cargo.toml b/src/common/native/Cargo.toml new file mode 100644 index 000000000000..0edc04df0480 --- /dev/null +++ b/src/common/native/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "databend-common-native" +version = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +edition = { workspace = true } + +[lib] +test = true + +[features] + +[dependencies] +databend-common-column = { workspace = true } +databend-common-expression = { workspace = true } + +ahash = { workspace = true } +bitpacking = { workspace = true } +bytemuck = { workspace = true } +byteorder = { workspace = true } +bytes = { workspace = true } +env_logger = { workspace = true } +ethnum = { workspace = true } +hashbrown_v0_14 = { workspace = true } +log = { workspace = true } +lz4 = { workspace = true } +match-template = { workspace = true } +num = { workspace = true, features = ["std"] } +opendal = { workspace = true } +rand = { workspace = true } +ringbuffer = { workspace = true } +roaring = { workspace = true } +serde = { workspace = true, features = ["rc"] } +serde_json = { workspace = true } +snap = { workspace = true } +zstd = { workspace = true } + +[dev-dependencies] +# used to test async readers + +[package.metadata.cargo-machete] +ignored = ["match-template"] + +[lints] +workspace = true diff --git a/src/common/arrow/src/native/compression/basic.rs b/src/common/native/src/compression/basic.rs similarity index 92% rename from src/common/arrow/src/native/compression/basic.rs rename to src/common/native/src/compression/basic.rs index 56e76da03c22..e2913e3621a9 100644 --- a/src/common/arrow/src/native/compression/basic.rs +++ b/src/common/native/src/compression/basic.rs @@ -13,8 +13,8 @@ // limitations under the License. use super::Compression; -use crate::arrow::error::Error; -use crate::arrow::error::Result; +use crate::error::Error; +use crate::error::Result; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum CommonCompression { @@ -98,7 +98,7 @@ pub fn decompress_snappy(input_buf: &[u8], output_buf: &mut [u8]) -> Result<()> .decompress(input_buf, output_buf) .map(|_| {}) .map_err(|e| { - crate::arrow::error::Error::External("decompress snappy failed".to_owned(), Box::new(e)) + crate::error::Error::External("decompress snappy failed".to_owned(), Box::new(e)) }) } @@ -110,7 +110,7 @@ pub fn compress_lz4(input_buf: &[u8], output_buf: &mut Vec) -> Result let s = unsafe { core::slice::from_raw_parts_mut(output_buf.as_mut_ptr().add(len), bound) }; let size = lz4::block::compress_to_buffer(input_buf, None, false, s).map_err(|e| { - crate::arrow::error::Error::External("Compress lz4 failed".to_owned(), Box::new(e)) + crate::error::Error::External("Compress lz4 failed".to_owned(), Box::new(e)) })?; unsafe { output_buf.set_len(size + len) }; @@ -125,7 +125,7 @@ pub fn compress_zstd(input_buf: &[u8], output_buf: &mut Vec) -> Result) -> Result( diff --git a/src/common/arrow/src/native/compression/binary/dict.rs b/src/common/native/src/compression/binary/dict.rs similarity index 74% rename from src/common/arrow/src/native/compression/binary/dict.rs rename to src/common/native/src/compression/binary/dict.rs index d335c7fb998c..aa82875dc995 100644 --- a/src/common/arrow/src/native/compression/binary/dict.rs +++ b/src/common/native/src/compression/binary/dict.rs @@ -16,30 +16,30 @@ use std::io::BufRead; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::binary::BinaryColumn; +use databend_common_column::types::Index; use super::BinaryCompression; use super::BinaryStats; -use crate::arrow::array::BinaryArray; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::Offset; +use crate::compression::get_bits_needed; +use crate::compression::integer::compress_integer; +use crate::compression::integer::decompress_integer; +use crate::compression::integer::Dict; +use crate::compression::integer::DictEncoder; +use crate::compression::is_valid; +use crate::compression::Compression; +use crate::error::Error; +use crate::error::Result; use crate::general_err; -use crate::native::compression::get_bits_needed; -use crate::native::compression::integer::compress_integer; -use crate::native::compression::integer::decompress_integer; -use crate::native::compression::integer::Dict; -use crate::native::compression::integer::DictEncoder; -use crate::native::compression::is_valid; -use crate::native::compression::Compression; -use crate::native::util::AsBytes; -use crate::native::write::WriteOptions; - -impl BinaryCompression for Dict { +use crate::util::AsBytes; +use crate::write::WriteOptions; + +impl BinaryCompression for Dict { fn to_compression(&self) -> Compression { Compression::Dict } - fn compress_ratio(&self, stats: &super::BinaryStats) -> f64 { + fn compress_ratio(&self, stats: &super::BinaryStats) -> f64 { const MIN_DICT_RATIO: usize = 3; if stats.unique_count * MIN_DICT_RATIO >= stats.tuple_count { return 0.0f64; @@ -53,19 +53,19 @@ impl BinaryCompression for Dict { fn compress( &self, - array: &BinaryArray, - _stats: &BinaryStats, + col: &BinaryColumn, + stats: &BinaryStats, write_options: &WriteOptions, output_buf: &mut Vec, ) -> Result { let start = output_buf.len(); - let mut encoder = DictEncoder::with_capacity(array.len()); + let mut encoder = DictEncoder::with_capacity(col.len()); - for (i, range) in array.offsets().buffer().windows(2).enumerate() { - if !is_valid(&array.validity(), i) && !encoder.is_empty() { + for (i, range) in col.offsets().windows(2).enumerate() { + if !is_valid(stats.validity.as_ref(), i) && !encoder.is_empty() { encoder.push_last_index(); } else { - let data = array.values().clone().sliced( + let data = col.data().clone().sliced( range[0].to_usize(), range[1].to_usize() - range[0].to_usize(), ); @@ -77,7 +77,7 @@ impl BinaryCompression for Dict { // dict data use custom encoding let mut write_options = write_options.clone(); write_options.forbidden_compressions.push(Compression::Dict); - compress_integer(&indices, write_options, output_buf)?; + compress_integer(&indices, stats.validity.clone(), write_options, output_buf)?; // data page use plain encoding let sets = encoder.get_sets(); @@ -95,7 +95,7 @@ impl BinaryCompression for Dict { &self, mut input: &[u8], length: usize, - offsets: &mut Vec, + offsets: &mut Vec, values: &mut Vec, ) -> Result<()> { let mut indices: Vec = Vec::new(); @@ -119,7 +119,7 @@ impl BinaryCompression for Dict { } last_offset = if offsets.is_empty() { - offsets.push(O::default()); + offsets.push(0); 0 } else { offsets.last().unwrap().to_usize() @@ -134,7 +134,7 @@ impl BinaryCompression for Dict { values.extend_from_slice(&data[off..end]); last_offset += end - off; - offsets.push(O::from_usize(last_offset).unwrap()); + offsets.push(last_offset as u64); } Ok(()) } diff --git a/src/common/arrow/src/native/compression/binary/freq.rs b/src/common/native/src/compression/binary/freq.rs similarity index 86% rename from src/common/arrow/src/native/compression/binary/freq.rs rename to src/common/native/src/compression/binary/freq.rs index 3a28eaf87207..b408a1a39d8c 100644 --- a/src/common/arrow/src/native/compression/binary/freq.rs +++ b/src/common/native/src/compression/binary/freq.rs @@ -17,28 +17,27 @@ use std::ops::Deref; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::binary::BinaryColumn; use roaring::RoaringBitmap; use super::BinaryCompression; use super::BinaryStats; -use crate::arrow::array::BinaryArray; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::Offset; +use crate::compression::integer::Freq; +use crate::compression::Compression; +use crate::error::Error; +use crate::error::Result; use crate::general_err; -use crate::native::compression::integer::Freq; -use crate::native::compression::Compression; -use crate::native::write::WriteOptions; +use crate::write::WriteOptions; -impl BinaryCompression for Freq { +impl BinaryCompression for Freq { fn to_compression(&self) -> Compression { Compression::Freq } fn compress( &self, - array: &BinaryArray, - stats: &BinaryStats, + col: &BinaryColumn, + stats: &BinaryStats, write_options: &WriteOptions, output: &mut Vec, ) -> Result { @@ -63,7 +62,7 @@ impl BinaryCompression for Freq { let mut exceptions_bitmap = RoaringBitmap::new(); let mut exceptions = Vec::with_capacity(stats.tuple_count - max_count); - for (i, val) in array.iter().enumerate() { + for (i, val) in col.option_iter(stats.validity.as_ref()).enumerate() { if let Some(val) = val { if top_value_is_null || val != top_value { exceptions_bitmap.insert(i as u32); @@ -97,7 +96,7 @@ impl BinaryCompression for Freq { &self, mut input: &[u8], length: usize, - offsets: &mut Vec, + offsets: &mut Vec, values: &mut Vec, ) -> Result<()> { let len = input.read_u64::()? as usize; @@ -114,7 +113,7 @@ impl BinaryCompression for Freq { input.consume(exceptions_bitmap_size as usize); if offsets.is_empty() { - offsets.push(O::default()); + offsets.push(0); } offsets.reserve(length); @@ -128,17 +127,17 @@ impl BinaryCompression for Freq { input.consume(len); values.extend_from_slice(val); - offsets.push(O::from_usize(values.len()).unwrap()); + offsets.push(values.len() as u64); } else { values.extend_from_slice(top_value); - offsets.push(O::from_usize(values.len()).unwrap()); + offsets.push(values.len() as u64); } } Ok(()) } - fn compress_ratio(&self, stats: &super::BinaryStats) -> f64 { + fn compress_ratio(&self, stats: &super::BinaryStats) -> f64 { if stats.unique_count <= 1 { return 0.0f64; } diff --git a/src/common/arrow/src/native/compression/binary/mod.rs b/src/common/native/src/compression/binary/mod.rs similarity index 78% rename from src/common/arrow/src/native/compression/binary/mod.rs rename to src/common/native/src/compression/binary/mod.rs index 0cf9875288ff..ce5fafc9896d 100644 --- a/src/common/arrow/src/native/compression/binary/mod.rs +++ b/src/common/native/src/compression/binary/mod.rs @@ -18,30 +18,32 @@ mod one_value; use std::collections::HashMap; use std::hash::Hash; -use std::marker::PhantomData; + +use databend_common_column::binary::BinaryColumn; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::types::Index; +use databend_common_expression::types::Buffer; use super::basic::CommonCompression; use super::integer::Dict; use super::integer::Freq; use super::integer::OneValue; use super::Compression; -use crate::arrow::array::BinaryArray; -use crate::arrow::buffer::Buffer; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::Offset; -use crate::native::read::read_basic::read_compress_header; -use crate::native::read::NativeReadBuf; -use crate::native::write::WriteOptions; - -pub fn compress_binary( - array: &BinaryArray, +use crate::error::Error; +use crate::error::Result; +use crate::read::read_basic::read_compress_header; +use crate::read::NativeReadBuf; +use crate::write::WriteOptions; + +pub fn compress_binary( + col: &BinaryColumn, + validity: Option, buf: &mut Vec, write_options: WriteOptions, ) -> Result<()> { // choose compressor - let stats = gen_stats(array); - let compressor = choose_compressor(array, &stats, &write_options); + let stats = gen_stats(col, validity); + let compressor = choose_compressor(col, &stats, &write_options); log::debug!( "choose binary compression : {:?}", @@ -53,11 +55,11 @@ pub fn compress_binary( match compressor { BinaryCompressor::Basic(c) => { // offsets - let offsets = array.offsets(); - let offsets = if offsets.first().is_zero() { - offsets.buffer().clone() + let offsets = col.offsets(); + let offsets = if *offsets.first().unwrap() == 0 { + offsets.clone() } else { - let first = offsets.first(); + let first = offsets.first().unwrap(); let mut zero_offsets = Vec::with_capacity(offsets.len()); for offset in offsets.iter() { zero_offsets.push(*offset - *first); @@ -76,10 +78,11 @@ pub fn compress_binary( buf[pos + 4..pos + 8].copy_from_slice(&(input_buf.len() as u32).to_le_bytes()); // values - let mut values = array.values().clone(); + let mut values = col.data().clone(); values.slice( - array.offsets().first().to_usize(), - array.offsets().last().to_usize() - array.offsets().first().to_usize(), + col.offsets().first().unwrap().to_usize(), + col.offsets().last().unwrap().to_usize() + - col.offsets().first().unwrap().to_usize(), ); let input_buf = bytemuck::cast_slice(&values); buf.extend_from_slice(&codec.to_le_bytes()); @@ -94,19 +97,19 @@ pub fn compress_binary( buf.extend_from_slice(&codec.to_le_bytes()); let pos = buf.len(); buf.extend_from_slice(&[0u8; 8]); - let compressed_size = c.compress(array, &stats, &write_options, buf)?; + let compressed_size = c.compress(col, &stats, &write_options, buf)?; buf[pos..pos + 4].copy_from_slice(&(compressed_size as u32).to_le_bytes()); - buf[pos + 4..pos + 8].copy_from_slice(&(array.values().len() as u32).to_le_bytes()); + buf[pos + 4..pos + 8].copy_from_slice(&(col.data().len() as u32).to_le_bytes()); } } Ok(()) } -pub fn decompress_binary( +pub fn decompress_binary( reader: &mut R, length: usize, - offsets: &mut Vec, + offsets: &mut Vec, values: &mut Vec, scratch: &mut Vec, ) -> Result<()> { @@ -124,7 +127,7 @@ pub fn decompress_binary( scratch.as_slice() }; - let encoder = BinaryCompressor::::from_compression(compression)?; + let encoder = BinaryCompressor::from_compression(compression)?; match encoder { BinaryCompressor::Basic(c) => { @@ -133,7 +136,7 @@ pub fn decompress_binary( let out_slice = unsafe { core::slice::from_raw_parts_mut( offsets.as_mut_ptr().add(offsets.len()) as *mut u8, - (length + 1) * std::mem::size_of::(), + (length + 1) * std::mem::size_of::(), ) }; c.decompress(&input[..compressed_size], out_slice)?; @@ -178,11 +181,11 @@ pub fn decompress_binary( Ok(()) } -pub trait BinaryCompression { +pub trait BinaryCompression { fn compress( &self, - array: &BinaryArray, - stats: &BinaryStats, + col: &BinaryColumn, + stats: &BinaryStats, write_options: &WriteOptions, output: &mut Vec, ) -> Result; @@ -191,20 +194,20 @@ pub trait BinaryCompression { &self, input: &[u8], length: usize, - offsets: &mut Vec, + offsets: &mut Vec, values: &mut Vec, ) -> Result<()>; - fn compress_ratio(&self, stats: &BinaryStats) -> f64; + fn compress_ratio(&self, stats: &BinaryStats) -> f64; fn to_compression(&self) -> Compression; } -enum BinaryCompressor { +enum BinaryCompressor { Basic(CommonCompression), - Extend(Box>), + Extend(Box), } -impl BinaryCompressor { +impl BinaryCompressor { fn to_compression(&self) -> Compression { match self { Self::Basic(c) => c.to_compression(), @@ -248,29 +251,32 @@ impl std::ops::Deref for U8Buffer { #[allow(dead_code)] #[derive(Debug)] -pub struct BinaryStats { +pub struct BinaryStats { tuple_count: usize, total_bytes: usize, unique_count: usize, total_unique_size: usize, + validity: Option, null_count: usize, distinct_values: HashMap, - _data: PhantomData, } -fn gen_stats(array: &BinaryArray) -> BinaryStats { +fn gen_stats(col: &BinaryColumn, validity: Option) -> BinaryStats { let mut stats = BinaryStats { - tuple_count: array.len(), - total_bytes: array.values().len() + (array.len() + 1) * std::mem::size_of::(), + tuple_count: col.len(), + total_bytes: col.data().len() + (col.len() + 1) * std::mem::size_of::(), unique_count: 0, total_unique_size: 0, - null_count: array.validity().map(|v| v.unset_bits()).unwrap_or_default(), + null_count: validity + .as_ref() + .map(|v| v.null_count()) + .unwrap_or_default(), + validity, distinct_values: HashMap::new(), - _data: PhantomData, }; - for o in array.offsets().windows(2) { - let mut values = array.values().clone(); + for o in col.offsets().windows(2) { + let mut values = col.data().clone(); values.slice(o[0].to_usize(), o[1].to_usize() - o[0].to_usize()); *stats.distinct_values.entry(U8Buffer(values)).or_insert(0) += 1; @@ -286,21 +292,21 @@ fn gen_stats(array: &BinaryArray) -> BinaryStats { stats } -fn choose_compressor( - _value: &BinaryArray, - stats: &BinaryStats, +fn choose_compressor( + _value: &BinaryColumn, + stats: &BinaryStats, write_options: &WriteOptions, -) -> BinaryCompressor { +) -> BinaryCompressor { #[cfg(debug_assertions)] { - if crate::native::util::env::check_freq_env() + if crate::util::env::check_freq_env() && !write_options .forbidden_compressions .contains(&Compression::Freq) { return BinaryCompressor::Extend(Box::new(Freq {})); } - if crate::native::util::env::check_dict_env() + if crate::util::env::check_dict_env() && !write_options .forbidden_compressions .contains(&Compression::Dict) @@ -314,7 +320,7 @@ fn choose_compressor( let mut max_ratio = ratio; let mut result = basic; - let compressors: Vec>> = vec![ + let compressors: Vec> = vec![ Box::new(OneValue {}) as _, Box::new(Freq {}) as _, Box::new(Dict {}) as _, diff --git a/src/common/arrow/src/native/compression/binary/one_value.rs b/src/common/native/src/compression/binary/one_value.rs similarity index 71% rename from src/common/arrow/src/native/compression/binary/one_value.rs rename to src/common/native/src/compression/binary/one_value.rs index dfdb5b6068a8..3a087a9c7c80 100644 --- a/src/common/arrow/src/native/compression/binary/one_value.rs +++ b/src/common/native/src/compression/binary/one_value.rs @@ -16,24 +16,23 @@ use std::io::BufRead; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::binary::BinaryColumn; use super::BinaryCompression; use super::BinaryStats; -use crate::arrow::array::BinaryArray; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::Offset; +use crate::compression::integer::OneValue; +use crate::compression::Compression; +use crate::error::Error; +use crate::error::Result; use crate::general_err; -use crate::native::compression::integer::OneValue; -use crate::native::compression::Compression; -use crate::native::write::WriteOptions; +use crate::write::WriteOptions; -impl BinaryCompression for OneValue { +impl BinaryCompression for OneValue { fn to_compression(&self) -> Compression { Compression::OneValue } - fn compress_ratio(&self, stats: &super::BinaryStats) -> f64 { + fn compress_ratio(&self, stats: &super::BinaryStats) -> f64 { if stats.unique_count <= 1 { stats.tuple_count as f64 } else { @@ -43,16 +42,12 @@ impl BinaryCompression for OneValue { fn compress( &self, - array: &BinaryArray, - _stats: &BinaryStats, + col: &BinaryColumn, + _stats: &BinaryStats, _write_options: &WriteOptions, output_buf: &mut Vec, ) -> Result { - let val = array.iter().find(|v| v.is_some()); - let val = match val { - Some(Some(v)) => v, - _ => &[], - }; + let val = col.iter().next().unwrap_or(&[]); let start = output_buf.len(); @@ -66,7 +61,7 @@ impl BinaryCompression for OneValue { &self, mut input: &[u8], length: usize, - offsets: &mut Vec, + offsets: &mut Vec, values: &mut Vec, ) -> Result<()> { let len = input.read_u32::()? as usize; @@ -79,7 +74,7 @@ impl BinaryCompression for OneValue { input.consume(len); if offsets.is_empty() { - offsets.push(O::zero()); + offsets.push(0); } offsets.reserve(length); @@ -87,7 +82,7 @@ impl BinaryCompression for OneValue { for _ in 0..length { values.extend_from_slice(val); - offsets.push(O::from_usize(values.len()).unwrap()); + offsets.push(values.len() as u64); } Ok(()) } diff --git a/src/common/arrow/src/native/compression/boolean/mod.rs b/src/common/native/src/compression/boolean/mod.rs similarity index 75% rename from src/common/arrow/src/native/compression/boolean/mod.rs rename to src/common/native/src/compression/boolean/mod.rs index c9487996264e..fc840eb018eb 100644 --- a/src/common/arrow/src/native/compression/boolean/mod.rs +++ b/src/common/native/src/compression/boolean/mod.rs @@ -15,6 +15,8 @@ mod one_value; mod rle; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_expression::types::Bitmap; use rand::thread_rng; use rand::Rng; @@ -22,24 +24,21 @@ use super::basic::CommonCompression; use super::integer::OneValue; use super::integer::Rle; use super::Compression; -use crate::arrow::array::BooleanArray; -use crate::arrow::array::MutableBooleanArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::native::read::read_basic::read_compress_header; -use crate::native::read::NativeReadBuf; -use crate::native::write::WriteOptions; +use crate::error::Error; +use crate::error::Result; +use crate::read::read_basic::read_compress_header; +use crate::read::NativeReadBuf; +use crate::write::WriteOptions; pub fn compress_boolean( - array: &BooleanArray, + col: &Bitmap, + validity: Option, buf: &mut Vec, write_options: WriteOptions, ) -> Result<()> { // choose compressor - let stats = gen_stats(array); - let compressor = choose_compressor(array, &stats, &write_options); + let stats = gen_stats(col, validity.clone()); + let compressor = choose_compressor(col, &stats, &write_options); log::debug!( "choose boolean compression : {:?}", @@ -53,22 +52,22 @@ pub fn compress_boolean( let compressed_size = match compressor { BooleanCompressor::Basic(c) => { - let bitmap = array.values(); - let (_, slice_offset, _) = bitmap.as_slice(); + let (_, slice_offset, _) = col.as_slice(); let bitmap = if slice_offset != 0 { // case where we can't slice the bitmap as the offsets are not multiple of 8 - Bitmap::from_trusted_len_iter(bitmap.iter()) + Bitmap::from_trusted_len_iter(col.iter()) } else { - bitmap.clone() + col.clone() }; + let (slice, _, _) = bitmap.as_slice(); c.compress(slice, buf) } - BooleanCompressor::Extend(c) => c.compress(array, buf), + BooleanCompressor::Extend(c) => c.compress(col, validity, buf), }?; buf[pos..pos + 4].copy_from_slice(&(compressed_size as u32).to_le_bytes()); - buf[pos + 4..pos + 8].copy_from_slice(&(array.len() as u32).to_le_bytes()); + buf[pos + 4..pos + 8].copy_from_slice(&(col.len() as u32).to_le_bytes()); Ok(()) } @@ -113,7 +112,12 @@ pub fn decompress_boolean( } pub trait BooleanCompression { - fn compress(&self, array: &BooleanArray, output: &mut Vec) -> Result; + fn compress( + &self, + col: &Bitmap, + validity: Option, + output: &mut Vec, + ) -> Result; fn decompress(&self, input: &[u8], length: usize, output: &mut MutableBitmap) -> Result<()>; fn to_compression(&self) -> Compression; @@ -150,16 +154,17 @@ impl BooleanCompressor { #[allow(dead_code)] #[derive(Debug, Clone)] pub struct BooleanStats { - pub src: BooleanArray, + pub src: Bitmap, pub total_bytes: usize, pub rows: usize, + pub validity: Option, pub null_count: usize, pub false_count: usize, pub true_count: usize, pub average_run_length: f64, } -fn gen_stats(array: &BooleanArray) -> BooleanStats { +fn gen_stats(col: &Bitmap, validity: Option) -> BooleanStats { let mut null_count = 0; let mut false_count = 0; let mut true_count = 0; @@ -168,7 +173,7 @@ fn gen_stats(array: &BooleanArray) -> BooleanStats { let mut last_value = false; let mut run_count = 0; - for v in array.iter() { + for v in col.option_iter(validity.as_ref()) { if !is_init_value_initialized { is_init_value_initialized = true; last_value = v.unwrap_or_default(); @@ -192,24 +197,25 @@ fn gen_stats(array: &BooleanArray) -> BooleanStats { } BooleanStats { - src: array.clone(), - rows: array.len(), - total_bytes: array.values().len() / 8, + src: col.clone(), + rows: col.len(), + total_bytes: col.len() / 8, + validity, null_count, false_count, true_count, - average_run_length: array.len() as f64 / 8.0f64 / run_count as f64, + average_run_length: col.len() as f64 / 8.0f64 / run_count as f64, } } fn choose_compressor( - _array: &BooleanArray, + _col: &Bitmap, stats: &BooleanStats, write_options: &WriteOptions, ) -> BooleanCompressor { #[cfg(debug_assertions)] { - if crate::native::util::env::check_rle_env() + if crate::util::env::check_rle_env() && !write_options .forbidden_compressions .contains(&Compression::Rle) @@ -260,10 +266,17 @@ fn compress_sample_ratio( let stats = if stats.src.len() / sample_count <= sample_size { stats.clone() } else { - let array = &stats.src; - let separator = array.len() / sample_count; - let remainder = array.len() % sample_count; - let mut builder = MutableBooleanArray::with_capacity(sample_count * sample_size); + let col = &stats.src; + let separator = col.len() / sample_count; + let remainder = col.len() % sample_count; + let mut builder = MutableBitmap::with_capacity(sample_count * sample_size); + + let mut validity = if stats.null_count > 0 { + Some(MutableBitmap::with_capacity(sample_count * sample_size)) + } else { + None + }; + for sample_i in 0..sample_count { let range_end = if sample_i == sample_count - 1 { separator + remainder @@ -273,16 +286,23 @@ fn compress_sample_ratio( let partition_begin = sample_i * separator + rng.gen_range(0..range_end); - let mut s = array.clone(); + let mut s = col.clone(); s.slice(partition_begin, sample_size); - builder.extend_trusted_len(s.into_iter()); + + if let (Some(b), Some(validity)) = (&mut validity, &stats.validity) { + let mut v = validity.clone(); + v.slice(partition_begin, sample_size); + b.extend_from_trusted_len_iter(v.into_iter()); + } + + builder.extend_from_trusted_len_iter(s.into_iter()); } - let sample_array: BooleanArray = builder.into(); - gen_stats(&sample_array) + let sample_col: Bitmap = builder.into(); + gen_stats(&sample_col, validity.map(|x| x.into())) }; let size = c - .compress(&stats.src, &mut vec![]) + .compress(&stats.src, stats.validity.clone(), &mut vec![]) .unwrap_or(stats.total_bytes); stats.total_bytes as f64 / size as f64 diff --git a/src/common/arrow/src/native/compression/boolean/one_value.rs b/src/common/native/src/compression/boolean/one_value.rs similarity index 73% rename from src/common/arrow/src/native/compression/boolean/one_value.rs rename to src/common/native/src/compression/boolean/one_value.rs index 85ae01583925..cb018cac6240 100644 --- a/src/common/arrow/src/native/compression/boolean/one_value.rs +++ b/src/common/native/src/compression/boolean/one_value.rs @@ -12,14 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; + use super::BooleanCompression; -use crate::arrow::array::BooleanArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::error::Error; -use crate::arrow::error::Result; +use crate::compression::integer::OneValue; +use crate::compression::Compression; +use crate::error::Error; +use crate::error::Result; use crate::general_err; -use crate::native::compression::integer::OneValue; -use crate::native::compression::Compression; impl BooleanCompression for OneValue { fn to_compression(&self) -> Compression { @@ -34,12 +35,13 @@ impl BooleanCompression for OneValue { } } - fn compress(&self, array: &BooleanArray, output_buf: &mut Vec) -> Result { - let val = array.iter().find(|v| v.is_some()); - let val = match val { - Some(Some(v)) => v, - _ => false, - }; + fn compress( + &self, + col: &Bitmap, + _validity: Option, + output_buf: &mut Vec, + ) -> Result { + let val = col.iter().last().unwrap_or_default(); output_buf.push(val as u8); Ok(1) } diff --git a/src/common/arrow/src/native/compression/boolean/rle.rs b/src/common/native/src/compression/boolean/rle.rs similarity index 68% rename from src/common/arrow/src/native/compression/boolean/rle.rs rename to src/common/native/src/compression/boolean/rle.rs index a8dbf038d145..6417384b617d 100644 --- a/src/common/arrow/src/native/compression/boolean/rle.rs +++ b/src/common/native/src/compression/boolean/rle.rs @@ -14,36 +14,37 @@ use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; use super::compress_sample_ratio; use super::BooleanCompression; use super::BooleanStats; -use crate::arrow::array::BooleanArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::error::Result; -use crate::native::compression::integer::Rle; -use crate::native::compression::Compression; -use crate::native::compression::SAMPLE_COUNT; -use crate::native::compression::SAMPLE_SIZE; +use crate::compression::integer::Rle; +use crate::compression::Compression; +use crate::compression::SAMPLE_COUNT; +use crate::compression::SAMPLE_SIZE; +use crate::error::Result; impl BooleanCompression for Rle { - fn compress(&self, array: &BooleanArray, output: &mut Vec) -> Result { + fn compress( + &self, + col: &Bitmap, + validity: Option, + output: &mut Vec, + ) -> Result { let size = output.len(); - self.compress_integer( - output, - array.values().iter().map(|v| v as u8), - array.validity(), - )?; + self.compress_integer(output, col.iter().map(|v| v as u8), validity)?; Ok(output.len() - size) } - fn decompress(&self, mut input: &[u8], length: usize, array: &mut MutableBitmap) -> Result<()> { + fn decompress(&self, mut input: &[u8], length: usize, col: &mut MutableBitmap) -> Result<()> { let mut num_values = 0; while !input.is_empty() { let len: u32 = input.read_u32::()?; let t = input.read_u8()? != 0; for _ in 0..len { - array.push(t); + col.push(t); } num_values += len as usize; if num_values >= length { diff --git a/src/common/arrow/src/native/compression/double/dict.rs b/src/common/native/src/compression/double/dict.rs similarity index 80% rename from src/common/arrow/src/native/compression/double/dict.rs rename to src/common/native/src/compression/double/dict.rs index 46f131c6f81f..1ef47f930b46 100644 --- a/src/common/arrow/src/native/compression/double/dict.rs +++ b/src/common/native/src/compression/double/dict.rs @@ -14,36 +14,36 @@ use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; use super::traits::DoubleType; use super::DoubleCompression; use super::DoubleStats; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Error; -use crate::arrow::error::Result; +use crate::compression::get_bits_needed; +use crate::compression::integer::compress_integer; +use crate::compression::integer::decompress_integer; +use crate::compression::integer::Dict; +use crate::compression::integer::DictEncoder; +use crate::compression::integer::RawNative; +use crate::compression::Compression; +use crate::error::Error; +use crate::error::Result; use crate::general_err; -use crate::native::compression::get_bits_needed; -use crate::native::compression::integer::compress_integer; -use crate::native::compression::integer::decompress_integer; -use crate::native::compression::integer::Dict; -use crate::native::compression::integer::DictEncoder; -use crate::native::compression::integer::RawNative; -use crate::native::compression::Compression; -use crate::native::write::WriteOptions; +use crate::write::WriteOptions; impl DoubleCompression for Dict { fn compress( &self, - array: &PrimitiveArray, - _stats: &DoubleStats, + col: &Buffer, + stats: &DoubleStats, write_options: &WriteOptions, output_buf: &mut Vec, ) -> Result { let start = output_buf.len(); - let mut encoder = DictEncoder::with_capacity(array.len()); - for val in array.iter() { + let mut encoder = DictEncoder::with_capacity(col.len()); + for val in col.option_iter(stats.validity.as_ref()) { match val { - Some(val) => encoder.push(&RawNative { inner: *val }), + Some(val) => encoder.push(&RawNative { inner: val }), None => { if encoder.is_empty() { encoder.push(&RawNative { @@ -60,7 +60,7 @@ impl DoubleCompression for Dict { // dict data use custom encoding let mut write_options = write_options.clone(); write_options.forbidden_compressions.push(Compression::Dict); - compress_integer(&indices, write_options, output_buf)?; + compress_integer(&indices, stats.validity.clone(), write_options, output_buf)?; let sets = encoder.get_sets(); output_buf.extend_from_slice(&(sets.len() as u32).to_le_bytes()); diff --git a/src/common/arrow/src/native/compression/double/freq.rs b/src/common/native/src/compression/double/freq.rs similarity index 82% rename from src/common/arrow/src/native/compression/double/freq.rs rename to src/common/native/src/compression/double/freq.rs index 6a69cf780d0b..38973bed0089 100644 --- a/src/common/arrow/src/native/compression/double/freq.rs +++ b/src/common/native/src/compression/double/freq.rs @@ -17,23 +17,23 @@ use std::io::Read; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; use roaring::RoaringBitmap; use super::compress_double; use super::DoubleCompression; use super::DoubleStats; use super::DoubleType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::native::compression::double::decompress_double; -use crate::native::compression::integer::Freq; -use crate::native::compression::Compression; -use crate::native::write::WriteOptions; +use crate::compression::double::decompress_double; +use crate::compression::integer::Freq; +use crate::compression::Compression; +use crate::error::Result; +use crate::write::WriteOptions; impl DoubleCompression for Freq { fn compress( &self, - array: &PrimitiveArray, + col: &Buffer, stats: &DoubleStats, write_options: &WriteOptions, output: &mut Vec, @@ -41,7 +41,7 @@ impl DoubleCompression for Freq { let size = output.len(); let mut top_value_is_null = false; - let mut top_value = T::default().as_order(); + let mut top_value = T::default(); let mut max_count = 0; if stats.null_count as f64 / stats.tuple_count as f64 >= 0.9 { @@ -58,17 +58,15 @@ impl DoubleCompression for Freq { let mut exceptions_bitmap = RoaringBitmap::new(); let mut exceptions = Vec::with_capacity(stats.tuple_count - max_count); - for (i, val) in array.iter().enumerate() { - if let Some(val) = val { - if top_value_is_null || val.as_order() != top_value { - exceptions_bitmap.insert(i as u32); - exceptions.push(*val); - } + for (i, val) in col.iter().enumerate() { + if top_value_is_null || *val != top_value { + exceptions_bitmap.insert(i as u32); + exceptions.push(*val); } } // Write TopValue - output.extend_from_slice(T::from_order(top_value).to_le_bytes().as_ref()); + output.extend_from_slice(top_value.to_le_bytes().as_ref()); // Write exceptions bitmap output.extend_from_slice(&(exceptions_bitmap.serialized_size() as u32).to_le_bytes()); @@ -81,8 +79,8 @@ impl DoubleCompression for Freq { let mut write_options = write_options.clone(); write_options.forbidden_compressions.push(Compression::Freq); - let exceptions = PrimitiveArray::::from_vec(exceptions); - compress_double(&exceptions, write_options, output)?; + let exceptions = exceptions.into(); + compress_double(&exceptions, stats.validity.clone(), write_options, output)?; Ok(output.len() - size) } diff --git a/src/common/arrow/src/native/compression/double/mod.rs b/src/common/native/src/compression/double/mod.rs similarity index 78% rename from src/common/arrow/src/native/compression/double/mod.rs rename to src/common/native/src/compression/double/mod.rs index 6ab743aeb8d9..322b631170db 100644 --- a/src/common/arrow/src/native/compression/double/mod.rs +++ b/src/common/native/src/compression/double/mod.rs @@ -21,6 +21,9 @@ mod traits; use std::collections::HashMap; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_column::buffer::Buffer; use rand::thread_rng; use rand::Rng; @@ -33,23 +36,21 @@ use super::integer::Freq; use super::integer::Rle; use super::is_valid; use super::Compression; -use crate::arrow::array::Array; -use crate::arrow::array::MutablePrimitiveArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::native::read::read_basic::read_compress_header; -use crate::native::read::NativeReadBuf; -use crate::native::write::WriteOptions; +use crate::error::Error; +use crate::error::Result; +use crate::read::read_basic::read_compress_header; +use crate::read::NativeReadBuf; +use crate::write::WriteOptions; pub fn compress_double( - array: &PrimitiveArray, + col: &Buffer, + validity: Option, write_options: WriteOptions, buf: &mut Vec, ) -> Result<()> { // choose compressor - let stats = gen_stats(array); - let compressor = choose_compressor(array, &stats, &write_options); + let stats = gen_stats(col, validity); + let compressor = choose_compressor(col, &stats, &write_options); log::debug!( "choose double compression : {:?}", @@ -63,14 +64,14 @@ pub fn compress_double( let compressed_size = match compressor { DoubleCompressor::Basic(c) => { - let input_buf = bytemuck::cast_slice(array.values()); + let input_buf = bytemuck::cast_slice(col.as_slice()); c.compress(input_buf, buf) } - DoubleCompressor::Extend(c) => c.compress(array, &stats, &write_options, buf), + DoubleCompressor::Extend(c) => c.compress(col, &stats, &write_options, buf), }?; buf[pos..pos + 4].copy_from_slice(&(compressed_size as u32).to_le_bytes()); buf[pos + 4..pos + 8] - .copy_from_slice(&((array.len() * std::mem::size_of::()) as u32).to_le_bytes()); + .copy_from_slice(&((col.len() * std::mem::size_of::()) as u32).to_le_bytes()); log::debug!( "double compress ratio {}", @@ -128,7 +129,7 @@ pub fn decompress_double( pub trait DoubleCompression { fn compress( &self, - array: &PrimitiveArray, + col: &Buffer, stats: &DoubleStats, write_options: &WriteOptions, output: &mut Vec, @@ -172,44 +173,43 @@ impl DoubleCompressor { #[derive(Debug, Clone)] pub struct DoubleStats { - pub src: PrimitiveArray, + pub src: Buffer, pub tuple_count: usize, pub total_bytes: usize, pub null_count: usize, + pub validity: Option, pub is_sorted: bool, - pub min: T::OrderType, - pub max: T::OrderType, + pub min: T, + pub max: T, pub average_run_length: f64, - pub distinct_values: HashMap, + pub distinct_values: HashMap, pub unique_count: usize, - pub set_count: usize, } -fn gen_stats(array: &PrimitiveArray) -> DoubleStats { +fn gen_stats(col: &Buffer, validity: Option) -> DoubleStats { + let null_count = validity.as_ref().map(|x| x.null_count()).unwrap_or(0); let mut stats = DoubleStats:: { - src: array.clone(), - tuple_count: array.len(), - total_bytes: array.len() * std::mem::size_of::(), - null_count: array.null_count(), + src: col.clone(), + tuple_count: col.len(), + total_bytes: col.len() * std::mem::size_of::(), + null_count, + validity, is_sorted: true, - min: T::default().as_order(), - max: T::default().as_order(), + min: T::default(), + max: T::default(), average_run_length: 0.0, distinct_values: HashMap::new(), unique_count: 0, - set_count: array.len() - array.null_count(), }; let mut is_init_value_initialized = false; - let mut last_value = T::default().as_order(); + let mut last_value = T::default(); let mut run_count = 0; - let validity = array.validity(); - for (i, current_value) in array.values().iter().cloned().enumerate() { - let current_value = current_value.as_order(); - if is_valid(&validity, i) { + for (i, current_value) in col.iter().cloned().enumerate() { + if is_valid(stats.validity.as_ref(), i) { if current_value < last_value { stats.is_sorted = false; } @@ -235,40 +235,40 @@ fn gen_stats(array: &PrimitiveArray) -> DoubleStats { *stats.distinct_values.entry(current_value).or_insert(0) += 1; } stats.unique_count = stats.distinct_values.len(); - stats.average_run_length = array.len() as f64 / run_count as f64; + stats.average_run_length = col.len() as f64 / run_count as f64; stats } fn choose_compressor( - _value: &PrimitiveArray, + _value: &Buffer, stats: &DoubleStats, write_options: &WriteOptions, ) -> DoubleCompressor { #[cfg(debug_assertions)] { - if crate::native::util::env::check_freq_env() + if crate::util::env::check_freq_env() && !write_options .forbidden_compressions .contains(&Compression::Freq) { return DoubleCompressor::Extend(Box::new(Freq {})); } - if crate::native::util::env::check_dict_env() + if crate::util::env::check_dict_env() && !write_options .forbidden_compressions .contains(&Compression::Dict) { return DoubleCompressor::Extend(Box::new(Dict {})); } - if crate::native::util::env::check_rle_env() + if crate::util::env::check_rle_env() && !write_options .forbidden_compressions .contains(&Compression::Rle) { return DoubleCompressor::Extend(Box::new(Rle {})); } - if crate::native::util::env::check_patas_env() + if crate::util::env::check_patas_env() && !write_options .forbidden_compressions .contains(&Compression::Patas) @@ -329,10 +329,15 @@ fn compress_sample_ratio>( let stats = if stats.src.len() / sample_count <= sample_size { stats.clone() } else { - let array = &stats.src; - let separator = array.len() / sample_count; - let remainder = array.len() % sample_count; - let mut builder = MutablePrimitiveArray::with_capacity(sample_count * sample_size); + let col = &stats.src; + let separator = col.len() / sample_count; + let remainder = col.len() % sample_count; + let mut builder = Vec::with_capacity(sample_count * sample_size); + let mut validity = if stats.null_count > 0 { + Some(MutableBitmap::with_capacity(sample_count * sample_size)) + } else { + None + }; for sample_i in 0..sample_count { let range_end = if sample_i == sample_count - 1 { @@ -343,12 +348,18 @@ fn compress_sample_ratio>( let partition_begin = sample_i * separator + rng.gen_range(0..range_end); - let mut s = array.clone(); + let mut s = col.clone(); s.slice(partition_begin, sample_size); - builder.extend_trusted_len(s.into_iter()); + + if let (Some(b), Some(validity)) = (&mut validity, &stats.validity) { + let mut v = validity.clone(); + v.slice(partition_begin, sample_size); + b.extend_from_trusted_len_iter(v.into_iter()); + } + builder.extend(s); } - let sample_array: PrimitiveArray = builder.into(); - gen_stats(&sample_array) + let sample_col: Buffer = builder.into(); + gen_stats(&sample_col, validity.map(|x| x.into())) }; let size = c diff --git a/src/common/arrow/src/native/compression/double/one_value.rs b/src/common/native/src/compression/double/one_value.rs similarity index 85% rename from src/common/arrow/src/native/compression/double/one_value.rs rename to src/common/native/src/compression/double/one_value.rs index 2dc7facc6b5a..cab98a02b573 100644 --- a/src/common/arrow/src/native/compression/double/one_value.rs +++ b/src/common/native/src/compression/double/one_value.rs @@ -15,13 +15,14 @@ use std::io::Read; use std::io::Write; +use databend_common_column::buffer::Buffer; + use super::DoubleCompression; use super::DoubleStats; use super::DoubleType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::native::compression::Compression; -use crate::native::write::WriteOptions; +use crate::compression::Compression; +use crate::error::Result; +use crate::write::WriteOptions; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct OneValue {} @@ -29,7 +30,7 @@ pub struct OneValue {} impl DoubleCompression for OneValue { fn compress( &self, - array: &PrimitiveArray, + array: &Buffer, _stats: &DoubleStats, _write_options: &WriteOptions, output: &mut Vec, @@ -61,13 +62,9 @@ impl OneValue { pub fn encode_native( &self, w: &mut W, - array: &PrimitiveArray, + array: &Buffer, ) -> Result<()> { - let val = array.iter().find(|v| v.is_some()); - let val = match val { - Some(Some(v)) => *v, - _ => T::default(), - }; + let val = array.iter().cloned().next().unwrap_or_default(); let _ = w.write(val.to_le_bytes().as_ref())?; Ok(()) } diff --git a/src/common/arrow/src/native/compression/double/patas.rs b/src/common/native/src/compression/double/patas.rs similarity index 93% rename from src/common/arrow/src/native/compression/double/patas.rs rename to src/common/native/src/compression/double/patas.rs index cf30f19fc71b..5692e5122ff6 100644 --- a/src/common/arrow/src/native/compression/double/patas.rs +++ b/src/common/native/src/compression/double/patas.rs @@ -18,6 +18,8 @@ use std::io::Read; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; +use databend_common_column::types::NativeType; use ringbuffer::AllocRingBuffer; use ringbuffer::RingBuffer; @@ -25,21 +27,19 @@ use super::compress_sample_ratio; use super::DoubleCompression; use super::DoubleStats; use super::DoubleType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; -use crate::native::compression::Compression; -use crate::native::compression::SAMPLE_COUNT; -use crate::native::compression::SAMPLE_SIZE; -use crate::native::util::ByteWriter; -use crate::native::write::WriteOptions; +use crate::compression::Compression; +use crate::compression::SAMPLE_COUNT; +use crate::compression::SAMPLE_SIZE; +use crate::error::Result; +use crate::util::ByteWriter; +use crate::write::WriteOptions; pub(crate) struct Patas {} impl DoubleCompression for Patas { fn compress( &self, - array: &PrimitiveArray, + array: &Buffer, _stats: &DoubleStats, _write_options: &WriteOptions, output: &mut Vec, @@ -52,7 +52,7 @@ impl DoubleCompression for Patas { let mut byte_writer = ByteWriter::::with_capacity(array.len() * (std::mem::size_of::() + 2)); - for (i, val) in array.values().iter().enumerate() { + for (i, val) in array.iter().enumerate() { let val = val.as_bits(); if !is_first { diff --git a/src/common/arrow/src/native/compression/double/rle.rs b/src/common/native/src/compression/double/rle.rs similarity index 77% rename from src/common/arrow/src/native/compression/double/rle.rs rename to src/common/native/src/compression/double/rle.rs index cfd1ea015bfc..eaa966e75af4 100644 --- a/src/common/arrow/src/native/compression/double/rle.rs +++ b/src/common/native/src/compression/double/rle.rs @@ -17,31 +17,31 @@ use std::io::Write; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; +use databend_common_expression::types::Bitmap; use super::compress_sample_ratio; use super::DoubleCompression; use super::DoubleStats; use super::DoubleType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::error::Result; -use crate::native::compression::integer::Rle; -use crate::native::compression::is_valid; -use crate::native::compression::Compression; -use crate::native::compression::SAMPLE_COUNT; -use crate::native::compression::SAMPLE_SIZE; -use crate::native::write::WriteOptions; +use crate::compression::integer::Rle; +use crate::compression::is_valid; +use crate::compression::Compression; +use crate::compression::SAMPLE_COUNT; +use crate::compression::SAMPLE_SIZE; +use crate::error::Result; +use crate::write::WriteOptions; impl DoubleCompression for Rle { fn compress( &self, - array: &PrimitiveArray, - _stats: &DoubleStats, + col: &Buffer, + stats: &DoubleStats, _write_options: &WriteOptions, output: &mut Vec, ) -> Result { let size = output.len(); - self.compress_double(output, array.values().clone(), array.validity())?; + self.compress_double(output, col.clone(), stats.validity.clone())?; Ok(output.len() - size) } @@ -64,17 +64,15 @@ impl Rle { &self, w: &mut W, values: impl IntoIterator, - validity: Option<&Bitmap>, + validity: Option, ) -> Result<()> { // help me generate RLE encode algorithm let mut seen_count: u32 = 0; - let mut last_value = T::default().as_order(); + let mut last_value = T::default(); let mut all_null = true; for (i, item) in values.into_iter().enumerate() { - let item = item.as_order(); - - if is_valid(&validity, i) { + if is_valid(validity.as_ref(), i) { if all_null { all_null = false; last_value = item; @@ -83,7 +81,7 @@ impl Rle { } else if last_value != item { // flush u32 cnt , value w.write_all(&seen_count.to_le_bytes())?; - w.write_all(T::from_order(last_value).to_le_bytes().as_ref())?; + w.write_all(last_value.to_le_bytes().as_ref())?; last_value = item; seen_count = 1; @@ -98,7 +96,7 @@ impl Rle { if seen_count != 0 { w.write_all(&seen_count.to_le_bytes())?; - w.write_all(T::from_order(last_value).to_le_bytes().as_ref())?; + w.write_all(last_value.to_le_bytes().as_ref())?; } Ok(()) @@ -108,12 +106,12 @@ impl Rle { &self, mut input: &'a [u8], length: usize, - array: &mut Vec, + col: &mut Vec, ) -> Result<&'a [u8]> { let mut bs = vec![0u8; std::mem::size_of::()]; let mut num_values = 0; - array.reserve(length); + col.reserve(length); loop { let len = input.read_u32::()?; @@ -125,7 +123,7 @@ impl Rle { }; let t = T::from_le_bytes(a); for _ in 0..len { - array.push(t); + col.push(t); } num_values += len as usize; diff --git a/src/common/arrow/src/native/compression/double/traits.rs b/src/common/native/src/compression/double/traits.rs similarity index 65% rename from src/common/arrow/src/native/compression/double/traits.rs rename to src/common/native/src/compression/double/traits.rs index 1aa615bea6ab..da03acad40c4 100644 --- a/src/common/arrow/src/native/compression/double/traits.rs +++ b/src/common/native/src/compression/double/traits.rs @@ -19,21 +19,17 @@ use std::ops::ShlAssign; use std::ops::Shr; use std::ops::ShrAssign; +use databend_common_column::types::NativeType; +use databend_common_expression::types::F32; +use databend_common_expression::types::F64; use num::Float; -use ordered_float::OrderedFloat; -use crate::arrow::types::NativeType; -use crate::native::util::AsBytes; +use crate::util::AsBytes; -pub trait DoubleType: AsBytes + Copy + Clone + NativeType + Float { - type OrderType: std::fmt::Debug - + std::fmt::Display - + Eq - + Hash - + PartialOrd - + Hash - + Copy - + Clone; +pub trait DoubleType: + AsBytes + Copy + Clone + NativeType + Float + Ord + Hash + PartialOrd +{ + type RawType: std::fmt::Debug + std::fmt::Display + Copy + Clone; type BitType: Eq + NativeType @@ -47,10 +43,6 @@ pub trait DoubleType: AsBytes + Copy + Clone + NativeType + Float { + Shr + ShrAssign; - fn as_order(&self) -> Self::OrderType; - - fn from_order(order: Self::OrderType) -> Self; - fn as_bits(&self) -> Self::BitType; fn from_bits_val(bits: Self::BitType) -> Self; @@ -59,25 +51,17 @@ pub trait DoubleType: AsBytes + Copy + Clone + NativeType + Float { } macro_rules! double_type { - ($type:ty, $order_type: ty, $bit_type: ty) => { + ($type:ty, $raw_type: ty, $bit_type: ty) => { impl DoubleType for $type { - type OrderType = $order_type; + type RawType = $raw_type; type BitType = $bit_type; - fn as_order(&self) -> Self::OrderType { - OrderedFloat(*self) - } - - fn from_order(order: Self::OrderType) -> Self { - order.0 - } - fn as_bits(&self) -> Self::BitType { - self.to_bits() + self.0.to_bits() } fn from_bits_val(bits: Self::BitType) -> Self { - Self::from_bits(bits) + Self::from(Self::RawType::from_bits(bits)) } fn leading_zeros(bit_value: &Self::BitType) -> u32 { @@ -91,8 +75,5 @@ macro_rules! double_type { }; } -type F32 = OrderedFloat; -type F64 = OrderedFloat; - -double_type!(f32, F32, u32); -double_type!(f64, F64, u64); +double_type!(F32, f32, u32); +double_type!(F64, f64, u64); diff --git a/src/common/arrow/src/native/compression/integer/bp.rs b/src/common/native/src/compression/integer/bp.rs similarity index 89% rename from src/common/arrow/src/native/compression/integer/bp.rs rename to src/common/native/src/compression/integer/bp.rs index abbe8dd53a6a..7deabf19fe66 100644 --- a/src/common/arrow/src/native/compression/integer/bp.rs +++ b/src/common/native/src/compression/integer/bp.rs @@ -17,17 +17,17 @@ use std::io::BufRead; use bitpacking::BitPacker; use bitpacking::BitPacker4x; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; use super::compress_sample_ratio; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::native::compression::Compression; -use crate::native::compression::SAMPLE_COUNT; -use crate::native::compression::SAMPLE_SIZE; -use crate::native::write::WriteOptions; +use crate::compression::Compression; +use crate::compression::SAMPLE_COUNT; +use crate::compression::SAMPLE_SIZE; +use crate::error::Result; +use crate::write::WriteOptions; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Bitpacking {} @@ -35,14 +35,14 @@ pub struct Bitpacking {} impl IntegerCompression for Bitpacking { fn compress( &self, - array: &PrimitiveArray, + array: &Buffer, _stats: &IntegerStats, _write_options: &WriteOptions, output: &mut Vec, ) -> Result { let start: usize = output.len(); let bitpacker = BitPacker4x::new(); - let my_data = bytemuck::cast_slice(array.values().as_slice()); + let my_data = bytemuck::cast_slice(array.as_slice()); for chunk in my_data.chunks(BitPacker4x::BLOCK_LEN) { let num_bits: u8 = bitpacker.num_bits(chunk); diff --git a/src/common/arrow/src/native/compression/integer/delta_bp.rs b/src/common/native/src/compression/integer/delta_bp.rs similarity index 90% rename from src/common/arrow/src/native/compression/integer/delta_bp.rs rename to src/common/native/src/compression/integer/delta_bp.rs index 5064d1ce40c5..bc6bde4bcaff 100644 --- a/src/common/arrow/src/native/compression/integer/delta_bp.rs +++ b/src/common/native/src/compression/integer/delta_bp.rs @@ -17,17 +17,17 @@ use std::io::BufRead; use bitpacking::BitPacker; use bitpacking::BitPacker4x; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; use super::compress_sample_ratio; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::native::compression::Compression; -use crate::native::compression::SAMPLE_COUNT; -use crate::native::compression::SAMPLE_SIZE; -use crate::native::write::WriteOptions; +use crate::compression::Compression; +use crate::compression::SAMPLE_COUNT; +use crate::compression::SAMPLE_SIZE; +use crate::error::Result; +use crate::write::WriteOptions; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct DeltaBitpacking {} @@ -35,14 +35,14 @@ pub struct DeltaBitpacking {} impl IntegerCompression for DeltaBitpacking { fn compress( &self, - array: &PrimitiveArray, + array: &Buffer, _stats: &IntegerStats, _write_options: &WriteOptions, output: &mut Vec, ) -> Result { let start: usize = output.len(); let bitpacker = BitPacker4x::new(); - let my_data = bytemuck::cast_slice(array.values().as_slice()); + let my_data = bytemuck::cast_slice(array.as_slice()); let mut initial = 0; for chunk in my_data.chunks(BitPacker4x::BLOCK_LEN) { diff --git a/src/common/arrow/src/native/compression/integer/dict.rs b/src/common/native/src/compression/integer/dict.rs similarity index 91% rename from src/common/arrow/src/native/compression/integer/dict.rs rename to src/common/native/src/compression/integer/dict.rs index e35ad72548e5..b9620ef82fde 100644 --- a/src/common/arrow/src/native/compression/integer/dict.rs +++ b/src/common/native/src/compression/integer/dict.rs @@ -16,16 +16,16 @@ use std::hash::Hash; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; +use databend_common_column::types::NativeType; use super::compress_integer; use super::decompress_integer; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::NativeType; +use crate::error::Error; +use crate::error::Result; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Dict {} @@ -33,16 +33,16 @@ pub struct Dict {} impl IntegerCompression for Dict { fn compress( &self, - array: &PrimitiveArray, - _stats: &IntegerStats, + array: &Buffer, + stats: &IntegerStats, write_options: &WriteOptions, output_buf: &mut Vec, ) -> Result { let start = output_buf.len(); let mut encoder = DictEncoder::with_capacity(array.len()); - for val in array.iter() { + for val in array.option_iter(stats.validity.as_ref()) { match val { - Some(val) => encoder.push(&RawNative { inner: *val }), + Some(val) => encoder.push(&RawNative { inner: val }), None => { if encoder.is_empty() { encoder.push(&RawNative { @@ -59,7 +59,7 @@ impl IntegerCompression for Dict { // dict data use custom encoding let mut write_options = write_options.clone(); write_options.forbidden_compressions.push(Compression::Dict); - compress_integer(&indices, write_options, output_buf)?; + compress_integer(&indices, stats.validity.clone(), write_options, output_buf)?; let sets = encoder.get_sets(); output_buf.extend_from_slice(&(sets.len() as u32).to_le_bytes()); @@ -162,20 +162,20 @@ where T: AsBytes + PartialEq + Clone &self.interner.sets } - pub fn take_indices(&mut self) -> PrimitiveArray { + pub fn take_indices(&mut self) -> Buffer { let indices = std::mem::take(&mut self.indices); - PrimitiveArray::::from_vec(indices) + indices.into() } } use hashbrown_v0_14::hash_map::RawEntryMut; use hashbrown_v0_14::HashMap; +use crate::compression::get_bits_needed; +use crate::compression::Compression; use crate::general_err; -use crate::native::compression::get_bits_needed; -use crate::native::compression::Compression; -use crate::native::util::AsBytes; -use crate::native::write::WriteOptions; +use crate::util::AsBytes; +use crate::write::WriteOptions; const DEFAULT_DEDUP_CAPACITY: usize = 4096; diff --git a/src/common/arrow/src/native/compression/integer/freq.rs b/src/common/native/src/compression/integer/freq.rs similarity index 90% rename from src/common/arrow/src/native/compression/integer/freq.rs rename to src/common/native/src/compression/integer/freq.rs index 0b0a2544101a..7ff97d20dcfb 100644 --- a/src/common/arrow/src/native/compression/integer/freq.rs +++ b/src/common/native/src/compression/integer/freq.rs @@ -17,6 +17,7 @@ use std::io::Read; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; use roaring::RoaringBitmap; use super::compress_integer; @@ -24,10 +25,9 @@ use super::decompress_integer; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::native::compression::Compression; -use crate::native::write::WriteOptions; +use crate::compression::Compression; +use crate::error::Result; +use crate::write::WriteOptions; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Freq {} @@ -35,7 +35,7 @@ pub struct Freq {} impl IntegerCompression for Freq { fn compress( &self, - array: &PrimitiveArray, + col: &Buffer, stats: &IntegerStats, write_options: &WriteOptions, output: &mut Vec, @@ -60,11 +60,11 @@ impl IntegerCompression for Freq { let mut exceptions_bitmap = RoaringBitmap::new(); let mut exceptions = Vec::with_capacity(stats.tuple_count - max_count); - for (i, val) in array.iter().enumerate() { + for (i, val) in col.option_iter(stats.validity.as_ref()).enumerate() { if let Some(val) = val { - if top_value_is_null || *val != top_value { + if top_value_is_null || val != top_value { exceptions_bitmap.insert(i as u32); - exceptions.push(*val); + exceptions.push(val); } } } @@ -81,8 +81,8 @@ impl IntegerCompression for Freq { let mut write_options = write_options.clone(); write_options.forbidden_compressions.push(Compression::Freq); - let exceptions = PrimitiveArray::::from_vec(exceptions); - compress_integer(&exceptions, write_options, output)?; + let exceptions = exceptions.into(); + compress_integer(&exceptions, stats.validity.clone(), write_options, output)?; Ok(output.len() - size) } diff --git a/src/common/arrow/src/native/compression/integer/mod.rs b/src/common/native/src/compression/integer/mod.rs similarity index 78% rename from src/common/arrow/src/native/compression/integer/mod.rs rename to src/common/native/src/compression/integer/mod.rs index 41972f81f200..a135ff13a873 100644 --- a/src/common/arrow/src/native/compression/integer/mod.rs +++ b/src/common/native/src/compression/integer/mod.rs @@ -22,6 +22,9 @@ mod traits; use std::collections::HashMap; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_column::buffer::Buffer; use rand::thread_rng; use rand::Rng; @@ -35,25 +38,22 @@ pub use self::one_value::OneValue; pub use self::rle::Rle; pub use self::traits::IntegerType; use super::basic::CommonCompression; -use super::is_valid; use super::Compression; -use crate::arrow::array::Array; -use crate::arrow::array::MutablePrimitiveArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::native::read::read_basic::read_compress_header; -use crate::native::read::NativeReadBuf; -use crate::native::write::WriteOptions; +use crate::error::Error; +use crate::error::Result; +use crate::read::read_basic::read_compress_header; +use crate::read::NativeReadBuf; +use crate::write::WriteOptions; pub fn compress_integer( - array: &PrimitiveArray, + col: &Buffer, + validity: Option, write_options: WriteOptions, buf: &mut Vec, ) -> Result<()> { // choose compressor - let stats = gen_stats(array); - let compressor = choose_compressor(array, &stats, &write_options); + let stats = gen_stats(col, validity); + let compressor = choose_compressor(col, &stats, &write_options); log::debug!( "choose integer compression : {:?}", @@ -67,14 +67,14 @@ pub fn compress_integer( let compressed_size = match compressor { IntCompressor::Basic(c) => { - let input_buf = bytemuck::cast_slice(array.values()); + let input_buf = bytemuck::cast_slice(col.as_slice()); c.compress(input_buf, buf) } - IntCompressor::Extend(c) => c.compress(array, &stats, &write_options, buf), + IntCompressor::Extend(c) => c.compress(col, &stats, &write_options, buf), }?; buf[pos..pos + 4].copy_from_slice(&(compressed_size as u32).to_le_bytes()); buf[pos + 4..pos + 8] - .copy_from_slice(&((array.len() * std::mem::size_of::()) as u32).to_le_bytes()); + .copy_from_slice(&((col.len() * std::mem::size_of::()) as u32).to_le_bytes()); log::debug!( "integer compress ratio {}", @@ -132,7 +132,7 @@ pub fn decompress_integer( pub trait IntegerCompression { fn compress( &self, - array: &PrimitiveArray, + col: &Buffer, stats: &IntegerStats, write_options: &WriteOptions, output: &mut Vec, @@ -176,52 +176,49 @@ impl IntCompressor { #[derive(Debug, Clone)] pub struct IntegerStats { - pub src: PrimitiveArray, + pub src: Buffer, pub tuple_count: usize, pub total_bytes: usize, pub null_count: usize, + validity: Option, pub average_run_length: f64, pub is_sorted: bool, pub min: T, pub max: T, pub distinct_values: HashMap, pub unique_count: usize, - pub set_count: usize, } -fn gen_stats(array: &PrimitiveArray) -> IntegerStats { +fn gen_stats(col: &Buffer, validity: Option) -> IntegerStats { + let null_count = validity.as_ref().map(|x| x.null_count()).unwrap_or(0); + let mut stats = IntegerStats:: { - src: array.clone(), - tuple_count: array.len(), - total_bytes: array.len() * std::mem::size_of::(), - null_count: array.null_count(), + src: col.clone(), + tuple_count: col.len(), + total_bytes: col.len() * std::mem::size_of::(), + null_count, + validity, average_run_length: 0.0, is_sorted: true, min: T::default(), max: T::default(), distinct_values: HashMap::new(), unique_count: 0, - set_count: array.len() - array.null_count(), }; let mut is_init_value_initialized = false; let mut last_value = T::default(); let mut run_count = 0; - let validity = array.validity(); - for (i, current_value) in array.values().iter().cloned().enumerate() { - if is_valid(&validity, i) { - if current_value < last_value { - stats.is_sorted = false; - } - - if last_value != current_value { - run_count += 1; - last_value = current_value; - } + for current_value in col.option_iter(stats.validity.as_ref()).flatten() { + if current_value < last_value { + stats.is_sorted = false; } - *stats.distinct_values.entry(current_value).or_insert(0) += 1; + if last_value != current_value { + run_count += 1; + last_value = current_value; + } if !is_init_value_initialized { is_init_value_initialized = true; @@ -234,42 +231,43 @@ fn gen_stats(array: &PrimitiveArray) -> IntegerStats { } else if current_value < stats.min { stats.min = current_value; } + *stats.distinct_values.entry(current_value).or_insert(0) += 1; } stats.unique_count = stats.distinct_values.len(); - stats.average_run_length = array.len() as f64 / run_count as f64; + stats.average_run_length = col.len() as f64 / run_count as f64; stats } fn choose_compressor( - _value: &PrimitiveArray, + _value: &Buffer, stats: &IntegerStats, write_options: &WriteOptions, ) -> IntCompressor { #[cfg(debug_assertions)] { - if crate::native::util::env::check_freq_env() + if crate::util::env::check_freq_env() && !write_options .forbidden_compressions .contains(&Compression::Freq) { return IntCompressor::Extend(Box::new(Freq {})); } - if crate::native::util::env::check_dict_env() + if crate::util::env::check_dict_env() && !write_options .forbidden_compressions .contains(&Compression::Dict) { return IntCompressor::Extend(Box::new(Dict {})); } - if crate::native::util::env::check_rle_env() + if crate::util::env::check_rle_env() && !write_options .forbidden_compressions .contains(&Compression::Rle) { return IntCompressor::Extend(Box::new(Rle {})); } - if crate::native::util::env::check_bitpack_env() + if crate::util::env::check_bitpack_env() && !write_options .forbidden_compressions .contains(&Compression::Bitpacking) @@ -331,10 +329,17 @@ fn compress_sample_ratio>( let stats = if stats.src.len() / sample_count <= sample_size { stats.clone() } else { - let array = &stats.src; - let separator = array.len() / sample_count; - let remainder = array.len() % sample_count; - let mut builder = MutablePrimitiveArray::with_capacity(sample_count * sample_size); + let col = &stats.src; + let separator = col.len() / sample_count; + let remainder = col.len() % sample_count; + let mut builder = Vec::with_capacity(sample_count * sample_size); + + let mut validity = if stats.null_count > 0 { + Some(MutableBitmap::with_capacity(sample_count * sample_size)) + } else { + None + }; + for sample_i in 0..sample_count { let range_end = if sample_i == sample_count - 1 { separator + remainder @@ -344,12 +349,19 @@ fn compress_sample_ratio>( let partition_begin = sample_i * separator + rng.gen_range(0..range_end); - let mut s = array.clone(); + let mut s = col.clone(); s.slice(partition_begin, sample_size); - builder.extend_trusted_len(s.into_iter()); + + if let (Some(b), Some(validity)) = (&mut validity, &stats.validity) { + let mut v = validity.clone(); + v.slice(partition_begin, sample_size); + b.extend_from_trusted_len_iter(v.into_iter()); + } + + builder.extend(s.into_iter()); } - let sample_array: PrimitiveArray = builder.into(); - gen_stats(&sample_array) + let sample_col: Buffer = builder.into(); + gen_stats(&sample_col, validity.map(|x| x.into())) }; let size = c diff --git a/src/common/arrow/src/native/compression/integer/one_value.rs b/src/common/native/src/compression/integer/one_value.rs similarity index 85% rename from src/common/arrow/src/native/compression/integer/one_value.rs rename to src/common/native/src/compression/integer/one_value.rs index 23262dd328b7..db28d8e1ccc1 100644 --- a/src/common/arrow/src/native/compression/integer/one_value.rs +++ b/src/common/native/src/compression/integer/one_value.rs @@ -15,13 +15,14 @@ use std::io::Read; use std::io::Write; +use databend_common_column::buffer::Buffer; + use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::error::Result; -use crate::native::compression::Compression; -use crate::native::write::WriteOptions; +use crate::compression::Compression; +use crate::error::Result; +use crate::write::WriteOptions; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct OneValue {} @@ -29,7 +30,7 @@ pub struct OneValue {} impl IntegerCompression for OneValue { fn compress( &self, - array: &PrimitiveArray, + array: &Buffer, _stats: &IntegerStats, _write_options: &WriteOptions, output: &mut Vec, @@ -61,13 +62,9 @@ impl OneValue { pub fn encode_native( &self, w: &mut W, - array: &PrimitiveArray, + array: &Buffer, ) -> Result<()> { - let val = array.iter().find(|v| v.is_some()); - let val = match val { - Some(Some(v)) => *v, - _ => T::default(), - }; + let val = array.iter().next().cloned().unwrap_or_default(); let _ = w.write(val.to_le_bytes().as_ref())?; Ok(()) } diff --git a/src/common/arrow/src/native/compression/integer/rle.rs b/src/common/native/src/compression/integer/rle.rs similarity index 86% rename from src/common/arrow/src/native/compression/integer/rle.rs rename to src/common/native/src/compression/integer/rle.rs index fdd9aceeecc2..ca21bc1c7b01 100644 --- a/src/common/arrow/src/native/compression/integer/rle.rs +++ b/src/common/native/src/compression/integer/rle.rs @@ -17,19 +17,19 @@ use std::io::Write; use byteorder::LittleEndian; use byteorder::ReadBytesExt; +use databend_common_column::buffer::Buffer; +use databend_common_expression::types::Bitmap; use super::compress_sample_ratio; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::error::Result; -use crate::native::compression::is_valid; -use crate::native::compression::Compression; -use crate::native::compression::SAMPLE_COUNT; -use crate::native::compression::SAMPLE_SIZE; -use crate::native::write::WriteOptions; +use crate::compression::is_valid; +use crate::compression::Compression; +use crate::compression::SAMPLE_COUNT; +use crate::compression::SAMPLE_SIZE; +use crate::error::Result; +use crate::write::WriteOptions; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Rle {} @@ -37,13 +37,13 @@ pub struct Rle {} impl IntegerCompression for Rle { fn compress( &self, - array: &PrimitiveArray, - _stats: &IntegerStats, + array: &Buffer, + stats: &IntegerStats, _write_options: &WriteOptions, output: &mut Vec, ) -> Result { let size = output.len(); - self.compress_integer(output, array.values().clone(), array.validity())?; + self.compress_integer(output, array.clone(), stats.validity.clone())?; Ok(output.len() - size) } @@ -66,7 +66,7 @@ impl Rle { &self, w: &mut W, values: impl IntoIterator, - validity: Option<&Bitmap>, + validity: Option, ) -> Result<()> { // help me generate RLE encode algorithm let mut seen_count: u32 = 0; @@ -74,7 +74,7 @@ impl Rle { let mut all_null = true; for (i, item) in values.into_iter().enumerate() { - if is_valid(&validity, i) { + if is_valid(validity.as_ref(), i) { if all_null { all_null = false; last_value = item; diff --git a/src/common/arrow/src/native/compression/integer/traits.rs b/src/common/native/src/compression/integer/traits.rs similarity index 90% rename from src/common/arrow/src/native/compression/integer/traits.rs rename to src/common/native/src/compression/integer/traits.rs index 822d71e5af3e..8a9fc4e9f7ea 100644 --- a/src/common/arrow/src/native/compression/integer/traits.rs +++ b/src/common/native/src/compression/integer/traits.rs @@ -14,8 +14,8 @@ use std::hash::Hash; -use crate::arrow::types::i256; -use crate::arrow::types::NativeType; +use databend_common_column::types::i256; +use databend_common_column::types::NativeType; pub trait IntegerType: NativeType + PartialOrd + Hash + Eq { fn as_i64(&self) -> i64; @@ -39,8 +39,6 @@ integer_type!(i8); integer_type!(i16); integer_type!(i32); integer_type!(i64); -// integer_type!(days_ms); -// integer_type!(months_days_ns); impl IntegerType for i128 { fn as_i64(&self) -> i64 { diff --git a/src/common/arrow/src/native/compression/mod.rs b/src/common/native/src/compression/mod.rs similarity index 92% rename from src/common/arrow/src/native/compression/mod.rs rename to src/common/native/src/compression/mod.rs index 47a1873500af..b9f57384e6cb 100644 --- a/src/common/arrow/src/native/compression/mod.rs +++ b/src/common/native/src/compression/mod.rs @@ -20,9 +20,9 @@ pub mod double; pub mod integer; pub use basic::CommonCompression; +use databend_common_expression::types::Bitmap; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::error::Result; +use crate::error::Result; // number of samples to take pub static SAMPLE_COUNT: usize = 10; @@ -74,7 +74,7 @@ impl Compression { 15 => Ok(Compression::DeltaBitpacking), 16 => Ok(Compression::Patas), - other => Err(crate::arrow::error::Error::OutOfSpec(format!( + other => Err(crate::error::Error::OutOfSpec(format!( "Unknown compression codec {other}", ))), } @@ -101,7 +101,7 @@ impl Compression { } #[inline] -pub(crate) fn is_valid(validity: &Option<&Bitmap>, i: usize) -> bool { +pub(crate) fn is_valid(validity: Option<&Bitmap>, i: usize) -> bool { match validity { Some(v) => v.get_bit(i), None => true, diff --git a/src/common/arrow/src/native/errors.rs b/src/common/native/src/error.rs similarity index 92% rename from src/common/arrow/src/native/errors.rs rename to src/common/native/src/error.rs index 6a1d29952016..dd04a00893ae 100644 --- a/src/common/arrow/src/native/errors.rs +++ b/src/common/native/src/error.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub use databend_common_column::error::Error; +pub use databend_common_column::error::Result; + #[macro_export] macro_rules! general_err { ($fmt:expr) => (Error::OutOfSpec($fmt.to_owned())); diff --git a/src/common/arrow/src/native/mod.rs b/src/common/native/src/lib.rs similarity index 97% rename from src/common/arrow/src/native/mod.rs rename to src/common/native/src/lib.rs index ee65c1b91973..38debadb17df 100644 --- a/src/common/arrow/src/native/mod.rs +++ b/src/common/native/src/lib.rs @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::useless_transmute)] + #[macro_use] -mod errors; +mod error; #[macro_use] mod util; diff --git a/src/common/native/src/nested.rs b/src/common/native/src/nested.rs new file mode 100644 index 000000000000..e9c135f4ac99 --- /dev/null +++ b/src/common/native/src/nested.rs @@ -0,0 +1,282 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ops::Range; + +use databend_common_expression::types::AnyType; +use databend_common_expression::types::ArrayColumn; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::error::Result; + +/// Descriptor of nested information of a field +#[derive(Debug, Clone, PartialEq)] +pub enum Nested { + /// A primitive column + Primitive(usize, bool, Option), + /// a list + LargeList(ListNested), + /// A struct column + Struct(usize, bool, Option), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ListNested { + pub is_nullable: bool, + pub offsets: Buffer, + pub validity: Option, +} + +impl ListNested { + pub fn new(offsets: Buffer, validity: Option, is_nullable: bool) -> Self { + Self { + is_nullable, + offsets, + validity, + } + } +} + +pub type NestedState = Vec; + +impl Nested { + pub fn length(&self) -> usize { + match self { + Nested::Primitive(len, _, _) => *len, + Nested::LargeList(l) => l.offsets.len(), + Nested::Struct(len, _, _) => *len, + } + } + + pub fn is_nullable(&self) -> bool { + match self { + Nested::Primitive(_, b, _) => *b, + Nested::LargeList(l) => l.is_nullable, + Nested::Struct(_, b, _) => *b, + } + } + + pub fn inner(&self) -> (Buffer, &Option) { + match self { + Nested::Primitive(_, _, v) => (Buffer::new(), v), + Nested::LargeList(l) => { + let start = *l.offsets.first().unwrap(); + let buffer = if start == 0 { + l.offsets.clone() + } else { + l.offsets.iter().map(|x| *x - start).collect() + }; + (buffer, &l.validity) + } + Nested::Struct(_, _, v) => (Buffer::new(), v), + } + } + + pub fn validity(&self) -> &Option { + match self { + Nested::Primitive(_, _, v) => v, + Nested::LargeList(l) => &l.validity, + Nested::Struct(_, _, v) => v, + } + } + + pub fn is_list(&self) -> bool { + matches!(self, Nested::LargeList(_)) + } +} + +/// Constructs the necessary `Vec>` to write the rep and def levels of `column` to parquet +pub fn to_nested(column: &Column) -> Result>> { + let mut nested = vec![]; + + to_nested_recursive(column, &mut nested, vec![])?; + Ok(nested) +} + +pub fn is_nested_type(t: &TableDataType) -> bool { + matches!( + t, + TableDataType::Tuple { .. } | TableDataType::Array(_) | TableDataType::Map(_) + ) +} + +/// Slices the [`column`] to `Column` and `Vec`. +pub fn slice_nest_column( + primitive_column: &mut Column, + nested: &mut [Nested], + mut current_offset: usize, + mut current_length: usize, +) { + for nested in nested.iter_mut() { + match nested { + Nested::LargeList(l_nested) => { + l_nested.offsets.slice(current_offset, current_length + 1); + if let Some(validity) = l_nested.validity.as_mut() { + validity.slice(current_offset, current_length) + }; + + let r = *l_nested.offsets.last().unwrap() - *l_nested.offsets.first().unwrap(); + current_length = r as usize; + current_offset = *l_nested.offsets.first().unwrap() as usize; + } + Nested::Struct(length, _, validity) => { + *length = current_length; + if let Some(validity) = validity.as_mut() { + validity.slice(current_offset, current_length) + }; + } + Nested::Primitive(length, _, validity) => { + *length = current_length; + if let Some(validity) = validity.as_mut() { + validity.slice(current_offset, current_length) + }; + *primitive_column = primitive_column.slice(Range { + start: current_offset, + end: current_offset + current_length, + }); + } + } + } +} + +fn to_nested_recursive( + column: &Column, + nested: &mut Vec>, + mut parents: Vec, +) -> Result<()> { + let nullable = column.as_nullable().is_some(); + let validity = column.validity().1.cloned(); + + match column.remove_nullable() { + Column::Tuple(values) => { + parents.push(Nested::Struct(column.len(), nullable, validity)); + for column in values { + to_nested_recursive(&column, nested, parents.clone())?; + } + } + Column::Array(inner) => { + parents.push(Nested::LargeList(ListNested { + is_nullable: nullable, + offsets: inner.offsets.clone(), + validity, + })); + to_nested_recursive(&inner.values, nested, parents)?; + } + Column::Map(inner) => { + parents.push(Nested::LargeList(ListNested { + is_nullable: nullable, + offsets: inner.offsets.clone(), + validity, + })); + to_nested_recursive(&inner.values, nested, parents)?; + } + _ => { + parents.push(Nested::Primitive(column.len(), nullable, validity)); + nested.push(parents); + } + } + + Ok(()) +} + +/// Convert [`column`] to `Vec` leaves in DFS order. +pub fn to_leaves(column: &Column) -> Vec { + let mut leaves = vec![]; + to_leaves_recursive(column, &mut leaves); + leaves +} + +fn to_leaves_recursive(column: &Column, leaves: &mut Vec) { + match column { + Column::Tuple(cs) => { + cs.iter().for_each(|a| to_leaves_recursive(a, leaves)); + } + Column::Array(col) => { + to_leaves_recursive(&col.values, leaves); + } + Column::Map(col) => { + to_leaves_recursive(&col.values, leaves); + } + // Handle nullable columns by recursing into their inner value + Column::Nullable(inner) => to_leaves_recursive(&inner.column, leaves), + // All primitive/leaf types + _ => leaves.push(column.clone()), + } +} + +/// The initial info of nested data types. +/// The initial info of nested data types. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InitNested { + /// Primitive data types + Primitive(bool), + /// List data types + List(bool), + /// Struct data types + Struct(bool), +} + +impl InitNested { + pub fn is_nullable(&self) -> bool { + match self { + InitNested::Primitive(b) => *b, + InitNested::List(b) => *b, + InitNested::Struct(b) => *b, + } + } +} + +pub fn create_list(data_type: TableDataType, nested: &mut NestedState, values: Column) -> Column { + let n = nested.pop().unwrap(); + let (offsets, validity) = n.inner(); + let col = Column::Array(Box::new(ArrayColumn:: { values, offsets })); + + if data_type.is_nullable() { + col.wrap_nullable(validity.clone()) + } else { + col + } +} + +/// Creates a new [`Mapcolumn`]. +pub fn create_map(data_type: TableDataType, nested: &mut NestedState, values: Column) -> Column { + let n = nested.pop().unwrap(); + let (offsets, validity) = n.inner(); + let col = Column::Map(Box::new(ArrayColumn:: { values, offsets })); + if data_type.is_nullable() { + col.wrap_nullable(validity.clone()) + } else { + col + } +} + +pub fn create_struct( + is_nullable: bool, + nested: &mut Vec, + values: Vec, +) -> (NestedState, Column) { + let mut nest = nested.pop().unwrap(); + let n = nest.pop().unwrap(); + let (_, validity) = n.inner(); + + let col = Column::Tuple(values); + if is_nullable { + (nest, col.wrap_nullable(validity.clone())) + } else { + (nest, col) + } +} diff --git a/src/common/native/src/read/array/binary.rs b/src/common/native/src/read/array/binary.rs new file mode 100644 index 000000000000..083972b8ee0c --- /dev/null +++ b/src/common/native/src/read/array/binary.rs @@ -0,0 +1,156 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Cursor; + +use databend_common_column::binary::BinaryColumn; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; +use databend_common_expression::types::GeographyColumn; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::compression::binary::decompress_binary; +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::read::read_basic::*; +use crate::read::BufReader; +use crate::read::NativeReadBuf; +use crate::read::PageIterator; +use crate::PageMeta; + +#[derive(Debug)] +pub struct BinaryNestedIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + iter: I, + data_type: TableDataType, + init: Vec, + scratch: Vec, +} + +impl BinaryNestedIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + pub fn new(iter: I, data_type: TableDataType, init: Vec) -> Self { + Self { + iter, + data_type, + init, + scratch: vec![], + } + } +} + +impl BinaryNestedIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result<(NestedState, Column)> { + let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); + let length = num_values as usize; + + let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; + let mut offsets: Vec = Vec::with_capacity(length + 1); + let mut values = Vec::with_capacity(0); + + decompress_binary( + &mut reader, + length, + &mut offsets, + &mut values, + &mut self.scratch, + )?; + + let column = + try_new_binary_column(&self.data_type, offsets.into(), values.into(), validity)?; + Ok((nested, column)) + } +} + +impl Iterator for BinaryNestedIter +where I: Iterator)>> + PageIterator + Send + Sync +{ + type Item = Result<(NestedState, Column)>; + + fn nth(&mut self, n: usize) -> Option { + match self.iter.nth(n) { + Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), + Some(Err(err)) => Some(Result::Err(err)), + None => None, + } + } + + fn next(&mut self) -> Option { + match self.iter.next() { + Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), + Some(Err(err)) => Some(Result::Err(err)), + None => None, + } + } +} + +pub fn read_nested_binary( + reader: &mut R, + data_type: TableDataType, + init: Vec, + page_metas: Vec, +) -> Result> { + let mut scratch = vec![]; + + let mut results = Vec::with_capacity(page_metas.len()); + + for page_meta in page_metas { + let num_values = page_meta.num_values as usize; + let (nested, validity) = read_nested(reader, &init, num_values)?; + let mut offsets: Vec = Vec::with_capacity(num_values + 1); + let mut values = Vec::with_capacity(0); + + decompress_binary(reader, num_values, &mut offsets, &mut values, &mut scratch)?; + + let column = try_new_binary_column(&data_type, offsets.into(), values.into(), validity)?; + results.push((nested, column)); + } + Ok(results) +} + +fn try_new_binary_column( + data_type: &TableDataType, + offsets: Buffer, + values: Buffer, + validity: Option, +) -> Result { + let column = BinaryColumn::new(values, offsets); + Ok(binary_column_to_column(data_type, column, validity)) +} + +fn binary_column_to_column( + data_type: &TableDataType, + column: BinaryColumn, + validity: Option, +) -> Column { + let col = match data_type.remove_nullable() { + TableDataType::Binary => Column::Binary(column), + TableDataType::Bitmap => Column::Bitmap(column), + TableDataType::Variant => Column::Variant(column), + TableDataType::Geometry => Column::Geometry(column), + TableDataType::Geography => Column::Geography(GeographyColumn(column)), + _ => unreachable!(), + }; + if data_type.is_nullable() { + col.wrap_nullable(validity) + } else { + col + } +} diff --git a/src/common/arrow/src/native/read/array/boolean.rs b/src/common/native/src/read/array/boolean.rs similarity index 66% rename from src/common/arrow/src/native/read/array/boolean.rs rename to src/common/native/src/read/array/boolean.rs index c507bc8b9dfa..b5edb6bdf9b2 100644 --- a/src/common/arrow/src/native/read/array/boolean.rs +++ b/src/common/native/src/read/array/boolean.rs @@ -14,26 +14,27 @@ use std::io::Cursor; -use crate::arrow::array::Array; -use crate::arrow::array::BooleanArray; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::native::compression::boolean::decompress_boolean; -use crate::native::nested::InitNested; -use crate::native::nested::NestedState; -use crate::native::read::read_basic::*; -use crate::native::read::BufReader; -use crate::native::read::NativeReadBuf; -use crate::native::read::PageIterator; -use crate::native::PageMeta; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::compression::boolean::decompress_boolean; +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::read::read_basic::*; +use crate::read::BufReader; +use crate::read::NativeReadBuf; +use crate::read::PageIterator; +use crate::PageMeta; #[derive(Debug)] pub struct BooleanNestedIter where I: Iterator)>> + PageIterator + Send + Sync { iter: I, - data_type: DataType, + data_type: TableDataType, init: Vec, scratch: Vec, } @@ -41,7 +42,7 @@ where I: Iterator)>> + PageIterator + Send + Sync impl BooleanNestedIter where I: Iterator)>> + PageIterator + Send + Sync { - pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { + pub fn new(iter: I, data_type: TableDataType, init: Vec) -> Self { Self { iter, data_type, @@ -54,11 +55,7 @@ where I: Iterator)>> + PageIterator + Send + Sync impl BooleanNestedIter where I: Iterator)>> + PageIterator + Send + Sync { - fn deserialize( - &mut self, - length: u64, - buffer: Vec, - ) -> Result<(NestedState, Box)> { + fn deserialize(&mut self, length: u64, buffer: Vec) -> Result<(NestedState, Column)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); let length = length as usize; let (nested, validity) = read_nested(&mut reader, &self.init, length)?; @@ -66,19 +63,20 @@ where I: Iterator)>> + PageIterator + Send + Sync let mut bitmap_builder = MutableBitmap::with_capacity(length); decompress_boolean(&mut reader, length, &mut bitmap_builder, &mut self.scratch)?; - let values = std::mem::take(&mut bitmap_builder).into(); - let mut buffer = reader.into_inner().into_inner(); - self.iter.swap_buffer(&mut buffer); - - let array = BooleanArray::try_new(self.data_type.clone(), values, validity)?; - Ok((nested, Box::new(array) as Box)) + let values: Bitmap = bitmap_builder.into(); + let col = if self.data_type.is_nullable() { + Column::Boolean(values).wrap_nullable(validity) + } else { + Column::Boolean(values) + }; + Ok((nested, col)) } } impl Iterator for BooleanNestedIter where I: Iterator)>> + PageIterator + Send + Sync { - type Item = Result<(NestedState, Box)>; + type Item = Result<(NestedState, Column)>; fn nth(&mut self, n: usize) -> Option { match self.iter.nth(n) { @@ -99,10 +97,10 @@ where I: Iterator)>> + PageIterator + Send + Sync pub fn read_nested_boolean( reader: &mut R, - data_type: DataType, + data_type: TableDataType, init: Vec, page_metas: Vec, -) -> Result)>> { +) -> Result> { let mut scratch = vec![]; let mut results = Vec::with_capacity(page_metas.len()); @@ -113,9 +111,13 @@ pub fn read_nested_boolean( decompress_boolean(reader, num_values, &mut bitmap_builder, &mut scratch)?; - let values = std::mem::take(&mut bitmap_builder).into(); - let array = BooleanArray::try_new(data_type.clone(), values, validity)?; - results.push((nested, Box::new(array) as Box)); + let values: Bitmap = bitmap_builder.into(); + let col = if data_type.is_nullable() { + Column::Boolean(values).wrap_nullable(validity) + } else { + Column::Boolean(values) + }; + results.push((nested, col)); } Ok(results) } diff --git a/src/common/arrow/src/native/read/array/integer.rs b/src/common/native/src/read/array/decimal.rs similarity index 59% rename from src/common/arrow/src/native/read/array/integer.rs rename to src/common/native/src/read/array/decimal.rs index f89c2e79db81..4fcd5437fc81 100644 --- a/src/common/arrow/src/native/read/array/integer.rs +++ b/src/common/native/src/read/array/decimal.rs @@ -16,60 +16,71 @@ use std::convert::TryInto; use std::io::Cursor; use std::marker::PhantomData; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::native::compression::integer::decompress_integer; -use crate::native::compression::integer::IntegerType; -use crate::native::nested::InitNested; -use crate::native::nested::NestedState; -use crate::native::read::read_basic::*; -use crate::native::read::BufReader; -use crate::native::read::NativeReadBuf; -use crate::native::read::PageIterator; -use crate::native::PageMeta; +use databend_common_column::buffer::Buffer; +use databend_common_expression::types::Decimal; +use databend_common_expression::types::DecimalSize; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::compression::integer::decompress_integer; +use crate::compression::integer::IntegerType; +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::read::read_basic::*; +use crate::read::BufReader; +use crate::read::NativeReadBuf; +use crate::read::PageIterator; +use crate::PageMeta; #[derive(Debug)] -pub struct IntegerNestedIter +pub struct DecimalNestedIter where I: Iterator)>> + PageIterator + Send + Sync, T: IntegerType, + F: Decimal, { iter: I, - data_type: DataType, + data_type: TableDataType, + decimal_size: DecimalSize, init: Vec, scratch: Vec, _phantom: PhantomData, + _phantom2: PhantomData, } -impl IntegerNestedIter +impl DecimalNestedIter where I: Iterator)>> + PageIterator + Send + Sync, T: IntegerType, + F: Decimal, { - pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { + pub fn new( + iter: I, + data_type: TableDataType, + decimal_size: DecimalSize, + init: Vec, + ) -> Self { Self { iter, data_type, + decimal_size, init, scratch: vec![], _phantom: PhantomData, + _phantom2: PhantomData, } } } -impl IntegerNestedIter +impl DecimalNestedIter where I: Iterator)>> + PageIterator + Send + Sync, T: IntegerType, + F: Decimal, Vec: TryInto, { - fn deserialize( - &mut self, - num_values: u64, - buffer: Vec, - ) -> Result<(NestedState, Box)> { + fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result<(NestedState, Column)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; let length = num_values as usize; @@ -81,19 +92,24 @@ where let mut buffer = reader.into_inner().into_inner(); self.iter.swap_buffer(&mut buffer); - let array = PrimitiveArray::::try_new(self.data_type.clone(), values.into(), validity)?; - - Ok((nested, Box::new(array) as Box)) + let column: Buffer = values.into(); + let column: Buffer = unsafe { std::mem::transmute(column) }; + let mut col = F::upcast_column(column, self.decimal_size); + if self.data_type.is_nullable() { + col = col.wrap_nullable(validity); + } + Ok((nested, col)) } } -impl Iterator for IntegerNestedIter +impl Iterator for DecimalNestedIter where I: Iterator)>> + PageIterator + Send + Sync, T: IntegerType, + F: Decimal, Vec: TryInto, { - type Item = Result<(NestedState, Box)>; + type Item = Result<(NestedState, Column)>; fn nth(&mut self, n: usize) -> Option { match self.iter.nth(n) { @@ -112,12 +128,13 @@ where } } -pub fn read_nested_integer( +pub fn read_nested_decimal( reader: &mut R, - data_type: DataType, + data_type: TableDataType, + decimal_size: DecimalSize, init: Vec, page_metas: Vec, -) -> Result)>> { +) -> Result> { let mut scratch = vec![]; let mut results = Vec::with_capacity(page_metas.len()); for page_meta in page_metas { @@ -127,8 +144,13 @@ pub fn read_nested_integer( let mut values = Vec::with_capacity(num_values); decompress_integer(reader, num_values, &mut values, &mut scratch)?; - let array = PrimitiveArray::::try_new(data_type.clone(), values.into(), validity)?; - results.push((nested, Box::new(array) as Box)); + let column: Buffer = values.into(); + let column: Buffer = unsafe { std::mem::transmute(column) }; + let mut col = F::upcast_column(column, decimal_size); + if data_type.is_nullable() { + col = col.wrap_nullable(validity); + } + results.push((nested, col)); } Ok(results) } diff --git a/src/common/arrow/src/native/read/array/double.rs b/src/common/native/src/read/array/double.rs similarity index 65% rename from src/common/arrow/src/native/read/array/double.rs rename to src/common/native/src/read/array/double.rs index 6be5f30f1f46..2838e675c92e 100644 --- a/src/common/arrow/src/native/read/array/double.rs +++ b/src/common/native/src/read/array/double.rs @@ -16,28 +16,32 @@ use std::convert::TryInto; use std::io::Cursor; use std::marker::PhantomData; -use crate::arrow::array::Array; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::native::compression::double::decompress_double; -use crate::native::compression::double::DoubleType; -use crate::native::nested::InitNested; -use crate::native::nested::NestedState; -use crate::native::read::read_basic::*; -use crate::native::read::BufReader; -use crate::native::read::NativeReadBuf; -use crate::native::read::PageIterator; -use crate::native::PageMeta; +use databend_common_column::buffer::Buffer; +use databend_common_expression::types::Number; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::ValueType; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::compression::double::decompress_double; +use crate::compression::double::DoubleType; +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::read::read_basic::*; +use crate::read::BufReader; +use crate::read::NativeReadBuf; +use crate::read::PageIterator; +use crate::PageMeta; #[derive(Debug)] pub struct DoubleNestedIter where I: Iterator)>> + PageIterator + Send + Sync, - T: DoubleType, + T: Number + DoubleType, { iter: I, - data_type: DataType, + data_type: TableDataType, init: Vec, scratch: Vec, _phantom: PhantomData, @@ -46,9 +50,9 @@ where impl DoubleNestedIter where I: Iterator)>> + PageIterator + Send + Sync, - T: DoubleType, + T: Number + DoubleType, { - pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { + pub fn new(iter: I, data_type: TableDataType, init: Vec) -> Self { Self { iter, data_type, @@ -62,14 +66,10 @@ where impl DoubleNestedIter where I: Iterator)>> + PageIterator + Send + Sync, - T: DoubleType, + T: Number + DoubleType, Vec: TryInto, { - fn deserialize( - &mut self, - num_values: u64, - buffer: Vec, - ) -> Result<(NestedState, Box)> { + fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result<(NestedState, Column)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; let length = num_values as usize; @@ -81,19 +81,23 @@ where let mut buffer = reader.into_inner().into_inner(); self.iter.swap_buffer(&mut buffer); - let array = PrimitiveArray::::try_new(self.data_type.clone(), values.into(), validity)?; + let column: Buffer = values.into(); + let mut col = NumberType::::upcast_column(column); + if self.data_type.is_nullable() { + col = col.wrap_nullable(validity); + } - Ok((nested, Box::new(array) as Box)) + Ok((nested, col)) } } impl Iterator for DoubleNestedIter where I: Iterator)>> + PageIterator + Send + Sync, - T: DoubleType, + T: Number + DoubleType, Vec: TryInto, { - type Item = Result<(NestedState, Box)>; + type Item = Result<(NestedState, Column)>; fn nth(&mut self, n: usize) -> Option { match self.iter.nth(n) { @@ -112,12 +116,12 @@ where } } -pub fn read_nested_primitive( +pub fn read_nested_primitive( reader: &mut R, - data_type: DataType, + data_type: TableDataType, init: Vec, page_metas: Vec, -) -> Result)>> { +) -> Result> { let mut scratch = vec![]; let mut results = Vec::with_capacity(page_metas.len()); for page_meta in page_metas { @@ -127,8 +131,12 @@ pub fn read_nested_primitive( let mut values = Vec::with_capacity(num_values); decompress_double(reader, num_values, &mut values, &mut scratch)?; - let array = PrimitiveArray::::try_new(data_type.clone(), values.into(), validity)?; - results.push((nested, Box::new(array) as Box)); + let column: Buffer = values.into(); + let mut col = NumberType::::upcast_column(column); + if data_type.is_nullable() { + col = col.wrap_nullable(validity); + } + results.push((nested, col)); } Ok(results) } diff --git a/src/common/native/src/read/array/integer.rs b/src/common/native/src/read/array/integer.rs new file mode 100644 index 000000000000..f0ad2e0d86e6 --- /dev/null +++ b/src/common/native/src/read/array/integer.rs @@ -0,0 +1,152 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::convert::TryInto; +use std::io::Cursor; +use std::marker::PhantomData; + +use databend_common_column::buffer::Buffer; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::Number; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::compression::integer::decompress_integer; +use crate::compression::integer::IntegerType; +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::read::read_basic::*; +use crate::read::BufReader; +use crate::read::NativeReadBuf; +use crate::read::PageIterator; +use crate::PageMeta; + +#[derive(Debug)] +pub struct IntegerNestedIter +where + I: Iterator)>> + PageIterator + Send + Sync, + V: ArgType, Scalar = T>, + T: Number + IntegerType, + Vec: TryInto, +{ + iter: I, + data_type: TableDataType, + init: Vec, + scratch: Vec, + _phantom: PhantomData, + _phantom2: PhantomData, +} + +impl IntegerNestedIter +where + I: Iterator)>> + PageIterator + Send + Sync, + V: ArgType, Scalar = T>, + T: Number + IntegerType, + Vec: TryInto, +{ + pub fn new(iter: I, data_type: TableDataType, init: Vec) -> Self { + Self { + iter, + data_type, + init, + scratch: vec![], + _phantom: PhantomData, + _phantom2: PhantomData, + } + } +} + +impl IntegerNestedIter +where + I: Iterator)>> + PageIterator + Send + Sync, + V: ArgType, Scalar = T>, + T: Number + IntegerType, + Vec: TryInto, +{ + fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result<(NestedState, Column)> { + let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); + let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; + let length = num_values as usize; + + let mut values = Vec::with_capacity(length); + decompress_integer(&mut reader, length, &mut values, &mut self.scratch)?; + assert_eq!(values.len(), length); + + let mut buffer = reader.into_inner().into_inner(); + self.iter.swap_buffer(&mut buffer); + + let column: Buffer = values.into(); + let mut col = V::upcast_column(column); + if self.data_type.is_nullable() { + col = col.wrap_nullable(validity); + } + Ok((nested, col)) + } +} + +impl Iterator for IntegerNestedIter +where + I: Iterator)>> + PageIterator + Send + Sync, + V: ArgType, Scalar = T>, + T: Number + IntegerType, + Vec: TryInto, +{ + type Item = Result<(NestedState, Column)>; + + fn nth(&mut self, n: usize) -> Option { + match self.iter.nth(n) { + Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), + Some(Err(err)) => Some(Result::Err(err)), + None => None, + } + } + + fn next(&mut self) -> Option { + match self.iter.next() { + Some(Ok((num_values, buffer))) => Some(self.deserialize(num_values, buffer)), + Some(Err(err)) => Some(Result::Err(err)), + None => None, + } + } +} + +pub fn read_nested_integer( + reader: &mut R, + data_type: TableDataType, + init: Vec, + page_metas: Vec, +) -> Result> +where + V: ArgType, Scalar = T>, + T: Number + IntegerType, +{ + let mut scratch = vec![]; + let mut results = Vec::with_capacity(page_metas.len()); + for page_meta in page_metas { + let num_values = page_meta.num_values as usize; + let (nested, validity) = read_nested(reader, &init, num_values)?; + + let mut values = Vec::with_capacity(num_values); + decompress_integer(reader, num_values, &mut values, &mut scratch)?; + + let column: Buffer = values.into(); + let mut col = V::upcast_column(column); + if data_type.is_nullable() { + col = col.wrap_nullable(validity); + } + results.push((nested, col)); + } + Ok(results) +} diff --git a/src/common/arrow/src/native/read/array/list.rs b/src/common/native/src/read/array/list.rs similarity index 63% rename from src/common/arrow/src/native/read/array/list.rs rename to src/common/native/src/read/array/list.rs index c478bb110dcb..3d68c232c424 100644 --- a/src/common/arrow/src/native/read/array/list.rs +++ b/src/common/native/src/read/array/list.rs @@ -12,43 +12,44 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::arrow::array::Array; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Result; -use crate::native::nested::create_list; -use crate::native::nested::NestedState; -use crate::native::read::deserialize::DynIter; - -/// An iterator adapter over [`DynIter`] assumed to be encoded as List arrays +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::error::Result; +use crate::nested::create_list; +use crate::nested::NestedState; +use crate::read::deserialize::DynIter; + +/// An iterator adapter over [`DynIter`] assumed to be encoded as List columns pub struct ListIterator<'a> { - iter: DynIter<'a, Result<(NestedState, Box)>>, - field: Field, + iter: DynIter<'a, Result<(NestedState, Column)>>, + data_type: TableDataType, } impl<'a> ListIterator<'a> { - /// Creates a new [`ListIterator`] with `iter` and `field`. - pub fn new(iter: DynIter<'a, Result<(NestedState, Box)>>, field: Field) -> Self { - Self { iter, field } + /// Creates a new [`ListIterator`] with `iter` and `data_type`. + pub fn new(iter: DynIter<'a, Result<(NestedState, Column)>>, data_type: TableDataType) -> Self { + Self { iter, data_type } } } impl<'a> ListIterator<'a> { fn deserialize( &mut self, - value: Option)>>, - ) -> Option)>> { + value: Option>, + ) -> Option> { let (mut nested, values) = match value { Some(Ok((nested, values))) => (nested, values), Some(Err(err)) => return Some(Err(err)), None => return None, }; - let array = create_list(self.field.data_type().clone(), &mut nested, values); + let array = create_list(self.data_type.clone(), &mut nested, values); Some(Ok((nested, array))) } } impl<'a> Iterator for ListIterator<'a> { - type Item = Result<(NestedState, Box)>; + type Item = Result<(NestedState, Column)>; fn nth(&mut self, n: usize) -> Option { let value = self.iter.nth(n); diff --git a/src/common/arrow/src/native/read/array/map.rs b/src/common/native/src/read/array/map.rs similarity index 66% rename from src/common/arrow/src/native/read/array/map.rs rename to src/common/native/src/read/array/map.rs index f8de5824169c..fa5845618152 100644 --- a/src/common/arrow/src/native/read/array/map.rs +++ b/src/common/native/src/read/array/map.rs @@ -12,43 +12,44 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::arrow::array::Array; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Result; -use crate::native::nested::create_map; -use crate::native::nested::NestedState; -use crate::native::read::deserialize::DynIter; - -/// An iterator adapter over [`DynIter`] assumed to be encoded as Map arrays +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::error::Result; +use crate::nested::create_map; +use crate::nested::NestedState; +use crate::read::deserialize::DynIter; + +/// An iterator adapter over [`DynIter`] assumed to be encoded as Map columns pub struct MapIterator<'a> { - iter: DynIter<'a, Result<(NestedState, Box)>>, - field: Field, + iter: DynIter<'a, Result<(NestedState, Column)>>, + data_type: TableDataType, } impl<'a> MapIterator<'a> { /// Creates a new [`MapIterator`] with `iter` and `field`. - pub fn new(iter: DynIter<'a, Result<(NestedState, Box)>>, field: Field) -> Self { - Self { iter, field } + pub fn new(iter: DynIter<'a, Result<(NestedState, Column)>>, data_type: TableDataType) -> Self { + Self { iter, data_type } } } impl<'a> MapIterator<'a> { fn deserialize( &mut self, - value: Option)>>, - ) -> Option)>> { + value: Option>, + ) -> Option> { let (mut nested, values) = match value { Some(Ok((nested, values))) => (nested, values), Some(Err(err)) => return Some(Err(err)), None => return None, }; - let array = create_map(self.field.data_type().clone(), &mut nested, values); + let array = create_map(self.data_type.clone(), &mut nested, values); Some(Ok((nested, array))) } } impl<'a> Iterator for MapIterator<'a> { - type Item = Result<(NestedState, Box)>; + type Item = Result<(NestedState, Column)>; fn nth(&mut self, n: usize) -> Option { let value = self.iter.nth(n); diff --git a/src/common/arrow/src/native/read/array/mod.rs b/src/common/native/src/read/array/mod.rs similarity index 96% rename from src/common/arrow/src/native/read/array/mod.rs rename to src/common/native/src/read/array/mod.rs index e3da3130c23a..2e424195b58b 100644 --- a/src/common/arrow/src/native/read/array/mod.rs +++ b/src/common/native/src/read/array/mod.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod decimal; mod integer; +pub use decimal::*; pub use integer::*; mod double; @@ -25,7 +27,6 @@ pub use binary::*; mod view; pub use view::*; -mod null; mod struct_; pub use struct_::*; mod list; diff --git a/src/common/arrow/src/native/read/array/null.rs b/src/common/native/src/read/array/null.rs similarity index 57% rename from src/common/arrow/src/native/read/array/null.rs rename to src/common/native/src/read/array/null.rs index 9105d27a0225..42fa8b3c5e0e 100644 --- a/src/common/arrow/src/native/read/array/null.rs +++ b/src/common/native/src/read/array/null.rs @@ -12,25 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::arrow::array::Array; -use crate::arrow::array::NullArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::native::read::PageIterator; -use crate::native::PageMeta; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::read::read_basic::read_nested; +use crate::read::NativeReadBuf; +use crate::read::PageIterator; +use crate::PageMeta; #[derive(Debug)] pub struct NullIter where I: Iterator)>> + PageIterator + Send + Sync { iter: I, - data_type: DataType, + data_type: TableDataType, } impl NullIter where I: Iterator)>> + PageIterator + Send + Sync { - pub fn new(iter: I, data_type: DataType) -> Self { + pub fn new(iter: I, data_type: TableDataType) -> Self { Self { iter, data_type } } } @@ -38,17 +42,16 @@ where I: Iterator)>> + PageIterator + Send + Sync impl NullIter where I: Iterator)>> + PageIterator + Send + Sync { - fn deserialize(&mut self, num_values: u64) -> Result> { + fn deserialize(&mut self, num_values: u64) -> Result { let length = num_values as usize; - let array = NullArray::try_new(self.data_type.clone(), length)?; - Ok(Box::new(array) as Box) + Ok(null_column(&self.data_type, length)) } } impl Iterator for NullIter where I: Iterator)>> + PageIterator + Send + Sync { - type Item = Result>; + type Item = Result; fn nth(&mut self, n: usize) -> Option { match self.iter.nth(n) { @@ -73,9 +76,28 @@ where I: Iterator)>> + PageIterator + Send + Sync } } -pub fn read_null(data_type: DataType, page_metas: Vec) -> Result> { - let length = page_metas.iter().map(|p| p.num_values as usize).sum(); +pub fn read_nested_null( + reader: &mut R, + data_type: &TableDataType, + init: Vec, + page_metas: Vec, +) -> Result> { + let mut results = Vec::with_capacity(page_metas.len()); + for page_meta in page_metas { + let length = page_meta.num_values as usize; + let (nested, _) = read_nested(reader, &init, length)?; - let array = NullArray::try_new(data_type, length)?; - Ok(Box::new(array) as Box) + let col = null_column(data_type, length); + results.push((nested, col)); + } + Ok(results) +} + +fn null_column(data_type: &TableDataType, length: usize) -> Column { + match data_type { + TableDataType::Null => Column::Null { len: length }, + TableDataType::EmptyArray => Column::EmptyArray { len: length }, + TableDataType::EmptyMap => Column::EmptyMap { len: length }, + _ => unreachable!(), + } } diff --git a/src/common/arrow/src/native/read/array/struct_.rs b/src/common/native/src/read/array/struct_.rs similarity index 73% rename from src/common/arrow/src/native/read/array/struct_.rs rename to src/common/native/src/read/array/struct_.rs index 320166b12e16..296a775439f7 100644 --- a/src/common/arrow/src/native/read/array/struct_.rs +++ b/src/common/native/src/read/array/struct_.rs @@ -12,34 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::arrow::array::Array; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Result; -use crate::native::nested::create_struct; -use crate::native::nested::NestedState; -use crate::native::read::deserialize::NestedIters; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; -type StructValues = Vec)>>>; +use crate::error::Result; +use crate::nested::create_struct; +use crate::nested::NestedState; +use crate::read::deserialize::NestedIters; -/// An iterator adapter over [`DynIter`] assumed to be encoded as Struct arrays +type StructValues = Vec>>; + +/// An iterator adapter over [`DynIter`] assumed to be encoded as Struct columns pub struct StructIterator<'a> { iters: Vec>, - fields: Vec, + is_nullable: bool, } impl<'a> StructIterator<'a> { /// Creates a new [`StructIterator`] with `iters` and `fields`. - pub fn new(iters: Vec>, fields: Vec) -> Self { - assert_eq!(iters.len(), fields.len()); - Self { iters, fields } + pub fn new( + is_nullable: bool, + iters: Vec>, + _fields: Vec, + ) -> Self { + Self { iters, is_nullable } } } impl<'a> StructIterator<'a> { - fn deserialize( - &mut self, - values: StructValues, - ) -> Option)>> { + fn deserialize(&mut self, values: StructValues) -> Option> { // This code is copied from arrow2 `StructIterator` and adds a custom `nth` method implementation // https://github.com/jorgecarleitao/arrow2/blob/main/src/io/parquet/read/deserialize/struct_.rs if values.iter().any(|x| x.is_none()) { @@ -59,13 +60,13 @@ impl<'a> StructIterator<'a> { } } - let array = create_struct(self.fields.clone(), &mut nested, new_values); + let array = create_struct(self.is_nullable, &mut nested, new_values); Some(Ok(array)) } } impl<'a> Iterator for StructIterator<'a> { - type Item = Result<(NestedState, Box)>; + type Item = Result<(NestedState, Column)>; fn nth(&mut self, n: usize) -> Option { let values = self diff --git a/src/common/arrow/src/native/read/array/view.rs b/src/common/native/src/read/array/view.rs similarity index 65% rename from src/common/arrow/src/native/read/array/view.rs rename to src/common/native/src/read/array/view.rs index e129d4146d87..c2dca02a8238 100644 --- a/src/common/arrow/src/native/read/array/view.rs +++ b/src/common/native/src/read/array/view.rs @@ -16,67 +16,61 @@ use std::io::Cursor; use byteorder::LittleEndian; use byteorder::ReadBytesExt; - -use crate::arrow::array::Array; -use crate::arrow::array::BinaryViewArray; -use crate::arrow::array::View; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::buffer::Buffer; -use crate::arrow::datatypes::DataType; -use crate::arrow::error::Result; -use crate::native::nested::InitNested; -use crate::native::nested::NestedState; -use crate::native::read::read_basic::*; -use crate::native::read::BufReader; -use crate::native::read::NativeReadBuf; -use crate::native::read::PageIterator; -use crate::native::CommonCompression; -use crate::native::PageMeta; +use databend_common_column::binview::Utf8ViewColumn; +use databend_common_column::binview::View; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::read::read_basic::*; +use crate::read::BufReader; +use crate::read::NativeReadBuf; +use crate::read::PageIterator; +use crate::CommonCompression; +use crate::PageMeta; #[derive(Debug)] -pub struct ViewArrayNestedIter +pub struct ViewColNestedIter where I: Iterator)>> + PageIterator + Send + Sync { iter: I, - data_type: DataType, + data_type: TableDataType, init: Vec, - scratch: Vec, } -impl ViewArrayNestedIter +impl ViewColNestedIter where I: Iterator)>> + PageIterator + Send + Sync { - pub fn new(iter: I, data_type: DataType, init: Vec) -> Self { + pub fn new(iter: I, data_type: TableDataType, init: Vec) -> Self { Self { iter, data_type, init, - scratch: vec![], } } } -impl ViewArrayNestedIter +impl ViewColNestedIter where I: Iterator)>> + PageIterator + Send + Sync { - fn deserialize( - &mut self, - num_values: u64, - buffer: Vec, - ) -> Result<(NestedState, Box)> { + fn deserialize(&mut self, num_values: u64, buffer: Vec) -> Result<(NestedState, Column)> { let mut reader = BufReader::with_capacity(buffer.len(), Cursor::new(buffer)); let (nested, validity) = read_nested(&mut reader, &self.init, num_values as usize)?; let length = num_values as usize; - let array = read_view_array(&mut reader, length, self.data_type.clone(), validity)?; - Ok((nested, array)) + let col = read_view_col(&mut reader, length, self.data_type.clone(), validity)?; + Ok((nested, col)) } } -impl Iterator for ViewArrayNestedIter +impl Iterator for ViewColNestedIter where I: Iterator)>> + PageIterator + Send + Sync { - type Item = Result<(NestedState, Box)>; + type Item = Result<(NestedState, Column)>; fn nth(&mut self, n: usize) -> Option { match self.iter.nth(n) { @@ -95,29 +89,29 @@ where I: Iterator)>> + PageIterator + Send + Sync } } -pub fn read_nested_view_array( +pub fn read_nested_view_col( reader: &mut R, - data_type: DataType, + data_type: TableDataType, init: Vec, page_metas: Vec, -) -> Result)>> { +) -> Result> { let mut results = Vec::with_capacity(page_metas.len()); for page_meta in page_metas { let num_values = page_meta.num_values as usize; let (nested, validity) = read_nested(reader, &init, num_values)?; - let array = read_view_array(reader, num_values, data_type.clone(), validity)?; - results.push((nested, array)); + let col = read_view_col(reader, num_values, data_type.clone(), validity)?; + results.push((nested, col)); } Ok(results) } -fn read_view_array( +fn read_view_col( reader: &mut R, length: usize, - data_type: DataType, + data_type: TableDataType, validity: Option, -) -> Result> { +) -> Result { let mut scratch = vec![0; 9]; let (_c, _compressed_size, _uncompressed_size) = read_compress_header(reader, &mut scratch)?; @@ -146,19 +140,17 @@ fn read_view_array( buffers.push(Buffer::from(buffer)); } - let array = unsafe { - BinaryViewArray::new_unchecked_unknown_md( - data_type.clone(), + let col = unsafe { + Column::String(Utf8ViewColumn::new_unchecked_unknown_md( views, buffers.into(), - validity, None, - ) + )) }; - if matches!(data_type, DataType::Utf8View) { - Ok(Box::new(array.to_utf8view()?)) + if data_type.is_nullable() { + Ok(col.wrap_nullable(validity)) } else { - Ok(Box::new(array)) + Ok(col) } } diff --git a/src/common/native/src/read/batch_read.rs b/src/common/native/src/read/batch_read.rs new file mode 100644 index 000000000000..884573450b5e --- /dev/null +++ b/src/common/native/src/read/batch_read.rs @@ -0,0 +1,197 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_expression::types::DateType; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::MAX_DECIMAL128_PRECISION; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; + +use super::array::*; +use super::NativeReadBuf; +use crate::error::Result; +use crate::nested::create_list; +use crate::nested::create_map; +use crate::nested::create_struct; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::util::n_columns; +use crate::PageMeta; + +pub fn read_nested_column( + mut readers: Vec, + data_type: TableDataType, + mut init: Vec, + mut page_metas: Vec>, +) -> Result> { + let is_nullable = data_type.is_nullable(); + use TableDataType::*; + let result = match data_type.remove_nullable() { + Null | EmptyArray | EmptyMap => { + unimplemented!("Can't store pure nulls") + } + Boolean => { + init.push(InitNested::Primitive(is_nullable)); + read_nested_boolean( + &mut readers.pop().unwrap(), + data_type.clone(), + init, + page_metas.pop().unwrap(), + )? + } + Number(number) => with_match_integer_double_type!(number, + |$T| { + init.push(InitNested::Primitive(is_nullable)); + read_nested_integer::, $T, _>( + &mut readers.pop().unwrap(), + data_type.clone(), + init, + page_metas.pop().unwrap(), + )? + }, + |$T| { + init.push(InitNested::Primitive(is_nullable)); + read_nested_primitive::<$T, _>( + &mut readers.pop().unwrap(), + data_type.clone(), + init, + page_metas.pop().unwrap(), + )? + } + ), + Decimal(decimal) if decimal.precision() > MAX_DECIMAL128_PRECISION => { + init.push(InitNested::Primitive(is_nullable)); + read_nested_decimal::( + &mut readers.pop().unwrap(), + data_type.clone(), + decimal.size(), + init, + page_metas.pop().unwrap(), + )? + } + Decimal(decimal) => { + init.push(InitNested::Primitive(is_nullable)); + + read_nested_decimal::( + &mut readers.pop().unwrap(), + data_type.clone(), + decimal.size(), + init, + page_metas.pop().unwrap(), + )? + } + Timestamp => { + init.push(InitNested::Primitive(is_nullable)); + read_nested_integer::( + &mut readers.pop().unwrap(), + data_type.clone(), + init, + page_metas.pop().unwrap(), + )? + } + Date => { + init.push(InitNested::Primitive(is_nullable)); + read_nested_integer::( + &mut readers.pop().unwrap(), + data_type.clone(), + init, + page_metas.pop().unwrap(), + )? + } + t if t.is_physical_binary() => { + init.push(InitNested::Primitive(is_nullable)); + read_nested_binary::<_>( + &mut readers.pop().unwrap(), + data_type.clone(), + init, + page_metas.pop().unwrap(), + )? + } + + String => { + init.push(InitNested::Primitive(is_nullable)); + read_nested_view_col::<_>( + &mut readers.pop().unwrap(), + data_type.clone(), + init, + page_metas.pop().unwrap(), + )? + } + Array(inner) => { + init.push(InitNested::List(is_nullable)); + let results = read_nested_column(readers, inner.as_ref().clone(), init, page_metas)?; + let mut columns = Vec::with_capacity(results.len()); + for (mut nested, values) in results { + let array = create_list(data_type.clone(), &mut nested, values); + columns.push((nested, array)); + } + columns + } + Map(inner) => { + init.push(InitNested::List(is_nullable)); + let results = read_nested_column(readers, inner.as_ref().clone(), init, page_metas)?; + let mut columns = Vec::with_capacity(results.len()); + for (mut nested, values) in results { + let array = create_map(data_type.clone(), &mut nested, values); + columns.push((nested, array)); + } + columns + } + Tuple { + fields_name: _, + fields_type, + } => { + let mut results = fields_type + .iter() + .map(|f| { + let mut init = init.clone(); + init.push(InitNested::Struct(is_nullable)); + let n = n_columns(f); + let readers = readers.drain(..n).collect(); + let page_metas = page_metas.drain(..n).collect(); + read_nested_column(readers, f.clone(), init, page_metas) + }) + .collect::>>()?; + let mut columns = Vec::with_capacity(results[0].len()); + while !results[0].is_empty() { + let mut nesteds = Vec::with_capacity(fields_type.len()); + let mut values = Vec::with_capacity(fields_type.len()); + for result in results.iter_mut() { + let (nested, value) = result.pop().unwrap(); + nesteds.push(nested); + values.push(value); + } + let array = create_struct(is_nullable, &mut nesteds, values); + columns.push(array); + } + columns.reverse(); + columns + } + other => unimplemented!("read datatype {} is not supported", other), + }; + Ok(result) +} + +/// Read all pages of column at once. +pub fn batch_read_column( + readers: Vec, + data_type: TableDataType, + page_metas: Vec>, +) -> Result { + let results = read_nested_column(readers, data_type, vec![], page_metas)?; + let columns: Vec = results.iter().map(|(_, v)| v.clone()).collect(); + let column = Column::concat_columns(columns.into_iter()).unwrap(); + Ok(column) +} diff --git a/src/common/native/src/read/deserialize.rs b/src/common/native/src/read/deserialize.rs new file mode 100644 index 000000000000..c9fffa9f1f48 --- /dev/null +++ b/src/common/native/src/read/deserialize.rs @@ -0,0 +1,237 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_expression::types::DateType; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::MAX_DECIMAL128_PRECISION; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; + +use super::array::*; +use super::PageIterator; +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; +use crate::util::n_columns; + +/// [`DynIter`] is an iterator adapter adds a custom `nth` method implementation. +pub struct DynIter<'a, V> { + iter: Box + Send + Sync + 'a>, +} + +impl<'a, V> Iterator for DynIter<'a, V> { + type Item = V; + + fn next(&mut self) -> Option { + self.iter.next() + } + + fn nth(&mut self, n: usize) -> Option { + self.iter.nth(n) + } +} + +impl<'a, V> DynIter<'a, V> { + pub fn new(iter: I) -> Self + where I: Iterator + Send + Sync + 'a { + Self { + iter: Box::new(iter), + } + } +} + +pub type ColumnIter<'a> = DynIter<'a, Result>; + +/// [`NestedIter`] is a wrapper iterator used to remove the `NestedState` from inner iterator +/// and return only the `Column` +#[derive(Debug)] +pub struct NestedIter +where I: Iterator> + Send + Sync +{ + iter: I, +} + +impl NestedIter +where I: Iterator> + Send + Sync +{ + pub fn new(iter: I) -> Self { + Self { iter } + } +} + +impl Iterator for NestedIter +where I: Iterator> + Send + Sync +{ + type Item = Result; + + fn next(&mut self) -> Option { + match self.iter.next() { + Some(Ok((_, item))) => Some(Ok(item)), + Some(Err(err)) => Some(Err(err)), + None => None, + } + } + + fn nth(&mut self, n: usize) -> Option { + match self.iter.nth(n) { + Some(Ok((_, item))) => Some(Ok(item)), + Some(Err(err)) => Some(Err(err)), + None => None, + } + } +} + +pub type NestedIters<'a> = DynIter<'a, Result<(NestedState, Column)>>; + +fn deserialize_nested<'a, I>( + mut readers: Vec, + data_type: TableDataType, + mut init: Vec, +) -> Result> +where + I: Iterator)>> + PageIterator + Send + Sync + 'a, +{ + let is_nullable = matches!(data_type, TableDataType::Nullable(_)); + Ok(match data_type.remove_nullable() { + TableDataType::Null | TableDataType::EmptyArray | TableDataType::EmptyMap => { + unimplemented!("Can't store pure nulls") + } + TableDataType::Boolean => { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(BooleanNestedIter::new( + readers.pop().unwrap(), + data_type.clone(), + init, + )) + } + TableDataType::Number(number) => with_match_integer_double_type!(number, + |$I| { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(IntegerNestedIter::<_, NumberType<$I>, $I>::new( + readers.pop().unwrap(), + data_type.clone(), + init, + )) + }, + |$T| { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(DoubleNestedIter::<_, $T>::new( + readers.pop().unwrap(), + data_type.clone(), + init, + )) + } + ), + TableDataType::Timestamp => { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(IntegerNestedIter::<_, TimestampType, i64>::new( + readers.pop().unwrap(), + data_type.clone(), + init, + )) + } + TableDataType::Date => { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(IntegerNestedIter::<_, DateType, i32>::new( + readers.pop().unwrap(), + data_type.clone(), + init, + )) + } + TableDataType::Decimal(t) if t.precision() > MAX_DECIMAL128_PRECISION => { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(DecimalNestedIter::< + _, + databend_common_column::types::i256, + ethnum::i256, + >::new( + readers.pop().unwrap(), data_type.clone(), t.size(), init + )) + } + TableDataType::Decimal(t) => { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(DecimalNestedIter::<_, i128, i128>::new( + readers.pop().unwrap(), + data_type.clone(), + t.size(), + init, + )) + } + t if t.is_physical_binary() => { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(BinaryNestedIter::<_>::new( + readers.pop().unwrap(), + data_type.clone(), + init, + )) + } + TableDataType::String => { + init.push(InitNested::Primitive(is_nullable)); + DynIter::new(ViewColNestedIter::<_>::new( + readers.pop().unwrap(), + data_type.clone(), + init, + )) + } + TableDataType::Array(inner) => { + init.push(InitNested::List(is_nullable)); + let iter = deserialize_nested(readers, inner.as_ref().clone(), init)?; + DynIter::new(ListIterator::new(iter, data_type.clone())) + } + TableDataType::Map(inner) => { + init.push(InitNested::List(is_nullable)); + let iter = deserialize_nested(readers, inner.as_ref().clone(), init)?; + DynIter::new(MapIterator::new(iter, data_type.clone())) + } + TableDataType::Tuple { + fields_name: _, + fields_type, + } => { + let columns = fields_type + .iter() + .rev() + .map(|f| { + let mut init = init.clone(); + init.push(InitNested::Struct(is_nullable)); + let n = n_columns(f); + let readers = readers.drain(readers.len().saturating_sub(n)..).collect(); + deserialize_nested(readers, f.clone(), init) + }) + .collect::>>()?; + let columns = columns.into_iter().rev().collect(); + DynIter::new(StructIterator::new( + is_nullable, + columns, + fields_type.clone(), + )) + } + other => unimplemented!("read datatype {} is not supported", other), + }) +} + +/// An iterator adapter that maps [`PageIterator`]s into an iterator of [`Array`]s. +pub fn column_iters<'a, I>( + readers: Vec, + field: TableField, + init: Vec, +) -> Result> +where + I: Iterator)>> + PageIterator + Send + Sync + 'a, +{ + let iter = deserialize_nested(readers, field.data_type().clone(), init)?; + let nested_iter = NestedIter::new(iter); + Ok(DynIter::new(nested_iter)) +} diff --git a/src/common/arrow/src/native/read/mod.rs b/src/common/native/src/read/mod.rs similarity index 75% rename from src/common/arrow/src/native/read/mod.rs rename to src/common/native/src/read/mod.rs index a00bc1f0af02..dd9d9cea6e84 100644 --- a/src/common/arrow/src/native/read/mod.rs +++ b/src/common/native/src/read/mod.rs @@ -15,19 +15,19 @@ mod array; pub mod batch_read; pub mod deserialize; -use batch_read::batch_read_array; -pub use deserialize::column_iter_to_arrays; -pub use deserialize::ArrayIter; +use batch_read::batch_read_column; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; +pub use deserialize::column_iters; +pub use deserialize::ColumnIter; -use crate::arrow::array::Array; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Result; +use crate::error::Result; pub(crate) mod read_basic; use std::io::BufReader; use super::nested::InitNested; use super::PageMeta; -use crate::arrow::datatypes::Schema; pub mod reader; pub trait NativeReadBuf: std::io::BufRead { @@ -70,35 +70,33 @@ pub trait PageIterator { } #[derive(Clone)] -pub struct NativeColumnsReader { - schema: Schema, -} +pub struct NativeColumnsReader {} impl NativeColumnsReader { - pub fn new(schema: Schema) -> Result { - Ok(Self { schema }) + pub fn new() -> Result { + Ok(Self {}) } /// An iterator adapter that maps [`PageIterator`]s into an iterator of [`Array`]s. - pub fn column_iter_to_arrays<'a, I>( + pub fn column_iters<'a, I>( &self, readers: Vec, - field: Field, + field: TableField, init: Vec, - ) -> Result> + ) -> Result> where I: Iterator)>> + PageIterator + Send + Sync + 'a, { - column_iter_to_arrays(readers, field, init) + column_iters(readers, field, init) } /// Read all pages of column at once. - pub fn batch_read_array( + pub fn batch_read_column( &self, readers: Vec, - field: Field, + data_type: TableDataType, page_metas: Vec>, - ) -> Result> { - batch_read_array(readers, field, page_metas) + ) -> Result { + batch_read_column(readers, data_type, page_metas) } } diff --git a/src/common/arrow/src/native/read/read_basic.rs b/src/common/native/src/read/read_basic.rs similarity index 87% rename from src/common/arrow/src/native/read/read_basic.rs rename to src/common/native/src/read/read_basic.rs index 17a1e355eb31..d45890a87f0b 100644 --- a/src/common/arrow/src/native/read/read_basic.rs +++ b/src/common/native/src/read/read_basic.rs @@ -15,15 +15,14 @@ use std::convert::TryInto; use std::io::Read; +use databend_common_expression::types::Bitmap; + use super::NativeReadBuf; -use crate::arrow::bitmap::Bitmap; -use crate::arrow::error::Result; -use crate::arrow::offset::Offsets; -use crate::arrow::offset::OffsetsBuffer; -use crate::native::compression::Compression; -use crate::native::nested::InitNested; -use crate::native::nested::ListNested; -use crate::native::nested::Nested; +use crate::compression::Compression; +use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::ListNested; +use crate::nested::Nested; pub fn read_validity(reader: &mut R) -> Result> { let mut buf = vec![0u8; 4]; @@ -71,13 +70,12 @@ pub fn read_nested( InitNested::List(_) => { let mut buf = vec![0u8; 4]; let length = read_u32(reader, &mut buf)?; - let mut values = vec![0i64; length as usize]; + let mut values = vec![0u64; length as usize]; let bytes: &mut [u8] = bytemuck::cast_slice_mut(values.as_mut()); reader.read_exact(bytes)?; - let offsets = Offsets::try_from(values).unwrap(); results.push(Nested::LargeList(ListNested::new( - OffsetsBuffer::from(offsets), + values.into(), bitmap, n.is_nullable(), ))) diff --git a/src/common/arrow/src/native/read/reader.rs b/src/common/native/src/read/reader.rs similarity index 90% rename from src/common/arrow/src/native/read/reader.rs rename to src/common/native/src/read/reader.rs index 640b6c30732f..e407e26f9003 100644 --- a/src/common/arrow/src/native/read/reader.rs +++ b/src/common/native/src/read/reader.rs @@ -16,39 +16,20 @@ use std::io::Read; use std::io::Seek; use std::io::SeekFrom; +use databend_common_expression::TableSchema; use opendal::Reader; use super::read_basic::read_u32; use super::read_basic::read_u64; use super::NativeReadBuf; use super::PageIterator; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::native::ColumnMeta; -use crate::native::PageMeta; +use crate::error::Error; +use crate::error::Result; +use crate::ColumnMeta; +use crate::PageMeta; const DEFAULT_FOOTER_SIZE: u64 = 64 * 1024; -pub fn is_primitive(data_type: &DataType) -> bool { - matches!( - data_type.to_physical_type(), - PhysicalType::Primitive(_) - | PhysicalType::Null - | PhysicalType::Boolean - | PhysicalType::Utf8 - | PhysicalType::LargeUtf8 - | PhysicalType::Binary - | PhysicalType::Utf8View - | PhysicalType::BinaryView - | PhysicalType::LargeBinary - | PhysicalType::FixedSizeBinary - | PhysicalType::Dictionary(_) - ) -} - #[derive(Debug)] pub struct NativeReader { page_reader: R, @@ -184,7 +165,7 @@ pub fn read_meta(reader: &mut Reader) -> Result(reader: &mut Reader) -> Result { +pub fn infer_schema(reader: &mut Reader) -> Result { // EOS(8 bytes) + meta_size(4 bytes) + schema_size(4bytes) = 16 bytes reader.seek(SeekFrom::End(-16))?; let mut buf = vec![0u8; 4]; @@ -203,7 +184,7 @@ pub fn infer_schema(reader: &mut Reader) -> Result pub async fn read_meta_async( reader: Reader, total_len: usize, -) -> Result<(Vec, Schema)> { +) -> Result<(Vec, TableSchema)> { // Pre-read footer data to reduce IO. let pre_read_len = total_len.min(DEFAULT_FOOTER_SIZE as usize); diff --git a/src/common/arrow/src/native/stat.rs b/src/common/native/src/stat.rs similarity index 62% rename from src/common/arrow/src/native/stat.rs rename to src/common/native/src/stat.rs index da8207bba62b..6fe7959df166 100644 --- a/src/common/arrow/src/native/stat.rs +++ b/src/common/native/src/stat.rs @@ -14,17 +14,19 @@ use std::io::BufRead; -use crate::arrow::datatypes::Field; -use crate::arrow::datatypes::PhysicalType; -use crate::arrow::error::Result; -use crate::arrow::types::PrimitiveType; -use crate::native::compression::Compression; -use crate::native::read::PageIterator; -use crate::native::CommonCompression; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::MAX_DECIMAL128_PRECISION; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; + +use crate::compression::Compression; +use crate::error::Result; +use crate::read::PageIterator; +use crate::CommonCompression; #[derive(Debug)] pub struct ColumnInfo { - pub field: Field, + pub field: TableField, pub pages: Vec, } @@ -60,7 +62,7 @@ pub struct DictPageBody { pub unique_num: u32, } -pub fn stat_simple<'a, I>(reader: I, field: Field) -> Result +pub fn stat_simple<'a, I>(reader: I, field: TableField) -> Result where I: Iterator)>> + PageIterator + Send + Sync + 'a { let mut pages = vec![]; @@ -69,7 +71,7 @@ where I: Iterator)>> + PageIterator + Send + Sync + let mut buffer = buffer.as_slice(); let mut opt_validity_size = None; - if field.is_nullable { + if field.data_type().is_nullable() { let validity_size = u32::from_le_bytes(buffer[0..4].try_into().unwrap()); debug_assert!(validity_size == 0 || validity_size as u64 == num_values); let consume_validity_size = 4 + ((validity_size + 7) / 8) as usize; @@ -79,8 +81,8 @@ where I: Iterator)>> + PageIterator + Send + Sync + } }; - let physical_type = field.data_type.to_physical_type(); - let page = stat_body(&mut buffer, opt_validity_size, physical_type)?; + let data_type = field.data_type(); + let page = stat_body(&mut buffer, opt_validity_size, data_type)?; pages.push(page); } Ok(ColumnInfo { field, pages }) @@ -89,7 +91,7 @@ where I: Iterator)>> + PageIterator + Send + Sync + fn stat_body( buffer: &mut &[u8], opt_validity_size: Option, - physical_type: PhysicalType, + data_type: &TableDataType, ) -> Result { let codec = Compression::from_codec(buffer[0])?; let compressed_size = u32::from_le_bytes(buffer[1..5].try_into().unwrap()); @@ -98,9 +100,9 @@ fn stat_body( let body = match codec { Compression::Rle => PageBody::Rle, - Compression::Dict => stat_dict_body(buffer, physical_type)?, + Compression::Dict => stat_dict_body(buffer, data_type)?, Compression::OneValue => PageBody::OneValue, - Compression::Freq => stat_freq_body(buffer, physical_type)?, + Compression::Freq => stat_freq_body(buffer, data_type)?, Compression::Bitpacking => PageBody::Bitpack, Compression::DeltaBitpacking => PageBody::DeltaBitpack, Compression::Patas => PageBody::Patas, @@ -115,23 +117,35 @@ fn stat_body( }) } -fn stat_freq_body(mut buffer: &[u8], physical_type: PhysicalType) -> Result { - match physical_type { - PhysicalType::Primitive(p) => { +fn stat_freq_body(mut buffer: &[u8], data_type: &TableDataType) -> Result { + match data_type { + TableDataType::Number(p) => { let top_value_size = size_of_primitive(p); buffer = &buffer[top_value_size..]; let exceptions_bitmap_size = u32::from_le_bytes(buffer[0..4].try_into().unwrap()); buffer = &buffer[4 + exceptions_bitmap_size as usize..]; - let exceptions = stat_body(&mut buffer, None, physical_type)?; + let exceptions = stat_body(&mut buffer, None, data_type)?; Ok(PageBody::Freq(FreqPageBody { exceptions: Some(Box::new(exceptions)), exceptions_bitmap_size, })) } - PhysicalType::Binary - | PhysicalType::LargeBinary - | PhysicalType::Utf8 - | PhysicalType::LargeUtf8 => { + TableDataType::Decimal(decimal_size) => { + let top_value_size = if decimal_size.precision() > MAX_DECIMAL128_PRECISION { + 32 + } else { + 16 + }; + buffer = &buffer[top_value_size..]; + let exceptions_bitmap_size = u32::from_le_bytes(buffer[0..4].try_into().unwrap()); + buffer = &buffer[4 + exceptions_bitmap_size as usize..]; + let exceptions = stat_body(&mut buffer, None, data_type)?; + Ok(PageBody::Freq(FreqPageBody { + exceptions: Some(Box::new(exceptions)), + exceptions_bitmap_size, + })) + } + TableDataType::Binary | TableDataType::String => { let len = u64::from_le_bytes(buffer[0..8].try_into().unwrap()); buffer = &buffer[8 + len as usize..]; let exceptions_bitmap_size = u32::from_le_bytes(buffer[0..4].try_into().unwrap()); @@ -140,12 +154,12 @@ fn stat_freq_body(mut buffer: &[u8], physical_type: PhysicalType) -> Result unreachable!("type {:?} not supported", physical_type), + _ => unreachable!("type {:?} not supported", data_type), } } -fn stat_dict_body(mut buffer: &[u8], physical_type: PhysicalType) -> Result { - let indices = stat_body(&mut buffer, None, physical_type)?; +fn stat_dict_body(mut buffer: &[u8], data_type: &TableDataType) -> Result { + let indices = stat_body(&mut buffer, None, data_type)?; let unique_num = u32::from_le_bytes(buffer[0..4].try_into().unwrap()); Ok(PageBody::Dict(DictPageBody { indices: Box::new(indices), @@ -153,23 +167,19 @@ fn stat_dict_body(mut buffer: &[u8], physical_type: PhysicalType) -> Result usize { +fn size_of_primitive(p: &NumberDataType) -> usize { match p { - PrimitiveType::Int8 => 1, - PrimitiveType::Int16 => 2, - PrimitiveType::Int32 => 4, - PrimitiveType::Int64 => 8, - PrimitiveType::Int128 | PrimitiveType::UInt128 => 16, - PrimitiveType::Int256 => 32, - PrimitiveType::UInt8 => 1, - PrimitiveType::UInt16 => 2, - PrimitiveType::UInt32 => 4, - PrimitiveType::UInt64 => 8, - PrimitiveType::Float16 => unimplemented!(), - PrimitiveType::Float32 => 4, - PrimitiveType::Float64 => 8, - PrimitiveType::DaysMs => unimplemented!(), - PrimitiveType::MonthDayNano => unimplemented!(), + NumberDataType::Int8 => 1, + NumberDataType::Int16 => 2, + NumberDataType::Int32 => 4, + NumberDataType::Int64 => 8, + + NumberDataType::UInt8 => 1, + NumberDataType::UInt16 => 2, + NumberDataType::UInt32 => 4, + NumberDataType::UInt64 => 8, + NumberDataType::Float32 => 4, + NumberDataType::Float64 => 8, } } @@ -177,30 +187,30 @@ fn size_of_primitive(p: PrimitiveType) -> usize { mod test { use std::io::BufRead; + use databend_common_column::binary::BinaryColumn; + use databend_common_expression::infer_schema_type; + use databend_common_expression::types::Int64Type; + use databend_common_expression::Column; + use databend_common_expression::FromData; + use databend_common_expression::TableField; + use databend_common_expression::TableSchema; + use super::stat_simple; use super::ColumnInfo; - use crate::arrow::array::Array; - use crate::arrow::array::BinaryArray; - use crate::arrow::array::PrimitiveArray; - use crate::arrow::chunk::Chunk; - use crate::arrow::datatypes::Field; - use crate::arrow::datatypes::Schema; - use crate::native::read::reader::is_primitive; - use crate::native::read::reader::NativeReader; - use crate::native::stat::PageBody; - use crate::native::util::env::remove_all_env; - use crate::native::util::env::set_dict_env; - use crate::native::util::env::set_freq_env; - use crate::native::write::NativeWriter; - use crate::native::write::WriteOptions; - use crate::native::CommonCompression; + use crate::read::reader::NativeReader; + use crate::stat::PageBody; + use crate::util::env::remove_all_env; + use crate::util::env::set_dict_env; + use crate::util::env::set_freq_env; + use crate::write::NativeWriter; + use crate::write::WriteOptions; + use crate::CommonCompression; const PAGE_SIZE: usize = 2048; const PAGE_PER_COLUMN: usize = 10; const COLUMN_SIZE: usize = PAGE_SIZE * PAGE_PER_COLUMN; - fn write_and_stat_simple_column(array: Box) -> ColumnInfo { - assert!(is_primitive(array.data_type())); + fn write_and_stat_simple_column(column: Column) -> ColumnInfo { let options = WriteOptions { default_compression: CommonCompression::Lz4, max_page_size: Some(PAGE_SIZE), @@ -209,16 +219,12 @@ mod test { }; let mut bytes = Vec::new(); - let field = Field::new( - "name", - array.data_type().clone(), - array.validity().is_some(), - ); - let schema = Schema::from(vec![field.clone()]); - let mut writer = NativeWriter::new(&mut bytes, schema, options).unwrap(); + let field = TableField::new("name", infer_schema_type(&column.data_type()).unwrap()); + let table_schema = TableSchema::new(vec![field.clone()]); + let mut writer = NativeWriter::new(&mut bytes, table_schema, options).unwrap(); writer.start().unwrap(); - writer.write(&Chunk::new(vec![array])).unwrap(); + writer.write(&[column]).unwrap(); writer.finish().unwrap(); let meta = writer.metas[0].clone(); @@ -237,18 +243,16 @@ mod test { let values: Vec> = (0..COLUMN_SIZE) .map(|d| if d % 3 == 0 { None } else { Some(d as i64) }) .collect(); - let array = Box::new(PrimitiveArray::::from_iter(values)); - let column_info = write_and_stat_simple_column(array.clone()); + let column = Int64Type::from_opt_data(values); + let column_info = write_and_stat_simple_column(column.clone()); assert_eq!(column_info.pages.len(), 10); for p in column_info.pages { assert_eq!(p.validity_size, Some(PAGE_SIZE as u32)); } - let array = Box::new(BinaryArray::::from_iter_values( - ["a"; COLUMN_SIZE].iter(), - )); - let column_info = write_and_stat_simple_column(array.clone()); + let column = Column::Binary(BinaryColumn::from_iter(["a"; COLUMN_SIZE].iter())); + let column_info = write_and_stat_simple_column(column.clone()); assert_eq!(column_info.pages.len(), 10); for p in column_info.pages { assert_eq!(p.validity_size, None); @@ -256,7 +260,7 @@ mod test { } set_dict_env(); - let column_info = write_and_stat_simple_column(array.clone()); + let column_info = write_and_stat_simple_column(column.clone()); assert_eq!(column_info.pages.len(), 10); for p in column_info.pages { assert_eq!(p.validity_size, None); @@ -272,7 +276,7 @@ mod test { remove_all_env(); set_freq_env(); - let column_info = write_and_stat_simple_column(array); + let column_info = write_and_stat_simple_column(column); assert_eq!(column_info.pages.len(), 10); for p in column_info.pages { assert_eq!(p.validity_size, None); diff --git a/src/common/arrow/src/native/util/bit_util.rs b/src/common/native/src/util/bit_util.rs similarity index 92% rename from src/common/arrow/src/native/util/bit_util.rs rename to src/common/native/src/util/bit_util.rs index ebadee3c6bf7..28d950b79ee2 100644 --- a/src/common/arrow/src/native/util/bit_util.rs +++ b/src/common/native/src/util/bit_util.rs @@ -15,9 +15,12 @@ use std::io::Write; use std::mem::size_of; -use crate::arrow::buffer::Buffer; -use crate::arrow::error::Error; -use crate::arrow::error::Result; +use databend_common_expression::types::Buffer; +use databend_common_expression::types::F32; +use databend_common_expression::types::F64; + +use crate::error::Error; +use crate::error::Result; #[inline] pub fn from_le_slice(bs: &[u8]) -> T { @@ -178,8 +181,28 @@ gen_as_bytes!(u8); gen_as_bytes!(u16); gen_as_bytes!(u32); gen_as_bytes!(u64); -gen_as_bytes!(f32); -gen_as_bytes!(f64); + +impl AsBytes for F32 { + fn as_bytes(&self) -> &[u8] { + unsafe { + std::slice::from_raw_parts( + &self.0 as *const f32 as *const u8, + std::mem::size_of::(), + ) + } + } +} + +impl AsBytes for F64 { + fn as_bytes(&self) -> &[u8] { + unsafe { + std::slice::from_raw_parts( + &self.0 as *const f64 as *const u8, + std::mem::size_of::(), + ) + } + } +} /// Reads `size` of bytes from `src`, and reinterprets them as type `ty`, in /// little-endian order. diff --git a/src/common/arrow/src/native/util/byte_writer.rs b/src/common/native/src/util/byte_writer.rs similarity index 100% rename from src/common/arrow/src/native/util/byte_writer.rs rename to src/common/native/src/util/byte_writer.rs diff --git a/src/common/arrow/src/native/util/env.rs b/src/common/native/src/util/env.rs similarity index 100% rename from src/common/arrow/src/native/util/env.rs rename to src/common/native/src/util/env.rs diff --git a/src/common/arrow/src/native/util/memory.rs b/src/common/native/src/util/memory.rs similarity index 100% rename from src/common/arrow/src/native/util/memory.rs rename to src/common/native/src/util/memory.rs diff --git a/src/common/native/src/util/mod.rs b/src/common/native/src/util/mod.rs new file mode 100644 index 000000000000..c8e6de31bef4 --- /dev/null +++ b/src/common/native/src/util/mod.rs @@ -0,0 +1,71 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[allow(dead_code)] +mod bit_util; +mod byte_writer; +#[allow(dead_code)] +pub mod env; +pub mod memory; + +pub use bit_util::*; +pub use byte_writer::ByteWriter; +use databend_common_expression::TableDataType; + +#[macro_export] +macro_rules! with_match_integer_double_type { + ( + $key_type:expr, | $_:tt $I:ident | $body_integer:tt, | $__ :tt $T:ident | $body_primitive:tt +) => {{ + macro_rules! __with_ty__ { + ( $_ $I:ident ) => { + $body_integer + }; + } + macro_rules! __with_ty_double__ { + ( $_ $T:ident ) => { + $body_primitive + }; + } + use databend_common_expression::types::NumberDataType::*; + use databend_common_expression::types::F32; + use databend_common_expression::types::F64; + + match $key_type { + Int8 => __with_ty__! { i8 }, + Int16 => __with_ty__! { i16 }, + Int32 => __with_ty__! { i32 }, + Int64 => __with_ty__! { i64 }, + UInt8 => __with_ty__! { u8 }, + UInt16 => __with_ty__! { u16 }, + UInt32 => __with_ty__! { u32 }, + UInt64 => __with_ty__! { u64 }, + + Float32 => __with_ty_double__! { F32 }, + Float64 => __with_ty_double__! { F64 }, + } + }}; +} + +/// Returns the number of (parquet) columns that a [`DataType`] contains. +pub fn n_columns(data_type: &TableDataType) -> usize { + use TableDataType::*; + + match data_type.remove_nullable() { + Array(inner) => n_columns(&inner), + Map(inner) => n_columns(&inner), + Tuple { fields_type, .. } => fields_type.iter().map(n_columns).sum(), + _ => 1, + } +} diff --git a/src/common/arrow/src/native/write/binary.rs b/src/common/native/src/write/binary.rs similarity index 71% rename from src/common/arrow/src/native/write/binary.rs rename to src/common/native/src/write/binary.rs index 0e7c89e2f7d5..536c1d45862e 100644 --- a/src/common/arrow/src/native/write/binary.rs +++ b/src/common/native/src/write/binary.rs @@ -14,20 +14,22 @@ use std::io::Write; +use databend_common_column::binary::BinaryColumn; +use databend_common_column::bitmap::Bitmap; + use super::WriteOptions; -use crate::arrow::array::BinaryArray; -use crate::arrow::error::Result; -use crate::arrow::types::Offset; -use crate::native::compression::binary::compress_binary; +use crate::compression::binary::compress_binary; +use crate::error::Result; -pub(crate) fn write_binary( +pub(crate) fn write_binary( w: &mut W, - array: &BinaryArray, + array: &BinaryColumn, + validity: Option, write_options: WriteOptions, scratch: &mut Vec, ) -> Result<()> { scratch.clear(); - compress_binary(array, scratch, write_options)?; + compress_binary(array, validity, scratch, write_options)?; w.write_all(scratch.as_slice())?; Ok(()) } diff --git a/src/common/arrow/src/native/write/boolean.rs b/src/common/native/src/write/boolean.rs similarity index 77% rename from src/common/arrow/src/native/write/boolean.rs rename to src/common/native/src/write/boolean.rs index 3c7070b4f195..e736d65b768d 100644 --- a/src/common/arrow/src/native/write/boolean.rs +++ b/src/common/native/src/write/boolean.rs @@ -14,19 +14,21 @@ use std::io::Write; +use databend_common_column::bitmap::Bitmap; + use super::WriteOptions; -use crate::arrow::array::BooleanArray; -use crate::arrow::error::Result; -use crate::native::compression::boolean::compress_boolean; +use crate::compression::boolean::compress_boolean; +use crate::error::Result; pub(crate) fn write_bitmap( w: &mut W, - array: &BooleanArray, + column: &Bitmap, + validity: Option, write_options: WriteOptions, scratch: &mut Vec, ) -> Result<()> { scratch.clear(); - compress_boolean(array, scratch, write_options)?; + compress_boolean(column, validity, scratch, write_options)?; w.write_all(scratch)?; Ok(()) } diff --git a/src/common/arrow/src/native/write/common.rs b/src/common/native/src/write/common.rs similarity index 67% rename from src/common/arrow/src/native/write/common.rs rename to src/common/native/src/write/common.rs index d3c0fef93f4e..0237a0acd736 100644 --- a/src/common/arrow/src/native/write/common.rs +++ b/src/common/native/src/write/common.rs @@ -14,19 +14,19 @@ use std::io::Write; +use databend_common_expression::Column; + use super::write; use super::NativeWriter; -use crate::arrow::array::*; -use crate::arrow::chunk::Chunk; -use crate::arrow::error::Result; -use crate::native::compression::CommonCompression; -use crate::native::compression::Compression; -use crate::native::nested::slice_nest_array; -use crate::native::nested::to_leaves; -use crate::native::nested::to_nested; -use crate::native::ColumnMeta; -use crate::native::PageMeta; -use crate::native::EOF_MARKER; +use crate::compression::CommonCompression; +use crate::compression::Compression; +use crate::error::Result; +use crate::nested::slice_nest_column; +use crate::nested::to_leaves; +use crate::nested::to_nested; +use crate::ColumnMeta; +use crate::PageMeta; +use crate::EOF_MARKER; /// Options declaring the behaviour of writing to IPC #[derive(Debug, Clone, PartialEq, Default)] @@ -41,22 +41,18 @@ pub struct WriteOptions { } impl NativeWriter { - /// Encode and write a [`Chunk`] to the file - pub fn encode_chunk(&mut self, chunk: &Chunk>) -> Result<()> { - let page_size = self - .options - .max_page_size - .unwrap_or(chunk.len()) - .min(chunk.len()); - - for (array, field) in chunk.arrays().iter().zip(self.schema.fields.iter()) { - let length = array.len(); - - let nested = to_nested(array.as_ref(), field)?; - let leaf_arrays = to_leaves(array.as_ref()); + /// Encode and write columns to the file + pub fn encode_chunk(&mut self, chunk: &[Column]) -> Result<()> { + assert!(!chunk.is_empty()); + let rows = chunk.first().map(|c| c.len()).unwrap(); + let page_size = self.options.max_page_size.unwrap_or(rows).min(rows); - for (leaf_array, nested) in leaf_arrays.iter().zip(nested.into_iter()) { - let leaf_array = leaf_array.to_boxed(); + for column in chunk.iter() { + let length = column.len(); + let nested = to_nested(column)?; + let leaf_columns = to_leaves(column); + for (leaf_column, nested) in leaf_columns.iter().zip(nested.into_iter()) { + let leaf_column = leaf_column.clone(); let mut page_metas = Vec::with_capacity((length + 1) / page_size + 1); let start = self.writer.offset; @@ -67,15 +63,15 @@ impl NativeWriter { page_size }; - let mut sub_array = leaf_array.clone(); + let mut sub_column = leaf_column.clone(); let mut sub_nested = nested.clone(); - slice_nest_array(sub_array.as_mut(), &mut sub_nested, offset, length); + slice_nest_column(&mut sub_column, &mut sub_nested, offset, length); { let page_start = self.writer.offset; write( &mut self.writer, - sub_array.as_ref(), + &sub_column, &sub_nested, self.options.clone(), &mut self.scratch, @@ -85,7 +81,7 @@ impl NativeWriter { let page_end = self.writer.offset; page_metas.push(PageMeta { length: (page_end - page_start), - num_values: sub_array.len() as u64, + num_values: sub_column.len() as u64, }); } } diff --git a/src/common/arrow/src/native/write/mod.rs b/src/common/native/src/write/mod.rs similarity index 100% rename from src/common/arrow/src/native/write/mod.rs rename to src/common/native/src/write/mod.rs diff --git a/src/common/native/src/write/primitive.rs b/src/common/native/src/write/primitive.rs new file mode 100644 index 000000000000..c11be5d3676f --- /dev/null +++ b/src/common/native/src/write/primitive.rs @@ -0,0 +1,95 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Write; + +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; +use databend_common_column::types::i256; +use databend_common_column::types::NativeType; +use databend_common_column::types::PrimitiveType; +use databend_common_expression::types::F32; +use databend_common_expression::types::F64; + +use super::WriteOptions; +use crate::compression::double::compress_double; +use crate::compression::integer::compress_integer; +use crate::error::Result; + +pub(crate) fn write_primitive( + w: &mut W, + col: &Buffer, + validity: Option, + write_options: WriteOptions, + scratch: &mut Vec, +) -> Result<()> { + scratch.clear(); + match T::PRIMITIVE { + PrimitiveType::Int8 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::Int16 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::Int32 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::Int64 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::UInt8 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::UInt16 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::UInt32 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::UInt64 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::Int128 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::Int256 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_integer(array, validity, write_options, scratch)?; + } + PrimitiveType::Float32 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_double(array, validity, write_options, scratch)?; + } + PrimitiveType::Float64 => { + let array: &Buffer = unsafe { std::mem::transmute(col) }; + compress_double(array, validity, write_options, scratch)?; + } + + PrimitiveType::Float16 => unimplemented!(), + PrimitiveType::DaysMs => unimplemented!(), + PrimitiveType::MonthDayNano => unimplemented!(), + PrimitiveType::UInt128 => unimplemented!(), + } + w.write_all(scratch.as_slice())?; + Ok(()) +} diff --git a/src/common/native/src/write/serialize.rs b/src/common/native/src/write/serialize.rs new file mode 100644 index 000000000000..8239ae9777ee --- /dev/null +++ b/src/common/native/src/write/serialize.rs @@ -0,0 +1,130 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Write; + +use databend_common_column::buffer::Buffer; +use databend_common_column::types::i256; +use databend_common_expression::types::DecimalColumn; +use databend_common_expression::types::GeographyColumn; +use databend_common_expression::types::NumberColumn; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; +use databend_common_expression::Column; + +use super::boolean::write_bitmap; +use super::WriteOptions; +use crate::error::Result; +use crate::nested::Nested; +use crate::util::encode_bool; +use crate::write::binary::write_binary; +use crate::write::primitive::write_primitive; +use crate::write::view::write_view; +/// Writes an [`Array`] to the file +pub fn write( + w: &mut W, + column: &Column, + nested: &[Nested], + write_options: WriteOptions, + scratch: &mut Vec, +) -> Result<()> { + write_nest_info::(w, nested)?; + + let (_, validity) = column.validity(); + let validity = validity.cloned(); + + match column.remove_nullable() { + Column::Null { .. } | Column::EmptyArray { .. } | Column::EmptyMap { .. } => Ok(()), + Column::Number(column) => { + with_number_mapped_type!(|NUM_TYPE| match column { + NumberColumn::NUM_TYPE(column) => { + write_primitive::(w, &column, validity, write_options, scratch) + } + }) + } + Column::Decimal(column) => with_decimal_mapped_type!(|DT| match column { + DecimalColumn::DT(column, _) => { + let column: Buffer

= unsafe { std::mem::transmute(column) }; + write_primitive::(w, &column, validity, write_options, scratch) + } + }), + Column::Boolean(column) => write_bitmap(w, &column, validity, write_options, scratch), + Column::String(column) => write_view::(w, &column.to_binview(), write_options, scratch), + Column::Timestamp(column) => { + write_primitive::(w, &column, validity, write_options, scratch) + } + Column::Date(column) => { + write_primitive::(w, &column, validity, write_options, scratch) + } + + Column::Binary(b) + | Column::Bitmap(b) + | Column::Variant(b) + | Column::Geography(GeographyColumn(b)) + | Column::Geometry(b) => write_binary::(w, &b, validity, write_options, scratch), + + Column::Tuple(_) | Column::Map(_) | Column::Array(_) | Column::Nullable(_) => { + unreachable!() + } + } +} + +fn write_nest_info(w: &mut W, nesteds: &[Nested]) -> Result<()> { + let is_simple = nesteds.len() == 1; + + if is_simple { + let nest = nesteds.last().unwrap(); + + if nest.is_nullable() { + let (_, validity) = nest.inner(); + if let Some(bitmap) = validity { + w.write_all(&(bitmap.len() as u32).to_le_bytes())?; + let (s, offset, _) = bitmap.as_slice(); + if offset == 0 { + w.write_all(s)?; + } else { + encode_bool(w, bitmap.iter())?; + } + } else { + w.write_all(&0u32.to_le_bytes())?; + } + } + } else { + for nested in nesteds { + let (values, validity) = nested.inner(); + + if nested.is_nullable() { + if let Some(bitmap) = validity { + w.write_all(&(bitmap.len() as u32).to_le_bytes())?; + let (s, offset, _) = bitmap.as_slice(); + if offset == 0 { + w.write_all(s)?; + } else { + encode_bool(w, bitmap.iter())?; + } + } else { + w.write_all(&0u32.to_le_bytes())?; + } + } + + if nested.is_list() { + w.write_all(&(values.len() as u32).to_le_bytes())?; + let input_buf: &[u8] = bytemuck::cast_slice(&values); + w.write_all(input_buf)?; + } + } + } + + Ok(()) +} diff --git a/src/common/arrow/src/native/write/view.rs b/src/common/native/src/write/view.rs similarity index 92% rename from src/common/arrow/src/native/write/view.rs rename to src/common/native/src/write/view.rs index fc5b416b04c1..cb9285a72c2e 100644 --- a/src/common/arrow/src/native/write/view.rs +++ b/src/common/native/src/write/view.rs @@ -14,14 +14,15 @@ use std::io::Write; +use databend_common_column::binview::BinaryViewColumn; +use databend_common_column::binview::View; + use super::WriteOptions; -use crate::arrow::array::BinaryViewArray; -use crate::arrow::array::View; -use crate::arrow::error::Result; +use crate::error::Result; pub(crate) fn write_view( w: &mut W, - array: &BinaryViewArray, + array: &BinaryViewColumn, write_options: WriteOptions, buf: &mut Vec, ) -> Result<()> { diff --git a/src/common/arrow/src/native/write/writer.rs b/src/common/native/src/write/writer.rs similarity index 90% rename from src/common/arrow/src/native/write/writer.rs rename to src/common/native/src/write/writer.rs index 37be134448fc..18e9f4371229 100644 --- a/src/common/arrow/src/native/write/writer.rs +++ b/src/common/native/src/write/writer.rs @@ -14,16 +14,16 @@ use std::io::Write; +use databend_common_expression::Column; +use databend_common_expression::TableSchema; + use super::common::write_eof; use super::common::WriteOptions; -use crate::arrow::array::Array; -use crate::arrow::chunk::Chunk; -use crate::arrow::datatypes::Schema; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::native::ColumnMeta; -use crate::native::STRAWBOAT_MAGIC; -use crate::native::STRAWBOAT_VERSION; +use crate::error::Error; +use crate::error::Result; +use crate::ColumnMeta; +use crate::STRAWBOAT_MAGIC; +use crate::STRAWBOAT_VERSION; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum State { @@ -40,7 +40,7 @@ pub struct NativeWriter { /// pa write options pub(crate) options: WriteOptions, /// A reference to the schema, used in validating record batches - pub(crate) schema: Schema, + pub(crate) schema: TableSchema, /// Record blocks that will be written as part of the strawboat footer pub metas: Vec, @@ -52,7 +52,7 @@ pub struct NativeWriter { impl NativeWriter { /// Creates a new [`NativeWriter`] and writes the header to `writer` - pub fn try_new(writer: W, schema: &Schema, options: WriteOptions) -> Result { + pub fn try_new(writer: W, schema: &TableSchema, options: WriteOptions) -> Result { let mut slf = Self::new(writer, schema.clone(), options)?; slf.start()?; @@ -60,7 +60,7 @@ impl NativeWriter { } /// Creates a new [`NativeWriter`]. - pub fn new(writer: W, schema: Schema, options: WriteOptions) -> Result { + pub fn new(writer: W, schema: TableSchema, options: WriteOptions) -> Result { let num_cols = schema.fields.len(); Ok(Self { writer: OffsetWriter { @@ -99,7 +99,7 @@ impl NativeWriter { } /// Writes [`Chunk`] to the file - pub fn write(&mut self, chunk: &Chunk>) -> Result<()> { + pub fn write(&mut self, chunk: &[Column]) -> Result<()> { if self.state == State::Written { return Err(Error::OutOfSpec( "The strawboat file can only accept one RowGroup in a single file".to_string(), @@ -110,7 +110,7 @@ impl NativeWriter { "The strawboat file must be started before it can be written to. Call `start` before `write`".to_string(), )); } - assert_eq!(chunk.arrays().len(), self.schema.fields.len()); + assert_eq!(chunk.len(), self.schema.fields.len()); self.encode_chunk(chunk)?; self.state = State::Written; diff --git a/src/common/arrow/tests/it/main.rs b/src/common/native/tests/it/main.rs similarity index 98% rename from src/common/arrow/tests/it/main.rs rename to src/common/native/tests/it/main.rs index 83c1d406d287..aae3f71b1d7e 100644 --- a/src/common/arrow/tests/it/main.rs +++ b/src/common/native/tests/it/main.rs @@ -12,5 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod arrow; mod native; diff --git a/src/common/native/tests/it/native/io.rs b/src/common/native/tests/it/native/io.rs new file mode 100644 index 000000000000..f905890c3725 --- /dev/null +++ b/src/common/native/tests/it/native/io.rs @@ -0,0 +1,295 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::BufRead; +use std::io::BufReader; + +use databend_common_expression::infer_schema_type; +use databend_common_expression::types::*; +use databend_common_expression::Column; +use databend_common_expression::FromData; +use databend_common_expression::TableField; +use databend_common_expression::TableSchema; +use databend_common_native::n_columns; +use databend_common_native::read::batch_read::batch_read_column; +use databend_common_native::read::deserialize::column_iters; +use databend_common_native::read::reader::NativeReader; +use databend_common_native::write::NativeWriter; +use databend_common_native::write::WriteOptions; +use databend_common_native::ColumnMeta; +use databend_common_native::CommonCompression; +use databend_common_native::PageMeta; + +pub const WRITE_PAGE: usize = 2048; +pub const SMALL_WRITE_PAGE: usize = 2; + +pub fn new_test_column() -> Vec { + vec![ + UInt8Type::from_data(vec![1, 2, 3, 4, 5, 6]), + BooleanType::from_data(vec![true, true, true, false, false, false]), + UInt16Type::from_data(vec![1, 2, 3, 4, 5, 6]), + UInt32Type::from_data(vec![1, 2, 3, 4, 5, 6]), + UInt64Type::from_data(vec![1, 2, 3, 4, 5, 6]), + Int8Type::from_data(vec![1, 2, 3, 4, 5, 6]), + Int16Type::from_data(vec![1, 2, 3, 4, 5, 6]), + Int32Type::from_data(vec![1, 2, 3, 4, 5, 6]), + Int64Type::from_data(vec![1, 2, 3, 4, 5, 6]), + Float32Type::from_data(vec![1.1, 2.2, 3.3, 4.4, 5.5, 6.6]), + Float64Type::from_data(vec![1.1, 2.2, 3.3, 4.4, 5.5, 6.6]), + Column::String(StringColumn::from_iter( + ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), + )), + Column::Binary(BinaryColumn::from_iter( + ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), + )), + Column::Variant(BinaryColumn::from_iter( + ["abcdefg", "mn", "11", "", "3456", "xyz"].iter(), + )), + ] +} + +#[test] +fn test_basic() { + test_write_read(new_test_column()); +} + +#[test] +fn test_random() { + let size = 1000; + let chunk = rand_columns_for_all_types(size); + test_write_read(chunk); +} + +#[test] +fn test_freq() { + let size = WRITE_PAGE * 5; + let mut values: Vec = Vec::with_capacity(size); + for _ in 0..5 { + values.extend_from_slice(&vec![20; WRITE_PAGE - 3]); + values.push(10000); + values.push(10000); + values.push(10000); + } + + let chunk = vec![UInt32Type::from_data(values)]; + test_write_read(chunk); +} + +#[test] +fn test_bitpacking() { + let size = WRITE_PAGE * 5; + let chunk = vec![ + Column::random(&DataType::Number(NumberDataType::Int32), size, None), + Column::random(&DataType::Number(NumberDataType::Int32), size, None), + ]; + test_write_read(chunk); +} + +#[test] +fn test_deleta_bitpacking() { + let size = WRITE_PAGE * 5; + let chunk = vec![ + UInt32Type::from_data((0..size as u32).collect()), + Int32Type::from_data((0..size as i32).collect()), + ]; + test_write_read(chunk); +} + +#[test] +fn test_onevalue() { + let size = 10000; + let chunk = vec![ + BooleanType::from_data((0..size).map(|_| true).collect()), + BooleanType::from_data((0..size).map(|_| false).collect()), + UInt32Type::from_data(vec![3; size]), + ]; + test_write_read(chunk); +} + +fn test_write_read(chunk: Vec) { + let _ = env_logger::try_init(); + + let compressions = vec![ + CommonCompression::Lz4, + CommonCompression::Zstd, + CommonCompression::Snappy, + CommonCompression::None, + ]; + let page_sizes = vec![WRITE_PAGE, SMALL_WRITE_PAGE]; + + for compression in compressions { + for page_size in &page_sizes { + test_write_read_with_options(chunk.clone(), WriteOptions { + default_compression: compression, + max_page_size: Some(*page_size), + default_compress_ratio: Some(2.0f64), + forbidden_compressions: vec![], + }); + } + } +} + +fn test_write_read_with_options(chunk: Vec, options: WriteOptions) { + let mut bytes = Vec::new(); + let fields: Vec = chunk + .iter() + .map(|col| TableField::new("name", infer_schema_type(&col.data_type()).unwrap())) + .collect(); + + let schema = TableSchema::new(fields); + let mut writer = NativeWriter::new(&mut bytes, schema.clone(), options).unwrap(); + + writer.start().unwrap(); + writer.write(&chunk).unwrap(); + writer.finish().unwrap(); + + log::info!("write finished, start to read"); + + let mut batch_metas = writer.metas.clone(); + + let mut metas = writer.metas.clone(); + + let mut results = Vec::with_capacity(schema.fields.len()); + for field in schema.fields.iter() { + let n = n_columns(&field.data_type); + + let curr_metas: Vec = metas.drain(..n).collect(); + + let mut native_readers = Vec::with_capacity(n); + for curr_meta in curr_metas.iter() { + let mut range_bytes = std::io::Cursor::new(bytes.clone()); + range_bytes.consume(curr_meta.offset as usize); + + let native_reader = NativeReader::new(range_bytes, curr_meta.pages.clone(), vec![]); + native_readers.push(native_reader); + } + + let mut col_iter = column_iters(native_readers, field.clone(), vec![]).unwrap(); + + let mut cols = vec![]; + for col in col_iter.by_ref() { + cols.push(col.unwrap()); + } + let result = Column::concat_columns(cols.into_iter()).unwrap(); + results.push(result); + } + + let result_chunk = results; + assert_eq!(chunk, result_chunk); + + // test batch read + let mut batch_results = Vec::with_capacity(schema.fields.len()); + for field in schema.fields.iter() { + let n = n_columns(&field.data_type); + + let curr_metas: Vec = batch_metas.drain(..n).collect(); + + let mut pages: Vec> = Vec::with_capacity(n); + let mut readers = Vec::with_capacity(n); + for curr_meta in curr_metas.iter() { + pages.push(curr_meta.pages.clone()); + let mut reader = std::io::Cursor::new(bytes.clone()); + reader.consume(curr_meta.offset as usize); + + let buffer_size = curr_meta.total_len().min(8192) as usize; + let reader = BufReader::with_capacity(buffer_size, reader); + + readers.push(reader); + } + let batch_result = batch_read_column(readers, field.data_type().clone(), pages).unwrap(); + batch_results.push(batch_result); + } + let batch_result_chunk = batch_results; + assert_eq!(chunk, batch_result_chunk); +} + +fn get_all_test_data_types() -> Vec { + vec![ + DataType::Boolean, + DataType::Binary, + DataType::String, + DataType::Bitmap, + DataType::Variant, + DataType::Timestamp, + DataType::Date, + DataType::Number(NumberDataType::UInt8), + DataType::Number(NumberDataType::UInt16), + DataType::Number(NumberDataType::UInt32), + DataType::Number(NumberDataType::UInt64), + DataType::Number(NumberDataType::Int8), + DataType::Number(NumberDataType::Int16), + DataType::Number(NumberDataType::Int32), + DataType::Number(NumberDataType::Int64), + DataType::Number(NumberDataType::Float32), + DataType::Number(NumberDataType::Float64), + DataType::Decimal(DecimalDataType::Decimal128(DecimalSize { + precision: 10, + scale: 2, + })), + DataType::Decimal(DecimalDataType::Decimal128(DecimalSize { + precision: 35, + scale: 3, + })), + DataType::Decimal(DecimalDataType::Decimal256(DecimalSize { + precision: 55, + scale: 3, + })), + DataType::Nullable(Box::new(DataType::Geography)), + DataType::Nullable(Box::new(DataType::Geometry)), + DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt32))), + DataType::Nullable(Box::new(DataType::String)), + DataType::Nullable(Box::new(DataType::Variant)), + DataType::Nullable(Box::new(DataType::Binary)), + DataType::Array(Box::new(DataType::Number(NumberDataType::UInt32))), + DataType::Array(Box::new( + DataType::Number(NumberDataType::UInt32).wrap_nullable(), + )), + DataType::Nullable(Box::new(DataType::Array(Box::new(DataType::Number( + NumberDataType::UInt32, + ))))), + DataType::Map(Box::new(DataType::Tuple(vec![ + DataType::Number(NumberDataType::UInt64), + DataType::String, + ]))), + DataType::Array(Box::new(DataType::Array(Box::new(DataType::Number( + NumberDataType::UInt32, + ))))), + DataType::Array(Box::new(DataType::Map(Box::new(DataType::Tuple(vec![ + DataType::Number(NumberDataType::UInt64), + DataType::String, + ]))))), + DataType::Tuple(vec![ + DataType::Tuple(vec![ + DataType::Number(NumberDataType::Int64), + DataType::Number(NumberDataType::Float64), + ]) + .wrap_nullable(), + DataType::Tuple(vec![ + DataType::Number(NumberDataType::Int64), + DataType::Number(NumberDataType::Int64), + ]) + .wrap_nullable(), + ]), + ] +} + +fn rand_columns_for_all_types(num_rows: usize) -> Vec { + let types = get_all_test_data_types(); + let mut columns = Vec::with_capacity(types.len()); + for data_type in types.iter() { + columns.push(Column::random(data_type, num_rows, None)); + } + + columns +} diff --git a/src/common/arrow/tests/it/native/mod.rs b/src/common/native/tests/it/native/mod.rs similarity index 100% rename from src/common/arrow/tests/it/native/mod.rs rename to src/common/native/tests/it/native/mod.rs diff --git a/src/common/arrow/tests/it/native/read_meta.rs b/src/common/native/tests/it/native/read_meta.rs similarity index 65% rename from src/common/arrow/tests/it/native/read_meta.rs rename to src/common/native/tests/it/native/read_meta.rs index d41d56d5187f..c3565d69ab5b 100644 --- a/src/common/arrow/tests/it/native/read_meta.rs +++ b/src/common/native/tests/it/native/read_meta.rs @@ -12,32 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::datatypes::Schema; -use databend_common_arrow::arrow::error::Result; -use databend_common_arrow::native::read::reader::read_meta; -use databend_common_arrow::native::write::NativeWriter; -use databend_common_arrow::native::write::WriteOptions; -use databend_common_arrow::native::ColumnMeta; -use databend_common_arrow::native::CommonCompression; +use databend_common_expression::infer_schema_type; +use databend_common_expression::TableField; +use databend_common_expression::TableSchema; +use databend_common_native::read::reader::read_meta; +use databend_common_native::write::NativeWriter; +use databend_common_native::write::WriteOptions; +use databend_common_native::ColumnMeta; +use databend_common_native::CommonCompression; -use crate::native::io::new_test_chunk; -use crate::native::io::WRITE_PAGE; +use super::io::new_test_column; +use super::io::WRITE_PAGE; fn write_data(dest: &mut Vec) -> Vec { - let chunk = new_test_chunk(); - let fields: Vec = chunk + let chunk = new_test_column(); + let fields: Vec = chunk .iter() - .map(|array| { - Field::new( - "name", - array.data_type().clone(), - array.validity().is_some(), - ) - }) + .map(|col| TableField::new("name", infer_schema_type(&col.data_type()).unwrap())) .collect(); - let mut writer = NativeWriter::new(dest, Schema::from(fields), WriteOptions { + let mut writer = NativeWriter::new(dest, TableSchema::new(fields), WriteOptions { default_compression: CommonCompression::Lz4, max_page_size: Some(WRITE_PAGE), ..Default::default() @@ -52,12 +46,12 @@ fn write_data(dest: &mut Vec) -> Vec { } #[test] -fn test_read_meta() -> Result<()> { +fn test_read_meta() -> std::io::Result<()> { let mut buf = Vec::new(); let expected_meta = write_data(&mut buf); let mut reader = std::io::Cursor::new(buf); - let meta = read_meta(&mut reader)?; + let meta = read_meta(&mut reader).unwrap(); assert_eq!(expected_meta, meta); diff --git a/src/common/storage/Cargo.toml b/src/common/storage/Cargo.toml index 0f197f4dddd2..bc47fa73a375 100644 --- a/src/common/storage/Cargo.toml +++ b/src/common/storage/Cargo.toml @@ -15,13 +15,13 @@ arrow-schema = { workspace = true } async-backtrace = { workspace = true } chrono = { workspace = true } dashmap = { workspace = true, features = ["serde"] } -databend-common-arrow = { workspace = true } databend-common-auth = { workspace = true } databend-common-base = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } databend-common-meta-app = { workspace = true } databend-common-metrics = { workspace = true } +databend-common-native = { workspace = true } databend-enterprise-storage-encryption = { workspace = true } flagset = { workspace = true } futures = { workspace = true } diff --git a/src/common/storage/src/column_node.rs b/src/common/storage/src/column_node.rs index 93860057b9d6..46619584ebac 100644 --- a/src/common/storage/src/column_node.rs +++ b/src/common/storage/src/column_node.rs @@ -15,15 +15,16 @@ //! This module provides data structures for build column indexes. //! It's used by Fuse Engine and Parquet Engine. -use databend_common_arrow::arrow::datatypes::DataType as ArrowType; -use databend_common_arrow::arrow::datatypes::Field as ArrowField; -use databend_common_arrow::arrow::datatypes::Schema as ArrowSchema; -use databend_common_arrow::native::nested::InitNested; +use arrow_schema::DataType as ArrowType; +use arrow_schema::Field; +use arrow_schema::Schema; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_common_expression::FieldIndex; +use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_native::nested::InitNested; #[derive(Debug, Clone)] pub struct ColumnNodes { @@ -31,7 +32,7 @@ pub struct ColumnNodes { } impl ColumnNodes { - pub fn new_from_schema(schema: &ArrowSchema, table_schema: Option<&TableSchema>) -> Self { + pub fn new_from_schema(schema: &Schema, table_schema: Option<&TableSchema>) -> Self { let mut leaf_id = 0; let mut column_nodes = Vec::with_capacity(schema.fields.len()); @@ -56,18 +57,18 @@ impl ColumnNodes { /// If we don't dfs into it, the inner columns information will be lost. /// and we can not construct the arrow-parquet reader correctly. fn traverse_fields_dfs( - field: &ArrowField, + field: &Field, is_nested: bool, init: Vec, leaf_id: &mut usize, ) -> ColumnNode { - match &field.data_type { + match &field.data_type() { ArrowType::Struct(inner_fields) => { let mut child_column_nodes = Vec::with_capacity(inner_fields.len()); let mut child_leaf_ids = Vec::with_capacity(inner_fields.len()); for inner_field in inner_fields { let mut inner_init = init.clone(); - inner_init.push(InitNested::Struct(field.is_nullable)); + inner_init.push(InitNested::Struct(field.is_nullable())); let child_column_node = Self::traverse_fields_dfs(inner_field, true, inner_init, leaf_id); @@ -86,7 +87,7 @@ impl ColumnNodes { | ArrowType::LargeList(inner_field) | ArrowType::FixedSizeList(inner_field, _) => { let mut inner_init = init.clone(); - inner_init.push(InitNested::List(field.is_nullable)); + inner_init.push(InitNested::List(field.is_nullable())); let mut child_column_nodes = Vec::with_capacity(1); let mut child_leaf_ids = Vec::with_capacity(1); @@ -104,7 +105,7 @@ impl ColumnNodes { } ArrowType::Map(inner_field, _) => { let mut inner_init = init.clone(); - inner_init.push(InitNested::List(field.is_nullable)); + inner_init.push(InitNested::List(field.is_nullable())); let mut child_column_nodes = Vec::with_capacity(1); let mut child_leaf_ids = Vec::with_capacity(1); @@ -172,7 +173,8 @@ impl ColumnNodes { /// For the nested types, it may contain more than one leaf column. #[derive(Debug, Clone)] pub struct ColumnNode { - pub field: ArrowField, + pub table_field: TableField, + pub field: Field, // Array/Struct column or inner column of nested data types. pub is_nested: bool, // The initial info of nested data types, used to read inner field of struct column. @@ -187,13 +189,15 @@ pub struct ColumnNode { impl ColumnNode { pub fn new( - field: ArrowField, + field: Field, is_nested: bool, init: Vec, leaf_indices: Vec, children: Option>, ) -> Self { + let table_field = TableField::try_from(&field).unwrap(); Self { + table_field, field, is_nested, init, diff --git a/src/common/storage/tests/it/column_node.rs b/src/common/storage/tests/it/column_node.rs index 15c87a087fa4..8567c6d8f6df 100644 --- a/src/common/storage/tests/it/column_node.rs +++ b/src/common/storage/tests/it/column_node.rs @@ -66,7 +66,7 @@ fn test_column_leaf_schema_from_struct() -> Result<()> { for (i, column_leaf) in column_leaves.column_nodes.iter().enumerate() { let expected_column_id = expected_column_ids[i]; - assert_eq!(expected_column_id.0.to_string(), column_leaf.field.name); + assert_eq!(expected_column_id.0, column_leaf.field.name()); assert_eq!(*expected_column_id.1, column_leaf.leaf_column_ids); } @@ -88,7 +88,7 @@ fn test_column_leaf_schema_from_struct_of_old_version() -> Result<()> { .iter() .zip(new_column_leaves.column_nodes.iter()) { - assert_eq!(old_leaf.field.name, new_leaf.field.name); + assert_eq!(old_leaf.field.name(), new_leaf.field.name()); assert_eq!(old_leaf.leaf_indices, new_leaf.leaf_indices); // assert new column node column ids equal to old column node leaf ids. diff --git a/src/meta/binaries/Cargo.toml b/src/meta/binaries/Cargo.toml index 66e746fd7b6b..bcb08fa0d7c9 100644 --- a/src/meta/binaries/Cargo.toml +++ b/src/meta/binaries/Cargo.toml @@ -8,12 +8,11 @@ publish = { workspace = true } edition = { workspace = true } [features] -default = ["simd", "jemalloc"] +default = ["jemalloc"] memory-profiling = [ "databend-meta/memory-profiling", "databend-common-base/memory-profiling", ] -simd = ["databend-meta/simd"] jemalloc = ["databend-common-base/jemalloc"] io-uring = [ "databend-meta/io-uring", diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index f10704003d81..c8fae89c3539 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -12,9 +12,8 @@ doctest = false test = true [features] -default = ["simd"] +default = [] memory-profiling = ["databend-common-base/memory-profiling", "databend-common-http/memory-profiling"] -simd = ["databend-common-arrow/simd"] io-uring = [ "databend-common-meta-sled-store/io-uring", "databend-common-meta-raft-store/io-uring", @@ -27,7 +26,6 @@ arrow-flight = { workspace = true } async-trait = { workspace = true } backon = { workspace = true } clap = { workspace = true } -databend-common-arrow = { workspace = true } databend-common-base = { workspace = true } databend-common-grpc = { workspace = true } databend-common-http = { workspace = true } diff --git a/src/meta/service/src/version.rs b/src/meta/service/src/version.rs index 6ea7d3bcf349..df1c81c0cf4b 100644 --- a/src/meta/service/src/version.rs +++ b/src/meta/service/src/version.rs @@ -23,10 +23,8 @@ pub static METASRV_COMMIT_VERSION: LazyLock = LazyLock::new(|| { let rustc_semver = option_env!("VERGEN_RUSTC_SEMVER"); let timestamp = option_env!("VERGEN_BUILD_TIMESTAMP"); + // simd is enabled by default now match (build_semver, git_sha, rustc_semver, timestamp) { - #[cfg(not(feature = "simd"))] - (Some(v1), Some(v2), Some(v3), Some(v4)) => format!("{}-{}({}-{})", v1, v2, v3, v4), - #[cfg(feature = "simd")] (Some(v1), Some(v2), Some(v3), Some(v4)) => { format!("{}-{}-simd({}-{})", v1, v2, v3, v4) } @@ -147,7 +145,7 @@ pub(crate) mod raft { del_require( ("install_snapshot", 0), "2024-05-21", (1, 2, 479)), del_require( ("install_snapshot", 1), "2024-07-02", (1, 2, 552)), add_require( ("install_snapshot", 3), "2024-07-02", (1, 2, 552)), - + ]; /// Feature set provided by raft client. diff --git a/src/meta/service/tests/it/meta_node/meta_node_replication.rs b/src/meta/service/tests/it/meta_node/meta_node_replication.rs index f4794de0f201..236fdfb6a9db 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_replication.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_replication.rs @@ -15,7 +15,6 @@ use std::fs; use std::io::Read; -use databend_common_arrow::arrow::array::ViewType; use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; use databend_common_meta_raft_store::state_machine_api_ext::StateMachineApiExt; @@ -234,7 +233,7 @@ async fn test_raft_service_install_snapshot_v1() -> anyhow::Result<()> { let mut offset = 0; for (i, line) in snapshot_data.into_iter().enumerate() { - let mut chunk = line.to_bytes().to_vec(); + let mut chunk = line.as_bytes().to_vec(); let done = i == snapshot_data.len() - 1; if !done { chunk.push(b'\n'); diff --git a/src/query/catalog/Cargo.toml b/src/query/catalog/Cargo.toml index fa5a499cae29..c9aa49ca9733 100644 --- a/src/query/catalog/Cargo.toml +++ b/src/query/catalog/Cargo.toml @@ -16,7 +16,7 @@ async-backtrace = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } dashmap = { workspace = true } -databend-common-arrow = { workspace = true } + databend-common-ast = { workspace = true } databend-common-base = { workspace = true } databend-common-config = { workspace = true } diff --git a/src/query/catalog/src/plan/internal_column.rs b/src/query/catalog/src/plan/internal_column.rs index 10438eb7f40c..7d804492f0cf 100644 --- a/src/query/catalog/src/plan/internal_column.rs +++ b/src/query/catalog/src/plan/internal_column.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::number::F32; @@ -21,6 +20,7 @@ use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::DecimalSize; use databend_common_expression::types::Float32Type; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; use databend_common_expression::types::UInt64Type; diff --git a/src/query/codegen/src/writes/register.rs b/src/query/codegen/src/writes/register.rs index b50588a32be0..f6ab9631a797 100644 --- a/src/query/codegen/src/writes/register.rs +++ b/src/query/codegen/src/writes/register.rs @@ -606,7 +606,7 @@ pub fn codegen_register() { .iter() .map(|n| format!("arg{}.validity", n + 1)) .reduce(|acc, item| { - format!("databend_common_arrow::arrow::bitmap::and(&{acc}, &{item})") + format!("databend_common_column::bitmap::and(&{acc}, &{item})") }) .unwrap(); let func_arg = (0..n_args) @@ -714,7 +714,7 @@ pub fn codegen_register() { .iter() .map(|n| format!("arg{}.validity", n + 1)) .reduce(|acc, item| { - format!("databend_common_arrow::arrow::bitmap::and(&{acc}, &{item})") + format!("databend_common_column::bitmap::and(&{acc}, &{item})") }) .unwrap(); let func_arg = (0..n_args) @@ -733,7 +733,7 @@ pub fn codegen_register() { let validity = ctx.validity.as_ref().map(|valid| valid & (&and_validity)).unwrap_or(and_validity); ctx.validity = Some(validity.clone()); let nullable_column = func({func_arg} ctx).into_column().unwrap(); - let combine_validity = databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + let combine_validity = databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new(nullable_column.column, combine_validity)) }}" ) diff --git a/src/query/expression/Cargo.toml b/src/query/expression/Cargo.toml index e44cabed9b12..bc49814fe8c2 100644 --- a/src/query/expression/Cargo.toml +++ b/src/query/expression/Cargo.toml @@ -11,6 +11,9 @@ test = true [dependencies] arrow-array = { workspace = true } +arrow-buffer = { workspace = true } +arrow-cast = { workspace = true } +arrow-data = { workspace = true } arrow-flight = { workspace = true } arrow-ipc = { workspace = true, features = ["lz4"] } arrow-schema = { workspace = true } @@ -23,9 +26,9 @@ chrono = { workspace = true } chrono-tz = { workspace = true } comfy-table = { workspace = true } dashmap = { workspace = true } -databend-common-arrow = { workspace = true } databend-common-ast = { workspace = true } databend-common-base = { workspace = true } +databend-common-column = { workspace = true } databend-common-datavalues = { workspace = true } databend-common-exception = { workspace = true } databend-common-grpc = { workspace = true } @@ -56,7 +59,6 @@ roaring = { workspace = true, features = ["serde"] } rust_decimal = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -simdutf8 = { workspace = true } strength_reduce = { workspace = true } terminal_size = { workspace = true } tonic = { workspace = true } diff --git a/src/query/expression/src/aggregate/aggregate_function.rs b/src/query/expression/src/aggregate/aggregate_function.rs index ccdd61371468..95da2a33f2fd 100755 --- a/src/query/expression/src/aggregate/aggregate_function.rs +++ b/src/query/expression/src/aggregate/aggregate_function.rs @@ -16,7 +16,7 @@ use std::alloc::Layout; use std::fmt; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use super::StateAddr; diff --git a/src/query/expression/src/aggregate/group_hash.rs b/src/query/expression/src/aggregate/group_hash.rs index 15f36900af8f..543767a9eb38 100644 --- a/src/query/expression/src/aggregate/group_hash.rs +++ b/src/query/expression/src/aggregate/group_hash.rs @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::types::Index; use databend_common_base::base::OrderedFloat; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; +use databend_common_column::types::Index; use databend_common_exception::Result; use ethnum::i256; @@ -267,10 +268,7 @@ where I: Index self.visit_indices(|i| column.index(i.to_usize()).unwrap().as_bytes().agg_hash()) } - fn visit_boolean( - &mut self, - bitmap: databend_common_arrow::arrow::bitmap::Bitmap, - ) -> Result<()> { + fn visit_boolean(&mut self, bitmap: Bitmap) -> Result<()> { self.visit_indices(|i| bitmap.get(i.to_usize()).unwrap().agg_hash()) } @@ -466,7 +464,7 @@ impl AggHash for ScalarRef<'_> { #[cfg(test)] mod tests { - use databend_common_arrow::arrow::bitmap::Bitmap; + use databend_common_column::bitmap::Bitmap; use super::*; use crate::types::ArgType; diff --git a/src/query/expression/src/aggregate/payload.rs b/src/query/expression/src/aggregate/payload.rs index 4396df667e37..d7f01dcf1e48 100644 --- a/src/query/expression/src/aggregate/payload.rs +++ b/src/query/expression/src/aggregate/payload.rs @@ -237,8 +237,8 @@ impl Payload { for col in group_columns.iter() { if let Column::Nullable(c) = col { let bitmap = &c.validity; - if bitmap.unset_bits() == 0 || bitmap.unset_bits() == bitmap.len() { - let val: u8 = if bitmap.unset_bits() == 0 { 1 } else { 0 }; + if bitmap.null_count() == 0 || bitmap.null_count() == bitmap.len() { + let val: u8 = if bitmap.null_count() == 0 { 1 } else { 0 }; // faster path for idx in select_vector.iter().take(new_group_rows).copied() { unsafe { diff --git a/src/query/expression/src/aggregate/payload_row.rs b/src/query/expression/src/aggregate/payload_row.rs index 3a171d2d8b1f..4c81a1371fd6 100644 --- a/src/query/expression/src/aggregate/payload_row.rs +++ b/src/query/expression/src/aggregate/payload_row.rs @@ -13,7 +13,7 @@ // limitations under the License. use bumpalo::Bump; -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_io::prelude::bincode_deserialize_from_slice; use databend_common_io::prelude::bincode_serialize_into_buf; use ethnum::i256; @@ -95,8 +95,8 @@ pub unsafe fn serialize_column_to_rowformat( }) } Column::Boolean(v) => { - if v.unset_bits() == 0 || v.unset_bits() == v.len() { - let val: u8 = if v.unset_bits() == 0 { 1 } else { 0 }; + if v.null_count() == 0 || v.null_count() == v.len() { + let val: u8 = if v.null_count() == 0 { 1 } else { 0 }; // faster path for index in select_vector.iter().take(rows).copied() { store(&val, address[index].add(offset) as *mut u8); @@ -360,7 +360,7 @@ unsafe fn row_match_binary_column( let mut equal: bool; if let Some(validity) = validity { - let is_all_set = validity.unset_bits() == 0; + let is_all_set = validity.null_count() == 0; for idx in select_vector[..*count].iter() { let idx = *idx; let validity_address = address[idx].add(validity_offset); @@ -441,7 +441,7 @@ unsafe fn row_match_string_column( let mut equal: bool; if let Some(validity) = validity { - let is_all_set = validity.unset_bits() == 0; + let is_all_set = validity.null_count() == 0; for idx in select_vector[..*count].iter() { let idx = *idx; let validity_address = address[idx].add(validity_offset); @@ -523,7 +523,7 @@ unsafe fn row_match_column_type( let mut equal: bool; if let Some(validity) = validity { - let is_all_set = validity.unset_bits() == 0; + let is_all_set = validity.null_count() == 0; for idx in select_vector[..*count].iter() { let idx = *idx; let validity_address = address[idx].add(validity_offset); diff --git a/src/query/expression/src/block.rs b/src/query/expression/src/block.rs index 47715b137db7..22f8375423ed 100644 --- a/src/query/expression/src/block.rs +++ b/src/query/expression/src/block.rs @@ -17,9 +17,7 @@ use std::collections::HashSet; use std::fmt::Debug; use std::ops::Range; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::chunk::Chunk as ArrowChunk; -use databend_common_arrow::ArrayRef; +use arrow_array::ArrayRef; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -455,38 +453,18 @@ impl DataBlock { self.meta } - pub fn from_arrow_chunk>( - arrow_chunk: &ArrowChunk, - schema: &DataSchema, - ) -> Result { - let cols = schema - .fields - .iter() - .zip(arrow_chunk.arrays()) - .map(|(field, col)| { - Ok(BlockEntry::new( - field.data_type().clone(), - Value::Column(Column::from_arrow(col.as_ref(), field.data_type())?), - )) - }) - .collect::>()?; - - Ok(DataBlock::new(cols, arrow_chunk.len())) - } - // If default_vals[i].is_some(), then DataBlock.column[i] = num_rows * default_vals[i]. - // Else, DataBlock.column[i] = chuck.column. + // Else, DataBlock.column[i] = self.column. // For example, Schema.field is [a,b,c] and default_vals is [Some("a"), None, Some("c")], // then the return block column will be ["a"*num_rows, chunk.column[0], "c"*num_rows]. - pub fn create_with_default_value_and_chunk>( + pub fn create_with_opt_default_value( + arrays: Vec, schema: &DataSchema, - chunk: &ArrowChunk, default_vals: &[Option], num_rows: usize, ) -> Result { let mut chunk_idx: usize = 0; let schema_fields = schema.fields(); - let chunk_columns = chunk.arrays(); let mut columns = Vec::with_capacity(default_vals.len()); for (i, default_val) in default_vals.iter().enumerate() { @@ -498,12 +476,9 @@ impl DataBlock { BlockEntry::new(data_type.clone(), Value::Scalar(default_val.to_owned())) } None => { - let chunk_column = &chunk_columns[chunk_idx]; + let col = Column::from_arrow_rs(arrays[chunk_idx].clone(), data_type)?; chunk_idx += 1; - BlockEntry::new( - data_type.clone(), - Value::Column(Column::from_arrow(chunk_column.as_ref(), data_type)?), - ) + BlockEntry::new(data_type.clone(), Value::Column(col)) } }; @@ -518,17 +493,18 @@ impl DataBlock { default_vals: &[Scalar], num_rows: usize, ) -> Result { - let default_opt_vals: Vec> = default_vals - .iter() - .map(|default_val| Some(default_val.to_owned())) - .collect(); + let schema_fields = schema.fields(); - Self::create_with_default_value_and_chunk( - schema, - &ArrowChunk::::new(vec![]), - &default_opt_vals[0..], - num_rows, - ) + let mut columns = Vec::with_capacity(default_vals.len()); + for (i, default_val) in default_vals.iter().enumerate() { + let field = &schema_fields[i]; + let data_type = field.data_type(); + + let column = BlockEntry::new(data_type.clone(), Value::Scalar(default_val.to_owned())); + columns.push(column); + } + + Ok(DataBlock::new(columns, num_rows)) } // If block_column_ids not contain schema.field[i].column_id, @@ -619,24 +595,6 @@ impl DataBlock { } } -impl TryFrom for ArrowChunk { - type Error = ErrorCode; - - fn try_from(v: DataBlock) -> Result> { - let arrays = v - .convert_to_full() - .columns() - .iter() - .map(|val| { - let column = val.value.clone().into_column().unwrap(); - column.as_arrow() - }) - .collect(); - - Ok(ArrowChunk::try_new(arrays)?) - } -} - impl BlockEntry { pub fn memory_size(&self) -> usize { match &self.value { diff --git a/src/query/expression/src/converts/arrow/from.rs b/src/query/expression/src/converts/arrow/from.rs index e644777caae9..c081fd9e098b 100644 --- a/src/query/expression/src/converts/arrow/from.rs +++ b/src/query/expression/src/converts/arrow/from.rs @@ -14,18 +14,39 @@ use std::sync::Arc; +use arrow_array::ArrayRef; use arrow_array::RecordBatch; +use arrow_schema::DataType as ArrowDataType; use arrow_schema::Field; -use arrow_schema::Schema as ArrowSchema; -use databend_common_arrow::arrow::datatypes::Field as Arrow2Field; +use arrow_schema::Schema; +use databend_common_column::binary::BinaryColumn; +use databend_common_column::binview::StringColumn; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use super::ARROW_EXT_TYPE_BITMAP; +use super::ARROW_EXT_TYPE_EMPTY_ARRAY; +use super::ARROW_EXT_TYPE_EMPTY_MAP; +use super::ARROW_EXT_TYPE_GEOGRAPHY; +use super::ARROW_EXT_TYPE_GEOMETRY; +use super::ARROW_EXT_TYPE_VARIANT; +use super::EXTENSION_KEY; +use crate::types::ArrayColumn; use crate::types::DataType; +use crate::types::DecimalColumn; +use crate::types::DecimalDataType; +use crate::types::DecimalSize; +use crate::types::GeographyColumn; +use crate::types::NullableColumn; +use crate::types::NumberColumn; +use crate::types::NumberDataType; use crate::Column; use crate::DataBlock; use crate::DataField; use crate::DataSchema; +use crate::TableDataType; use crate::TableField; use crate::TableSchema; @@ -39,40 +60,133 @@ impl TryFrom<&Field> for DataField { impl TryFrom<&Field> for TableField { type Error = ErrorCode; fn try_from(arrow_f: &Field) -> Result { - TableField::try_from(&Arrow2Field::from(arrow_f)) + let mut data_type = match arrow_f + .metadata() + .get(EXTENSION_KEY) + .map(|x| x.as_str()) + .unwrap_or("") + { + ARROW_EXT_TYPE_EMPTY_ARRAY => TableDataType::EmptyArray, + ARROW_EXT_TYPE_EMPTY_MAP => TableDataType::EmptyMap, + ARROW_EXT_TYPE_BITMAP => TableDataType::Bitmap, + ARROW_EXT_TYPE_VARIANT => TableDataType::Variant, + ARROW_EXT_TYPE_GEOMETRY => TableDataType::Geometry, + ARROW_EXT_TYPE_GEOGRAPHY => TableDataType::Geography, + _ => match arrow_f.data_type() { + ArrowDataType::Null => TableDataType::Null, + ArrowDataType::Boolean => TableDataType::Boolean, + ArrowDataType::Int8 => TableDataType::Number(NumberDataType::Int8), + ArrowDataType::Int16 => TableDataType::Number(NumberDataType::Int16), + ArrowDataType::Int32 => TableDataType::Number(NumberDataType::Int32), + ArrowDataType::Int64 => TableDataType::Number(NumberDataType::Int64), + ArrowDataType::UInt8 => TableDataType::Number(NumberDataType::UInt8), + ArrowDataType::UInt16 => TableDataType::Number(NumberDataType::UInt16), + ArrowDataType::UInt32 => TableDataType::Number(NumberDataType::UInt32), + ArrowDataType::UInt64 => TableDataType::Number(NumberDataType::UInt64), + ArrowDataType::Float32 => TableDataType::Number(NumberDataType::Float32), + ArrowDataType::Float64 => TableDataType::Number(NumberDataType::Float64), + + ArrowDataType::FixedSizeBinary(_) + | ArrowDataType::Binary + | ArrowDataType::LargeBinary => TableDataType::Binary, + ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 | ArrowDataType::Utf8View => { + TableDataType::String + } + ArrowDataType::Decimal128(precision, scale) => { + TableDataType::Decimal(DecimalDataType::Decimal128(DecimalSize { + precision: *precision, + scale: *scale as u8, + })) + } + ArrowDataType::Decimal256(precision, scale) => { + TableDataType::Decimal(DecimalDataType::Decimal256(DecimalSize { + precision: *precision, + scale: *scale as u8, + })) + } + ArrowDataType::Timestamp(_, _) => TableDataType::Timestamp, + ArrowDataType::Date32 => TableDataType::Date, + ArrowDataType::Date64 => TableDataType::Date, + ArrowDataType::List(field) => { + let inner_type = TableField::try_from(field.as_ref())?; + TableDataType::Array(Box::new(inner_type.data_type)) + } + ArrowDataType::LargeList(field) => { + let inner_type = TableField::try_from(field.as_ref())?; + TableDataType::Array(Box::new(inner_type.data_type)) + } + ArrowDataType::Map(field, _) => { + if let ArrowDataType::Struct(fields) = field.data_type() { + let fields_name: Vec = + fields.iter().map(|f| f.name().clone()).collect(); + let fields_type: Vec = fields + .iter() + .map(|f| TableField::try_from(f.as_ref()).map(|f| f.data_type)) + .collect::>>()?; + TableDataType::Map(Box::new(TableDataType::Tuple { + fields_name, + fields_type, + })) + } else { + return Err(ErrorCode::Internal(format!( + "Invalid map field type: {:?}", + field.data_type() + ))); + } + } + ArrowDataType::Struct(fields) => { + let fields_name: Vec = + fields.iter().map(|f| f.name().clone()).collect(); + let fields_type: Vec = fields + .iter() + .map(|f| TableField::try_from(f.as_ref()).map(|f| f.data_type)) + .collect::>>()?; + TableDataType::Tuple { + fields_name, + fields_type, + } + } + arrow_type => { + return Err(ErrorCode::Internal(format!( + "Unsupported Arrow type: {:?}", + arrow_type + ))); + } + }, + }; + if arrow_f.is_nullable() { + data_type = data_type.wrap_nullable(); + } + Ok(TableField::new(arrow_f.name(), data_type)) } } -impl TryFrom<&ArrowSchema> for DataSchema { +impl TryFrom<&Schema> for DataSchema { type Error = ErrorCode; - fn try_from(schema: &ArrowSchema) -> Result { + fn try_from(schema: &Schema) -> Result { let fields = schema - .fields + .fields() .iter() - .map(|arrow_f| { - Ok(DataField::from(&TableField::try_from(&Arrow2Field::from( - arrow_f, - ))?)) - }) + .map(|arrow_f| DataField::try_from(arrow_f.as_ref())) .collect::>>()?; Ok(DataSchema::new_from( fields, - schema.metadata.clone().into_iter().collect(), + schema.metadata().clone().into_iter().collect(), )) } } -impl TryFrom<&ArrowSchema> for TableSchema { +impl TryFrom<&Schema> for TableSchema { type Error = ErrorCode; - fn try_from(schema: &ArrowSchema) -> Result { + fn try_from(schema: &Schema) -> Result { let fields = schema - .fields + .fields() .iter() - .map(|arrow_f| TableField::try_from(&Arrow2Field::from(arrow_f))) + .map(|arrow_f| TableField::try_from(arrow_f.as_ref())) .collect::>>()?; Ok(TableSchema::new_from( fields, - schema.metadata.clone().into_iter().collect(), + schema.metadata().clone().into_iter().collect(), )) } } @@ -112,8 +226,139 @@ impl DataBlock { } impl Column { - pub fn from_arrow_rs(array: Arc, data_type: &DataType) -> Result { - let arrow2_array: Box = array.into(); - Column::from_arrow(arrow2_array.as_ref(), data_type) + pub fn arrow_field(&self) -> Field { + let f = DataField::new("DUMMY", self.data_type()); + Field::from(&f) + } + + pub fn from_arrow_rs(array: ArrayRef, data_type: &DataType) -> Result { + let column = match data_type { + DataType::Null => Column::Null { len: array.len() }, + DataType::EmptyArray => Column::EmptyArray { len: array.len() }, + DataType::EmptyMap => Column::EmptyMap { len: array.len() }, + DataType::Number(_ty) => { + let col = NumberColumn::try_from_arrow_data(array.to_data())?; + Column::Number(col) + } + DataType::Boolean => Column::Boolean(Bitmap::from_array_data(array.to_data())), + DataType::String => Column::String(try_to_string_column(array)?), + DataType::Decimal(_) => { + Column::Decimal(DecimalColumn::try_from_arrow_data(array.to_data())?) + } + DataType::Timestamp => { + let array = arrow_cast::cast( + array.as_ref(), + &ArrowDataType::Timestamp(arrow_schema::TimeUnit::Microsecond, None), + )?; + let buffer: Buffer = array.to_data().buffers()[0].clone().into(); + Column::Timestamp(buffer) + } + DataType::Date => { + let array = arrow_cast::cast(array.as_ref(), &ArrowDataType::Date32)?; + let buffer: Buffer = array.to_data().buffers()[0].clone().into(); + Column::Date(buffer) + } + DataType::Nullable(_) => { + let validity = match array.nulls() { + Some(nulls) => Bitmap::from_null_buffer(nulls.clone()), + None => Bitmap::new_constant(true, array.len()), + }; + let column = Column::from_arrow_rs(array, &data_type.remove_nullable())?; + NullableColumn::new_column(column, validity) + } + DataType::Array(inner) => { + let f = DataField::new("DUMMY", *inner.clone()); + let inner_f = Field::from(&f); + let array = + arrow_cast::cast(array.as_ref(), &ArrowDataType::LargeList(inner_f.into()))?; + + let array = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + ErrorCode::Internal(format!( + "Cannot downcast to LargeListArray from array: {:?}", + array + )) + })?; + let values = Column::from_arrow_rs(array.values().clone(), inner.as_ref())?; + let offsets: Buffer = array.offsets().inner().inner().clone().into(); + + let inner_col = ArrayColumn { values, offsets }; + Column::Array(Box::new(inner_col)) + } + DataType::Map(inner) => { + let array = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + ErrorCode::Internal(format!( + "Cannot downcast to MapArray from array: {:?}", + array + )) + })?; + let entries = Arc::new(array.entries().clone()); + let values = Column::from_arrow_rs(entries, inner.as_ref())?; + let offsets: Buffer = array.offsets().inner().inner().clone().into(); + let offsets = offsets.into_iter().map(|x| x as u64).collect(); + + let inner_col = ArrayColumn { values, offsets }; + Column::Map(Box::new(inner_col)) + } + DataType::Tuple(ts) => { + let array = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + ErrorCode::Internal(format!( + "Cannot downcast to StructArray from array: {:?}", + array + )) + })?; + let columns = array + .columns() + .iter() + .zip(ts.iter()) + .map(|(array, ty)| Column::from_arrow_rs(array.clone(), ty)) + .collect::>>()?; + Column::Tuple(columns) + } + + DataType::Binary => Column::Binary(try_to_binary_column(array)?), + DataType::Bitmap => Column::Bitmap(try_to_binary_column(array)?), + DataType::Variant => Column::Variant(try_to_binary_column(array)?), + DataType::Geometry => Column::Geometry(try_to_binary_column(array)?), + DataType::Geography => Column::Geography(GeographyColumn(try_to_binary_column(array)?)), + DataType::Generic(_) => unreachable!("Generic type is not supported"), + }; + + Ok(column) } } + +// Convert from `ArrayData` into BinaryColumn ignores the validity +fn try_to_binary_column(array: ArrayRef) -> Result { + let array = if !matches!(array.data_type(), ArrowDataType::LargeBinary) { + arrow_cast::cast(array.as_ref(), &ArrowDataType::LargeBinary)? + } else { + array + }; + + let data = array.to_data(); + let offsets = data.buffers()[0].clone(); + let values = data.buffers()[1].clone(); + + Ok(BinaryColumn::new(values.into(), offsets.into())) +} + +// Convert from `ArrayData` into BinaryColumn ignores the validity +fn try_to_string_column(array: ArrayRef) -> Result { + let array = if !matches!(array.data_type(), ArrowDataType::Utf8View) { + arrow_cast::cast(array.as_ref(), &ArrowDataType::Utf8View)? + } else { + array + }; + + let data = array.to_data(); + Ok(data.into()) +} diff --git a/src/query/expression/src/converts/arrow/mod.rs b/src/query/expression/src/converts/arrow/mod.rs index 1774ba3ed7d3..bacab47f7a70 100644 --- a/src/query/expression/src/converts/arrow/mod.rs +++ b/src/query/expression/src/converts/arrow/mod.rs @@ -16,5 +16,9 @@ mod from; mod to; pub const EXTENSION_KEY: &str = "Extension"; - -pub use to::table_schema_to_arrow_schema; +pub const ARROW_EXT_TYPE_EMPTY_ARRAY: &str = "EmptyArray"; +pub const ARROW_EXT_TYPE_EMPTY_MAP: &str = "EmptyMap"; +pub const ARROW_EXT_TYPE_VARIANT: &str = "Variant"; +pub const ARROW_EXT_TYPE_BITMAP: &str = "Bitmap"; +pub const ARROW_EXT_TYPE_GEOMETRY: &str = "Geometry"; +pub const ARROW_EXT_TYPE_GEOGRAPHY: &str = "Geography"; diff --git a/src/query/expression/src/converts/arrow/to.rs b/src/query/expression/src/converts/arrow/to.rs index 34f929edbd2a..b2c486a7f156 100644 --- a/src/query/expression/src/converts/arrow/to.rs +++ b/src/query/expression/src/converts/arrow/to.rs @@ -12,79 +12,190 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::sync::Arc; use arrow_array::cast::AsArray; -use arrow_array::Array; -use arrow_array::LargeListArray; -use arrow_array::MapArray; -use arrow_array::RecordBatch; -use arrow_array::RecordBatchOptions; -use arrow_array::StructArray; +use arrow_array::*; +use arrow_data::ArrayData; +use arrow_data::ArrayDataBuilder; use arrow_schema::DataType as ArrowDataType; -use arrow_schema::Field as ArrowField; +use arrow_schema::Field; use arrow_schema::Fields; -use arrow_schema::Schema as ArrowSchema; -use databend_common_arrow::arrow::datatypes::Field as Arrow2Field; +use arrow_schema::Schema; +use arrow_schema::TimeUnit; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::buffer_to_array_data; use databend_common_exception::Result; +use super::ARROW_EXT_TYPE_BITMAP; +use super::ARROW_EXT_TYPE_EMPTY_ARRAY; +use super::ARROW_EXT_TYPE_EMPTY_MAP; +use super::ARROW_EXT_TYPE_GEOGRAPHY; +use super::ARROW_EXT_TYPE_GEOMETRY; +use super::ARROW_EXT_TYPE_VARIANT; +use super::EXTENSION_KEY; use crate::infer_table_schema; +use crate::types::DataType; +use crate::types::DecimalDataType; +use crate::types::GeographyColumn; +use crate::types::NumberDataType; +use crate::with_number_type; use crate::Column; use crate::DataBlock; use crate::DataField; use crate::DataSchema; +use crate::TableDataType; use crate::TableField; use crate::TableSchema; -impl From<&DataSchema> for ArrowSchema { +impl From<&DataSchema> for Schema { fn from(schema: &DataSchema) -> Self { - let fields = schema - .fields + let fields = schema.fields().iter().map(Field::from).collect::>(); + let metadata = schema + .metadata .iter() - .map(|f| ArrowField::from(Arrow2Field::from(f))) - .collect::>(); - ArrowSchema { - fields: Fields::from(fields), - metadata: schema.metadata.clone().into_iter().collect(), - } + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + Schema::new(fields).with_metadata(metadata) } } -impl From<&TableSchema> for ArrowSchema { +impl From<&TableSchema> for Schema { fn from(schema: &TableSchema) -> Self { - let fields = schema - .fields + let fields = schema.fields().iter().map(Field::from).collect::>(); + let metadata = schema + .metadata .iter() - .map(|f| ArrowField::from(Arrow2Field::from(f))) - .collect::>(); - ArrowSchema { - fields: Fields::from(fields), - metadata: schema.metadata.clone().into_iter().collect(), - } + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + + Schema::new(fields).with_metadata(metadata) } } -pub fn table_schema_to_arrow_schema(schema: &TableSchema) -> ArrowSchema { - let fields = schema - .fields - .iter() - .map(|f| ArrowField::from(Arrow2Field::from(f))) - .collect::>(); - ArrowSchema { - fields: Fields::from(fields), - metadata: schema.metadata.clone().into_iter().collect(), +impl From<&DataType> for ArrowDataType { + fn from(ty: &DataType) -> Self { + let fields = DataField::new("dummy", ty.clone()); + let f = Field::from(&fields); + f.data_type().clone() } } -impl From<&TableField> for ArrowField { - fn from(field: &TableField) -> Self { - ArrowField::from(Arrow2Field::from(field)) +impl From<&TableField> for Field { + fn from(f: &TableField) -> Self { + let mut metadata = HashMap::new(); + + let ty = match &f.data_type { + TableDataType::Null => ArrowDataType::Null, + TableDataType::EmptyArray => { + metadata.insert( + EXTENSION_KEY.to_string(), + ARROW_EXT_TYPE_EMPTY_ARRAY.to_string(), + ); + ArrowDataType::Boolean + } + TableDataType::EmptyMap => { + metadata.insert( + EXTENSION_KEY.to_string(), + ARROW_EXT_TYPE_EMPTY_MAP.to_string(), + ); + ArrowDataType::Boolean + } + TableDataType::Boolean => ArrowDataType::Boolean, + TableDataType::Binary => ArrowDataType::LargeBinary, + TableDataType::String => ArrowDataType::Utf8View, + TableDataType::Number(ty) => with_number_type!(|TYPE| match ty { + NumberDataType::TYPE => ArrowDataType::TYPE, + }), + TableDataType::Decimal(DecimalDataType::Decimal128(size)) => { + ArrowDataType::Decimal128(size.precision, size.scale as i8) + } + TableDataType::Decimal(DecimalDataType::Decimal256(size)) => { + ArrowDataType::Decimal256(size.precision, size.scale as i8) + } + TableDataType::Timestamp => ArrowDataType::Timestamp(TimeUnit::Microsecond, None), + TableDataType::Date => ArrowDataType::Date32, + TableDataType::Nullable(ty) => { + let mut f = f.clone(); + f.data_type = *ty.clone(); + return Field::from(&f).with_nullable(true); + } + TableDataType::Array(ty) => { + let f = TableField::new("_array", *ty.clone()); + let arrow_f = Field::from(&f); + ArrowDataType::LargeList(Arc::new(arrow_f)) + } + TableDataType::Map(ty) => { + let inner_ty = match ty.as_ref() { + TableDataType::Tuple { + fields_name, + fields_type, + } => { + let key = TableField::new(&fields_name[0], fields_type[0].clone()); + let arrow_key = Field::from(&key); + + let value = TableField::new(&fields_name[1], fields_type[1].clone()); + let arrow_value = Field::from(&value); + + ArrowDataType::Struct(Fields::from(vec![arrow_key, arrow_value])) + } + _ => unreachable!(), + }; + ArrowDataType::Map( + Arc::new(Field::new("entries", inner_ty, ty.is_nullable())), + false, + ) + } + TableDataType::Bitmap => { + metadata.insert(EXTENSION_KEY.to_string(), ARROW_EXT_TYPE_BITMAP.to_string()); + ArrowDataType::LargeBinary + } + TableDataType::Tuple { + fields_name, + fields_type, + } => { + let fields: Vec = fields_name + .iter() + .zip(fields_type) + .map(|(name, ty)| { + let f = TableField::new(name, ty.clone()); + let f = Field::from(&f); + f.with_nullable(ty.is_nullable_or_null()) + }) + .collect(); + ArrowDataType::Struct(Fields::from(fields)) + } + TableDataType::Variant => { + metadata.insert( + EXTENSION_KEY.to_string(), + ARROW_EXT_TYPE_VARIANT.to_string(), + ); + ArrowDataType::LargeBinary + } + TableDataType::Geometry => { + metadata.insert( + EXTENSION_KEY.to_string(), + ARROW_EXT_TYPE_GEOMETRY.to_string(), + ); + ArrowDataType::LargeBinary + } + TableDataType::Geography => { + metadata.insert( + EXTENSION_KEY.to_string(), + ARROW_EXT_TYPE_GEOGRAPHY.to_string(), + ); + ArrowDataType::LargeBinary + } + }; + + Field::new(f.name(), ty, f.is_nullable()).with_metadata(metadata) } } -impl From<&DataField> for ArrowField { - fn from(field: &DataField) -> Self { - ArrowField::from(Arrow2Field::from(field)) +impl From<&DataField> for Field { + fn from(f: &DataField) -> Self { + Field::from(&TableField::from(f)) } } @@ -98,13 +209,13 @@ impl DataBlock { pub fn to_record_batch(self, table_schema: &TableSchema) -> Result { if table_schema.num_fields() == 0 { return Ok(RecordBatch::try_new_with_options( - Arc::new(ArrowSchema::empty()), + Arc::new(Schema::empty()), vec![], &RecordBatchOptions::default().with_row_count(Some(self.num_rows())), )?); } - let arrow_schema = table_schema_to_arrow_schema(table_schema); + let arrow_schema = Schema::from(table_schema); let mut arrays = Vec::with_capacity(self.columns().len()); for (entry, arrow_field) in self .consume_convert_to_full() @@ -122,7 +233,7 @@ impl DataBlock { Ok(RecordBatch::try_new(Arc::new(arrow_schema), arrays)?) } - fn adjust_nested_array(array: Arc, arrow_field: &ArrowField) -> Arc { + fn adjust_nested_array(array: Arc, arrow_field: &Field) -> Arc { if let ArrowDataType::Struct(fs) = arrow_field.data_type() { let array = array.as_ref().as_struct(); let inner_arrays = array @@ -166,10 +277,73 @@ impl DataBlock { } } +impl From<&Column> for ArrayData { + fn from(value: &Column) -> Self { + let arrow_type = ArrowDataType::from(&value.data_type()); + match value { + Column::Null { len } => { + let builder = ArrayDataBuilder::new(arrow_type).len(*len); + unsafe { builder.build_unchecked() } + } + Column::EmptyArray { len } => Bitmap::new_constant(true, *len).into(), + Column::EmptyMap { len } => Bitmap::new_constant(true, *len).into(), + Column::Boolean(col) => col.into(), + Column::Number(c) => c.arrow_data(arrow_type), + Column::Decimal(c) => c.arrow_data(arrow_type), + Column::String(col) => col.clone().into(), + Column::Timestamp(col) => buffer_to_array_data((col.clone(), arrow_type)), + Column::Date(col) => buffer_to_array_data((col.clone(), arrow_type)), + Column::Array(col) => { + let child_data = ArrayData::from(&col.values); + let builder = ArrayDataBuilder::new(arrow_type) + .len(value.len()) + .buffers(vec![col.offsets.clone().into()]) + .child_data(vec![child_data]); + + unsafe { builder.build_unchecked() } + } + Column::Nullable(col) => { + let data = ArrayData::from(&col.column); + let builder = data.into_builder(); + let nulls = col.validity.clone().into(); + unsafe { builder.nulls(Some(nulls)).build_unchecked() } + } + Column::Map(col) => { + let child_data = ArrayData::from(&col.values); + let offsets: Vec = col.offsets.iter().map(|x| *x as i32).collect(); + let builder = ArrayDataBuilder::new(arrow_type) + .len(value.len()) + .buffers(vec![offsets.into()]) + .child_data(vec![child_data]); + unsafe { builder.build_unchecked() } + } + Column::Tuple(fields) => { + let child_data = fields.iter().map(ArrayData::from).collect::>(); + let builder = ArrayDataBuilder::new(arrow_type) + .len(value.len()) + .child_data(child_data); + + unsafe { builder.build_unchecked() } + } + + Column::Binary(col) + | Column::Bitmap(col) + | Column::Variant(col) + | Column::Geometry(col) + | Column::Geography(GeographyColumn(col)) => col.clone().into(), + } + } +} + +impl From<&Column> for Arc { + fn from(col: &Column) -> Self { + let data = ArrayData::from(col); + arrow_array::make_array(data) + } +} + impl Column { pub fn into_arrow_rs(self) -> Arc { - let arrow2_array: Box = self.as_arrow(); - let arrow_array: Arc = arrow2_array.into(); - arrow_array + (&self).into() } } diff --git a/src/query/expression/src/converts/arrow2/from.rs b/src/query/expression/src/converts/arrow2/from.rs deleted file mode 100644 index 87822a5045b2..000000000000 --- a/src/query/expression/src/converts/arrow2/from.rs +++ /dev/null @@ -1,781 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::BinaryArray; -use databend_common_arrow::arrow::array::FixedSizeBinaryArray; -use databend_common_arrow::arrow::array::Utf8Array; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType as ArrowDataType; -use databend_common_arrow::arrow::datatypes::Field as ArrowField; -use databend_common_arrow::arrow::datatypes::Schema as ArrowSchema; -use databend_common_arrow::arrow::datatypes::TimeUnit; -use databend_common_arrow::arrow::types::Offset; -use databend_common_exception::ErrorCode; -use databend_common_exception::Result; - -use super::ARROW_EXT_TYPE_BITMAP; -use super::ARROW_EXT_TYPE_EMPTY_ARRAY; -use super::ARROW_EXT_TYPE_EMPTY_MAP; -use super::ARROW_EXT_TYPE_GEOMETRY; -use super::ARROW_EXT_TYPE_VARIANT; -use crate::types::array::ArrayColumn; -use crate::types::binary::BinaryColumn; -use crate::types::binary::BinaryColumnBuilder; -use crate::types::decimal::DecimalColumn; -use crate::types::geography::GeographyColumn; -use crate::types::nullable::NullableColumn; -use crate::types::string::StringColumn; -use crate::types::string::StringColumnBuilder; -use crate::types::DataType; -use crate::types::DecimalDataType; -use crate::types::DecimalSize; -use crate::types::NumberColumn; -use crate::types::NumberDataType; -use crate::types::F32; -use crate::types::F64; -use crate::with_number_type; -use crate::Column; -use crate::DataField; -use crate::DataSchema; -use crate::TableDataType; -use crate::TableField; -use crate::TableSchema; - -impl TryFrom<&ArrowSchema> for TableSchema { - type Error = ErrorCode; - - fn try_from(schema: &ArrowSchema) -> Result { - let fields = schema - .fields - .iter() - .map(|f| f.try_into()) - .collect::>>()?; - - Ok(TableSchema::new_from(fields, schema.metadata.clone())) - } -} - -impl TryFrom<&ArrowSchema> for DataSchema { - type Error = ErrorCode; - - fn try_from(schema: &ArrowSchema) -> Result { - let fields = schema - .fields - .iter() - .map(|f| f.try_into()) - .collect::>>()?; - - Ok(DataSchema::new_from(fields, schema.metadata.clone())) - } -} - -impl TryFrom<&ArrowField> for TableField { - type Error = ErrorCode; - - fn try_from(f: &ArrowField) -> Result { - let ty = arrow_type_to_table_type(&f.data_type, f.is_nullable)?; - Ok(TableField::new(&f.name, ty)) - } -} - -impl TryFrom<&ArrowField> for DataField { - type Error = ErrorCode; - - fn try_from(f: &ArrowField) -> Result { - Ok(DataField::from(&TableField::try_from(f)?)) - } -} - -fn arrow_type_to_table_type(ty: &ArrowDataType, is_nullable: bool) -> Result { - let ty = with_number_type!(|TYPE| match ty { - ArrowDataType::TYPE => TableDataType::Number(NumberDataType::TYPE), - - ArrowDataType::Decimal(precision, scale) => - TableDataType::Decimal(DecimalDataType::Decimal128(DecimalSize { - precision: *precision as u8, - scale: *scale as u8, - })), - ArrowDataType::Decimal256(precision, scale) => - TableDataType::Decimal(DecimalDataType::Decimal256(DecimalSize { - precision: *precision as u8, - scale: *scale as u8, - })), - - ArrowDataType::Null => return Ok(TableDataType::Null), - ArrowDataType::Boolean => TableDataType::Boolean, - - ArrowDataType::List(f) - | ArrowDataType::LargeList(f) - | ArrowDataType::FixedSizeList(f, _) => TableDataType::Array(Box::new( - arrow_type_to_table_type(&f.data_type, f.is_nullable)? - )), - - ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => - TableDataType::Binary, - - ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 | ArrowDataType::Utf8View => - TableDataType::String, - - ArrowDataType::Timestamp(_, _) => TableDataType::Timestamp, - ArrowDataType::Date32 | ArrowDataType::Date64 => TableDataType::Date, - ArrowDataType::Map(f, _) => { - let inner_ty = arrow_type_to_table_type(&f.data_type, f.is_nullable)?; - TableDataType::Map(Box::new(inner_ty)) - } - ArrowDataType::Struct(fields) => { - let mut fields_name = vec![]; - let mut fields_type = vec![]; - for f in fields { - fields_name.push(f.name.to_string()); - fields_type.push(arrow_type_to_table_type(&f.data_type, f.is_nullable)?); - } - TableDataType::Tuple { - fields_name, - fields_type, - } - } - ArrowDataType::Extension(custom_name, data_type, _) => match custom_name.as_str() { - ARROW_EXT_TYPE_EMPTY_ARRAY => TableDataType::EmptyArray, - ARROW_EXT_TYPE_EMPTY_MAP => TableDataType::EmptyMap, - ARROW_EXT_TYPE_BITMAP => TableDataType::Bitmap, - ARROW_EXT_TYPE_VARIANT => TableDataType::Variant, - ARROW_EXT_TYPE_GEOMETRY => TableDataType::Geometry, - _ => arrow_type_to_table_type(data_type, is_nullable)?, - }, - _ => { - return Err(ErrorCode::UnknownFormat(format!( - "unsupported arrow data type: {:?}", - ty - ))); - } - }); - - if is_nullable { - Ok(TableDataType::Nullable(Box::new(ty))) - } else { - Ok(ty) - } -} - -impl Column { - pub fn from_arrow( - arrow_col: &dyn databend_common_arrow::arrow::array::Array, - data_type: &DataType, - ) -> Result { - fn from_arrow_with_arrow_type( - arrow_col: &dyn databend_common_arrow::arrow::array::Array, - arrow_type: &ArrowDataType, - data_type: &DataType, - ) -> Result { - let column = match (data_type, arrow_type) { - (DataType::Null, ArrowDataType::Null) => Column::Null { - len: arrow_col.len(), - }, - (DataType::EmptyArray, _) => Column::EmptyArray { - len: arrow_col.len(), - }, - (DataType::EmptyMap, _) => Column::EmptyMap { - len: arrow_col.len(), - }, - (DataType::Number(NumberDataType::UInt8), ArrowDataType::UInt8) => { - Column::Number(NumberColumn::UInt8( - arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `UInt8` from arrow: array should be `UInt8Array`") - .values() - .clone(), - )) - } - (DataType::Number(NumberDataType::UInt16), ArrowDataType::UInt16) => { - Column::Number(NumberColumn::UInt16( - arrow_col - .as_any() - .downcast_ref::() - .expect( - "fail to read `UInt16` from arrow: array should be `UInt16Array`", - ) - .values() - .clone(), - )) - } - (DataType::Number(NumberDataType::UInt32), ArrowDataType::UInt32) => { - Column::Number(NumberColumn::UInt32( - arrow_col - .as_any() - .downcast_ref::() - .expect( - "fail to read `UInt32` from arrow: array should be `UInt32Array`", - ) - .values() - .clone(), - )) - } - (DataType::Number(NumberDataType::UInt64), ArrowDataType::UInt64) => { - Column::Number(NumberColumn::UInt64( - arrow_col - .as_any() - .downcast_ref::() - .expect( - "fail to read `UInt64` from arrow: array should be `UInt64Array`", - ) - .values() - .clone(), - )) - } - (DataType::Number(NumberDataType::Int8), ArrowDataType::Int8) => { - Column::Number(NumberColumn::Int8( - arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Int8` from arrow: array should be `Int8Array`") - .values() - .clone(), - )) - } - (DataType::Number(NumberDataType::Int16), ArrowDataType::Int16) => { - Column::Number(NumberColumn::Int16( - arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Int16` from arrow: array should be `Int16Array`") - .values() - .clone(), - )) - } - (DataType::Number(NumberDataType::Int32), ArrowDataType::Int32) => { - Column::Number(NumberColumn::Int32( - arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Int32` from arrow: array should be `Int32Array`") - .values() - .clone(), - )) - } - (DataType::Number(NumberDataType::Int64), ArrowDataType::Int64) => { - Column::Number(NumberColumn::Int64( - arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Int64` from arrow: array should be `Int64Array`") - .values() - .clone(), - )) - } - (DataType::Number(NumberDataType::Float32), ArrowDataType::Float32) => { - let col = arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Float32` from arrow: array should be `Float32Array`") - .values() - .clone(); - let col = unsafe { std::mem::transmute::, Buffer>(col) }; - Column::Number(NumberColumn::Float32(col)) - } - (DataType::Number(NumberDataType::Float64), ArrowDataType::Float64) => { - let col = arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Float64` from arrow: array should be `Float64Array`") - .values() - .clone(); - let col = unsafe { std::mem::transmute::, Buffer>(col) }; - Column::Number(NumberColumn::Float64(col)) - } - ( - DataType::Decimal(DecimalDataType::Decimal128(size)), - ArrowDataType::Decimal(precision, scale), - ) if size.precision as usize == *precision && size.scale as usize == *scale => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect("fail to read `Decimal128` from arrow: array should be `PrimitiveArray`"); - Column::Decimal(DecimalColumn::Decimal128( - arrow_col.values().clone(), - DecimalSize { - precision: *precision as u8, - scale: *scale as u8, - }, - )) - } - ( - DataType::Decimal(DecimalDataType::Decimal256(size)), - ArrowDataType::Decimal256(precision, scale), - ) if size.precision as usize == *precision && size.scale as usize == *scale => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect("fail to read `Decimal256` from arrow: array should be `PrimitiveArray`"); - let values = unsafe { - std::mem::transmute::< - Buffer, - Buffer, - >(arrow_col.values().clone()) - }; - Column::Decimal(DecimalColumn::Decimal256(values, DecimalSize { - precision: *precision as u8, - scale: *scale as u8, - })) - } - (DataType::Boolean, ArrowDataType::Boolean) => Column::Boolean( - arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Boolean` from arrow: array should be `BooleanArray`") - .values() - .clone(), - ), - (DataType::Binary, ArrowDataType::Binary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Binary` from arrow: array should be `BinaryArray`", - ); - let offsets = arrow_col - .offsets() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Binary(BinaryColumn { - data: arrow_col.values().clone(), - offsets: offsets.into(), - }) - } - (DataType::Binary, ArrowDataType::LargeBinary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Binary` from arrow: array should be `BinaryArray`", - ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Binary(BinaryColumn { - data: arrow_col.values().clone(), - offsets, - }) - } - (DataType::Binary, ArrowDataType::FixedSizeBinary(_)) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::() - .expect( - "fail to read `Binary` from arrow: array should be `FixedSizeBinaryArray`", - ); - Column::Binary(fixed_size_binary_array_to_binary_column(arrow_col)) - } - (DataType::Binary, ArrowDataType::Utf8) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Binary` from arrow: array should be `Utf8Array`", - ); - let offsets = arrow_col - .offsets() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Binary(BinaryColumn { - data: arrow_col.values().clone(), - offsets: offsets.into(), - }) - } - (DataType::Binary, ArrowDataType::LargeUtf8) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Binary` from arrow: array should be `Utf8Array`", - ); - - let offsets = unsafe { - std::mem::transmute::, Buffer>( - arrow_col.offsets().clone().into_inner(), - ) - }; - Column::Binary(BinaryColumn { - data: arrow_col.values().clone(), - offsets, - }) - } - (DataType::String, ArrowDataType::Binary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `String` from arrow: array should be `BinaryArray`", - ); - let col = binary_array_to_string_column(arrow_col); - Column::String(col) - } - (DataType::String, ArrowDataType::LargeBinary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `String` from arrow: array should be `BinaryArray`", - ); - let col = binary_array_to_string_column(arrow_col); - Column::String(col) - } - (DataType::String, ArrowDataType::FixedSizeBinary(_)) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::() - .expect( - "fail to read `String` from arrow: array should be `FixedSizeBinaryArray`", - ); - let col = fixed_size_binary_array_to_string_column(arrow_col); - Column::String(col) - } - (DataType::String, ArrowDataType::Utf8) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `String` from arrow: array should be `Utf8Array`", - ); - let col = utf8_array_to_string_column(arrow_col); - Column::String(col) - } - (DataType::String, ArrowDataType::LargeUtf8) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `String` from arrow: array should be `Utf8Array`", - ); - let col = utf8_array_to_string_column(arrow_col); - Column::String(col) - } - (DataType::String, ArrowDataType::Utf8View) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::() - .expect( - "fail to read `String` from arrow: array should be `Utf8ViewArray`", - ); - Column::String(StringColumn::new(arrow_col.clone())) - } - (DataType::Timestamp, ArrowDataType::Timestamp(uint, _)) => { - let values = arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Timestamp` from arrow: array should be `Int64Array`") - .values(); - let convert = match uint { - TimeUnit::Second => (1_000_000, 1), - TimeUnit::Millisecond => (1_000, 1), - TimeUnit::Microsecond => (1, 1), - TimeUnit::Nanosecond => (1, 1_000), - }; - let values = if convert.0 == 1 && convert.1 == 1 { - values.clone() - } else { - let values = values - .iter() - .map(|x| x * convert.0 / convert.1) - .collect::>(); - values.into() - }; - Column::Timestamp(values) - } - (DataType::Date, ArrowDataType::Date32) => Column::Date( - arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Date` from arrow: array should be `Int32Array`") - .values() - .clone(), - ), - ( - DataType::Variant, - ArrowDataType::Extension(name, box ArrowDataType::Binary, None), - ) if name == ARROW_EXT_TYPE_VARIANT => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect("fail to read from arrow: array should be `BinaryArray`"); - let offsets = arrow_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Variant(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) - } - (DataType::Variant, ArrowDataType::Binary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect("fail to read from arrow: array should be `BinaryArray`"); - let offsets = arrow_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Variant(BinaryColumn::new( - arrow_col.values().clone(), - offsets.into(), - )) - } - ( - DataType::Variant, - ArrowDataType::Extension(name, box ArrowDataType::LargeBinary, None), - ) if name == ARROW_EXT_TYPE_VARIANT => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Variant` from arrow: array should be `BinaryArray`", - ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Variant(BinaryColumn::new(arrow_col.values().clone(), offsets)) - } - (DataType::Variant, ArrowDataType::LargeBinary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Variant` from arrow: array should be `BinaryArray`", - ); - let offsets = arrow_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Variant(BinaryColumn::new(arrow_col.values().clone(), offsets)) - } - (DataType::Array(ty), ArrowDataType::List(_)) => { - let values_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Array` from arrow: array should be `ListArray`", - ); - let values = Column::from_arrow(&**values_col.values(), ty)?; - let offsets = values_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Array(Box::new(ArrayColumn { - values, - offsets: offsets.into(), - })) - } - (DataType::Array(ty), ArrowDataType::LargeList(_)) => { - let values_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Array` from arrow: array should be `ListArray`", - ); - let values = Column::from_arrow(&**values_col.values(), ty)?; - let offsets = values_col.offsets().clone().into_inner(); - let offsets = - unsafe { std::mem::transmute::, Buffer>(offsets) }; - Column::Array(Box::new(ArrayColumn { values, offsets })) - } - (DataType::Map(ty), ArrowDataType::Map(_, _)) => { - let map_col = arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read `Map` from arrow: array should be `MapArray`"); - let values = Column::from_arrow(&**map_col.field(), ty)?; - let offsets = map_col - .offsets() - .buffer() - .iter() - .map(|x| *x as u64) - .collect::>(); - Column::Map(Box::new(ArrayColumn { - values, - offsets: offsets.into(), - })) - } - (DataType::Tuple(fields), ArrowDataType::Struct(_)) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::() - .expect("fail to read from arrow: array should be `StructArray`"); - let field_cols = arrow_col - .values() - .iter() - .zip(fields) - .map(|(field_col, f)| Column::from_arrow(&**field_col, f)) - .collect::>>()?; - Column::Tuple(field_cols) - } - ( - DataType::Bitmap, - ArrowDataType::Extension(name, box ArrowDataType::LargeBinary, None), - ) if name == ARROW_EXT_TYPE_BITMAP => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Bitmap` from arrow: array should be `BinaryArray`", - ); - let offsets = unsafe { - std::mem::transmute::, Buffer>( - arrow_col.offsets().clone().into_inner(), - ) - }; - Column::Bitmap(BinaryColumn { - data: arrow_col.values().clone(), - offsets, - }) - } - (DataType::Bitmap, ArrowDataType::LargeBinary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Bitmap` from arrow: array should be `BinaryArray`", - ); - let offsets = unsafe { - std::mem::transmute::, Buffer>( - arrow_col.offsets().clone().into_inner(), - ) - }; - Column::Bitmap(BinaryColumn { - data: arrow_col.values().clone(), - offsets, - }) - } - ( - DataType::Geometry, - ArrowDataType::Extension(name, box ArrowDataType::LargeBinary, None), - ) if name == ARROW_EXT_TYPE_GEOMETRY => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Geometry` from arrow: array should be `BinaryArray`", - ); - let offsets = unsafe { - std::mem::transmute::, Buffer>( - arrow_col.offsets().clone().into_inner(), - ) - }; - Column::Geometry(BinaryColumn { - data: arrow_col.values().clone(), - offsets, - }) - } - (DataType::Geometry, ArrowDataType::LargeBinary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Geometry` from arrow: array should be `BinaryArray`", - ); - - let offsets = unsafe { - std::mem::transmute::, Buffer>( - arrow_col.offsets().clone().into_inner(), - ) - }; - Column::Geometry(BinaryColumn { - data: arrow_col.values().clone(), - offsets, - }) - } - (DataType::Geography, ArrowDataType::LargeBinary) => { - let arrow_col = arrow_col - .as_any() - .downcast_ref::>() - .expect( - "fail to read `Geography` from arrow: array should be `BinaryArray`", - ); - - let offsets = unsafe { - std::mem::transmute::, Buffer>( - arrow_col.offsets().clone().into_inner(), - ) - }; - Column::Geography(GeographyColumn(BinaryColumn { - data: arrow_col.values().clone(), - offsets, - })) - } - (data_type, ArrowDataType::Extension(_, arrow_type, _)) => { - from_arrow_with_arrow_type(arrow_col, arrow_type, data_type)? - } - (DataType::Nullable(ty), _) => { - let column = Column::from_arrow(arrow_col, ty)?; - let validity = arrow_col - .validity() - .cloned() - .unwrap_or_else(|| Bitmap::new_constant(true, arrow_col.len())); - NullableColumn::new_column(column, validity) - } - (ty, arrow_ty) => { - return Err(ErrorCode::Unimplemented(format!( - "conversion from arrow type {arrow_ty:?} to {ty:?} is not supported" - ))); - } - }; - Ok(column) - } - - from_arrow_with_arrow_type(arrow_col, arrow_col.data_type(), data_type) - } -} - -fn binary_array_to_string_column(array: &BinaryArray) -> StringColumn { - let mut builder = StringColumnBuilder::with_capacity(array.len()); - for value in array.values_iter() { - builder.put_and_commit(std::str::from_utf8(value).unwrap()); - } - builder.build() -} - -fn utf8_array_to_string_column(array: &Utf8Array) -> StringColumn { - let mut builder = StringColumnBuilder::with_capacity(array.len()); - for value in array.values_iter() { - builder.put_and_commit(value); - } - builder.build() -} - -fn fixed_size_binary_array_to_string_column(array: &FixedSizeBinaryArray) -> StringColumn { - let mut builder = StringColumnBuilder::with_capacity(array.len()); - for value in array.values_iter() { - builder.put_and_commit(std::str::from_utf8(value).unwrap()); - } - builder.build() -} - -fn fixed_size_binary_array_to_binary_column(array: &FixedSizeBinaryArray) -> BinaryColumn { - let mut builder = BinaryColumnBuilder::with_capacity(array.len(), array.len() * array.size()); - for value in array.values_iter() { - builder.put_slice(value); - builder.commit_row(); - } - builder.build() -} diff --git a/src/query/expression/src/converts/arrow2/mod.rs b/src/query/expression/src/converts/arrow2/mod.rs deleted file mode 100644 index 36210f307103..000000000000 --- a/src/query/expression/src/converts/arrow2/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod from; -mod to; - -pub const ARROW_EXT_TYPE_EMPTY_ARRAY: &str = "EmptyArray"; -pub const ARROW_EXT_TYPE_EMPTY_MAP: &str = "EmptyMap"; -pub const ARROW_EXT_TYPE_VARIANT: &str = "Variant"; -pub const ARROW_EXT_TYPE_BITMAP: &str = "Bitmap"; -pub const ARROW_EXT_TYPE_GEOMETRY: &str = "Geometry"; -pub const ARROW_EXT_TYPE_GEOGRAPHY: &str = "Geography"; - -pub use to::set_validities; diff --git a/src/query/expression/src/converts/arrow2/to.rs b/src/query/expression/src/converts/arrow2/to.rs deleted file mode 100644 index 89d51b70126a..000000000000 --- a/src/query/expression/src/converts/arrow2/to.rs +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::datatypes::DataType as ArrowDataType; -use databend_common_arrow::arrow::datatypes::Field as ArrowField; -use databend_common_arrow::arrow::datatypes::Schema as ArrowSchema; -use databend_common_arrow::arrow::datatypes::TimeUnit; -use databend_common_arrow::arrow::offset::OffsetsBuffer; - -use super::ARROW_EXT_TYPE_BITMAP; -use super::ARROW_EXT_TYPE_EMPTY_ARRAY; -use super::ARROW_EXT_TYPE_EMPTY_MAP; -use super::ARROW_EXT_TYPE_GEOGRAPHY; -use super::ARROW_EXT_TYPE_GEOMETRY; -use super::ARROW_EXT_TYPE_VARIANT; -use crate::types::decimal::DecimalColumn; -use crate::types::geography::GeographyColumn; -use crate::types::DecimalDataType; -use crate::types::NumberColumn; -use crate::types::NumberDataType; -use crate::types::F32; -use crate::types::F64; -use crate::with_number_type; -use crate::Column; -use crate::DataField; -use crate::DataSchema; -use crate::TableDataType; -use crate::TableField; -use crate::TableSchema; - -impl From<&TableSchema> for ArrowSchema { - fn from(schema: &TableSchema) -> Self { - let fields = schema - .fields() - .iter() - .map(ArrowField::from) - .collect::>(); - ArrowSchema::from(fields).with_metadata(schema.metadata.clone()) - } -} - -impl From<&DataSchema> for ArrowSchema { - fn from(schema: &DataSchema) -> Self { - let fields = schema - .fields() - .iter() - .map(ArrowField::from) - .collect::>(); - ArrowSchema::from(fields).with_metadata(schema.metadata.clone()) - } -} - -impl From<&TableField> for ArrowField { - fn from(f: &TableField) -> Self { - let ty = table_type_to_arrow_type(&f.data_type); - ArrowField::new(f.name(), ty, f.is_nullable()) - } -} - -impl From<&DataField> for ArrowField { - fn from(f: &DataField) -> Self { - ArrowField::from(&TableField::from(f)) - } -} - -// Note: Arrow's data type is not nullable, so we need to explicitly -// add nullable information to Arrow's field afterwards. -fn table_type_to_arrow_type(ty: &TableDataType) -> ArrowDataType { - match ty { - TableDataType::Null => ArrowDataType::Null, - TableDataType::EmptyArray => ArrowDataType::Extension( - ARROW_EXT_TYPE_EMPTY_ARRAY.to_string(), - Box::new(ArrowDataType::Boolean), - None, - ), - TableDataType::EmptyMap => ArrowDataType::Extension( - ARROW_EXT_TYPE_EMPTY_MAP.to_string(), - Box::new(ArrowDataType::Boolean), - None, - ), - TableDataType::Boolean => ArrowDataType::Boolean, - TableDataType::Binary => ArrowDataType::LargeBinary, - TableDataType::String => ArrowDataType::Utf8View, - TableDataType::Number(ty) => with_number_type!(|TYPE| match ty { - NumberDataType::TYPE => ArrowDataType::TYPE, - }), - TableDataType::Decimal(DecimalDataType::Decimal128(size)) => { - ArrowDataType::Decimal(size.precision as usize, size.scale as usize) - } - TableDataType::Decimal(DecimalDataType::Decimal256(size)) => { - ArrowDataType::Decimal256(size.precision as usize, size.scale as usize) - } - TableDataType::Timestamp => ArrowDataType::Timestamp(TimeUnit::Microsecond, None), - TableDataType::Date => ArrowDataType::Date32, - TableDataType::Nullable(ty) => table_type_to_arrow_type(ty.as_ref()), - TableDataType::Array(ty) => { - let arrow_ty = table_type_to_arrow_type(ty.as_ref()); - ArrowDataType::LargeList(Box::new(ArrowField::new( - "_array", - arrow_ty, - ty.is_nullable(), - ))) - } - TableDataType::Map(ty) => { - let inner_ty = match ty.as_ref() { - TableDataType::Tuple { - fields_name: _fields_name, - fields_type, - } => { - let key_ty = table_type_to_arrow_type(&fields_type[0]); - let val_ty = table_type_to_arrow_type(&fields_type[1]); - let key_field = ArrowField::new("key", key_ty, fields_type[0].is_nullable()); - let val_field = ArrowField::new("value", val_ty, fields_type[1].is_nullable()); - ArrowDataType::Struct(vec![key_field, val_field]) - } - _ => unreachable!(), - }; - ArrowDataType::Map( - Box::new(ArrowField::new("entries", inner_ty, ty.is_nullable())), - false, - ) - } - TableDataType::Bitmap => ArrowDataType::Extension( - ARROW_EXT_TYPE_BITMAP.to_string(), - Box::new(ArrowDataType::LargeBinary), - None, - ), - TableDataType::Tuple { - fields_name, - fields_type, - } => { - let fields = fields_name - .iter() - .zip(fields_type) - .map(|(name, ty)| { - ArrowField::new( - name.as_str(), - table_type_to_arrow_type(ty), - // null in tuple must be nullable - ty.is_nullable_or_null(), - ) - }) - .collect(); - ArrowDataType::Struct(fields) - } - TableDataType::Variant => ArrowDataType::Extension( - ARROW_EXT_TYPE_VARIANT.to_string(), - Box::new(ArrowDataType::LargeBinary), - None, - ), - TableDataType::Geometry => ArrowDataType::Extension( - ARROW_EXT_TYPE_GEOMETRY.to_string(), - Box::new(ArrowDataType::LargeBinary), - None, - ), - TableDataType::Geography => ArrowDataType::Extension( - ARROW_EXT_TYPE_GEOGRAPHY.to_string(), - Box::new(ArrowDataType::LargeBinary), - None, - ), - } -} - -impl Column { - pub fn arrow_field(&self) -> ArrowField { - ArrowField::from(&DataField::new("DUMMY", self.data_type())) - } - - pub fn as_arrow(&self) -> Box { - let arrow_type = self.arrow_field().data_type; - match self { - Column::Null { len } => Box::new( - databend_common_arrow::arrow::array::NullArray::new_null(arrow_type, *len), - ), - Column::EmptyArray { len } => Box::new( - databend_common_arrow::arrow::array::BooleanArray::try_new( - arrow_type, - Bitmap::new_constant(true, *len), - None, - ) - .unwrap(), - ), - Column::EmptyMap { len } => Box::new( - databend_common_arrow::arrow::array::BooleanArray::try_new( - arrow_type, - Bitmap::new_constant(true, *len), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::UInt8(col)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::UInt16(col)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::UInt32(col)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::UInt64(col)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::Int8(col)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::Int16(col)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::Int32(col)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::Int64(col)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Number(NumberColumn::Float32(col)) => { - let values = - unsafe { std::mem::transmute::, Buffer>(col.clone()) }; - Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, values, None, - ) - .unwrap(), - ) - } - Column::Number(NumberColumn::Float64(col)) => { - let values = - unsafe { std::mem::transmute::, Buffer>(col.clone()) }; - Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, values, None, - ) - .unwrap(), - ) - } - Column::Decimal(DecimalColumn::Decimal128(col, _)) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Decimal(DecimalColumn::Decimal256(col, _)) => { - let values = unsafe { - std::mem::transmute::< - Buffer, - Buffer, - >(col.clone()) - }; - Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::< - databend_common_arrow::arrow::types::i256, - >::try_new(arrow_type, values, None) - .unwrap(), - ) - } - Column::Boolean(col) => Box::new( - databend_common_arrow::arrow::array::BooleanArray::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::String(col) => Box::new(col.clone().into_inner()), - Column::Timestamp(col) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Date(col) => Box::new( - databend_common_arrow::arrow::array::PrimitiveArray::::try_new( - arrow_type, - col.clone(), - None, - ) - .unwrap(), - ), - Column::Array(col) => { - let offsets: Buffer = - col.offsets.iter().map(|offset| *offset as i64).collect(); - Box::new( - databend_common_arrow::arrow::array::ListArray::::try_new( - arrow_type, - unsafe { OffsetsBuffer::new_unchecked(offsets) }, - col.values.as_arrow(), - None, - ) - .unwrap(), - ) - } - Column::Map(col) => { - let offsets: Buffer = - col.offsets.iter().map(|offset| *offset as i32).collect(); - let values = match (&arrow_type, &col.values) { - (ArrowDataType::Map(inner_field, _), Column::Tuple(fields)) => { - let inner_type = inner_field.data_type.clone(); - Box::new( - databend_common_arrow::arrow::array::StructArray::try_new( - inner_type, - fields.iter().map(|field| field.as_arrow()).collect(), - None, - ) - .unwrap(), - ) - } - (_, _) => unreachable!(), - }; - Box::new( - databend_common_arrow::arrow::array::MapArray::try_new( - arrow_type, - unsafe { OffsetsBuffer::new_unchecked(offsets) }, - values, - None, - ) - .unwrap(), - ) - } - Column::Nullable(col) => { - let arrow_array = col.column.as_arrow(); - set_validities(arrow_array.clone(), &col.validity) - } - Column::Tuple(fields) => Box::new( - databend_common_arrow::arrow::array::StructArray::try_new( - arrow_type, - fields.iter().map(|field| field.as_arrow()).collect(), - None, - ) - .unwrap(), - ), - - Column::Binary(col) - | Column::Bitmap(col) - | Column::Variant(col) - | Column::Geometry(col) - | Column::Geography(GeographyColumn(col)) => { - let offsets: Buffer = - col.offsets().iter().map(|offset| *offset as i64).collect(); - Box::new( - databend_common_arrow::arrow::array::BinaryArray::::try_new( - arrow_type, - unsafe { OffsetsBuffer::new_unchecked(offsets) }, - col.data().clone(), - None, - ) - .unwrap(), - ) - } - } - } -} - -pub fn set_validities( - arrow_array: Box, - validity: &Bitmap, -) -> Box { - match arrow_array.data_type() { - ArrowDataType::Null => arrow_array.clone(), - ArrowDataType::Extension(_, t, _) if **t == ArrowDataType::Null => arrow_array.clone(), - _ => arrow_array.with_validity(Some(validity.clone())), - } -} diff --git a/src/query/expression/src/converts/meta/bincode.rs b/src/query/expression/src/converts/meta/bincode.rs index e839a10d99b0..b1cb2d656dfc 100644 --- a/src/query/expression/src/converts/meta/bincode.rs +++ b/src/query/expression/src/converts/meta/bincode.rs @@ -15,8 +15,8 @@ // DO NOT EDIT. // This crate keeps some legacy codes for compatibility, it's locked by bincode of meta's v3 version -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; use enum_as_inner::EnumAsInner; use serde::Deserialize; use serde::Deserializer; @@ -112,18 +112,15 @@ impl From for Scalar { impl From for BinaryColumn { fn from(value: LegacyBinaryColumn) -> Self { - BinaryColumn { - data: value.data, - offsets: value.offsets, - } + BinaryColumn::new(value.data, value.offsets) } } impl From for LegacyBinaryColumn { fn from(value: BinaryColumn) -> Self { LegacyBinaryColumn { - data: value.data, - offsets: value.offsets, + data: value.data().clone(), + offsets: value.offsets().clone(), } } } @@ -138,7 +135,7 @@ impl From for Column { LegacyColumn::Decimal(dec_col) => Column::Decimal(dec_col), LegacyColumn::Boolean(bmp) => Column::Boolean(bmp), LegacyColumn::String(str_col) => { - Column::String(StringColumn::try_from_binary(BinaryColumn::from(str_col)).unwrap()) + Column::String(StringColumn::try_from(BinaryColumn::from(str_col)).unwrap()) } LegacyColumn::Timestamp(buf) => Column::Timestamp(buf), LegacyColumn::Date(buf) => Column::Date(buf), diff --git a/src/query/expression/src/converts/mod.rs b/src/query/expression/src/converts/mod.rs index c67b7addf2c8..501b0e37ad81 100644 --- a/src/query/expression/src/converts/mod.rs +++ b/src/query/expression/src/converts/mod.rs @@ -13,6 +13,5 @@ // limitations under the License. pub mod arrow; -pub mod arrow2; pub mod datavalues; pub mod meta; diff --git a/src/query/expression/src/evaluator.rs b/src/query/expression/src/evaluator.rs index 85ec550fc8f5..a3625ecaed9a 100644 --- a/src/query/expression/src/evaluator.rs +++ b/src/query/expression/src/evaluator.rs @@ -17,10 +17,9 @@ use std::collections::HashMap; use std::collections::HashSet; use std::ops::Not; -use databend_common_arrow::arrow::bitmap; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_ast::Span; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use itertools::Itertools; @@ -34,6 +33,7 @@ use crate::type_check::check_function; use crate::type_check::get_simple_cast_function; use crate::types::any::AnyType; use crate::types::array::ArrayColumn; +use crate::types::boolean; use crate::types::boolean::BooleanDomain; use crate::types::nullable::NullableColumn; use crate::types::nullable::NullableDomain; @@ -379,7 +379,7 @@ impl<'a> Evaluator<'a> { (DataType::Nullable(inner_src_ty), _) => match value { Value::Scalar(Scalar::Null) => { let has_valid = validity - .map(|validity| validity.unset_bits() < validity.len()) + .map(|validity| validity.null_count() < validity.len()) .unwrap_or(true); if has_valid { Err(ErrorCode::BadArguments(format!( @@ -397,9 +397,9 @@ impl<'a> Evaluator<'a> { let has_valid_nulls = validity .as_ref() .map(|validity| { - (validity & (&col.validity)).unset_bits() > validity.unset_bits() + (validity & (&col.validity)).null_count() > validity.null_count() }) - .unwrap_or_else(|| col.validity.unset_bits() > 0); + .unwrap_or_else(|| col.validity.null_count() > 0); if has_valid_nulls { return Err(ErrorCode::Internal(format!( "unable to cast `NULL` to type `{dest_type}`" @@ -465,7 +465,7 @@ impl<'a> Evaluator<'a> { (DataType::Array(inner_src_ty), DataType::Array(inner_dest_ty)) => match value { Value::Scalar(Scalar::Array(array)) => { let validity = validity.map(|validity| { - Bitmap::new_constant(validity.unset_bits() != validity.len(), array.len()) + Bitmap::new_constant(validity.null_count() != validity.len(), array.len()) }); let new_array = self .run_cast( @@ -710,7 +710,7 @@ impl<'a> Evaluator<'a> { (DataType::Map(inner_src_ty), DataType::Map(inner_dest_ty)) => match value { Value::Scalar(Scalar::Map(array)) => { let validity = validity.map(|validity| { - Bitmap::new_constant(validity.unset_bits() != validity.len(), array.len()) + Bitmap::new_constant(validity.null_count() != validity.len(), array.len()) }); let new_array = self .run_cast( @@ -858,7 +858,7 @@ impl<'a> Evaluator<'a> { .unwrap() .into_nullable() .unwrap(); - let validity = bitmap::and(&col.validity, &new_col.validity); + let validity = boolean::and(&col.validity, &new_col.validity); Ok(Value::Column(NullableColumn::new_column( new_col.column, validity, diff --git a/src/query/expression/src/filter/filter_executor.rs b/src/query/expression/src/filter/filter_executor.rs index 321d9c68731a..b1a7fb8000b9 100644 --- a/src/query/expression/src/filter/filter_executor.rs +++ b/src/query/expression/src/filter/filter_executor.rs @@ -15,7 +15,7 @@ use core::ops::Range; use std::collections::HashSet; -use databend_common_arrow::arrow::bitmap::MutableBitmap; +use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::Result; use crate::filter::SelectExpr; diff --git a/src/query/expression/src/filter/select_value/mod.rs b/src/query/expression/src/filter/select_value/mod.rs index ff37ee693869..e33c591ec81e 100644 --- a/src/query/expression/src/filter/select_value/mod.rs +++ b/src/query/expression/src/filter/select_value/mod.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use crate::types::string::StringColumn; diff --git a/src/query/expression/src/filter/select_value/select_column.rs b/src/query/expression/src/filter/select_value/select_column.rs index e2aa34137cc0..f43da27d0c5c 100644 --- a/src/query/expression/src/filter/select_value/select_column.rs +++ b/src/query/expression/src/filter/select_value/select_column.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use crate::filter::SelectStrategy; diff --git a/src/query/expression/src/filter/select_value/select_column_scalar.rs b/src/query/expression/src/filter/select_value/select_column_scalar.rs index 8c2bad57e108..2663b60d63ff 100644 --- a/src/query/expression/src/filter/select_value/select_column_scalar.rs +++ b/src/query/expression/src/filter/select_value/select_column_scalar.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use crate::filter::SelectStrategy; diff --git a/src/query/expression/src/function.rs b/src/query/expression/src/function.rs index 620fe0f11b79..0db5ea63bce6 100755 --- a/src/query/expression/src/function.rs +++ b/src/query/expression/src/function.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_ast::Span; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_io::GeometryDataType; @@ -679,7 +679,7 @@ where F: Fn(&[ValueRef], &mut EvalContext) -> Value { // If the original value is NULL, we can ignore the error. let rhs: Bitmap = bitmap.clone().not().into(); let res = error_bitmap.clone().bitor(&rhs); - if res.unset_bits() == 0 { + if res.null_count() == 0 { ctx.errors = None; } else { *error_bitmap = res; @@ -697,8 +697,8 @@ where F: Fn(&[ValueRef], &mut EvalContext) -> Value { Value::Column(column) => { let result = match column { Column::Nullable(box nullable_column) => { - let validity = bitmap.into(); - let validity = databend_common_arrow::arrow::bitmap::and( + let validity: Bitmap = bitmap.into(); + let validity = databend_common_column::bitmap::and( &nullable_column.validity, &validity, ); diff --git a/src/query/expression/src/kernels/concat.rs b/src/query/expression/src/kernels/concat.rs index 782031de7c10..51e2292311fa 100644 --- a/src/query/expression/src/kernels/concat.rs +++ b/src/query/expression/src/kernels/concat.rs @@ -13,12 +13,11 @@ // limitations under the License. use std::iter::TrustedLen; +use std::sync::Arc; -use databend_common_arrow::arrow::array::growable::make_growable; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::BooleanArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; +use arrow_array::Array; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use ethnum::i256; @@ -182,7 +181,7 @@ impl Column { ); let (key_builder, val_builder) = match builder { ColumnBuilder::Tuple(fields) => (fields[0].clone(), fields[1].clone()), - _ => unreachable!(), + ty => unreachable!("ty: {}", ty.data_type()), }; let builder = KvColumnBuilder { keys: key_builder, @@ -222,7 +221,7 @@ impl Column { | Column::Binary(_) | Column::String(_) | Column::Bitmap(_) => { - Self::concat_use_grows(columns, first_column.data_type(), capacity) + Self::concat_use_arrow(columns, first_column.data_type(), capacity) } }; Ok(column) @@ -242,38 +241,22 @@ impl Column { builder.into() } - pub fn concat_use_grows( + pub fn concat_use_arrow( cols: impl Iterator, data_type: DataType, - num_rows: usize, + _num_rows: usize, ) -> Column { - let arrays: Vec> = - cols.map(|c| c.as_arrow()).collect(); - + let arrays: Vec> = cols.map(|c| c.into_arrow_rs()).collect(); let arrays = arrays.iter().map(|c| c.as_ref()).collect::>(); - let mut grow = make_growable(&arrays, false, num_rows); - - for (idx, array) in arrays.iter().enumerate() { - grow.extend(idx, 0, array.len()); - } - let array = grow.as_box(); - Column::from_arrow(array.as_ref(), &data_type).unwrap() + let result = arrow_select::concat::concat(&arrays).unwrap(); + Column::from_arrow_rs(result, &data_type).unwrap() } pub fn concat_boolean_types(bitmaps: impl Iterator, num_rows: usize) -> Bitmap { - use databend_common_arrow::arrow::datatypes::DataType as ArrowType; - let arrays: Vec = bitmaps - .map(|c| BooleanArray::new(ArrowType::Boolean, c, None)) - .collect(); - let arrays = arrays.iter().map(|c| c as &dyn Array).collect::>(); - let mut grow = make_growable(&arrays, false, num_rows); - - for (idx, array) in arrays.iter().enumerate() { - grow.extend(idx, 0, array.len()); - } - let array = grow.as_box(); - let array = array.as_any().downcast_ref::().unwrap(); - array.values().clone() + let cols = bitmaps.map(Column::Boolean); + Self::concat_use_arrow(cols, DataType::Boolean, num_rows) + .into_boolean() + .unwrap() } fn concat_value_types( diff --git a/src/query/expression/src/kernels/filter.rs b/src/query/expression/src/kernels/filter.rs index 1af12f7b046f..c24ad3d1554d 100644 --- a/src/query/expression/src/kernels/filter.rs +++ b/src/query/expression/src/kernels/filter.rs @@ -13,13 +13,11 @@ // limitations under the License. use binary::BinaryColumnBuilder; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::Utf8ViewArray; -use databend_common_arrow::arrow::bitmap::utils::SlicesIterator; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::bitmap::TrueIdxIter; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::bitmap::utils::SlicesIterator; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_column::bitmap::TrueIdxIter; +use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use string::StringColumnBuilder; @@ -40,7 +38,7 @@ impl DataBlock { return Ok(self); } - let count_zeros = bitmap.unset_bits(); + let count_zeros = bitmap.null_count(); match count_zeros { 0 => Ok(self), _ => { @@ -145,7 +143,7 @@ pub struct FilterVisitor<'a> { impl<'a> FilterVisitor<'a> { pub fn new(filter: &'a Bitmap) -> Self { - let filter_rows = filter.len() - filter.unset_bits(); + let filter_rows = filter.len() - filter.null_count(); let strategy = IterationStrategy::default_strategy(filter.len(), filter_rows); Self { filter, @@ -268,7 +266,7 @@ impl<'a> ValueVisitor for FilterVisitor<'a> { fn visit_boolean(&mut self, mut bitmap: Bitmap) -> Result<()> { // faster path for all bits set - if bitmap.unset_bits() == 0 { + if bitmap.null_count() == 0 { bitmap.slice(0, self.filter_rows); self.result = Some(Value::Column(BooleanType::upcast_column(bitmap))); return Ok(()); @@ -344,24 +342,21 @@ impl<'a> FilterVisitor<'a> { let iter = TrueIdxIter::new(self.original_rows, Some(self.filter)); for i in iter { unsafe { - builder.put_and_commit(values.index_unchecked(i)); + builder.put_and_commit(values.value_unchecked(i)); } } builder.build() } _ => { // reuse the buffers - let new_views = self.filter_primitive_types(values.data.views().clone()); - let new_col = unsafe { - Utf8ViewArray::new_unchecked_unknown_md( - values.data.data_type().clone(), + let new_views = self.filter_primitive_types(values.views().clone()); + unsafe { + StringColumn::new_unchecked_unknown_md( new_views, - values.data.data_buffers().clone(), - None, + values.data_buffers().clone(), None, ) - }; - StringColumn::new(new_col) + } } } } diff --git a/src/query/expression/src/kernels/group_by_hash/method.rs b/src/query/expression/src/kernels/group_by_hash/method.rs index b8215a0d213f..4dfb1af8b5be 100644 --- a/src/query/expression/src/kernels/group_by_hash/method.rs +++ b/src/query/expression/src/kernels/group_by_hash/method.rs @@ -16,7 +16,7 @@ use std::fmt::Debug; use std::iter::TrustedLen; use std::ptr::NonNull; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use databend_common_hashtable::DictionaryKeys; use databend_common_hashtable::FastHash; diff --git a/src/query/expression/src/kernels/group_by_hash/method_fixed_keys.rs b/src/query/expression/src/kernels/group_by_hash/method_fixed_keys.rs index a50ca23a9746..de433cfbf9f8 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_fixed_keys.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_fixed_keys.rs @@ -16,8 +16,8 @@ use std::fmt::Debug; use std::marker::PhantomData; use std::ops::Not; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_hashtable::FastHash; diff --git a/src/query/expression/src/kernels/group_by_hash/method_serializer.rs b/src/query/expression/src/kernels/group_by_hash/method_serializer.rs index b11f028e07c6..ca27d14cee5d 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_serializer.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_serializer.rs @@ -16,7 +16,7 @@ use databend_common_exception::Result; use databend_common_hashtable::hash_join_fast_string_hash; use super::utils::serialize_group_columns; -use crate::types::binary::BinaryIterator; +use crate::types::binary::BinaryColumnIter; use crate::Column; use crate::HashMethod; use crate::InputColumns; @@ -29,7 +29,7 @@ pub struct HashMethodSerializer {} impl HashMethod for HashMethodSerializer { type HashKey = [u8]; - type HashKeyIter<'a> = BinaryIterator<'a>; + type HashKeyIter<'a> = BinaryColumnIter<'a>; fn name(&self) -> String { "Serializer".to_string() diff --git a/src/query/expression/src/kernels/group_by_hash/method_single_string.rs b/src/query/expression/src/kernels/group_by_hash/method_single_string.rs index f2791534e55a..ce88c061bdf1 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_single_string.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_single_string.rs @@ -15,7 +15,7 @@ use databend_common_exception::Result; use databend_common_hashtable::hash_join_fast_string_hash; -use crate::types::binary::BinaryIterator; +use crate::types::binary::BinaryColumnIter; use crate::types::BinaryColumn; use crate::Column; use crate::HashMethod; @@ -29,7 +29,7 @@ pub struct HashMethodSingleBinary {} impl HashMethod for HashMethodSingleBinary { type HashKey = [u8]; - type HashKeyIter<'a> = BinaryIterator<'a>; + type HashKeyIter<'a> = BinaryColumnIter<'a>; fn name(&self) -> String { "SingleBinary".to_string() @@ -78,9 +78,7 @@ impl KeyAccessor for BinaryColumn { /// # Safety /// Calling this method with an out-of-bounds index is *[undefined behavior]*. unsafe fn key_unchecked(&self, index: usize) -> &Self::Key { - debug_assert!(index + 1 < self.offsets.len()); - - &self.data[*self.offsets.get_unchecked(index) as usize - ..*self.offsets.get_unchecked(index + 1) as usize] + debug_assert!(index + 1 < self.offsets().len()); + self.index_unchecked(index) } } diff --git a/src/query/expression/src/kernels/scatter.rs b/src/query/expression/src/kernels/scatter.rs index 1d690ac929a8..f6a581d46011 100644 --- a/src/query/expression/src/kernels/scatter.rs +++ b/src/query/expression/src/kernels/scatter.rs @@ -19,7 +19,7 @@ use crate::DataBlock; impl DataBlock { pub fn scatter(&self, indices: &[I], scatter_size: usize) -> Result> - where I: databend_common_arrow::arrow::types::Index { + where I: databend_common_column::types::Index { if indices.is_empty() { let mut result = Vec::with_capacity(scatter_size); result.push(self.clone()); @@ -41,7 +41,7 @@ impl DataBlock { } pub fn divide_indices_by_scatter_size(indices: &[I], scatter_size: usize) -> Vec> - where I: databend_common_arrow::arrow::types::Index { + where I: databend_common_column::types::Index { let mut scatter_indices: Vec> = Vec::with_capacity(scatter_size); unsafe { let mut scatter_num_rows = vec![0usize; scatter_size]; diff --git a/src/query/expression/src/kernels/sort_compare.rs b/src/query/expression/src/kernels/sort_compare.rs index 210054addf61..66dc55eee8af 100644 --- a/src/query/expression/src/kernels/sort_compare.rs +++ b/src/query/expression/src/kernels/sort_compare.rs @@ -15,8 +15,8 @@ use std::cmp::Ordering; use std::ops::Range; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use memchr::memchr; @@ -317,7 +317,7 @@ impl ValueVisitor for SortCompare { } fn visit_nullable(&mut self, column: Box>) -> Result<()> { - if column.validity.unset_bits() > 0 { + if column.validity.null_count() > 0 { self.validity = Some(column.validity.clone()); } self.visit_column(column.column.clone()) diff --git a/src/query/expression/src/kernels/take.rs b/src/query/expression/src/kernels/take.rs index 0828c27c9643..9b404bbe5e48 100644 --- a/src/query/expression/src/kernels/take.rs +++ b/src/query/expression/src/kernels/take.rs @@ -15,10 +15,8 @@ use std::sync::Arc; use binary::BinaryColumnBuilder; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::Utf8ViewArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use string::StringColumnBuilder; @@ -38,7 +36,7 @@ pub const BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; impl DataBlock { pub fn take(&self, indices: &[I]) -> Result - where I: databend_common_arrow::arrow::types::Index { + where I: databend_common_column::types::Index { if indices.is_empty() { return Ok(self.slice(0..0)); } @@ -48,7 +46,7 @@ impl DataBlock { } pub fn take_with_optimize_size(&self, indices: &[I]) -> Result - where I: databend_common_arrow::arrow::types::Index { + where I: databend_common_column::types::Index { if indices.is_empty() { return Ok(self.slice(0..0)); } @@ -58,7 +56,7 @@ impl DataBlock { } fn take_inner(&self, mut taker: TakeVisitor) -> Result - where I: databend_common_arrow::arrow::types::Index { + where I: databend_common_column::types::Index { let after_columns = self .columns() .iter() @@ -81,7 +79,7 @@ impl DataBlock { } struct TakeVisitor<'a, I> -where I: databend_common_arrow::arrow::types::Index +where I: databend_common_column::types::Index { indices: &'a [I], result: Option>, @@ -89,7 +87,7 @@ where I: databend_common_arrow::arrow::types::Index } impl<'a, I> TakeVisitor<'a, I> -where I: databend_common_arrow::arrow::types::Index +where I: databend_common_column::types::Index { fn new(indices: &'a [I]) -> Self { Self { @@ -111,7 +109,7 @@ where I: databend_common_arrow::arrow::types::Index } impl<'a, I> ValueVisitor for TakeVisitor<'a, I> -where I: databend_common_arrow::arrow::types::Index +where I: databend_common_column::types::Index { fn visit_scalar(&mut self, scalar: crate::Scalar) -> Result<()> { self.result = Some(Value::Scalar(scalar)); @@ -189,7 +187,7 @@ where I: databend_common_arrow::arrow::types::Index // Fast path: avoid iterating column to generate a new bitmap. // If this [`Bitmap`] is all true or all false and `num_rows <= bitmap.len()``, // we can just slice it. - if num_rows <= col.len() && (col.unset_bits() == 0 || col.unset_bits() == col.len()) { + if num_rows <= col.len() && (col.null_count() == 0 || col.null_count() == col.len()) { self.result = Some(Value::Column(BooleanType::upcast_column( col.sliced(0, num_rows), ))); @@ -250,7 +248,7 @@ where I: databend_common_arrow::arrow::types::Index } impl<'a, I> TakeVisitor<'a, I> -where I: databend_common_arrow::arrow::types::Index +where I: databend_common_column::types::Index { fn take_primitive_types(&mut self, buffer: Buffer) -> Buffer { let col = buffer.as_slice(); @@ -284,17 +282,10 @@ where I: databend_common_arrow::arrow::types::Index } builder.build() } else { - let new_views = self.take_primitive_types(col.data.views().clone()); - let new_col = unsafe { - Utf8ViewArray::new_unchecked_unknown_md( - col.data.data_type().clone(), - new_views, - col.data.data_buffers().clone(), - None, - None, - ) - }; - StringColumn::new(new_col) + let new_views = self.take_primitive_types(col.views().clone()); + unsafe { + StringColumn::new_unchecked_unknown_md(new_views, col.data_buffers().clone(), None) + } } } } @@ -302,11 +293,7 @@ where I: databend_common_arrow::arrow::types::Index impl Column { pub fn maybe_gc(self) -> Self { match self { - Column::String(c) => { - let data = c.data.maybe_gc(); - let c = StringColumn::new(data); - Column::String(c) - } + Column::String(c) => Column::String(c.maybe_gc()), Column::Nullable(n) => { let c = n.column.maybe_gc(); NullableColumn::new_column(c, n.validity) diff --git a/src/query/expression/src/kernels/take_chunks.rs b/src/query/expression/src/kernels/take_chunks.rs index 90bafc210d2f..6573e1d0fd09 100644 --- a/src/query/expression/src/kernels/take_chunks.rs +++ b/src/query/expression/src/kernels/take_chunks.rs @@ -15,8 +15,8 @@ use std::sync::Arc; use binary::BinaryColumnBuilder; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; use databend_common_hashtable::RowPtr; use itertools::Itertools; use string::StringColumnBuilder; @@ -811,7 +811,7 @@ impl Column { let mut total_len = 0; let mut unset_bits = 0; for bitmap in col.iter() { - unset_bits += bitmap.unset_bits(); + unset_bits += bitmap.null_count(); total_len += bitmap.len(); } if unset_bits == total_len || unset_bits == 0 { diff --git a/src/query/expression/src/kernels/take_compact.rs b/src/query/expression/src/kernels/take_compact.rs index 3353f467c5fe..403918e1752d 100644 --- a/src/query/expression/src/kernels/take_compact.rs +++ b/src/query/expression/src/kernels/take_compact.rs @@ -13,10 +13,8 @@ // limitations under the License. use binary::BinaryColumnBuilder; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::Utf8ViewArray; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_base::vec_ext::VecExt; +use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use crate::types::binary::BinaryColumn; @@ -231,16 +229,9 @@ impl<'a> TakeCompactVisitor<'a> { } fn take_string_types(&mut self, col: &StringColumn) -> StringColumn { - let new_views = self.take_primitive_types(col.data.views().clone()); - let new_col = unsafe { - Utf8ViewArray::new_unchecked_unknown_md( - col.data.data_type().clone(), - new_views, - col.data.data_buffers().clone(), - None, - None, - ) - }; - StringColumn::new(new_col) + let new_views = self.take_primitive_types(col.views().clone()); + unsafe { + StringColumn::new_unchecked_unknown_md(new_views, col.data_buffers().clone(), None) + } } } diff --git a/src/query/expression/src/kernels/take_ranges.rs b/src/query/expression/src/kernels/take_ranges.rs index f8ed6e1d84f5..a330c3c13181 100644 --- a/src/query/expression/src/kernels/take_ranges.rs +++ b/src/query/expression/src/kernels/take_ranges.rs @@ -15,12 +15,10 @@ use core::ops::Range; use binary::BinaryColumnBuilder; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::array::Utf8ViewArray; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_base::vec_ext::VecExt; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use crate::types::binary::BinaryColumn; @@ -163,7 +161,7 @@ impl<'a> ValueVisitor for TakeRangeVisitor<'a> { // If this [`Bitmap`] is all true or all false and `num_rows <= bitmap.len()``, // we can just slice it. if self.num_rows <= bitmap.len() - && (bitmap.unset_bits() == 0 || bitmap.unset_bits() == bitmap.len()) + && (bitmap.null_count() == 0 || bitmap.null_count() == bitmap.len()) { self.result = Some(Value::Column(BooleanType::upcast_column( bitmap.sliced(0, self.num_rows), @@ -232,16 +230,9 @@ impl<'a> TakeRangeVisitor<'a> { } fn take_string_types(&mut self, col: &StringColumn) -> StringColumn { - let new_views = self.take_primitive_types(col.data.views().clone()); - let new_col = unsafe { - Utf8ViewArray::new_unchecked_unknown_md( - col.data.data_type().clone(), - new_views, - col.data.data_buffers().clone(), - None, - None, - ) - }; - StringColumn::new(new_col) + let new_views = self.take_primitive_types(col.views().clone()); + unsafe { + StringColumn::new_unchecked_unknown_md(new_views, col.data_buffers().clone(), None) + } } } diff --git a/src/query/expression/src/kernels/topk.rs b/src/query/expression/src/kernels/topk.rs index 8877b3dfe410..e798a3077649 100644 --- a/src/query/expression/src/kernels/topk.rs +++ b/src/query/expression/src/kernels/topk.rs @@ -18,8 +18,8 @@ use std::intrinsics::assume; use std::mem; use std::ptr; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_base::runtime::drop_guard; +use databend_common_column::bitmap::MutableBitmap; use crate::types::*; use crate::with_number_mapped_type; diff --git a/src/query/expression/src/lib.rs b/src/query/expression/src/lib.rs index f6b23648378e..72cee61f8269 100755 --- a/src/query/expression/src/lib.rs +++ b/src/query/expression/src/lib.rs @@ -14,6 +14,7 @@ #![allow(clippy::uninlined_format_args)] #![allow(clippy::len_without_is_empty)] +#![allow(clippy::missing_transmute_annotations)] #![allow(clippy::arc_with_non_send_sync)] #![allow(internal_features)] // FIXME: we should avoid this by implementing Ord correctly. diff --git a/src/query/expression/src/register.rs b/src/query/expression/src/register.rs index b5c24aea7485..498b5a60ec8e 100755 --- a/src/query/expression/src/register.rs +++ b/src/query/expression/src/register.rs @@ -3518,8 +3518,7 @@ pub fn passthrough_nullable_2_arg( Value::Column(NullableColumn::new(column, validity)) } (ValueRef::Column(arg1), ValueRef::Column(arg2)) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg2.validity); let validity = ctx .validity .as_ref() @@ -3612,8 +3611,7 @@ pub fn passthrough_nullable_3_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg2.validity); let validity = ctx .validity .as_ref() @@ -3649,8 +3647,7 @@ pub fn passthrough_nullable_3_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -3668,8 +3665,7 @@ pub fn passthrough_nullable_3_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -3687,8 +3683,8 @@ pub fn passthrough_nullable_3_arg { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ); let validity = ctx @@ -3812,8 +3808,7 @@ pub fn passthrough_nullable_4_arg< ValueRef::Scalar(Some(arg3)), ValueRef::Scalar(Some(arg4)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg2.validity); let validity = ctx .validity .as_ref() @@ -3861,8 +3856,7 @@ pub fn passthrough_nullable_4_arg< ValueRef::Column(arg3), ValueRef::Scalar(Some(arg4)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -3886,8 +3880,7 @@ pub fn passthrough_nullable_4_arg< ValueRef::Column(arg3), ValueRef::Scalar(Some(arg4)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -3911,8 +3904,8 @@ pub fn passthrough_nullable_4_arg< ValueRef::Column(arg3), ValueRef::Scalar(Some(arg4)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ); let validity = ctx @@ -3962,8 +3955,7 @@ pub fn passthrough_nullable_4_arg< ValueRef::Scalar(Some(arg3)), ValueRef::Column(arg4), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -3987,8 +3979,7 @@ pub fn passthrough_nullable_4_arg< ValueRef::Scalar(Some(arg3)), ValueRef::Column(arg4), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -4012,8 +4003,8 @@ pub fn passthrough_nullable_4_arg< ValueRef::Scalar(Some(arg3)), ValueRef::Column(arg4), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg4.validity, ); let validity = ctx @@ -4039,8 +4030,7 @@ pub fn passthrough_nullable_4_arg< ValueRef::Column(arg3), ValueRef::Column(arg4), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg3.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg3.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -4064,8 +4054,8 @@ pub fn passthrough_nullable_4_arg< ValueRef::Column(arg3), ValueRef::Column(arg4), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg3.validity), &arg4.validity, ); let validity = ctx @@ -4091,8 +4081,8 @@ pub fn passthrough_nullable_4_arg< ValueRef::Column(arg3), ValueRef::Column(arg4), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg3.validity), &arg4.validity, ); let validity = ctx @@ -4118,9 +4108,9 @@ pub fn passthrough_nullable_4_arg< ValueRef::Column(arg3), ValueRef::Column(arg4), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ), &arg4.validity, @@ -4258,8 +4248,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg2.validity); let validity = ctx .validity .as_ref() @@ -4311,8 +4300,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -4338,8 +4326,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -4365,8 +4352,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ); let validity = ctx @@ -4420,8 +4407,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -4447,8 +4433,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -4474,8 +4459,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg4.validity, ); let validity = ctx @@ -4503,8 +4488,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg3.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg3.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -4530,8 +4514,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg3.validity), &arg4.validity, ); let validity = ctx @@ -4559,8 +4543,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg3.validity), &arg4.validity, ); let validity = ctx @@ -4588,9 +4572,9 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ), &arg4.validity, @@ -4646,8 +4630,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg5.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg5.validity); let validity = ctx .validity .as_ref() @@ -4673,8 +4656,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg5.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg5.validity); let validity = ctx .validity .as_ref() @@ -4700,8 +4682,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg5.validity, ); let validity = ctx @@ -4729,8 +4711,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg3.validity, &arg5.validity); + let and_validity = databend_common_column::bitmap::and(&arg3.validity, &arg5.validity); let validity = ctx .validity .as_ref() @@ -4756,8 +4737,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg3.validity), &arg5.validity, ); let validity = ctx @@ -4785,8 +4766,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg3.validity), &arg5.validity, ); let validity = ctx @@ -4814,9 +4795,9 @@ pub fn passthrough_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ), &arg5.validity, @@ -4846,8 +4827,7 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg4.validity, &arg5.validity); + let and_validity = databend_common_column::bitmap::and(&arg4.validity, &arg5.validity); let validity = ctx .validity .as_ref() @@ -4873,8 +4853,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg4.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg4.validity), &arg5.validity, ); let validity = ctx @@ -4902,8 +4882,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg4.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg4.validity), &arg5.validity, ); let validity = ctx @@ -4931,9 +4911,9 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg4.validity, ), &arg5.validity, @@ -4963,8 +4943,8 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg3.validity, &arg4.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg3.validity, &arg4.validity), &arg5.validity, ); let validity = ctx @@ -4992,9 +4972,9 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg3.validity), &arg4.validity, ), &arg5.validity, @@ -5024,9 +5004,9 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg3.validity), &arg4.validity, ), &arg5.validity, @@ -5056,10 +5036,10 @@ pub fn passthrough_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ), &arg4.validity, @@ -5113,7 +5093,7 @@ pub fn combine_nullable_1_arg( .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -5158,7 +5138,7 @@ pub fn combine_nullable_2_arg( .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -5176,15 +5156,14 @@ pub fn combine_nullable_2_arg( .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, )) } (ValueRef::Column(arg1), ValueRef::Column(arg2)) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg2.validity); let validity = ctx .validity .as_ref() @@ -5199,7 +5178,7 @@ pub fn combine_nullable_2_arg( .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -5262,7 +5241,7 @@ pub fn combine_nullable_3_arg .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -5285,15 +5264,14 @@ pub fn combine_nullable_3_arg .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, )) } (ValueRef::Column(arg1), ValueRef::Column(arg2), ValueRef::Scalar(Some(arg3))) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg2.validity); let validity = ctx .validity .as_ref() @@ -5309,7 +5287,7 @@ pub fn combine_nullable_3_arg .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -5332,15 +5310,14 @@ pub fn combine_nullable_3_arg .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, )) } (ValueRef::Column(arg1), ValueRef::Scalar(Some(arg2)), ValueRef::Column(arg3)) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -5356,15 +5333,14 @@ pub fn combine_nullable_3_arg .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, )) } (ValueRef::Scalar(Some(arg1)), ValueRef::Column(arg2), ValueRef::Column(arg3)) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -5380,15 +5356,15 @@ pub fn combine_nullable_3_arg .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, )) } (ValueRef::Column(arg1), ValueRef::Column(arg2), ValueRef::Column(arg3)) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ); let validity = ctx @@ -5406,7 +5382,7 @@ pub fn combine_nullable_3_arg .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -5480,7 +5456,7 @@ pub fn combine_nullable_4_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg2.validity); let validity = ctx .validity .as_ref() @@ -5539,7 +5514,7 @@ pub fn combine_nullable_4_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -5598,7 +5572,7 @@ pub fn combine_nullable_4_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -5628,7 +5601,7 @@ pub fn combine_nullable_4_arg { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ); let validity = ctx @@ -5660,7 +5633,7 @@ pub fn combine_nullable_4_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -5719,7 +5691,7 @@ pub fn combine_nullable_4_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -5749,7 +5720,7 @@ pub fn combine_nullable_4_arg { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg4.validity, ); let validity = ctx @@ -5781,7 +5752,7 @@ pub fn combine_nullable_4_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg3.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg3.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -5811,7 +5781,7 @@ pub fn combine_nullable_4_arg { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg3.validity), &arg4.validity, ); let validity = ctx @@ -5843,7 +5813,7 @@ pub fn combine_nullable_4_arg { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg3.validity), &arg4.validity, ); let validity = ctx @@ -5875,7 +5845,7 @@ pub fn combine_nullable_4_arg { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ), &arg4.validity, @@ -5910,7 +5880,7 @@ pub fn combine_nullable_4_arg { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg2.validity); let validity = ctx .validity .as_ref() @@ -6061,7 +6030,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6092,7 +6061,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6105,8 +6074,7 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -6124,7 +6092,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6137,8 +6105,7 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg3.validity); let validity = ctx .validity .as_ref() @@ -6156,7 +6123,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6169,8 +6136,8 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ); let validity = ctx @@ -6190,7 +6157,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6221,7 +6188,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6234,8 +6201,7 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -6253,7 +6219,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6266,8 +6232,7 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -6285,7 +6250,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6298,8 +6263,8 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg4.validity, ); let validity = ctx @@ -6319,7 +6284,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6332,8 +6297,7 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg3.validity, &arg4.validity); + let and_validity = databend_common_column::bitmap::and(&arg3.validity, &arg4.validity); let validity = ctx .validity .as_ref() @@ -6351,7 +6315,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6364,8 +6328,8 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg3.validity), &arg4.validity, ); let validity = ctx @@ -6385,7 +6349,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6398,8 +6362,8 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg3.validity), &arg4.validity, ); let validity = ctx @@ -6419,7 +6383,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6432,9 +6396,9 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Scalar(Some(arg5)), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ), &arg4.validity, @@ -6456,7 +6420,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6487,7 +6451,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6500,8 +6464,7 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg5.validity); + let and_validity = databend_common_column::bitmap::and(&arg1.validity, &arg5.validity); let validity = ctx .validity .as_ref() @@ -6519,7 +6482,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6532,8 +6495,7 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg5.validity); + let and_validity = databend_common_column::bitmap::and(&arg2.validity, &arg5.validity); let validity = ctx .validity .as_ref() @@ -6551,7 +6513,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6564,8 +6526,8 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg5.validity, ); let validity = ctx @@ -6585,7 +6547,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6598,8 +6560,7 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg3.validity, &arg5.validity); + let and_validity = databend_common_column::bitmap::and(&arg3.validity, &arg5.validity); let validity = ctx .validity .as_ref() @@ -6617,7 +6578,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6630,8 +6591,8 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg3.validity), &arg5.validity, ); let validity = ctx @@ -6651,7 +6612,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6664,8 +6625,8 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg3.validity), &arg5.validity, ); let validity = ctx @@ -6685,7 +6646,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6698,9 +6659,9 @@ pub fn combine_nullable_5_arg< ValueRef::Scalar(Some(arg4)), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ), &arg5.validity, @@ -6722,7 +6683,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6735,8 +6696,7 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = - databend_common_arrow::arrow::bitmap::and(&arg4.validity, &arg5.validity); + let and_validity = databend_common_column::bitmap::and(&arg4.validity, &arg5.validity); let validity = ctx .validity .as_ref() @@ -6754,7 +6714,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6767,8 +6727,8 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg4.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg4.validity), &arg5.validity, ); let validity = ctx @@ -6788,7 +6748,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6801,8 +6761,8 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg4.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg4.validity), &arg5.validity, ); let validity = ctx @@ -6822,7 +6782,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6835,9 +6795,9 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg4.validity, ), &arg5.validity, @@ -6859,7 +6819,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6872,8 +6832,8 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg3.validity, &arg4.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg3.validity, &arg4.validity), &arg5.validity, ); let validity = ctx @@ -6893,7 +6853,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6906,9 +6866,9 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg3.validity), &arg4.validity, ), &arg5.validity, @@ -6930,7 +6890,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6943,9 +6903,9 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg2.validity, &arg3.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg2.validity, &arg3.validity), &arg4.validity, ), &arg5.validity, @@ -6967,7 +6927,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, @@ -6980,10 +6940,10 @@ pub fn combine_nullable_5_arg< ValueRef::Column(arg4), ValueRef::Column(arg5), ) => { - let and_validity = databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and( - &databend_common_arrow::arrow::bitmap::and(&arg1.validity, &arg2.validity), + let and_validity = databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and( + &databend_common_column::bitmap::and(&arg1.validity, &arg2.validity), &arg3.validity, ), &arg4.validity, @@ -7007,7 +6967,7 @@ pub fn combine_nullable_5_arg< .into_column() .unwrap(); let combine_validity = - databend_common_arrow::arrow::bitmap::and(&validity, &nullable_column.validity); + databend_common_column::bitmap::and(&validity, &nullable_column.validity); Value::Column(NullableColumn::new( nullable_column.column, combine_validity, diff --git a/src/query/expression/src/row/fixed.rs b/src/query/expression/src/row/fixed.rs index df5a6b5a72e7..74c7e27f598e 100644 --- a/src/query/expression/src/row/fixed.rs +++ b/src/query/expression/src/row/fixed.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use ethnum::i256; use super::row_converter::null_sentinel; diff --git a/src/query/expression/src/row/variable.rs b/src/query/expression/src/row/variable.rs index 99680ab58a91..f7949c3cf2ef 100644 --- a/src/query/expression/src/row/variable.rs +++ b/src/query/expression/src/row/variable.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use super::row_converter::null_sentinel; use crate::types::binary::BinaryColumnBuilder; diff --git a/src/query/expression/src/schema.rs b/src/query/expression/src/schema.rs index b87a17c64c9d..92ca3ecefb08 100644 --- a/src/query/expression/src/schema.rs +++ b/src/query/expression/src/schema.rs @@ -18,7 +18,6 @@ use std::collections::HashSet; use std::sync::Arc; use std::sync::LazyLock; -use databend_common_arrow::arrow::datatypes::Schema as ArrowSchema; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use itertools::Itertools; @@ -341,12 +340,6 @@ impl DataSchema { pub fn project_by_fields(&self, fields: Vec) -> Self { Self::new_from(fields, self.meta().clone()) } - - pub fn to_arrow(&self) -> ArrowSchema { - let fields = self.fields().iter().map(|f| f.into()).collect::>(); - - ArrowSchema::from(fields).with_metadata(self.metadata.clone()) - } } impl TableSchema { @@ -1320,6 +1313,17 @@ impl TableDataType { _ => 1, } } + + pub fn is_physical_binary(&self) -> bool { + matches!( + self, + TableDataType::Binary + | TableDataType::Bitmap + | TableDataType::Variant + | TableDataType::Geometry + | TableDataType::Geography + ) + } } // for merge into not matched clauses, when there are multi inserts, they maybe diff --git a/src/query/expression/src/types.rs b/src/query/expression/src/types.rs index 06837d55f3b4..4ac3f6079a89 100755 --- a/src/query/expression/src/types.rs +++ b/src/query/expression/src/types.rs @@ -35,9 +35,9 @@ pub mod variant; use std::cmp::Ordering; use std::fmt::Debug; +use std::iter::TrustedLen; use std::ops::Range; -use databend_common_arrow::arrow::trusted_len::TrustedLen; pub use databend_common_io::deserialize_bitmap; use enum_as_inner::EnumAsInner; use serde::Deserialize; @@ -49,7 +49,9 @@ pub use self::array::ArrayType; pub use self::binary::BinaryColumn; pub use self::binary::BinaryType; pub use self::bitmap::BitmapType; +pub use self::boolean::Bitmap; pub use self::boolean::BooleanType; +pub use self::boolean::MutableBitmap; pub use self::date::DateType; pub use self::decimal::*; pub use self::empty_array::EmptyArrayType; @@ -314,6 +316,17 @@ impl DataType { _ => None, } } + + pub fn is_physical_binary(&self) -> bool { + matches!( + self, + DataType::Binary + | DataType::Bitmap + | DataType::Variant + | DataType::Geometry + | DataType::Geography + ) + } } pub trait ValueType: Debug + Clone + PartialEq + Sized + 'static { diff --git a/src/query/expression/src/types/array.rs b/src/query/expression/src/types/array.rs index 058954d323b0..31e3fd44dc98 100755 --- a/src/query/expression/src/types/array.rs +++ b/src/query/expression/src/types/array.rs @@ -13,11 +13,11 @@ // limitations under the License. use std::iter::once; +use std::iter::TrustedLen; use std::marker::PhantomData; use std::ops::Range; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::trusted_len::TrustedLen; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; diff --git a/src/query/expression/src/types/binary.rs b/src/query/expression/src/types/binary.rs index cf2708bfc1d7..8b125cd4e7d0 100644 --- a/src/query/expression/src/types/binary.rs +++ b/src/query/expression/src/types/binary.rs @@ -13,29 +13,23 @@ // limitations under the License. use std::cmp::Ordering; -use std::iter::once; -use std::marker::PhantomData; use std::ops::Range; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::trusted_len::TrustedLen; -use databend_common_exception::ErrorCode; -use databend_common_exception::Result; -use serde::Deserialize; -use serde::Serialize; - use crate::property::Domain; use crate::types::ArgType; use crate::types::DataType; use crate::types::DecimalSize; use crate::types::GenericMap; use crate::types::ValueType; -use crate::utils::arrow::buffer_into_mut; use crate::values::Column; use crate::values::Scalar; use crate::ColumnBuilder; use crate::ScalarRef; +pub type BinaryColumn = databend_common_column::binary::BinaryColumn; +pub type BinaryColumnBuilder = databend_common_column::binary::BinaryColumnBuilder; +pub type BinaryColumnIter<'a> = databend_common_column::binary::BinaryColumnIter<'a>; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct BinaryType; @@ -44,7 +38,7 @@ impl ValueType for BinaryType { type ScalarRef<'a> = &'a [u8]; type Column = BinaryColumn; type Domain = (); - type ColumnIterator<'a> = BinaryIterator<'a>; + type ColumnIterator<'a> = BinaryColumnIter<'a>; type ColumnBuilder = BinaryColumnBuilder; #[inline] @@ -187,342 +181,3 @@ impl ArgType for BinaryType { BinaryColumnBuilder::with_capacity(capacity, 0) } } - -#[derive(Clone, PartialEq)] -pub struct BinaryColumn { - pub(crate) data: Buffer, - pub(crate) offsets: Buffer, -} - -impl BinaryColumn { - pub fn new(data: Buffer, offsets: Buffer) -> Self { - debug_assert!({ offsets.windows(2).all(|w| w[0] <= w[1]) }); - - BinaryColumn { data, offsets } - } - - pub fn len(&self) -> usize { - self.offsets.len() - 1 - } - - pub fn current_buffer_len(&self) -> usize { - (*self.offsets().last().unwrap() - *self.offsets().first().unwrap()) as _ - } - - pub fn data(&self) -> &Buffer { - &self.data - } - - pub fn offsets(&self) -> &Buffer { - &self.offsets - } - - pub fn memory_size(&self) -> usize { - let offsets = self.offsets.as_slice(); - let len = offsets.len(); - len * 8 + (offsets[len - 1] - offsets[0]) as usize - } - - pub fn index(&self, index: usize) -> Option<&[u8]> { - if index + 1 < self.offsets.len() { - Some(&self.data[(self.offsets[index] as usize)..(self.offsets[index + 1] as usize)]) - } else { - None - } - } - - /// # Safety - /// - /// Calling this method with an out-of-bounds index is *[undefined behavior]* - #[inline] - pub unsafe fn index_unchecked(&self, index: usize) -> &[u8] { - let start = *self.offsets.get_unchecked(index) as usize; - let end = *self.offsets.get_unchecked(index + 1) as usize; - self.data.get_unchecked(start..end) - } - - pub fn slice(&self, range: Range) -> Self { - let offsets = self - .offsets - .clone() - .sliced(range.start, range.end - range.start + 1); - BinaryColumn { - data: self.data.clone(), - offsets, - } - } - - pub fn iter(&self) -> BinaryIterator { - BinaryIterator { - data: &self.data, - offsets: self.offsets.windows(2), - _t: PhantomData, - } - } - - pub fn into_buffer(self) -> (Buffer, Buffer) { - (self.data, self.offsets) - } - - pub fn check_valid(&self) -> Result<()> { - let offsets = self.offsets.as_slice(); - let len = offsets.len(); - if len < 1 { - return Err(ErrorCode::Internal(format!( - "BinaryColumn offsets length must be equal or greater than 1, but got {}", - len - ))); - } - - for i in 1..len { - if offsets[i] < offsets[i - 1] { - return Err(ErrorCode::Internal(format!( - "BinaryColumn offsets value must be equal or greater than previous value, but got {}", - offsets[i] - ))); - } - } - Ok(()) - } -} - -pub type BinaryIterator<'a> = BinaryLikeIterator<'a, &'a [u8]>; - -pub trait BinaryLike<'a> { - fn from(value: &'a [u8]) -> Self; -} - -impl<'a> BinaryLike<'a> for &'a [u8] { - fn from(value: &'a [u8]) -> Self { - value - } -} - -pub struct BinaryLikeIterator<'a, T> -where T: BinaryLike<'a> -{ - pub(crate) data: &'a [u8], - pub(crate) offsets: std::slice::Windows<'a, u64>, - pub(crate) _t: PhantomData, -} - -impl<'a, T: BinaryLike<'a>> Iterator for BinaryLikeIterator<'a, T> { - type Item = T; - - fn next(&mut self) -> Option { - self.offsets - .next() - .map(|range| T::from(&self.data[(range[0] as usize)..(range[1] as usize)])) - } - - fn size_hint(&self) -> (usize, Option) { - self.offsets.size_hint() - } -} - -unsafe impl<'a, T: BinaryLike<'a>> TrustedLen for BinaryLikeIterator<'a, T> {} - -unsafe impl<'a, T: BinaryLike<'a>> std::iter::TrustedLen for BinaryLikeIterator<'a, T> {} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct BinaryColumnBuilder { - // if the BinaryColumnBuilder is created with `data_capacity`, need_estimated is false - pub need_estimated: bool, - pub data: Vec, - pub offsets: Vec, -} - -impl BinaryColumnBuilder { - pub fn with_capacity(len: usize, data_capacity: usize) -> Self { - let mut offsets = Vec::with_capacity(len + 1); - offsets.push(0); - BinaryColumnBuilder { - need_estimated: data_capacity == 0 && len > 0, - data: Vec::with_capacity(data_capacity), - offsets, - } - } - - pub fn from_column(col: BinaryColumn) -> Self { - BinaryColumnBuilder { - need_estimated: col.data.is_empty(), - data: buffer_into_mut(col.data), - offsets: col.offsets.to_vec(), - } - } - - pub fn from_data(data: Vec, offsets: Vec) -> Self { - debug_assert!({ offsets.windows(2).all(|w| w[0] <= w[1]) }); - - BinaryColumnBuilder { - need_estimated: false, - data, - offsets, - } - } - - pub fn repeat(scalar: &[u8], n: usize) -> Self { - let len = scalar.len(); - let data = scalar.repeat(n); - let offsets = once(0) - .chain((0..n).map(|i| (len * (i + 1)) as u64)) - .collect(); - BinaryColumnBuilder { - data, - offsets, - need_estimated: false, - } - } - - pub fn repeat_default(n: usize) -> Self { - BinaryColumnBuilder { - data: vec![], - offsets: vec![0; n + 1], - need_estimated: false, - } - } - - pub fn len(&self) -> usize { - self.offsets.len() - 1 - } - - pub fn memory_size(&self) -> usize { - self.offsets.len() * 8 + self.data.len() - } - - pub fn put_u8(&mut self, item: u8) { - self.data.push(item); - } - - pub fn put_char(&mut self, item: char) { - self.data - .extend_from_slice(item.encode_utf8(&mut [0; 4]).as_bytes()); - } - - #[inline] - pub fn put_str(&mut self, item: &str) { - self.data.extend_from_slice(item.as_bytes()); - } - - #[inline] - pub fn put_slice(&mut self, item: &[u8]) { - self.data.extend_from_slice(item); - } - - pub fn put_char_iter(&mut self, iter: impl Iterator) { - for c in iter { - let mut buf = [0; 4]; - let result = c.encode_utf8(&mut buf); - self.data.extend_from_slice(result.as_bytes()); - } - } - - pub fn put(&mut self, item: &[u8]) { - self.data.extend_from_slice(item); - } - - #[inline] - pub fn commit_row(&mut self) { - self.offsets.push(self.data.len() as u64); - - if self.need_estimated - && self.offsets.len() - 1 == 64 - && self.offsets.len() < self.offsets.capacity() - { - let bytes_per_row = self.data.len() / 64 + 1; - let bytes_estimate = bytes_per_row * self.offsets.capacity(); - - const MAX_HINT_SIZE: usize = 1_000_000_000; - // if we are more than 10% over the capacity, we reserve more - if bytes_estimate < MAX_HINT_SIZE - && bytes_estimate as f64 > self.data.capacity() as f64 * 1.10f64 - { - self.data.reserve(bytes_estimate - self.data.capacity()); - } - } - } - - pub fn append_column(&mut self, other: &BinaryColumn) { - // the first offset of other column may not be zero - let other_start = *other.offsets.first().unwrap(); - let other_last = *other.offsets.last().unwrap(); - let start = self.offsets.last().cloned().unwrap(); - self.data - .extend_from_slice(&other.data[(other_start as usize)..(other_last as usize)]); - self.offsets.extend( - other - .offsets - .iter() - .skip(1) - .map(|offset| start + offset - other_start), - ); - } - - pub fn build(self) -> BinaryColumn { - BinaryColumn::new(self.data.into(), self.offsets.into()) - } - - pub fn build_scalar(self) -> Vec { - assert_eq!(self.offsets.len(), 2); - - self.data[(self.offsets[0] as usize)..(self.offsets[1] as usize)].to_vec() - } - - #[inline] - pub fn may_resize(&self, add_size: usize) -> bool { - self.data.len() + add_size > self.data.capacity() - } - - /// # Safety - /// - /// Calling this method with an out-of-bounds index is *[undefined behavior]* - pub unsafe fn index_unchecked(&self, row: usize) -> &[u8] { - debug_assert!(row + 1 < self.offsets.len()); - - let start = *self.offsets.get_unchecked(row) as usize; - let end = *self.offsets.get_unchecked(row + 1) as usize; - self.data.get_unchecked(start..end) - } - - pub fn push_repeat(&mut self, item: &[u8], n: usize) { - self.data.reserve(item.len() * n); - if self.need_estimated && self.offsets.len() - 1 < 64 { - for _ in 0..n { - self.data.extend_from_slice(item); - self.commit_row(); - } - } else { - let start = self.data.len(); - let len = item.len(); - for _ in 0..n { - self.data.extend_from_slice(item) - } - self.offsets - .extend((1..=n).map(|i| (start + len * i) as u64)); - } - } - - pub fn pop(&mut self) -> Option> { - if self.len() > 0 { - let index = self.len() - 1; - let start = unsafe { *self.offsets.get_unchecked(index) as usize }; - self.offsets.pop(); - let val = self.data.split_off(start); - Some(val) - } else { - None - } - } -} - -impl<'a> FromIterator<&'a [u8]> for BinaryColumnBuilder { - fn from_iter>(iter: T) -> Self { - let iter = iter.into_iter(); - let mut builder = BinaryColumnBuilder::with_capacity(iter.size_hint().0, 0); - for item in iter { - builder.put_slice(item); - builder.commit_row(); - } - builder - } -} diff --git a/src/query/expression/src/types/bitmap.rs b/src/query/expression/src/types/bitmap.rs index ab411346980a..1bd9e6386e7d 100644 --- a/src/query/expression/src/types/bitmap.rs +++ b/src/query/expression/src/types/bitmap.rs @@ -17,7 +17,7 @@ use std::ops::Range; use super::binary::BinaryColumn; use super::binary::BinaryColumnBuilder; -use super::binary::BinaryIterator; +use super::binary::BinaryColumnIter; use crate::property::Domain; use crate::types::ArgType; use crate::types::DataType; @@ -37,7 +37,7 @@ impl ValueType for BitmapType { type ScalarRef<'a> = &'a [u8]; type Column = BinaryColumn; type Domain = (); - type ColumnIterator<'a> = BinaryIterator<'a>; + type ColumnIterator<'a> = BinaryColumnIter<'a>; type ColumnBuilder = BinaryColumnBuilder; #[inline] diff --git a/src/query/expression/src/types/boolean.rs b/src/query/expression/src/types/boolean.rs index ed579f263fc9..4724d930dde2 100644 --- a/src/query/expression/src/types/boolean.rs +++ b/src/query/expression/src/types/boolean.rs @@ -15,8 +15,7 @@ use std::cmp::Ordering; use std::ops::Range; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; +pub use databend_common_column::bitmap::*; use crate::property::Domain; use crate::types::ArgType; @@ -38,7 +37,7 @@ impl ValueType for BooleanType { type ScalarRef<'a> = bool; type Column = Bitmap; type Domain = BooleanDomain; - type ColumnIterator<'a> = databend_common_arrow::arrow::bitmap::utils::BitmapIter<'a>; + type ColumnIterator<'a> = databend_common_column::bitmap::utils::BitmapIter<'a>; type ColumnBuilder = MutableBitmap; #[inline] diff --git a/src/query/expression/src/types/date.rs b/src/query/expression/src/types/date.rs index d14f52dc0b51..5a7e8ab989a5 100644 --- a/src/query/expression/src/types/date.rs +++ b/src/query/expression/src/types/date.rs @@ -19,7 +19,7 @@ use std::ops::Range; use chrono::NaiveDate; use chrono_tz::Tz; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::ReadBytesExt; diff --git a/src/query/expression/src/types/decimal.rs b/src/query/expression/src/types/decimal.rs index 0271d2d7fcc8..a54ec70792bb 100644 --- a/src/query/expression/src/types/decimal.rs +++ b/src/query/expression/src/types/decimal.rs @@ -17,9 +17,11 @@ use std::fmt::Debug; use std::marker::PhantomData; use std::ops::Range; +use arrow_data::ArrayData; +use arrow_data::ArrayDataBuilder; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_io::display_decimal_128; @@ -1144,6 +1146,55 @@ impl DecimalColumn { } }) } + + pub fn arrow_buffer(&self) -> arrow_buffer::Buffer { + match self { + DecimalColumn::Decimal128(col, _) => col.clone().into(), + DecimalColumn::Decimal256(col, _) => { + let col = unsafe { + std::mem::transmute::<_, Buffer>( + col.clone(), + ) + }; + col.into() + } + } + } + + pub fn arrow_data(&self, arrow_type: arrow_schema::DataType) -> ArrayData { + let buffer = self.arrow_buffer(); + let builder = ArrayDataBuilder::new(arrow_type) + .len(self.len()) + .buffers(vec![buffer]); + unsafe { builder.build_unchecked() } + } + + pub fn try_from_arrow_data(array: ArrayData) -> Result { + let buffer = array.buffers()[0].clone(); + match array.data_type() { + arrow_schema::DataType::Decimal128(p, s) => { + let decimal_size = DecimalSize { + precision: *p, + scale: *s as u8, + }; + Ok(Self::Decimal128(buffer.into(), decimal_size)) + } + arrow_schema::DataType::Decimal256(p, s) => { + let buffer: Buffer = buffer.into(); + let buffer = unsafe { std::mem::transmute::<_, Buffer>(buffer) }; + + let decimal_size = DecimalSize { + precision: *p, + scale: *s as u8, + }; + Ok(Self::Decimal256(buffer, decimal_size)) + } + data_type => Err(ErrorCode::Unimplemented(format!( + "Unsupported data type: {:?} into decimal column", + data_type + ))), + } + } } impl DecimalColumnBuilder { diff --git a/src/query/expression/src/types/geography.rs b/src/query/expression/src/types/geography.rs index 1a559eab0895..5e8c39fe0b59 100644 --- a/src/query/expression/src/types/geography.rs +++ b/src/query/expression/src/types/geography.rs @@ -19,7 +19,6 @@ use std::ops::Range; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::trusted_len::TrustedLen; use databend_common_exception::Result; use databend_common_io::geography::*; use databend_common_io::wkb::make_point; @@ -30,7 +29,7 @@ use geozero::ToWkt; use serde::Deserialize; use serde::Serialize; -use super::binary::BinaryIterator; +use super::binary::BinaryColumnIter; use crate::property::Domain; use crate::types::binary::BinaryColumn; use crate::types::binary::BinaryColumnBuilder; @@ -287,12 +286,12 @@ impl GeographyColumn { } pub fn check_valid(&self) -> Result<()> { - self.0.check_valid() + Ok(self.0.check_valid()?) } } pub struct GeographyIterator<'a> { - inner: BinaryIterator<'a>, + inner: BinaryColumnIter<'a>, } impl<'a> Iterator for GeographyIterator<'a> { @@ -303,6 +302,4 @@ impl<'a> Iterator for GeographyIterator<'a> { } } -unsafe impl<'a> TrustedLen for GeographyIterator<'a> {} - unsafe impl<'a> std::iter::TrustedLen for GeographyIterator<'a> {} diff --git a/src/query/expression/src/types/geometry.rs b/src/query/expression/src/types/geometry.rs index d1dfe01911cc..fac3c4ad4c06 100644 --- a/src/query/expression/src/types/geometry.rs +++ b/src/query/expression/src/types/geometry.rs @@ -21,7 +21,7 @@ use geozero::wkt::Ewkt; use super::binary::BinaryColumn; use super::binary::BinaryColumnBuilder; -use super::binary::BinaryIterator; +use super::binary::BinaryColumnIter; use crate::property::Domain; use crate::types::ArgType; use crate::types::DataType; @@ -41,7 +41,7 @@ impl ValueType for GeometryType { type ScalarRef<'a> = &'a [u8]; type Column = BinaryColumn; type Domain = (); - type ColumnIterator<'a> = BinaryIterator<'a>; + type ColumnIterator<'a> = BinaryColumnIter<'a>; type ColumnBuilder = BinaryColumnBuilder; #[inline] diff --git a/src/query/expression/src/types/map.rs b/src/query/expression/src/types/map.rs index b281b0aa11f8..15cba0a69738 100755 --- a/src/query/expression/src/types/map.rs +++ b/src/query/expression/src/types/map.rs @@ -12,11 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::iter::TrustedLen; use std::marker::PhantomData; use std::ops::Range; -use databend_common_arrow::arrow::trusted_len::TrustedLen; - use super::ArrayType; use super::DecimalSize; use crate::property::Domain; diff --git a/src/query/expression/src/types/nullable.rs b/src/query/expression/src/types/nullable.rs index a5c291136dde..891494b72602 100755 --- a/src/query/expression/src/types/nullable.rs +++ b/src/query/expression/src/types/nullable.rs @@ -13,12 +13,12 @@ // limitations under the License. use std::cmp::Ordering; +use std::iter::TrustedLen; use std::marker::PhantomData; use std::ops::Range; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::trusted_len::TrustedLen; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; use super::AnyType; use super::DecimalSize; @@ -357,7 +357,7 @@ impl NullableColumn { pub struct NullableIterator<'a, T: ValueType> { iter: T::ColumnIterator<'a>, - validity: databend_common_arrow::arrow::bitmap::utils::BitmapIter<'a>, + validity: databend_common_column::bitmap::utils::BitmapIter<'a>, } impl<'a, T: ValueType> Iterator for NullableIterator<'a, T> { diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index 879817e13d4a..d0734ed73270 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -17,10 +17,13 @@ use std::fmt::Debug; use std::marker::PhantomData; use std::ops::Range; +use arrow_data::ArrayData; +use arrow_data::ArrayDataBuilder; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_base::base::OrderedFloat; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; use enum_as_inner::EnumAsInner; use itertools::Itertools; use lexical_core::ToLexicalWithOptions; @@ -43,6 +46,7 @@ use crate::ScalarRef; pub type F32 = OrderedFloat; pub type F64 = OrderedFloat; +pub use databend_common_column::buffer::Buffer; pub const ALL_UNSIGNED_INTEGER_TYPES: &[NumberDataType] = &[ NumberDataType::UInt8, @@ -638,6 +642,63 @@ impl NumberColumn { } }) } + + pub fn arrow_buffer(&self) -> arrow_buffer::Buffer { + match self { + NumberColumn::UInt8(buffer) => buffer.clone().into(), + NumberColumn::UInt16(buffer) => buffer.clone().into(), + NumberColumn::UInt32(buffer) => buffer.clone().into(), + NumberColumn::UInt64(buffer) => buffer.clone().into(), + NumberColumn::Int8(buffer) => buffer.clone().into(), + NumberColumn::Int16(buffer) => buffer.clone().into(), + NumberColumn::Int32(buffer) => buffer.clone().into(), + NumberColumn::Int64(buffer) => buffer.clone().into(), + NumberColumn::Float32(buffer) => { + let r = unsafe { std::mem::transmute::, Buffer>(buffer.clone()) }; + r.into() + } + NumberColumn::Float64(buffer) => { + let r = unsafe { std::mem::transmute::, Buffer>(buffer.clone()) }; + r.into() + } + } + } + + pub fn arrow_data(&self, arrow_type: arrow_schema::DataType) -> ArrayData { + let buffer = self.arrow_buffer(); + let builder = ArrayDataBuilder::new(arrow_type) + .len(self.len()) + .buffers(vec![buffer]); + unsafe { builder.build_unchecked() } + } + + pub fn try_from_arrow_data(array: ArrayData) -> Result { + let buffer = array.buffers()[0].clone(); + match array.data_type() { + arrow_schema::DataType::UInt8 => Ok(NumberColumn::UInt8(buffer.into())), + arrow_schema::DataType::UInt16 => Ok(NumberColumn::UInt16(buffer.into())), + arrow_schema::DataType::UInt32 => Ok(NumberColumn::UInt32(buffer.into())), + arrow_schema::DataType::UInt64 => Ok(NumberColumn::UInt64(buffer.into())), + arrow_schema::DataType::Int8 => Ok(NumberColumn::Int8(buffer.into())), + arrow_schema::DataType::Int16 => Ok(NumberColumn::Int16(buffer.into())), + arrow_schema::DataType::Int32 => Ok(NumberColumn::Int32(buffer.into())), + arrow_schema::DataType::Int64 => Ok(NumberColumn::Int64(buffer.into())), + arrow_schema::DataType::Float32 => { + let buffer = buffer.into(); + let buffer = unsafe { std::mem::transmute::, Buffer>(buffer) }; + Ok(NumberColumn::Float32(buffer)) + } + arrow_schema::DataType::Float64 => { + let buffer = buffer.into(); + let buffer = unsafe { std::mem::transmute::, Buffer>(buffer) }; + Ok(NumberColumn::Float64(buffer)) + } + data_type => Err(ErrorCode::Unimplemented(format!( + "Unsupported data type: {:?} into number column", + data_type + ))), + } + } } impl NumberColumnBuilder { diff --git a/src/query/expression/src/types/string.rs b/src/query/expression/src/types/string.rs index 185e2a991fbc..0da330c0df77 100644 --- a/src/query/expression/src/types/string.rs +++ b/src/query/expression/src/types/string.rs @@ -15,15 +15,13 @@ use std::cmp::Ordering; use std::ops::Range; -use databend_common_arrow::arrow::array::MutableBinaryViewArray; -use databend_common_arrow::arrow::array::Utf8ViewArray; -use databend_common_arrow::arrow::trusted_len::TrustedLen; -use databend_common_exception::ErrorCode; +use databend_common_column::binview::BinaryViewColumnBuilder; +use databend_common_column::binview::BinaryViewColumnIter; +use databend_common_column::binview::Utf8ViewColumn; use databend_common_exception::Result; -use super::binary::BinaryColumn; -use super::binary::BinaryColumnBuilder; use crate::property::Domain; +use crate::types::binary::BinaryColumn; use crate::types::ArgType; use crate::types::DataType; use crate::types::DecimalSize; @@ -113,11 +111,11 @@ impl ValueType for StringType { #[inline] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + col.value_unchecked(index) } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { - col.slice(range) + col.clone().sliced(range.start, range.end - range.start) } fn iter_column(col: &Self::Column) -> Self::ColumnIterator<'_> { @@ -217,213 +215,20 @@ impl ArgType for StringType { } } -#[derive(Clone)] -pub struct StringColumn { - pub(crate) data: Utf8ViewArray, -} - -impl StringColumn { - pub fn new(data: Utf8ViewArray) -> Self { - Self { data } - } - - pub fn len(&self) -> usize { - self.data.len() - } - - pub fn current_buffer_len(&self) -> usize { - self.data.total_bytes_len() - } - - pub fn memory_size(&self) -> usize { - self.data.total_buffer_len() + self.len() * 12 - } - - pub fn index(&self, index: usize) -> Option<&str> { - if index >= self.len() { - return None; - } - - Some(unsafe { self.index_unchecked(index) }) - } - - /// # Safety - /// - /// Calling this method with an out-of-bounds index is *[undefined behavior]* - #[inline] - pub unsafe fn index_unchecked(&self, index: usize) -> &str { - debug_assert!(index < self.data.len()); - - self.data.value_unchecked(index) - } - - /// # Safety - /// - /// Calling this method with an out-of-bounds index is *[undefined behavior]* - #[inline] - pub unsafe fn index_unchecked_bytes(&self, index: usize) -> &[u8] { - debug_assert!(index < self.data.len()); - - self.data.value_unchecked(index).as_bytes() - } - - pub fn slice(&self, range: Range) -> Self { - let data = self - .data - .clone() - .sliced(range.start, range.end - range.start); - Self { data } - } - - pub fn iter(&self) -> StringIterator { - StringIterator { - col: self, - index: 0, - } - } - - pub fn into_inner(self) -> Utf8ViewArray { - self.data - } - - pub fn try_from_binary(col: BinaryColumn) -> Result { - let builder = StringColumnBuilder::try_from_bin_column(col)?; - Ok(builder.build()) - } - - pub fn compare(col_i: &Self, i: usize, col_j: &Self, j: usize) -> Ordering { - let view_i = unsafe { col_i.data.views().as_slice().get_unchecked(i) }; - let view_j = unsafe { col_j.data.views().as_slice().get_unchecked(j) }; - - if view_i.prefix == view_j.prefix { - unsafe { - let value_i = col_i.data.value_unchecked(i); - let value_j = col_j.data.value_unchecked(j); - value_i.cmp(value_j) - } - } else { - view_i - .prefix - .to_le_bytes() - .cmp(&view_j.prefix.to_le_bytes()) - } - } - - pub fn compare_str(col: &Self, i: usize, value: &str) -> Ordering { - let view = unsafe { col.data.views().as_slice().get_unchecked(i) }; - let prefix = load_prefix(value.as_bytes()); - - if view.prefix == prefix { - let value_i = unsafe { col.data.value_unchecked(i) }; - value_i.cmp(value) - } else { - view.prefix.to_le_bytes().as_slice().cmp(value.as_bytes()) - } - } -} - -// Loads (up to) the first 4 bytes of s as little-endian, padded with zeros. -#[inline] -fn load_prefix(s: &[u8]) -> u32 { - let start = &s[..s.len().min(4)]; - let mut tmp = [0u8; 4]; - tmp[..start.len()].copy_from_slice(start); - u32::from_le_bytes(tmp) -} - -impl PartialEq for StringColumn { - fn eq(&self, other: &Self) -> bool { - self.cmp(other) == Ordering::Equal - } -} +pub type StringColumn = Utf8ViewColumn; +pub type StringIterator<'a> = BinaryViewColumnIter<'a, str>; -impl Eq for StringColumn {} - -impl PartialOrd for StringColumn { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for StringColumn { - fn cmp(&self, other: &Self) -> Ordering { - for i in 0..self.len().max(other.len()) { - match (self.data.views().get(i), other.data.views().get(i)) { - (Some(left), Some(right)) => { - match left.prefix.to_le_bytes().cmp(&right.prefix.to_le_bytes()) { - Ordering::Equal => unsafe { - let left = self.data.value_unchecked(i); - let right = other.data.value_unchecked(i); - match left.cmp(right) { - Ordering::Equal => continue, - non_eq => return non_eq, - } - }, - non_eq => return non_eq, - } - } - (Some(_), None) => return Ordering::Greater, - (None, Some(_)) => return Ordering::Less, - (None, None) => return Ordering::Equal, - } - } - - Ordering::Equal - } -} - -impl TryFrom for StringColumn { - type Error = ErrorCode; - - fn try_from(col: BinaryColumn) -> Result { - StringColumn::try_from_binary(col) - } -} - -impl From for BinaryColumn { - fn from(col: StringColumn) -> BinaryColumn { - BinaryColumnBuilder::from_iter(col.iter().map(|x| x.as_bytes())).build() - } -} - -pub struct StringIterator<'a> { - col: &'a StringColumn, - index: usize, -} - -impl<'a> Iterator for StringIterator<'a> { - type Item = &'a str; - - fn next(&mut self) -> Option { - if self.index >= self.col.len() { - return None; - } - let value = self.col.index(self.index)?; - self.index += 1; - Some(value) - } - - fn size_hint(&self) -> (usize, Option) { - let remaining = self.col.len() - self.index; - (remaining, Some(remaining)) - } -} - -unsafe impl<'a> TrustedLen for StringIterator<'a> {} - -unsafe impl<'a> std::iter::TrustedLen for StringIterator<'a> {} - -type MutableUtf8ViewArray = MutableBinaryViewArray; +type Utf8ViewColumnBuilder = BinaryViewColumnBuilder; #[derive(Debug, Clone)] pub struct StringColumnBuilder { - pub data: MutableUtf8ViewArray, + pub data: Utf8ViewColumnBuilder, pub row_buffer: Vec, } impl StringColumnBuilder { pub fn with_capacity(len: usize) -> Self { - let data = MutableUtf8ViewArray::with_capacity(len); + let data = Utf8ViewColumnBuilder::with_capacity(len); StringColumnBuilder { data, row_buffer: Vec::new(), @@ -431,7 +236,7 @@ impl StringColumnBuilder { } pub fn from_column(col: StringColumn) -> Self { - let data = col.data.make_mut(); + let data = col.make_mut(); StringColumnBuilder { data, row_buffer: Vec::new(), @@ -439,12 +244,7 @@ impl StringColumnBuilder { } pub fn try_from_bin_column(col: BinaryColumn) -> Result { - let mut data = MutableUtf8ViewArray::with_capacity(col.len()); - col.data.as_slice().check_utf8()?; - for v in col.iter() { - data.push_value(unsafe { std::str::from_utf8_unchecked(v) }); - } - + let data = Utf8ViewColumnBuilder::try_from_bin_column(col)?; Ok(StringColumnBuilder { data, row_buffer: Vec::new(), @@ -452,8 +252,8 @@ impl StringColumnBuilder { } pub fn repeat(scalar: &str, n: usize) -> Self { - let mut data = MutableUtf8ViewArray::with_capacity(n); - data.extend_constant(n, Some(scalar)); + let mut data = Utf8ViewColumnBuilder::with_capacity(n); + data.extend_constant(n, scalar); StringColumnBuilder { data, row_buffer: Vec::new(), @@ -461,8 +261,8 @@ impl StringColumnBuilder { } pub fn repeat_default(n: usize) -> Self { - let mut data = MutableUtf8ViewArray::with_capacity(n); - data.extend_constant(n, Some("")); + let mut data = Utf8ViewColumnBuilder::with_capacity(n); + data.extend_constant(n, ""); StringColumnBuilder { data, row_buffer: Vec::new(), @@ -493,7 +293,7 @@ impl StringColumnBuilder { #[inline] pub fn put_and_commit>(&mut self, item: V) { - self.data.push_value_ignore_validity(item); + self.data.push_value(item); } #[inline] @@ -520,9 +320,7 @@ impl StringColumnBuilder { } pub fn build(self) -> StringColumn { - StringColumn { - data: self.data.into(), - } + self.data.into() } pub fn build_scalar(self) -> String { @@ -539,7 +337,7 @@ impl StringColumnBuilder { } pub fn push_repeat(&mut self, item: &str, n: usize) { - self.data.extend_constant(n, Some(item)); + self.data.extend_constant(n, item); } pub fn pop(&mut self) -> Option { @@ -560,7 +358,7 @@ impl<'a> FromIterator<&'a str> for StringColumnBuilder { impl PartialEq for StringColumnBuilder { fn eq(&self, other: &Self) -> bool { - self.data.values_iter().eq(other.data.values_iter()) + self.data.iter().eq(other.data.iter()) } } @@ -572,102 +370,3 @@ pub struct StringDomain { // max value is None for full domain pub max: Option, } - -pub trait CheckUTF8 { - fn check_utf8(&self) -> Result<()>; -} - -impl CheckUTF8 for &[u8] { - fn check_utf8(&self) -> Result<()> { - simdutf8::basic::from_utf8(self).map_err(|_| { - ErrorCode::InvalidUtf8String(format!( - "Encountered invalid utf8 data for string type, \ - if you were reading column with string type from a table, \ - it's recommended to alter the column type to `BINARY`.\n\ - Example: `ALTER TABLE
MODIFY COLUMN BINARY;`\n\ - Invalid utf8 data: `{}`", - hex::encode_upper(self) - )) - })?; - Ok(()) - } -} - -impl CheckUTF8 for Vec { - fn check_utf8(&self) -> Result<()> { - self.as_slice().check_utf8() - } -} - -impl CheckUTF8 for BinaryColumn { - fn check_utf8(&self) -> Result<()> { - for bytes in self.iter() { - bytes.check_utf8()?; - } - Ok(()) - } -} - -impl CheckUTF8 for BinaryColumnBuilder { - fn check_utf8(&self) -> Result<()> { - check_utf8_column(&self.offsets, &self.data) - } -} - -/// # Check if any slice of `values` between two consecutive pairs from `offsets` is invalid `utf8` -fn check_utf8_column(offsets: &[u64], data: &[u8]) -> Result<()> { - let res: Option<()> = try { - if offsets.len() == 1 { - return Ok(()); - } - - if data.is_ascii() { - return Ok(()); - } - - simdutf8::basic::from_utf8(data).ok()?; - - let last = if let Some(last) = offsets.last() { - if *last as usize == data.len() { - return Ok(()); - } else { - *last as usize - } - } else { - // given `l = data.len()`, this branch is hit iff either: - // * `offsets = [0, l, l, ...]`, which was covered by `from_utf8(data)` above - // * `offsets = [0]`, which never happens because offsets.len() == 1 is short-circuited above - return Ok(()); - }; - - // truncate to relevant offsets. Note: `=last` because last was computed skipping the first item - // following the example: starts = [0, 5] - let starts = unsafe { offsets.get_unchecked(..=last) }; - - let mut any_invalid = false; - for start in starts { - let start = *start as usize; - - // Safety: `try_check_offsets_bounds` just checked for bounds - let b = *unsafe { data.get_unchecked(start) }; - - // A valid code-point iff it does not start with 0b10xxxxxx - // Bit-magic taken from `std::str::is_char_boundary` - if (b as i8) < -0x40 { - any_invalid = true - } - } - if any_invalid { - None?; - } - }; - res.ok_or_else(|| { - ErrorCode::InvalidUtf8String( - "Encountered invalid utf8 data for string type, \ - if you were reading column with string type from a table, \ - it's recommended to alter the column type to `BINARY`.\n\ - Example: `ALTER TABLE
MODIFY COLUMN BINARY;`" - .to_string(), - ) - }) -} diff --git a/src/query/expression/src/types/timestamp.rs b/src/query/expression/src/types/timestamp.rs index 24c6dfcbb01c..636f84cdee61 100644 --- a/src/query/expression/src/types/timestamp.rs +++ b/src/query/expression/src/types/timestamp.rs @@ -19,7 +19,7 @@ use std::ops::Range; use chrono::DateTime; use chrono_tz::Tz; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::DateTimeResType; diff --git a/src/query/expression/src/types/variant.rs b/src/query/expression/src/types/variant.rs index 262d2f36aa72..1f20d4c14c7d 100644 --- a/src/query/expression/src/types/variant.rs +++ b/src/query/expression/src/types/variant.rs @@ -23,7 +23,7 @@ use jsonb::Value; use super::binary::BinaryColumn; use super::binary::BinaryColumnBuilder; -use super::binary::BinaryIterator; +use super::binary::BinaryColumnIter; use super::date::date_to_string; use super::number::NumberScalar; use super::timestamp::timestamp_to_string; @@ -52,7 +52,7 @@ impl ValueType for VariantType { type ScalarRef<'a> = &'a [u8]; type Column = BinaryColumn; type Domain = (); - type ColumnIterator<'a> = BinaryIterator<'a>; + type ColumnIterator<'a> = BinaryColumnIter<'a>; type ColumnBuilder = BinaryColumnBuilder; #[inline] diff --git a/src/query/expression/src/utils/arrow.rs b/src/query/expression/src/utils/arrow.rs index c73f45933533..71f6d84ccb71 100644 --- a/src/query/expression/src/utils/arrow.rs +++ b/src/query/expression/src/utils/arrow.rs @@ -24,18 +24,14 @@ use arrow_ipc::writer::FileWriter; use arrow_ipc::writer::IpcWriteOptions; use arrow_ipc::CompressionType; use arrow_schema::Schema; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use crate::BlockEntry; use crate::Column; -use crate::ColumnBuilder; use crate::DataField; -use crate::Value; pub fn bitmap_into_mut(bitmap: Bitmap) -> MutableBitmap { bitmap @@ -79,7 +75,7 @@ pub fn write_column( col: &Column, w: &mut impl Write, ) -> std::result::Result<(), arrow_schema::ArrowError> { - let field: arrow_schema::Field = col.arrow_field().into(); + let field = col.arrow_field(); let schema = Schema::new(vec![field]); let mut writer = FileWriter::try_new_with_options( w, @@ -111,17 +107,6 @@ pub fn read_column(r: &mut R) -> Result { Column::from_arrow_rs(col, f.data_type()) } -/// Convert a column to a arrow array. -pub fn column_to_arrow_array(column: &BlockEntry, num_rows: usize) -> Box { - match &column.value { - Value::Scalar(v) => { - let builder = ColumnBuilder::repeat(&v.as_ref(), num_rows, &column.data_type); - builder.build().as_arrow() - } - Value::Column(c) => c.as_arrow(), - } -} - pub fn and_validities(lhs: Option, rhs: Option) -> Option { match (lhs, rhs) { (Some(lhs), None) => Some(lhs), diff --git a/src/query/expression/src/utils/column_from.rs b/src/query/expression/src/utils/column_from.rs index fc3038652edd..ad8f5e3f9b0f 100755 --- a/src/query/expression/src/utils/column_from.rs +++ b/src/query/expression/src/utils/column_from.rs @@ -14,7 +14,7 @@ use std::iter::Iterator; -use databend_common_arrow::arrow::bitmap::MutableBitmap; +use databend_common_column::bitmap::MutableBitmap; use itertools::Itertools; use crate::types::decimal::*; diff --git a/src/query/expression/src/utils/display.rs b/src/query/expression/src/utils/display.rs index 32287b84c8e0..94d898fdc2f7 100755 --- a/src/query/expression/src/utils/display.rs +++ b/src/query/expression/src/utils/display.rs @@ -40,7 +40,6 @@ use crate::function::Function; use crate::function::FunctionSignature; use crate::property::Domain; use crate::property::FunctionProperty; -use crate::types::binary::BinaryColumn; use crate::types::boolean::BooleanDomain; use crate::types::date::date_to_string; use crate::types::decimal::DecimalColumn; @@ -54,7 +53,6 @@ use crate::types::number::NumberDataType; use crate::types::number::NumberDomain; use crate::types::number::NumberScalar; use crate::types::number::SimpleDomain; -use crate::types::string::StringColumn; use crate::types::string::StringDomain; use crate::types::timestamp::timestamp_to_string; use crate::types::AnyType; @@ -414,26 +412,6 @@ impl Debug for DecimalColumn { } } -impl Debug for BinaryColumn { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("BinaryColumn") - .field( - "data", - &format_args!("0x{}", &hex::encode(self.data().as_slice())), - ) - .field("offsets", &self.offsets()) - .finish() - } -} - -impl Debug for StringColumn { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("StringColumn") - .field("data", &format_args!("{:?}", self.data)) - .finish() - } -} - impl Display for RawExpr { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { diff --git a/src/query/expression/src/utils/filter_helper.rs b/src/query/expression/src/utils/filter_helper.rs index b3f1e2db1ccd..85ff6f3d275d 100644 --- a/src/query/expression/src/utils/filter_helper.rs +++ b/src/query/expression/src/utils/filter_helper.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::MutableBitmap; +use databend_common_column::bitmap::MutableBitmap; use crate::arrow::bitmap_into_mut; use crate::types::BooleanType; @@ -25,7 +25,7 @@ impl FilterHelpers { pub fn is_all_unset(predicate: &Value) -> bool { match &predicate { Value::Scalar(v) => !v, - Value::Column(bitmap) => bitmap.unset_bits() == bitmap.len(), + Value::Column(bitmap) => bitmap.null_count() == bitmap.len(), } } diff --git a/src/query/expression/src/utils/mod.rs b/src/query/expression/src/utils/mod.rs index bb0ba0cd7967..13ebfe080cf3 100644 --- a/src/query/expression/src/utils/mod.rs +++ b/src/query/expression/src/utils/mod.rs @@ -25,8 +25,8 @@ pub mod udf_client; pub mod variant_transform; pub mod visitor; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_ast::Span; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use ethnum::i256; diff --git a/src/query/expression/src/utils/visitor.rs b/src/query/expression/src/utils/visitor.rs index 231127ac8254..b3b9e06b211d 100755 --- a/src/query/expression/src/utils/visitor.rs +++ b/src/query/expression/src/utils/visitor.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use decimal::DecimalType; use geometry::GeometryType; diff --git a/src/query/expression/src/values.rs b/src/query/expression/src/values.rs index 6c63fd6e5392..516856bd9f3f 100755 --- a/src/query/expression/src/values.rs +++ b/src/query/expression/src/values.rs @@ -16,6 +16,7 @@ use std::cmp::Ordering; use std::hash::Hash; use std::io::Read; use std::io::Write; +use std::iter::TrustedLen; use std::ops::Range; use base64::engine::general_purpose; @@ -23,11 +24,10 @@ use base64::prelude::*; use binary::BinaryColumnBuilder; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_arrow::arrow::trusted_len::TrustedLen; use databend_common_base::base::OrderedFloat; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_io::prelude::BinaryRead; @@ -946,7 +946,7 @@ impl Column { Column::Decimal(col) => Some(ScalarRef::Decimal(col.index(index)?)), Column::Boolean(col) => Some(ScalarRef::Boolean(col.get(index)?)), Column::Binary(col) => Some(ScalarRef::Binary(col.index(index)?)), - Column::String(col) => Some(ScalarRef::String(col.index(index)?)), + Column::String(col) => Some(ScalarRef::String(col.value(index))), Column::Timestamp(col) => Some(ScalarRef::Timestamp(col.get(index).cloned()?)), Column::Date(col) => Some(ScalarRef::Date(col.get(index).cloned()?)), Column::Array(col) => Some(ScalarRef::Array(col.index(index)?)), @@ -1025,7 +1025,9 @@ impl Column { Column::Boolean(col.clone().sliced(range.start, range.end - range.start)) } Column::Binary(col) => Column::Binary(col.slice(range)), - Column::String(col) => Column::String(col.slice(range)), + Column::String(col) => { + Column::String(col.clone().sliced(range.start, range.end - range.start)) + } Column::Timestamp(col) => { Column::Timestamp(col.clone().sliced(range.start, range.end - range.start)) } @@ -1070,8 +1072,8 @@ impl Column { Column::Number(col) => Domain::Number(col.domain()), Column::Decimal(col) => Domain::Decimal(col.domain()), Column::Boolean(col) => Domain::Boolean(BooleanDomain { - has_false: col.unset_bits() > 0, - has_true: col.len() - col.unset_bits() > 0, + has_false: col.null_count() > 0, + has_true: col.len() - col.null_count() > 0, }), Column::String(col) => { let (min, max) = StringType::iter_column(col).minmax().into_option().unwrap(); @@ -1113,7 +1115,7 @@ impl Column { Column::Nullable(col) => { let inner_domain = col.column.domain(); Domain::Nullable(NullableDomain { - has_null: col.validity.unset_bits() > 0, + has_null: col.validity.null_count() > 0, value: Some(Box::new(inner_domain)), }) } @@ -1171,11 +1173,11 @@ impl Column { pub fn check_valid(&self) -> Result<()> { match self { - Column::Binary(x) => x.check_valid(), - Column::Variant(x) => x.check_valid(), - Column::Geometry(x) => x.check_valid(), - Column::Geography(x) => x.check_valid(), - Column::Bitmap(x) => x.check_valid(), + Column::Binary(x) => Ok(x.check_valid()?), + Column::Variant(x) => Ok(x.check_valid()?), + Column::Geometry(x) => Ok(x.check_valid()?), + Column::Geography(x) => Ok(x.check_valid()?), + Column::Bitmap(x) => Ok(x.check_valid()?), Column::Map(x) => { for y in x.iter() { y.check_valid()?; @@ -1446,7 +1448,7 @@ impl Column { | Column::Bitmap(col) | Column::Variant(col) | Column::Geometry(col) => col.memory_size(), - Column::String(col) => col.len() * 8 + col.current_buffer_len(), + Column::String(col) => col.len() * 8 + col.total_bytes_len(), Column::Array(col) | Column::Map(col) => col.values.serialize_size() + col.len() * 8, Column::Nullable(c) => c.column.serialize_size() + c.len(), Column::Tuple(fields) => fields.iter().map(|f| f.serialize_size()).sum(), @@ -1458,7 +1460,7 @@ impl Column { match self { Column::Null { .. } => (true, None), Column::Nullable(c) => { - if c.validity.unset_bits() == c.validity.len() { + if c.validity.null_count() == c.validity.len() { (true, Some(&c.validity)) } else { (false, Some(&c.validity)) @@ -2047,7 +2049,8 @@ impl ColumnBuilder { reader.read_exact(&mut builder.row_buffer)?; #[cfg(debug_assertions)] - string::CheckUTF8::check_utf8(&builder.row_buffer).unwrap(); + databend_common_column::binview::CheckUTF8::check_utf8(&builder.row_buffer) + .unwrap(); builder.commit_row(); } @@ -2144,7 +2147,7 @@ impl ColumnBuilder { let bytes = &reader[step * row..]; #[cfg(debug_assertions)] - string::CheckUTF8::check_utf8(&bytes).unwrap(); + databend_common_column::binview::CheckUTF8::check_utf8(&bytes).unwrap(); let s = unsafe { std::str::from_utf8_unchecked(bytes) }; builder.put_and_commit(s); diff --git a/src/query/expression/tests/it/column.rs b/src/query/expression/tests/it/column.rs deleted file mode 100644 index 9ca865ed9b3b..000000000000 --- a/src/query/expression/tests/it/column.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2022 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_arrow::arrow::array::new_empty_array; -use databend_common_arrow::arrow::datatypes::DataType as ArrowDataType; -use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::Column; - -#[test] -fn test_from_arrow_extension_to_column() -> Result<()> { - let data_type = DataType::Number(NumberDataType::Int8); - let extension_data_type = - ArrowDataType::Extension("a".to_string(), Box::new(ArrowDataType::Int8), None); - - let arrow_col = new_empty_array(extension_data_type); - let _ = Column::from_arrow(arrow_col.as_ref(), &data_type); - - Ok(()) -} diff --git a/src/query/expression/tests/it/fill_field_default_value.rs b/src/query/expression/tests/it/fill_field_default_value.rs index 9d4db03eef20..166eddf0dfb1 100644 --- a/src/query/expression/tests/it/fill_field_default_value.rs +++ b/src/query/expression/tests/it/fill_field_default_value.rs @@ -14,13 +14,10 @@ use std::collections::HashSet; -use databend_common_arrow::arrow::chunk::Chunk; -use databend_common_arrow::ArrayRef; use databend_common_exception::Result; use databend_common_expression::types::number::*; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; use databend_common_expression::*; use goldenfile::Mint; @@ -31,42 +28,6 @@ fn test_data_block_create_with_default_value_functions() -> Result<()> { let mut mint = Mint::new("tests/it/testdata"); let mut file = mint.new_goldenfile("fill_field_default_value.txt").unwrap(); - // test create_with_default_value_and_chunk - { - let schema = DataSchemaRefExt::create(vec![ - DataField::new("a", DataType::Number(NumberDataType::Int32)), - DataField::new("b", DataType::Number(NumberDataType::Int32)), - DataField::new("c", DataType::Number(NumberDataType::Float32)), - DataField::new("d", DataType::Number(NumberDataType::Int32)), - DataField::new("e", DataType::String), - ]); - - let num_rows = 3; - let chunk_block = new_block(&[ - Int32Type::from_data(vec![1i32, 2, 3]), - Float32Type::from_data(vec![1.0, 2.0, 3.0]), - StringType::from_data(vec!["x1", "x2", "x3"]), - ]); - - let chunks: Chunk = chunk_block.try_into().unwrap(); - let default_vals = vec![ - None, - Some(Scalar::Number(NumberScalar::Int32(2))), - None, - Some(Scalar::Number(NumberScalar::Int32(4))), - None, - ]; - let new_block: DataBlock = DataBlock::create_with_default_value_and_chunk( - &schema, - &chunks, - &default_vals, - num_rows, - ) - .unwrap(); - - run_take(&mut file, &[0, 1, 2], &new_block); - } - // test create_with_default_value { let fields = vec![ diff --git a/src/query/expression/tests/it/kernel.rs b/src/query/expression/tests/it/kernel.rs index d374f553fb6e..ba18f8ed077f 100644 --- a/src/query/expression/tests/it/kernel.rs +++ b/src/query/expression/tests/it/kernel.rs @@ -14,7 +14,7 @@ use core::ops::Range; -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_expression::block_debug::assert_block_value_eq; use databend_common_expression::types::number::*; use databend_common_expression::types::AnyType; @@ -626,7 +626,7 @@ fn test_builder() { } } - for (start, len) in databend_common_arrow::arrow::bitmap::utils::SlicesIterator::new(&bitmap) { + for (start, len) in databend_common_column::bitmap::utils::SlicesIterator::new(&bitmap) { let sub_col = col.slice(start..start + len); AnyType::append_column(&mut builder2, &sub_col); } diff --git a/src/query/expression/tests/it/main.rs b/src/query/expression/tests/it/main.rs index 3038e4460b8f..8b3bb44ea3f0 100644 --- a/src/query/expression/tests/it/main.rs +++ b/src/query/expression/tests/it/main.rs @@ -25,7 +25,6 @@ use databend_common_expression::DataBlock; extern crate core; mod block; -mod column; mod common; mod decimal; mod fill_field_default_value; diff --git a/src/query/expression/tests/it/row.rs b/src/query/expression/tests/it/row.rs index 07b9c2e196c9..425921ae0353 100644 --- a/src/query/expression/tests/it/row.rs +++ b/src/query/expression/tests/it/row.rs @@ -12,14 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - +use arrow_array::ArrayRef; use arrow_ord::sort::LexicographicalComparator; use arrow_ord::sort::SortColumn; use arrow_schema::SortOptions; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_base::base::OrderedFloat; -use databend_common_expression::converts::arrow2::set_validities; +use databend_common_column::bitmap::MutableBitmap; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::decimal::*; use databend_common_expression::types::nullable::NullableColumn; @@ -518,21 +516,9 @@ fn fuzz_test() { let order_columns = columns .iter() - .map(|col| { - let arrow2 = match col { - // arrow_ord does not support LargeBinary converted from Databend String - Column::Nullable(c) => match &c.column { - Column::String(sc) => { - let array = Box::new(sc.clone().into_inner()); - set_validities(array, &c.validity) - } - _ => col.as_arrow(), - }, - col => col.as_arrow(), - }; - arrow2.into() - }) - .collect::>>(); + .map(|col| col.clone().into_arrow_rs()) + .collect::>(); + let sort_columns = options .iter() .zip(order_columns.iter()) diff --git a/src/query/expression/tests/it/schema.rs b/src/query/expression/tests/it/schema.rs index 03537bcc5d8d..7c2af0649a1e 100644 --- a/src/query/expression/tests/it/schema.rs +++ b/src/query/expression/tests/it/schema.rs @@ -14,8 +14,6 @@ use std::collections::BTreeMap; -use databend_common_arrow::arrow::datatypes::DataType as ArrowDataType; -use databend_common_arrow::arrow::datatypes::Field as ArrowField; use databend_common_exception::Result; use databend_common_expression::create_test_complex_schema; use databend_common_expression::types::NumberDataType; @@ -26,15 +24,6 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchema; use pretty_assertions::assert_eq; -#[test] -fn test_from_arrow_field_to_table_field() -> Result<()> { - let extension_data_type = - ArrowDataType::Extension("a".to_string(), Box::new(ArrowDataType::Int8), None); - let arrow_field = ArrowField::new("".to_string(), extension_data_type, false); - let _: TableField = (&arrow_field).try_into().unwrap(); - Ok(()) -} - #[test] fn test_project_schema_from_tuple() -> Result<()> { let b1 = TableDataType::Tuple { @@ -666,8 +655,8 @@ fn test_geography_as_arrow() { builder.put_slice(&make_point(4.0, 5.0)); builder.commit_row(); let col = Column::Geography(GeographyColumn(builder.build())); - - let arr = col.as_arrow(); - let got = Column::from_arrow(&*arr, &col.data_type()).unwrap(); + let data_type = col.data_type(); + let arr = col.clone().into_arrow_rs(); + let got = Column::from_arrow_rs(arr, &data_type).unwrap(); assert_eq!(col, got) } diff --git a/src/query/expression/tests/it/testdata/fill_field_default_value.txt b/src/query/expression/tests/it/testdata/fill_field_default_value.txt index e08a98334334..dfa0f89dfa53 100644 --- a/src/query/expression/tests/it/testdata/fill_field_default_value.txt +++ b/src/query/expression/tests/it/testdata/fill_field_default_value.txt @@ -1,22 +1,3 @@ -Take: [0, 1, 2] -Source: -+----------+----------+----------+----------+----------+ -| Column 0 | Column 1 | Column 2 | Column 3 | Column 4 | -+----------+----------+----------+----------+----------+ -| 1 | 2 | 1 | 4 | 'x1' | -| 2 | 2 | 2 | 4 | 'x2' | -| 3 | 2 | 3 | 4 | 'x3' | -+----------+----------+----------+----------+----------+ -Result: -+----------+----------+----------+----------+----------+ -| Column 0 | Column 1 | Column 2 | Column 3 | Column 4 | -+----------+----------+----------+----------+----------+ -| 1 | 2 | 1 | 4 | 'x1' | -| 2 | 2 | 2 | 4 | 'x2' | -| 3 | 2 | 3 | 4 | 'x3' | -+----------+----------+----------+----------+----------+ - - Take: [0, 1, 2] Source: +----------+----------+----------+----------+----------+ diff --git a/src/query/expression/tests/it/testdata/kernel-pass.txt b/src/query/expression/tests/it/testdata/kernel-pass.txt index cfe2e20d5bda..f8fb4fd35710 100644 --- a/src/query/expression/tests/it/testdata/kernel-pass.txt +++ b/src/query/expression/tests/it/testdata/kernel-pass.txt @@ -19,25 +19,25 @@ Result: Concat-Column 0: -+-----------+----------------+----------------------------------------------------------------------------------------------------------------+ -| Column ID | Type | Column Data | -+-----------+----------------+----------------------------------------------------------------------------------------------------------------+ -| 0 | Int32 | Column(Int32([0, 1, 2, 3, -4])) | -| 1 | UInt8 NULL | Column(NullableColumn { column: UInt8([10, 11, 12, 13, 14]), validity: [0b___00010] }) | -| 2 | NULL | Column(Null { len: 5 }) | -| 3 | Array(Nothing) | Column(EmptyArray { len: 5 }) | -| 4 | String NULL | Column(NullableColumn { column: StringColumn { data: Utf8ViewArray[x, y, z, a, b] }, validity: [0b___00110] }) | -+-----------+----------------+----------------------------------------------------------------------------------------------------------------+ ++-----------+----------------+----------------------------------------------------------------------------------------+ +| Column ID | Type | Column Data | ++-----------+----------------+----------------------------------------------------------------------------------------+ +| 0 | Int32 | Column(Int32([0, 1, 2, 3, -4])) | +| 1 | UInt8 NULL | Column(NullableColumn { column: UInt8([10, 11, 12, 13, 14]), validity: [0b___00010] }) | +| 2 | NULL | Column(Null { len: 5 }) | +| 3 | Array(Nothing) | Column(EmptyArray { len: 5 }) | +| 4 | String NULL | Column(NullableColumn { column: StringColumn[x, y, z, a, b], validity: [0b___00110] }) | ++-----------+----------------+----------------------------------------------------------------------------------------+ Concat-Column 1: -+-----------+----------------+-------------------------------------------------------------------------------------------------------+ -| Column ID | Type | Column Data | -+-----------+----------------+-------------------------------------------------------------------------------------------------------+ -| 0 | Int32 | Column(Int32([5, 6])) | -| 1 | UInt8 NULL | Column(NullableColumn { column: UInt8([15, 16]), validity: [0b______10] }) | -| 2 | NULL | Column(Null { len: 2 }) | -| 3 | Array(Nothing) | Column(EmptyArray { len: 2 }) | -| 4 | String NULL | Column(NullableColumn { column: StringColumn { data: Utf8ViewArray[x, y] }, validity: [0b______10] }) | -+-----------+----------------+-------------------------------------------------------------------------------------------------------+ ++-----------+----------------+-------------------------------------------------------------------------------+ +| Column ID | Type | Column Data | ++-----------+----------------+-------------------------------------------------------------------------------+ +| 0 | Int32 | Column(Int32([5, 6])) | +| 1 | UInt8 NULL | Column(NullableColumn { column: UInt8([15, 16]), validity: [0b______10] }) | +| 2 | NULL | Column(Null { len: 2 }) | +| 3 | Array(Nothing) | Column(EmptyArray { len: 2 }) | +| 4 | String NULL | Column(NullableColumn { column: StringColumn[x, y], validity: [0b______10] }) | ++-----------+----------------+-------------------------------------------------------------------------------+ Result: +----------+----------+----------+----------+----------+ | Column 0 | Column 1 | Column 2 | Column 3 | Column 4 | diff --git a/src/query/expression/tests/it/types.rs b/src/query/expression/tests/it/types.rs index bfeb2b9e792d..0ba121d87569 100644 --- a/src/query/expression/tests/it/types.rs +++ b/src/query/expression/tests/it/types.rs @@ -50,12 +50,11 @@ fn test_convert_types() { assert_eq!(schema, schema2); let random_block = rand_block_for_all_types(1024); - - for c in random_block.columns() { + for (idx, c) in random_block.columns().iter().enumerate() { let c = c.value.as_column().unwrap().clone(); + let data = serialize_column(&c); let c2 = deserialize_column(&data).unwrap(); - - assert_eq!(c, c2); + assert_eq!(c, c2, "in {idx} | datatype: {}", c.data_type()); } } diff --git a/src/query/formats/Cargo.toml b/src/query/formats/Cargo.toml index a6132cfab396..6f2de788af28 100644 --- a/src/query/formats/Cargo.toml +++ b/src/query/formats/Cargo.toml @@ -11,7 +11,7 @@ doctest = false test = true [dependencies] -databend-common-arrow = { workspace = true } + databend-common-base = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } @@ -39,7 +39,7 @@ roaring = { workspace = true, features = ["serde"] } serde_json = { workspace = true } [dev-dependencies] -databend-common-arrow = { workspace = true } + pretty_assertions = { workspace = true } tokio = { workspace = true } diff --git a/src/query/formats/src/field_decoder/fast_values.rs b/src/query/formats/src/field_decoder/fast_values.rs index c3a5c16ebd55..7e3efd72e7f6 100644 --- a/src/query/formats/src/field_decoder/fast_values.rs +++ b/src/query/formats/src/field_decoder/fast_values.rs @@ -22,7 +22,6 @@ use std::sync::LazyLock; use aho_corasick::AhoCorasick; use bstr::ByteSlice; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::serialize::read_decimal_with_size; @@ -38,6 +37,7 @@ use databend_common_expression::types::number::Number; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::timestamp::clamp_timestamp; use databend_common_expression::types::AnyType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberColumnBuilder; use databend_common_expression::with_decimal_type; use databend_common_expression::with_number_mapped_type; diff --git a/src/query/formats/src/field_decoder/json_ast.rs b/src/query/formats/src/field_decoder/json_ast.rs index fa017db41e06..ef3dc809f091 100644 --- a/src/query/formats/src/field_decoder/json_ast.rs +++ b/src/query/formats/src/field_decoder/json_ast.rs @@ -16,7 +16,6 @@ use std::any::Any; use std::io::Cursor; use chrono_tz::Tz; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::serialize::read_decimal_from_json; @@ -32,6 +31,7 @@ use databend_common_expression::types::number::Number; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::timestamp::clamp_timestamp; use databend_common_expression::types::AnyType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberColumnBuilder; use databend_common_expression::with_decimal_type; use databend_common_expression::with_number_mapped_type; diff --git a/src/query/formats/src/field_decoder/nested.rs b/src/query/formats/src/field_decoder/nested.rs index 1b7b5ba958f7..6d6b515aead3 100644 --- a/src/query/formats/src/field_decoder/nested.rs +++ b/src/query/formats/src/field_decoder/nested.rs @@ -17,7 +17,6 @@ use std::io::BufRead; use std::io::Cursor; use bstr::ByteSlice; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::serialize::read_decimal_with_size; @@ -33,6 +32,7 @@ use databend_common_expression::types::number::Number; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::timestamp::clamp_timestamp; use databend_common_expression::types::AnyType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberColumnBuilder; use databend_common_expression::with_decimal_type; use databend_common_expression::with_number_mapped_type; diff --git a/src/query/formats/src/field_decoder/separated_text.rs b/src/query/formats/src/field_decoder/separated_text.rs index 31f0226f032b..50e7dd9a5cf6 100644 --- a/src/query/formats/src/field_decoder/separated_text.rs +++ b/src/query/formats/src/field_decoder/separated_text.rs @@ -16,7 +16,6 @@ use std::any::Any; use std::io::Cursor; use bstr::ByteSlice; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::serialize::read_decimal_with_size; @@ -30,6 +29,7 @@ use databend_common_expression::types::decimal::DecimalSize; use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_expression::types::timestamp::clamp_timestamp; use databend_common_expression::types::AnyType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::Number; use databend_common_expression::types::NumberColumnBuilder; use databend_common_expression::with_decimal_type; diff --git a/src/query/formats/src/field_encoder/values.rs b/src/query/formats/src/field_encoder/values.rs index 55483a3af6e6..6c7a13d2e09b 100644 --- a/src/query/formats/src/field_encoder/values.rs +++ b/src/query/formats/src/field_encoder/values.rs @@ -14,17 +14,17 @@ use bstr::ByteSlice; use chrono_tz::Tz; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_base::base::OrderedFloat; use databend_common_expression::types::array::ArrayColumn; -use databend_common_expression::types::binary::BinaryColumn; use databend_common_expression::types::date::date_to_string; use databend_common_expression::types::decimal::DecimalColumn; use databend_common_expression::types::geography::GeographyColumn; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::string::StringColumn; use databend_common_expression::types::timestamp::timestamp_to_string; +use databend_common_expression::types::BinaryColumn; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; use databend_common_expression::types::NumberColumn; use databend_common_expression::types::ValueType; use databend_common_expression::Column; diff --git a/src/query/formats/tests/it/output_format_utils.rs b/src/query/formats/tests/it/output_format_utils.rs index 33afd9946e44..8efb3ccdc298 100644 --- a/src/query/formats/tests/it/output_format_utils.rs +++ b/src/query/formats/tests/it/output_format_utils.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::number::Float64Type; use databend_common_expression::types::number::Int32Type; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DateType; use databend_common_expression::types::NumberDataType; diff --git a/src/query/functions/Cargo.toml b/src/query/functions/Cargo.toml index a0a3d1872c8c..dfac09efd4e8 100644 --- a/src/query/functions/Cargo.toml +++ b/src/query/functions/Cargo.toml @@ -19,7 +19,7 @@ chrono = { workspace = true } chrono-tz = { workspace = true } crc32fast = { workspace = true } ctor = { workspace = true } -databend-common-arrow = { workspace = true } + databend-common-base = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_null_unary_adaptor.rs b/src/query/functions/src/aggregates/adaptors/aggregate_null_unary_adaptor.rs index 609a0c7a9a14..7afa226ca77b 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_null_unary_adaptor.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_null_unary_adaptor.rs @@ -16,8 +16,8 @@ use std::alloc::Layout; use std::fmt; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::utils::column_merge_validity; use databend_common_expression::ColumnBuilder; @@ -122,7 +122,7 @@ impl AggregateFunction for AggregateNullUnaryAdapto if validity .as_ref() - .map(|c| c.unset_bits() != input_rows) + .map(|c| c.null_count() != input_rows) .unwrap_or(true) { self.set_flag(place, 1); @@ -144,9 +144,9 @@ impl AggregateFunction for AggregateNullUnaryAdapto let not_null_columns = not_null_columns.into(); match validity { - Some(v) if v.unset_bits() > 0 => { + Some(v) if v.null_count() > 0 => { // all nulls - if v.unset_bits() == v.len() { + if v.null_count() == v.len() { return Ok(()); } @@ -177,9 +177,9 @@ impl AggregateFunction for AggregateNullUnaryAdapto let not_null_columns = not_null_columns.into(); match validity { - Some(v) if v.unset_bits() > 0 => { + Some(v) if v.null_count() > 0 => { // all nulls - if v.unset_bits() == v.len() { + if v.null_count() == v.len() { return Ok(()); } diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_null_variadic_adaptor.rs b/src/query/functions/src/aggregates/adaptors/aggregate_null_variadic_adaptor.rs index b02b2fdff89b..57209ec9c4b2 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_null_variadic_adaptor.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_null_variadic_adaptor.rs @@ -16,8 +16,8 @@ use std::alloc::Layout; use std::fmt; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::utils::column_merge_validity; use databend_common_expression::ColumnBuilder; @@ -127,7 +127,7 @@ impl AggregateFunction if validity .as_ref() - .map(|c| c.unset_bits() != input_rows) + .map(|c| c.null_count() != input_rows) .unwrap_or(true) { self.set_flag(place, 1); @@ -151,9 +151,9 @@ impl AggregateFunction let not_null_columns = (¬_null_columns).into(); match validity { - Some(v) if v.unset_bits() > 0 => { + Some(v) if v.null_count() > 0 => { // all nulls - if v.unset_bits() == v.len() { + if v.null_count() == v.len() { return Ok(()); } for (valid, (row, place)) in v.iter().zip(places.iter().enumerate()) { @@ -185,9 +185,9 @@ impl AggregateFunction let not_null_columns = (¬_null_columns).into(); match validity { - Some(v) if v.unset_bits() > 0 => { + Some(v) if v.null_count() > 0 => { // all nulls - if v.unset_bits() == v.len() { + if v.null_count() == v.len() { return Ok(()); } diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_ornull_adaptor.rs b/src/query/functions/src/aggregates/adaptors/aggregate_ornull_adaptor.rs index 95dcdd0a6d56..f6c1e915e793 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_ornull_adaptor.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_ornull_adaptor.rs @@ -16,8 +16,8 @@ use std::alloc::Layout; use std::fmt; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::InputColumns; @@ -120,7 +120,7 @@ impl AggregateFunction for AggregateFunctionOrNullAdaptor { if validity .as_ref() - .map(|c| c.unset_bits() != input_rows) + .map(|c| c.null_count() != input_rows) .unwrap_or(true) { self.set_flag(place, 1); @@ -142,9 +142,9 @@ impl AggregateFunction for AggregateFunctionOrNullAdaptor { let if_cond = self.inner.get_if_condition(columns); match if_cond { - Some(v) if v.unset_bits() > 0 => { + Some(v) if v.null_count() > 0 => { // all nulls - if v.unset_bits() == v.len() { + if v.null_count() == v.len() { return Ok(()); } diff --git a/src/query/functions/src/aggregates/aggregate_arg_min_max.rs b/src/query/functions/src/aggregates/aggregate_arg_min_max.rs index 215036cee4b1..289512ccdceb 100644 --- a/src/query/functions/src/aggregates/aggregate_arg_min_max.rs +++ b/src/query/functions/src/aggregates/aggregate_arg_min_max.rs @@ -19,10 +19,10 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::number::*; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::*; use databend_common_expression::with_number_mapped_type; use databend_common_expression::ColumnBuilder; @@ -117,7 +117,7 @@ where return Ok(()); } let acc = if let Some(bit) = validity { - if bit.unset_bits() == column_len { + if bit.null_count() == column_len { return Ok(()); } diff --git a/src/query/functions/src/aggregates/aggregate_array_agg.rs b/src/query/functions/src/aggregates/aggregate_array_agg.rs index 609bfad92800..23f07b011290 100644 --- a/src/query/functions/src/aggregates/aggregate_array_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_array_agg.rs @@ -19,10 +19,10 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; use databend_common_expression::types::decimal::*; use databend_common_expression::types::number::*; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::ValueType; use databend_common_expression::types::*; diff --git a/src/query/functions/src/aggregates/aggregate_array_moving.rs b/src/query/functions/src/aggregates/aggregate_array_moving.rs index 3ca323a35536..39fc115fd60e 100644 --- a/src/query/functions/src/aggregates/aggregate_array_moving.rs +++ b/src/query/functions/src/aggregates/aggregate_array_moving.rs @@ -19,14 +19,14 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::check_number; use databend_common_expression::types::decimal::*; use databend_common_expression::types::number::Number; use databend_common_expression::types::ArgType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::Float64Type; diff --git a/src/query/functions/src/aggregates/aggregate_bitmap.rs b/src/query/functions/src/aggregates/aggregate_bitmap.rs index b0317d4a83ce..1713d1d02dc2 100644 --- a/src/query/functions/src/aggregates/aggregate_bitmap.rs +++ b/src/query/functions/src/aggregates/aggregate_bitmap.rs @@ -22,12 +22,12 @@ use std::ops::BitXorAssign; use std::ops::SubAssign; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::check_number; use databend_common_expression::types::decimal::DecimalType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::*; use databend_common_expression::with_number_mapped_type; use databend_common_expression::ColumnBuilder; @@ -230,7 +230,7 @@ where _input_rows: usize, ) -> Result<()> { let column = BitmapType::try_downcast_column(&columns[0]).unwrap(); - if column.len() == 0 { + if column.is_empty() { return Ok(()); } @@ -238,7 +238,7 @@ where let state = place.get::(); if let Some(validity) = validity { - if validity.unset_bits() == column.len() { + if validity.null_count() == column.len() { return Ok(()); } @@ -406,7 +406,7 @@ where } fn filter_place(places: &[StateAddr], predicate: &Bitmap) -> StateAddrs { - if predicate.unset_bits() == 0 { + if predicate.null_count() == 0 { return places.to_vec(); } let it = predicate @@ -468,7 +468,7 @@ where let new_places = Self::filter_place(places, &predicate); let new_places_slice = new_places.as_slice(); - let row_size = predicate.len() - predicate.unset_bits(); + let row_size = predicate.len() - predicate.null_count(); let input = [column]; self.inner diff --git a/src/query/functions/src/aggregates/aggregate_combinator_distinct.rs b/src/query/functions/src/aggregates/aggregate_combinator_distinct.rs index 81d2aadd6e44..c3bdd668f498 100644 --- a/src/query/functions/src/aggregates/aggregate_combinator_distinct.rs +++ b/src/query/functions/src/aggregates/aggregate_combinator_distinct.rs @@ -17,9 +17,9 @@ use std::fmt; use std::marker::PhantomData; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; use databend_common_expression::types::number::NumberColumnBuilder; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::with_number_mapped_type; diff --git a/src/query/functions/src/aggregates/aggregate_combinator_if.rs b/src/query/functions/src/aggregates/aggregate_combinator_if.rs index 9dd1971b889b..490976762be1 100644 --- a/src/query/functions/src/aggregates/aggregate_combinator_if.rs +++ b/src/query/functions/src/aggregates/aggregate_combinator_if.rs @@ -16,9 +16,9 @@ use std::alloc::Layout; use std::fmt; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; use databend_common_expression::types::ValueType; @@ -196,13 +196,13 @@ impl AggregateIfCombinator { .map(|c| c.filter(predicate)) .collect::>(); - let rows = predicate.len() - predicate.unset_bits(); + let rows = predicate.len() - predicate.null_count(); (columns, rows) } fn filter_place(places: &[StateAddr], predicate: &Bitmap) -> StateAddrs { - if predicate.unset_bits() == 0 { + if predicate.null_count() == 0 { return places.to_vec(); } let it = predicate diff --git a/src/query/functions/src/aggregates/aggregate_combinator_state.rs b/src/query/functions/src/aggregates/aggregate_combinator_state.rs index 413a8b2293ce..5163b0dee77b 100644 --- a/src/query/functions/src/aggregates/aggregate_combinator_state.rs +++ b/src/query/functions/src/aggregates/aggregate_combinator_state.rs @@ -16,8 +16,8 @@ use std::alloc::Layout; use std::fmt; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::InputColumns; diff --git a/src/query/functions/src/aggregates/aggregate_count.rs b/src/query/functions/src/aggregates/aggregate_count.rs index c3eec671e6ac..7cabda3a9d1a 100644 --- a/src/query/functions/src/aggregates/aggregate_count.rs +++ b/src/query/functions/src/aggregates/aggregate_count.rs @@ -16,9 +16,9 @@ use std::alloc::Layout; use std::fmt; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; use databend_common_expression::types::number::NumberColumnBuilder; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::utils::column_merge_validity; @@ -93,14 +93,14 @@ impl AggregateFunction for AggregateCountFunction { ) -> Result<()> { let state = place.get::(); let nulls = if columns.is_empty() { - validity.map(|v| v.unset_bits()).unwrap_or(0) + validity.map(|v| v.null_count()).unwrap_or(0) } else { match &columns[0] { Column::Nullable(c) => validity .map(|v| v & (&c.validity)) .unwrap_or_else(|| c.validity.clone()) - .unset_bits(), - _ => validity.map(|v| v.unset_bits()).unwrap_or(0), + .null_count(), + _ => validity.map(|v| v.null_count()).unwrap_or(0), } }; state.count += (input_rows - nulls) as u64; @@ -121,7 +121,7 @@ impl AggregateFunction for AggregateCountFunction { match validity { Some(v) => { // all nulls - if v.unset_bits() == v.len() { + if v.null_count() == v.len() { return Ok(()); } for (valid, place) in v.iter().zip(places.iter()) { diff --git a/src/query/functions/src/aggregates/aggregate_covariance.rs b/src/query/functions/src/aggregates/aggregate_covariance.rs index dd4c99e9429b..230b964b24b1 100644 --- a/src/query/functions/src/aggregates/aggregate_covariance.rs +++ b/src/query/functions/src/aggregates/aggregate_covariance.rs @@ -19,11 +19,11 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::number::Number; use databend_common_expression::types::number::F64; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; diff --git a/src/query/functions/src/aggregates/aggregate_distinct_state.rs b/src/query/functions/src/aggregates/aggregate_distinct_state.rs index 4db1b13b5b2f..fc03157245b5 100644 --- a/src/query/functions/src/aggregates/aggregate_distinct_state.rs +++ b/src/query/functions/src/aggregates/aggregate_distinct_state.rs @@ -23,12 +23,12 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; use bumpalo::Bump; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::Result; use databend_common_expression::types::number::Number; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::AnyType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; diff --git a/src/query/functions/src/aggregates/aggregate_json_array_agg.rs b/src/query/functions/src/aggregates/aggregate_json_array_agg.rs index 937d10e46f52..b65f7c321b8a 100644 --- a/src/query/functions/src/aggregates/aggregate_json_array_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_json_array_agg.rs @@ -19,10 +19,10 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; use databend_common_expression::date_helper::TzLUT; use databend_common_expression::types::variant::cast_scalar_to_variant; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::ValueType; use databend_common_expression::types::*; diff --git a/src/query/functions/src/aggregates/aggregate_json_object_agg.rs b/src/query/functions/src/aggregates/aggregate_json_object_agg.rs index 6acbd1800738..ac4eaeb6eff6 100644 --- a/src/query/functions/src/aggregates/aggregate_json_object_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_json_object_agg.rs @@ -20,13 +20,12 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::date_helper::TzLUT; use databend_common_expression::types::string::StringColumn; use databend_common_expression::types::variant::cast_scalar_to_variant; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::ValueType; use databend_common_expression::types::*; @@ -364,7 +363,7 @@ where }; let validity = match (key_validity, val_validity) { (Some(key_validity), Some(val_validity)) => { - let and_validity = bitmap::and(&key_validity, &val_validity); + let and_validity = boolean::and(&key_validity, &val_validity); Some(and_validity) } (Some(key_validity), None) => Some(key_validity.clone()), diff --git a/src/query/functions/src/aggregates/aggregate_min_max_any.rs b/src/query/functions/src/aggregates/aggregate_min_max_any.rs index bbe7325b93d2..cc8a975d6d1d 100644 --- a/src/query/functions/src/aggregates/aggregate_min_max_any.rs +++ b/src/query/functions/src/aggregates/aggregate_min_max_any.rs @@ -17,11 +17,11 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::decimal::*; use databend_common_expression::types::number::*; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::*; use databend_common_expression::with_number_mapped_type; use databend_common_expression::Scalar; @@ -98,7 +98,7 @@ where C: ChangeIf + Default let column_iter = 0..other.len(); if let Some(validity) = validity { - if validity.unset_bits() == column_len { + if validity.null_count() == column_len { return Ok(()); } let v = column_iter @@ -213,7 +213,7 @@ where let column_iter = T::iter_column(&other); if let Some(validity) = validity { - if validity.unset_bits() == column_len { + if validity.null_count() == column_len { return Ok(()); } for (data, valid) in column_iter.zip(validity.iter()) { diff --git a/src/query/functions/src/aggregates/aggregate_null_result.rs b/src/query/functions/src/aggregates/aggregate_null_result.rs index b3f67f779413..522e6dd4824a 100644 --- a/src/query/functions/src/aggregates/aggregate_null_result.rs +++ b/src/query/functions/src/aggregates/aggregate_null_result.rs @@ -16,9 +16,9 @@ use std::alloc::Layout; use std::fmt; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; use databend_common_expression::types::AnyType; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::ValueType; use databend_common_expression::ColumnBuilder; diff --git a/src/query/functions/src/aggregates/aggregate_quantile_tdigest.rs b/src/query/functions/src/aggregates/aggregate_quantile_tdigest.rs index fa03827aa9e6..0cd2ab611086 100644 --- a/src/query/functions/src/aggregates/aggregate_quantile_tdigest.rs +++ b/src/query/functions/src/aggregates/aggregate_quantile_tdigest.rs @@ -21,11 +21,11 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::check_number; use databend_common_expression::types::number::*; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::*; use databend_common_expression::with_number_mapped_type; use databend_common_expression::ColumnBuilder; diff --git a/src/query/functions/src/aggregates/aggregate_quantile_tdigest_weighted.rs b/src/query/functions/src/aggregates/aggregate_quantile_tdigest_weighted.rs index c6f3dbaf6b63..c46226ec64a9 100644 --- a/src/query/functions/src/aggregates/aggregate_quantile_tdigest_weighted.rs +++ b/src/query/functions/src/aggregates/aggregate_quantile_tdigest_weighted.rs @@ -18,11 +18,11 @@ use std::fmt::Formatter; use std::marker::PhantomData; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::check_number; use databend_common_expression::types::number::*; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::*; use databend_common_expression::with_number_mapped_type; use databend_common_expression::with_unsigned_integer_mapped_type; diff --git a/src/query/functions/src/aggregates/aggregate_retention.rs b/src/query/functions/src/aggregates/aggregate_retention.rs index 98380e69857a..d3d2d427beb9 100644 --- a/src/query/functions/src/aggregates/aggregate_retention.rs +++ b/src/query/functions/src/aggregates/aggregate_retention.rs @@ -18,9 +18,9 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; diff --git a/src/query/functions/src/aggregates/aggregate_scalar_state.rs b/src/query/functions/src/aggregates/aggregate_scalar_state.rs index 26ccc51d1f91..9150a14bd637 100644 --- a/src/query/functions/src/aggregates/aggregate_scalar_state.rs +++ b/src/query/functions/src/aggregates/aggregate_scalar_state.rs @@ -17,8 +17,8 @@ use std::marker::PhantomData; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::ValueType; use databend_common_expression::ColumnBuilder; @@ -206,7 +206,7 @@ where } if let Some(validity) = validity { - if validity.unset_bits() == column_len { + if validity.null_count() == column_len { return Ok(()); } diff --git a/src/query/functions/src/aggregates/aggregate_string_agg.rs b/src/query/functions/src/aggregates/aggregate_string_agg.rs index 3bb9259d11a9..2dc27dff33c9 100644 --- a/src/query/functions/src/aggregates/aggregate_string_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_string_agg.rs @@ -18,9 +18,9 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::StringType; use databend_common_expression::types::ValueType; diff --git a/src/query/functions/src/aggregates/aggregate_sum.rs b/src/query/functions/src/aggregates/aggregate_sum.rs index 355d8dfa8a41..5dc570f72ece 100644 --- a/src/query/functions/src/aggregates/aggregate_sum.rs +++ b/src/query/functions/src/aggregates/aggregate_sum.rs @@ -14,12 +14,12 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::decimal::*; use databend_common_expression::types::number::*; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; use databend_common_expression::types::*; use databend_common_expression::utils::arithmetics_type::ResultTypeOfUnary; use databend_common_expression::with_number_mapped_type; @@ -89,7 +89,7 @@ where TSum: Number + std::ops::AddAssign, { match validity { - Some(v) if v.unset_bits() > 0 => { + Some(v) if v.null_count() > 0 => { let mut sum = TSum::default(); inner.iter().zip(v.iter()).for_each(|(t, b)| { if b { diff --git a/src/query/functions/src/aggregates/aggregate_unary.rs b/src/query/functions/src/aggregates/aggregate_unary.rs index 5ac0bc5a4d22..66aaa29e2ff1 100644 --- a/src/query/functions/src/aggregates/aggregate_unary.rs +++ b/src/query/functions/src/aggregates/aggregate_unary.rs @@ -19,9 +19,9 @@ use std::fmt::Formatter; use std::marker::PhantomData; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_base::base::take_mut; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalSize; use databend_common_expression::types::ValueType; diff --git a/src/query/functions/src/aggregates/aggregate_window_funnel.rs b/src/query/functions/src/aggregates/aggregate_window_funnel.rs index 4fc76b93872d..88ab9c04cf4c 100644 --- a/src/query/functions/src/aggregates/aggregate_window_funnel.rs +++ b/src/query/functions/src/aggregates/aggregate_window_funnel.rs @@ -21,13 +21,13 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::check_number; use databend_common_expression::types::number::Number; use databend_common_expression::types::number::UInt8Type; use databend_common_expression::types::ArgType; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; diff --git a/src/query/functions/src/scalars/arithmetic.rs b/src/query/functions/src/scalars/arithmetic.rs index 081cba3653d4..a004d95855c7 100644 --- a/src/query/functions/src/scalars/arithmetic.rs +++ b/src/query/functions/src/scalars/arithmetic.rs @@ -20,7 +20,6 @@ use std::ops::BitXor; use std::str::FromStr; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_expression::serialize::read_decimal_with_size; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::decimal::DecimalDomain; @@ -32,6 +31,7 @@ use databend_common_expression::types::number::NumberType; use databend_common_expression::types::number::F64; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::AnyType; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::NullableType; diff --git a/src/query/functions/src/scalars/binary.rs b/src/query/functions/src/scalars/binary.rs index 1e865888b98b..9bc5489f0d46 100644 --- a/src/query/functions/src/scalars/binary.rs +++ b/src/query/functions/src/scalars/binary.rs @@ -15,7 +15,6 @@ use std::io::Write; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_expression::error_to_null; use databend_common_expression::passthrough_nullable; use databend_common_expression::types::binary::BinaryColumn; @@ -25,6 +24,7 @@ use databend_common_expression::types::string::StringColumn; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::AnyType; use databend_common_expression::types::BinaryType; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; @@ -89,7 +89,7 @@ pub fn register(registry: &mut FunctionRegistry) { "to_hex", |_, _| FunctionDomain::Full, vectorize_binary_to_string( - |col| col.current_buffer_len() * 2, + |col| col.total_bytes_len() * 2, |val, output, _| { let extra_len = val.len() * 2; output.row_buffer.resize(extra_len, 0); @@ -115,7 +115,7 @@ pub fn register(registry: &mut FunctionRegistry) { "to_base64", |_, _| FunctionDomain::Full, vectorize_binary_to_string( - |col| col.current_buffer_len() * 4 / 3 + col.len() * 4, + |col| col.total_bytes_len() * 4 / 3 + col.len() * 4, |val, output, _| { base64::write::EncoderWriter::new( &mut output.row_buffer, @@ -190,7 +190,7 @@ pub fn register(registry: &mut FunctionRegistry) { fn eval_binary_to_string(val: ValueRef, ctx: &mut EvalContext) -> Value { vectorize_binary_to_string( - |col| col.current_buffer_len(), + |col| col.total_bytes_len(), |val, output, ctx| { if let Ok(val) = simdutf8::basic::from_utf8(val) { output.put_str(val); @@ -204,7 +204,7 @@ fn eval_binary_to_string(val: ValueRef, ctx: &mut EvalContext) -> Va fn eval_unhex(val: ValueRef, ctx: &mut EvalContext) -> Value { vectorize_string_to_binary( - |col| col.current_buffer_len() / 2, + |col| col.total_bytes_len() / 2, |val, output, ctx| { let old_len = output.data.len(); let extra_len = val.len() / 2; @@ -219,7 +219,7 @@ fn eval_unhex(val: ValueRef, ctx: &mut EvalContext) -> Value, ctx: &mut EvalContext) -> Value { vectorize_string_to_binary( - |col| col.current_buffer_len() * 4 / 3 + col.len() * 4, + |col| col.total_bytes_len() * 4 / 3 + col.len() * 4, |val, output, ctx| { if let Err(err) = base64::Engine::decode_vec( &base64::engine::general_purpose::STANDARD, diff --git a/src/query/functions/src/scalars/boolean.rs b/src/query/functions/src/scalars/boolean.rs index 4235e6fa47e0..fa8a3b237746 100644 --- a/src/query/functions/src/scalars/boolean.rs +++ b/src/query/functions/src/scalars/boolean.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use databend_common_base::base::OrderedFloat; use databend_common_expression::error_to_null; +use databend_common_expression::types::boolean; use databend_common_expression::types::boolean::BooleanDomain; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::nullable::NullableDomain; @@ -224,9 +225,7 @@ pub fn register(registry: &mut FunctionRegistry) { (ValueRef::Scalar(true), ValueRef::Column(other)) | (ValueRef::Column(other), ValueRef::Scalar(true)) => Value::Column(!&other), (ValueRef::Scalar(false), other) | (other, ValueRef::Scalar(false)) => other.to_owned(), - (ValueRef::Column(a), ValueRef::Column(b)) => { - Value::Column(databend_common_arrow::arrow::bitmap::xor(&a, &b)) - } + (ValueRef::Column(a), ValueRef::Column(b)) => Value::Column(boolean::xor(&a, &b)), }, ); diff --git a/src/query/functions/src/scalars/comparison.rs b/src/query/functions/src/scalars/comparison.rs index 8346066d128e..1f6a200f57c6 100644 --- a/src/query/functions/src/scalars/comparison.rs +++ b/src/query/functions/src/scalars/comparison.rs @@ -16,7 +16,6 @@ use std::cmp::Ordering; use std::collections::HashMap; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_expression::generate_like_pattern; use databend_common_expression::types::boolean::BooleanDomain; use databend_common_expression::types::string::StringDomain; @@ -28,6 +27,7 @@ use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; use databend_common_expression::types::EmptyArrayType; use databend_common_expression::types::GenericType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberClass; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringColumn; @@ -589,7 +589,7 @@ fn vectorize_like( (ValueRef::Column(arg1), ValueRef::Scalar(arg2)) => { let arg1_iter = StringType::iter_column(&arg1); let mut builder = MutableBitmap::with_capacity(arg1.len()); - let pattern_type = generate_like_pattern(arg2.as_bytes(), arg1.current_buffer_len()); + let pattern_type = generate_like_pattern(arg2.as_bytes(), arg1.total_bytes_len()); if let LikePattern::SurroundByPercent(searcher) = pattern_type { for arg1 in arg1_iter { builder.push(searcher.search(arg1.as_bytes()).is_some()); @@ -636,7 +636,7 @@ fn variant_vectorize_like( (ValueRef::Column(arg1), ValueRef::Scalar(arg2)) => { let arg1_iter = VariantType::iter_column(&arg1); - let pattern_type = generate_like_pattern(arg2.as_bytes(), arg1.current_buffer_len()); + let pattern_type = generate_like_pattern(arg2.as_bytes(), arg1.total_bytes_len()); let mut builder = MutableBitmap::with_capacity(arg1.len()); for arg1 in arg1_iter { builder.push(func(arg1, &pattern_type)); diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index b4ed0508f772..730f4fa6c9af 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -22,8 +22,6 @@ use chrono::Datelike; use chrono::Duration; use chrono::MappedLocalTime; use chrono_tz::Tz; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; use databend_common_exception::ErrorCode; use databend_common_expression::error_to_null; use databend_common_expression::types::date::clamp_date; @@ -45,6 +43,7 @@ use databend_common_expression::types::timestamp::string_to_timestamp; use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_expression::types::timestamp::MICROS_PER_MILLI; use databend_common_expression::types::timestamp::MICROS_PER_SEC; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DateType; use databend_common_expression::types::Float64Type; use databend_common_expression::types::Int32Type; @@ -54,6 +53,7 @@ use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::F64; use databend_common_expression::utils::date_helper::*; +use databend_common_expression::utils::serialize::EPOCH_DAYS_FROM_CE; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_1_arg; diff --git a/src/query/functions/src/scalars/string_multi_args.rs b/src/query/functions/src/scalars/string_multi_args.rs index cb5547b9ae7f..11349c95b43e 100644 --- a/src/query/functions/src/scalars/string_multi_args.rs +++ b/src/query/functions/src/scalars/string_multi_args.rs @@ -14,12 +14,12 @@ use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_expression::passthrough_nullable; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::number::Int64Type; use databend_common_expression::types::number::NumberScalar; use databend_common_expression::types::string::StringDomain; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberColumn; use databend_common_expression::types::*; use databend_common_expression::Column; diff --git a/src/query/functions/src/scalars/variant.rs b/src/query/functions/src/scalars/variant.rs index 79e156ed8cb3..cd7c66552a9b 100644 --- a/src/query/functions/src/scalars/variant.rs +++ b/src/query/functions/src/scalars/variant.rs @@ -20,9 +20,6 @@ use std::sync::Arc; use bstr::ByteSlice; use chrono::Datelike; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::date::string_to_date; use databend_common_expression::types::nullable::NullableColumn; @@ -35,10 +32,12 @@ use databend_common_expression::types::variant::cast_scalar_to_variant; use databend_common_expression::types::variant::cast_scalars_to_variants; use databend_common_expression::types::AnyType; use databend_common_expression::types::ArrayType; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; use databend_common_expression::types::GenericType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; @@ -46,6 +45,7 @@ use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::VariantType; use databend_common_expression::types::ALL_NUMERICS_TYPES; +use databend_common_expression::utils::serialize::EPOCH_DAYS_FROM_CE; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; diff --git a/src/query/functions/src/scalars/vector.rs b/src/query/functions/src/scalars/vector.rs index f64032e5ca9a..bec069210957 100644 --- a/src/query/functions/src/scalars/vector.rs +++ b/src/query/functions/src/scalars/vector.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_expression::types::ArrayType; +use databend_common_expression::types::Buffer; use databend_common_expression::types::Float32Type; use databend_common_expression::types::Float64Type; use databend_common_expression::types::StringType; diff --git a/src/query/functions/src/srfs/variant.rs b/src/query/functions/src/srfs/variant.rs index f6d73b140577..7344e0709814 100644 --- a/src/query/functions/src/srfs/variant.rs +++ b/src/query/functions/src/srfs/variant.rs @@ -15,13 +15,13 @@ use std::collections::BTreeMap; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::AnyType; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberDataType; diff --git a/src/query/functions/tests/it/aggregates/testdata/agg.txt b/src/query/functions/tests/it/aggregates/testdata/agg.txt index 777b5b540fa0..3f2f302db43a 100644 --- a/src/query/functions/tests/it/aggregates/testdata/agg.txt +++ b/src/query/functions/tests/it/aggregates/testdata/agg.txt @@ -941,12 +941,12 @@ evaluation (internal): ast: array_agg('a') evaluation (internal): -+--------+-------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------+ -| a | Int64([4, 3, 2, 1]) | -| Output | ArrayColumn { values: StringColumn { data: Utf8ViewArray[a, a, a, a] }, offsets: [0, 4] } | -+--------+-------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------+ +| a | Int64([4, 3, 2, 1]) | +| Output | ArrayColumn { values: StringColumn[a, a, a, a], offsets: [0, 4] } | ++--------+-------------------------------------------------------------------+ ast: array_agg(NULL) @@ -1031,42 +1031,42 @@ evaluation (internal): ast: string_agg(s) evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abcdefopqxyz] }, validity: [0b_______1] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------+ +| s | StringColumn[abc, def, opq, xyz] | +| Output | NullableColumn { column: StringColumn[abcdefopqxyz], validity: [0b_______1] } | ++--------+-------------------------------------------------------------------------------+ ast: string_agg(s_null) evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------+ -| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[acd] }, validity: [0b_______1] } | -+--------+----------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------+ +| s_null | NullableColumn { column: StringColumn[a, , c, d], validity: [0b____1101] } | +| Output | NullableColumn { column: StringColumn[acd], validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------+ ast: string_agg(s, '|') evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc|def|opq|xyz] }, validity: [0b_______1] } | -+--------+----------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------+ +| s | StringColumn[abc, def, opq, xyz] | +| Output | NullableColumn { column: StringColumn[abc|def|opq|xyz], validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------------+ ast: string_agg(s_null, '-') evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------+ -| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a-c-d] }, validity: [0b_______1] } | -+--------+----------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------+ +| s_null | NullableColumn { column: StringColumn[a, , c, d], validity: [0b____1101] } | +| Output | NullableColumn { column: StringColumn[a-c-d], validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------+ ast: bitmap_and_count(bm) @@ -1317,42 +1317,42 @@ evaluation (internal): ast: histogram(all_null) evaluation (internal): -+----------+-------------------------------------------------------------------------------------------+ -| Column | Data | -+----------+-------------------------------------------------------------------------------------------+ -| all_null | NullableColumn { column: UInt64([1, 2, 3, 4]), validity: [0b____0000] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[] }, validity: [0b_______0] } | -+----------+-------------------------------------------------------------------------------------------+ ++----------+-------------------------------------------------------------------------+ +| Column | Data | ++----------+-------------------------------------------------------------------------+ +| all_null | NullableColumn { column: UInt64([1, 2, 3, 4]), validity: [0b____0000] } | +| Output | NullableColumn { column: StringColumn[], validity: [0b_______0] } | ++----------+-------------------------------------------------------------------------+ ast: histogram(x_null) evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| x_null | NullableColumn { column: UInt64([1, 2, 3, 4]), validity: [0b____0011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1}]] }, validity: [0b_______1] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| x_null | NullableColumn { column: UInt64([1, 2, 3, 4]), validity: [0b____0011] } | +| Output | NullableColumn { column: StringColumn[[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1}]], validity: [0b_______1] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast: histogram(a) evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | Int64([4, 3, 2, 1]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1},{"lower":"3","upper":"3","ndv":1,"count":1,"pre_sum":2},{"lower":"4","upper":"4","ndv":1,"count":1,"pre_sum":3}]] }, validity: [0b_______1] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | Int64([4, 3, 2, 1]) | +| Output | NullableColumn { column: StringColumn[[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1},{"lower":"3","upper":"3","ndv":1,"count":1,"pre_sum":2},{"lower":"4","upper":"4","ndv":1,"count":1,"pre_sum":3}]], validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast: histogram(a, 1) evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | Int64([4, 3, 2, 1]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[[{"lower":"1","upper":"4","ndv":4,"count":4,"pre_sum":0}]] }, validity: [0b_______1] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------+ +| a | Int64([4, 3, 2, 1]) | +| Output | NullableColumn { column: StringColumn[[{"lower":"1","upper":"4","ndv":4,"count":4,"pre_sum":0}]], validity: [0b_______1] } | ++--------+----------------------------------------------------------------------------------------------------------------------------+ ast: json_array_agg(1) @@ -1463,7 +1463,7 @@ evaluation (internal): | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ | a | Int64([4, 3, 2, 1]) | -| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | +| s | StringColumn[abc, def, opq, xyz] | | Output | BinaryColumn { data: 0x4000000410000003100000031000000310000003200000022000000220000002200000026162636465666f707178797a4004400340024001, offsets: [0, 56] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1474,7 +1474,7 @@ evaluation (internal): | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------+ | b | UInt64([1, 2, 3, 4]) | -| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | +| s_null | NullableColumn { column: StringColumn[a, , c, d], validity: [0b____1101] } | | Output | BinaryColumn { data: 0x40000003100000011000000110000001200000022000000220000002616364500150035004, offsets: [0, 37] } | +--------+-----------------------------------------------------------------------------------------------------------------------+ @@ -1486,7 +1486,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | +| s | StringColumn[abc, def, opq, xyz] | | dec | NullableColumn { column: Decimal128([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | | Output | BinaryColumn { data: 0x4000000310000003100000031000000320000009200000092000000961626364656678797a603ff199999999999a60400199999999999a60400a666666666666, offsets: [0, 64] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt b/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt index 1b69c2840d76..8195a1b83138 100644 --- a/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt +++ b/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt @@ -919,12 +919,12 @@ evaluation (internal): ast: array_agg('a') evaluation (internal): -+--------+----------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------+ -| a | Int64([4, 3, 2, 1]) | -| Output | ArrayColumn { values: StringColumn { data: Utf8ViewArray[a, a, a, a] }, offsets: [0, 2, 4] } | -+--------+----------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| a | Int64([4, 3, 2, 1]) | +| Output | ArrayColumn { values: StringColumn[a, a, a, a], offsets: [0, 2, 4] } | ++--------+----------------------------------------------------------------------+ ast: array_agg(NULL) @@ -1009,42 +1009,42 @@ evaluation (internal): ast: string_agg(s) evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abcopq, defxyz] }, validity: [0b______11] } | -+--------+---------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------+ +| s | StringColumn[abc, def, opq, xyz] | +| Output | NullableColumn { column: StringColumn[abcopq, defxyz], validity: [0b______11] } | ++--------+---------------------------------------------------------------------------------+ ast: string_agg(s_null) evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------+ -| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[ac, d] }, validity: [0b______11] } | -+--------+----------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------+ +| s_null | NullableColumn { column: StringColumn[a, , c, d], validity: [0b____1101] } | +| Output | NullableColumn { column: StringColumn[ac, d], validity: [0b______11] } | ++--------+----------------------------------------------------------------------------+ ast: string_agg(s, '|') evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc|opq, def|xyz] }, validity: [0b______11] } | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------+ +| s | StringColumn[abc, def, opq, xyz] | +| Output | NullableColumn { column: StringColumn[abc|opq, def|xyz], validity: [0b______11] } | ++--------+-----------------------------------------------------------------------------------+ ast: string_agg(s_null, '-') evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------+ -| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a-c, d] }, validity: [0b______11] } | -+--------+----------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------+ +| s_null | NullableColumn { column: StringColumn[a, , c, d], validity: [0b____1101] } | +| Output | NullableColumn { column: StringColumn[a-c, d], validity: [0b______11] } | ++--------+----------------------------------------------------------------------------+ ast: bitmap_and_count(bm) @@ -1401,7 +1401,7 @@ evaluation (internal): | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ | a | Int64([4, 3, 2, 1]) | -| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | +| s | StringColumn[abc, def, opq, xyz] | | Output | BinaryColumn { data: 0x4000000410000003100000031000000310000003200000022000000220000002200000026162636465666f707178797a4004400340024001, offsets: [0, 56] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1412,7 +1412,7 @@ evaluation (internal): | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------+ | b | UInt64([1, 2, 3, 4]) | -| s_null | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, , c, d] }, validity: [0b____1101] } | +| s_null | NullableColumn { column: StringColumn[a, , c, d], validity: [0b____1101] } | | Output | BinaryColumn { data: 0x40000003100000011000000110000001200000022000000220000002616364500150035004, offsets: [0, 37] } | +--------+-----------------------------------------------------------------------------------------------------------------------+ @@ -1424,7 +1424,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[abc, def, opq, xyz] } | +| s | StringColumn[abc, def, opq, xyz] | | dec | NullableColumn { column: Decimal128([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | | Output | BinaryColumn { data: 0x4000000310000003100000031000000320000009200000092000000961626364656678797a603ff199999999999a60400199999999999a60400a666666666666, offsets: [0, 64] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/mod.rs b/src/query/functions/tests/it/scalars/mod.rs index 5306a9a707e2..918e29b767b2 100644 --- a/src/query/functions/tests/it/scalars/mod.rs +++ b/src/query/functions/tests/it/scalars/mod.rs @@ -222,8 +222,8 @@ pub fn run_ast(file: &mut impl Write, text: impl AsRef, columns: &[(&str, C } fn test_arrow_conversion(col: &Column) { - let arrow_col = col.as_arrow(); - let new_col = Column::from_arrow(&*arrow_col, &col.data_type()).unwrap(); + let arrow_col = col.clone().into_arrow_rs(); + let new_col = Column::from_arrow_rs(arrow_col, &col.data_type()).unwrap(); assert_eq!(col, &new_col, "arrow conversion went wrong"); } diff --git a/src/query/functions/tests/it/scalars/testdata/arithmetic.txt b/src/query/functions/tests/it/scalars/testdata/arithmetic.txt index 281ddc8e843d..0b34995caf9a 100644 --- a/src/query/functions/tests/it/scalars/testdata/arithmetic.txt +++ b/src/query/functions/tests/it/scalars/testdata/arithmetic.txt @@ -1617,12 +1617,12 @@ evaluation: | Row 2 | 3 | '3' | +--------+---------+--------+ evaluation (internal): -+--------+-----------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------+ -| a | Int8([1, 2, 3]) | -| Output | StringColumn { data: Utf8ViewArray[1, 2, 3] } | -+--------+-----------------------------------------------+ ++--------+-----------------------+ +| Column | Data | ++--------+-----------------------+ +| a | Int8([1, 2, 3]) | +| Output | StringColumn[1, 2, 3] | ++--------+-----------------------+ ast : to_string(a2) @@ -1639,12 +1639,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+------------------+-----------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 2, 3] }, validity: [0b_____011] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn[1, 2, 3], validity: [0b_____011] } | ++--------+--------------------------------------------------------------------------+ ast : to_string(b) @@ -1661,12 +1661,12 @@ evaluation: | Row 2 | 6 | '6' | +--------+---------+--------+ evaluation (internal): -+--------+-----------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------+ -| b | Int16([2, 4, 6]) | -| Output | StringColumn { data: Utf8ViewArray[2, 4, 6] } | -+--------+-----------------------------------------------+ ++--------+-----------------------+ +| Column | Data | ++--------+-----------------------+ +| b | Int16([2, 4, 6]) | +| Output | StringColumn[2, 4, 6] | ++--------+-----------------------+ ast : to_string(c) @@ -1683,12 +1683,12 @@ evaluation: | Row 2 | 30 | '30' | +--------+-----------+--------+ evaluation (internal): -+--------+--------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------+ -| c | UInt32([10, 20, 30]) | -| Output | StringColumn { data: Utf8ViewArray[10, 20, 30] } | -+--------+--------------------------------------------------+ ++--------+--------------------------+ +| Column | Data | ++--------+--------------------------+ +| c | UInt32([10, 20, 30]) | +| Output | StringColumn[10, 20, 30] | ++--------+--------------------------+ ast : to_string(d) @@ -1705,12 +1705,12 @@ evaluation: | Row 2 | 30 | '30' | +--------+------------+--------+ evaluation (internal): -+--------+---------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------+ -| d | Float64([10, -20, 30]) | -| Output | StringColumn { data: Utf8ViewArray[10, -20, 30] } | -+--------+---------------------------------------------------+ ++--------+---------------------------+ +| Column | Data | ++--------+---------------------------+ +| d | Float64([10, -20, 30]) | +| Output | StringColumn[10, -20, 30] | ++--------+---------------------------+ ast : to_string(d2) @@ -1727,12 +1727,12 @@ evaluation: | Row 2 | 3 | '3' | +--------+------------------+-----------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| d2 | NullableColumn { column: UInt8([1, 0, 3]), validity: [0b_____101] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 0, 3] }, validity: [0b_____101] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| d2 | NullableColumn { column: UInt8([1, 0, 3]), validity: [0b_____101] } | +| Output | NullableColumn { column: StringColumn[1, 0, 3], validity: [0b_____101] } | ++--------+--------------------------------------------------------------------------+ ast : to_string(e) @@ -1749,12 +1749,12 @@ evaluation: | Row 2 | 188.8 | '188.8' | +--------+----------------+---------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| e | Decimal128([3.1, 33.5, 188.8]) | -| Output | StringColumn { data: Utf8ViewArray[3.1, 33.5, 188.8] } | -+--------+--------------------------------------------------------+ ++--------+--------------------------------+ +| Column | Data | ++--------+--------------------------------+ +| e | Decimal128([3.1, 33.5, 188.8]) | +| Output | StringColumn[3.1, 33.5, 188.8] | ++--------+--------------------------------+ ast : to_string(f) @@ -1771,12 +1771,12 @@ evaluation: | Row 2 | 12.34 | '12.34' | +--------+----------------+---------+ evaluation (internal): -+--------+---------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------+ -| f | Decimal256([0.50, 0.92, 12.34]) | -| Output | StringColumn { data: Utf8ViewArray[0.50, 0.92, 12.34] } | -+--------+---------------------------------------------------------+ ++--------+---------------------------------+ +| Column | Data | ++--------+---------------------------------+ +| f | Decimal256([0.50, 0.92, 12.34]) | +| Output | StringColumn[0.50, 0.92, 12.34] | ++--------+---------------------------------+ ast : a ^ 2 diff --git a/src/query/functions/tests/it/scalars/testdata/array.txt b/src/query/functions/tests/it/scalars/testdata/array.txt index 024e45bd3476..b22c7b293f83 100644 --- a/src/query/functions/tests/it/scalars/testdata/array.txt +++ b/src/query/functions/tests/it/scalars/testdata/array.txt @@ -473,12 +473,12 @@ evaluation: | Row 3 | '1234' | false | +--------+-------------+---------+ evaluation (internal): -+------------+-----------------------------------------------------+ -| Column | Data | -+------------+-----------------------------------------------------+ -| string_col | StringColumn { data: Utf8ViewArray[1, 2, 5, 1234] } | -| Output | Boolean([0b____0000]) | -+------------+-----------------------------------------------------+ ++------------+-----------------------------+ +| Column | Data | ++------------+-----------------------------+ +| string_col | StringColumn[1, 2, 5, 1234] | +| Output | Boolean([0b____0000]) | ++------------+-----------------------------+ ast : contains(['1', '5'], string_col) @@ -497,12 +497,12 @@ evaluation: | Row 3 | '1234' | false | +--------+-------------+---------------+ evaluation (internal): -+------------+-----------------------------------------------------+ -| Column | Data | -+------------+-----------------------------------------------------+ -| string_col | StringColumn { data: Utf8ViewArray[1, 2, 5, 1234] } | -| Output | Boolean([0b____0101]) | -+------------+-----------------------------------------------------+ ++------------+-----------------------------+ +| Column | Data | ++------------+-----------------------------+ +| string_col | StringColumn[1, 2, 5, 1234] | +| Output | Boolean([0b____0101]) | ++------------+-----------------------------+ ast : contains(['15000', '6000', '7000'], string_col) @@ -521,12 +521,12 @@ evaluation: | Row 3 | '1234' | false | +--------+-------------+---------------+ evaluation (internal): -+------------+-----------------------------------------------------+ -| Column | Data | -+------------+-----------------------------------------------------+ -| string_col | StringColumn { data: Utf8ViewArray[1, 2, 5, 1234] } | -| Output | Boolean([0b____0000]) | -+------------+-----------------------------------------------------+ ++------------+-----------------------------+ +| Column | Data | ++------------+-----------------------------+ +| string_col | StringColumn[1, 2, 5, 1234] | +| Output | Boolean([0b____0000]) | ++------------+-----------------------------+ ast : contains([1,2,null], nullable_col) diff --git a/src/query/functions/tests/it/scalars/testdata/binary.txt b/src/query/functions/tests/it/scalars/testdata/binary.txt index 84a4e2f3e3fc..4a63e71eb167 100644 --- a/src/query/functions/tests/it/scalars/testdata/binary.txt +++ b/src/query/functions/tests/it/scalars/testdata/binary.txt @@ -78,12 +78,12 @@ evaluation: | Row 1 | '123' | 'MTIz' | +--------+-----------------+--------+ evaluation (internal): -+--------+--------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, 123] } | -| Output | StringColumn { data: Utf8ViewArray[QWJj, MTIz] } | -+--------+--------------------------------------------------+ ++--------+--------------------------+ +| Column | Data | ++--------+--------------------------+ +| a | StringColumn[Abc, 123] | +| Output | StringColumn[QWJj, MTIz] | ++--------+--------------------------+ ast : to_hex(to_binary('abc')) @@ -109,12 +109,12 @@ evaluation: | Row 2 | 'databend' | '6461746162656e64' | +--------+-----------------+--------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[abc, def, databend] } | -| Output | StringColumn { data: Utf8ViewArray[616263, 646566, 6461746162656e64] } | -+--------+------------------------------------------------------------------------+ ++--------+------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------+ +| a | StringColumn[abc, def, databend] | +| Output | StringColumn[616263, 646566, 6461746162656e64] | ++--------+------------------------------------------------+ ast : from_base64('QWJj')::String @@ -148,12 +148,12 @@ evaluation: | Row 1 | 'MTIz' | '123' | +--------+-------------------+---------+ evaluation (internal): -+--------+--------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[QWJj, MTIz] } | -| Output | StringColumn { data: Utf8ViewArray[Abc, 123] } | -+--------+--------------------------------------------------+ ++--------+--------------------------+ +| Column | Data | ++--------+--------------------------+ +| a | StringColumn[QWJj, MTIz] | +| Output | StringColumn[Abc, 123] | ++--------+--------------------------+ error: @@ -195,12 +195,12 @@ evaluation: | Row 2 | '6461746162656e64' | 'databend' | +--------+-----------------------+------------+ evaluation (internal): -+--------+------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[616263, 646566, 6461746162656e64] } | -| Output | StringColumn { data: Utf8ViewArray[abc, def, databend] } | -+--------+------------------------------------------------------------------------+ ++--------+------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------+ +| s | StringColumn[616263, 646566, 6461746162656e64] | +| Output | StringColumn[abc, def, databend] | ++--------+------------------------------------------------+ ast : TRY_from_base64('QWJj')::String @@ -234,12 +234,12 @@ evaluation: | Row 1 | 'MTIz' | '123' | +--------+-------------------+---------+ evaluation (internal): -+--------+--------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[QWJj, MTIz] } | -| Output | StringColumn { data: Utf8ViewArray[Abc, 123] } | -+--------+--------------------------------------------------+ ++--------+--------------------------+ +| Column | Data | ++--------+--------------------------+ +| a | StringColumn[QWJj, MTIz] | +| Output | StringColumn[Abc, 123] | ++--------+--------------------------+ ast : TRY_from_base64('!@#') @@ -283,11 +283,11 @@ evaluation: | Row 2 | '6461746162656e64' | 'databend' | +--------+-----------------------+------------+ evaluation (internal): -+--------+------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[616263, 646566, 6461746162656e64] } | -| Output | StringColumn { data: Utf8ViewArray[abc, def, databend] } | -+--------+------------------------------------------------------------------------+ ++--------+------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------+ +| s | StringColumn[616263, 646566, 6461746162656e64] | +| Output | StringColumn[abc, def, databend] | ++--------+------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/cast.txt b/src/query/functions/tests/it/scalars/testdata/cast.txt index 4e17bf4bbf6a..0355c8e2c42d 100644 --- a/src/query/functions/tests/it/scalars/testdata/cast.txt +++ b/src/query/functions/tests/it/scalars/testdata/cast.txt @@ -820,12 +820,12 @@ evaluation: | Row 4 | '9223372036854775807' | 9223372036854775807 | +--------+--------------------------------+----------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------+ -| str | StringColumn { data: Utf8ViewArray[-9223372036854775808, -1, 0, 1, 9223372036854775807] } | -| Output | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | -+--------+-------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------+ +| str | StringColumn[-9223372036854775808, -1, 0, 1, 9223372036854775807] | +| Output | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | ++--------+-------------------------------------------------------------------+ error: @@ -852,12 +852,12 @@ evaluation: | Row 4 | 9223372036854775807 | '9223372036854775807' | +--------+----------------------------------------------+------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------+ -| num | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | -| Output | StringColumn { data: Utf8ViewArray[-9223372036854775808, -1, 0, 1, 9223372036854775807] } | -+--------+-------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------+ +| num | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | +| Output | StringColumn[-9223372036854775808, -1, 0, 1, 9223372036854775807] | ++--------+-------------------------------------------------------------------+ ast : CAST(num AS STRING) @@ -874,12 +874,12 @@ evaluation: | Row 2 | 18446744073709551615 | '18446744073709551615' | +--------+----------------------------+------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------+ -| num | UInt64([0, 1, 18446744073709551615]) | -| Output | StringColumn { data: Utf8ViewArray[0, 1, 18446744073709551615] } | -+--------+------------------------------------------------------------------+ ++--------+------------------------------------------+ +| Column | Data | ++--------+------------------------------------------+ +| num | UInt64([0, 1, 18446744073709551615]) | +| Output | StringColumn[0, 1, 18446744073709551615] | ++--------+------------------------------------------+ error: @@ -963,12 +963,12 @@ evaluation: | Row 1 | true | 'true' | +--------+---------------+---------+ evaluation (internal): -+--------+---------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------+ -| bool | Boolean([0b______10]) | -| Output | StringColumn { data: Utf8ViewArray[false, true] } | -+--------+---------------------------------------------------+ ++--------+---------------------------+ +| Column | Data | ++--------+---------------------------+ +| bool | Boolean([0b______10]) | +| Output | StringColumn[false, true] | ++--------+---------------------------+ ast : CAST('010.010' AS DECIMAL(5,3)) @@ -1482,12 +1482,12 @@ evaluation: | Row 4 | '2022-01-02T01' | '2022-01-02 01:00:00.000000' | +--------+-----------------------------------------------------+------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] } | -| Output | [1641081600000000, 1641119102868894, 1641088811000000, 1641111120000000, 1641085200000000] | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] | +| Output | [1641081600000000, 1641119102868894, 1641088811000000, 1641111120000000, 1641085200000000] | ++--------+---------------------------------------------------------------------------------------------------------------------------+ ast : CAST(TO_TIMESTAMP(-315360000000000) AS VARCHAR) @@ -1580,12 +1580,12 @@ evaluation: | Row 6 | '1979-12-30 00:00:00.000000' | '1979-12-30 00:00:00.000000' | +--------+--------------------------------------+------------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | [-315360000000000, -315360000000, -100, 0, 100, 315360000000, 315360000000000] | -| Output | StringColumn { data: Utf8ViewArray[1960-01-04 00:00:00.000000, 1969-12-28 08:24:00.000000, 1969-12-31 23:59:59.999900, 1970-01-01 00:00:00.000000, 1970-01-01 00:00:00.000100, 1970-01-04 15:36:00.000000, 1979-12-30 00:00:00.000000] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | [-315360000000000, -315360000000, -100, 0, 100, 315360000000, 315360000000000] | +| Output | StringColumn[1960-01-04 00:00:00.000000, 1969-12-28 08:24:00.000000, 1969-12-31 23:59:59.999900, 1970-01-01 00:00:00.000000, 1970-01-01 00:00:00.000100, 1970-01-04 15:36:00.000000, 1979-12-30 00:00:00.000000] | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ error: @@ -1682,12 +1682,12 @@ evaluation: | Row 4 | '2022-01-02T01' | '2022-01-02' | +--------+-----------------------------------------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] } | -| Output | [18994, 18994, 18994, 18994, 18994] | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] | +| Output | [18994, 18994, 18994, 18994, 18994] | ++--------+---------------------------------------------------------------------------------------------------------------------------+ ast : CAST(TO_DATE(-354285) AS VARCHAR) @@ -1760,12 +1760,12 @@ evaluation: | Row 4 | '9999-12-31' | '9999-12-31' | +--------+---------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| a | [-354285, -100, 0, 100, 2932896] | -| Output | StringColumn { data: Utf8ViewArray[1000-01-01, 1969-09-23, 1970-01-01, 1970-04-11, 9999-12-31] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| a | [-354285, -100, 0, 100, 2932896] | +| Output | StringColumn[1000-01-01, 1969-09-23, 1970-01-01, 1970-04-11, 9999-12-31] | ++--------+--------------------------------------------------------------------------+ error: @@ -1921,7 +1921,7 @@ evaluation (internal): +--------+------------------------------------------------------------------------------------------------+ | Column | Data | +--------+------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| a | StringColumn[Abc, Dobrý den, ß😀山] | | Output | BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] } | +--------+------------------------------------------------------------------------------------------------+ @@ -1951,7 +1951,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | +| a | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2014,12 +2014,12 @@ evaluation: | Row 2 | 'ß😀山' | 'ß😀山' | +--------+-------------------+-------------+ evaluation (internal): -+--------+-------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | -| Output | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | -+--------+-------------------------------------------------------------+ ++--------+-------------------------------------+ +| Column | Data | ++--------+-------------------------------------+ +| a | StringColumn[Abc, Dobrý den, ß😀山] | +| Output | StringColumn[Abc, Dobrý den, ß😀山] | ++--------+-------------------------------------+ error: @@ -2044,12 +2044,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+----------------------------+-------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | -+--------+----------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | ++--------+----------------------------------------------------------------------------------------+ ast : TRY_CAST(0 AS UINT8) @@ -2539,7 +2539,7 @@ evaluation (internal): +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":"v"}, [1,2,3]] }, validity: [0b_____101] } | +| a | NullableColumn { column: StringColumn[true, {"k":"v"}, [1,2,3]], validity: [0b_____101] } | | Output | NullableColumn { column: BinaryColumn { data: 0x200000004000000080000003200000022000000220000002500150025003, offsets: [0, 8, 8, 30] }, validity: [0b_____101] } | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3019,7 +3019,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------+ -| str | StringColumn { data: Utf8ViewArray[-9223372036854775808, -1, 0, 1, 9223372036854775807] } | +| str | StringColumn[-9223372036854775808, -1, 0, 1, 9223372036854775807] | | Output | NullableColumn { column: Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]), validity: [0b___11111] } | +--------+-----------------------------------------------------------------------------------------------------------------+ @@ -3039,12 +3039,12 @@ evaluation: | Row 3 | NULL | NULL | +--------+------------------------+-------------------------------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------+ -| str | NullableColumn { column: StringColumn { data: Utf8ViewArray[foo, foo, 0, 0] }, validity: [0b____0101] } | -| Output | NullableColumn { column: Int64([0, 0, 0, 0]), validity: [0b____0100] } | -+--------+---------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------+ +| str | NullableColumn { column: StringColumn[foo, foo, 0, 0], validity: [0b____0101] } | +| Output | NullableColumn { column: Int64([0, 0, 0, 0]), validity: [0b____0100] } | ++--------+---------------------------------------------------------------------------------+ ast : TRY_CAST(num AS STRING) @@ -3063,12 +3063,12 @@ evaluation: | Row 4 | 9223372036854775807 | '9223372036854775807' | +--------+----------------------------------------------+------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------+ -| num | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[-9223372036854775808, -1, 0, 1, 9223372036854775807] }, validity: [0b___11111] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------+ +| num | Int64([-9223372036854775808, -1, 0, 1, 9223372036854775807]) | +| Output | NullableColumn { column: StringColumn[-9223372036854775808, -1, 0, 1, 9223372036854775807], validity: [0b___11111] } | ++--------+----------------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(num AS STRING) @@ -3085,12 +3085,12 @@ evaluation: | Row 2 | 18446744073709551615 | '18446744073709551615' | +--------+----------------------------+------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------+ -| num | UInt64([0, 1, 18446744073709551615]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[0, 1, 18446744073709551615] }, validity: [0b_____111] } | -+--------+---------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------+ +| num | UInt64([0, 1, 18446744073709551615]) | +| Output | NullableColumn { column: StringColumn[0, 1, 18446744073709551615], validity: [0b_____111] } | ++--------+---------------------------------------------------------------------------------------------+ ast : TRY_CAST('t' AS BOOLEAN) @@ -3178,12 +3178,12 @@ evaluation: | Row 1 | true | 'true' | +--------+---------------+-----------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------+ -| bool | Boolean([0b______10]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[false, true] }, validity: [0b______11] } | -+--------+------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------+ +| bool | Boolean([0b______10]) | +| Output | NullableColumn { column: StringColumn[false, true], validity: [0b______11] } | ++--------+------------------------------------------------------------------------------+ ast : TRY_CAST('010.010' AS DECIMAL(5,3)) @@ -3702,12 +3702,12 @@ evaluation: | Row 4 | '2022-01-02T01' | '2022-01-02 01:00:00.000000' | +--------+-----------------------------------------------------+----------------------------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] } | -| Output | NullableColumn { column: [1641081600000000, 1641119102868894, 1641088811000000, 1641111120000000, 1641085200000000], validity: [0b___11111] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] | +| Output | NullableColumn { column: [1641081600000000, 1641119102868894, 1641088811000000, 1641111120000000, 1641085200000000], validity: [0b___11111] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(TO_TIMESTAMP(-315360000000000) AS VARCHAR) @@ -3800,12 +3800,12 @@ evaluation: | Row 6 | '1979-12-30 00:00:00.000000' | '1979-12-30 00:00:00.000000' | +--------+--------------------------------------+------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | [-315360000000000, -315360000000, -100, 0, 100, 315360000000, 315360000000000] | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1960-01-04 00:00:00.000000, 1969-12-28 08:24:00.000000, 1969-12-31 23:59:59.999900, 1970-01-01 00:00:00.000000, 1970-01-01 00:00:00.000100, 1970-01-04 15:36:00.000000, 1979-12-30 00:00:00.000000] }, validity: [0b_1111111] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | [-315360000000000, -315360000000, -100, 0, 100, 315360000000, 315360000000000] | +| Output | NullableColumn { column: StringColumn[1960-01-04 00:00:00.000000, 1969-12-28 08:24:00.000000, 1969-12-31 23:59:59.999900, 1970-01-01 00:00:00.000000, 1970-01-01 00:00:00.000100, 1970-01-04 15:36:00.000000, 1979-12-30 00:00:00.000000], validity: [0b_1111111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : TRY_TO_DATE('2022') @@ -3905,12 +3905,12 @@ evaluation: | Row 4 | '2022-01-02T01' | '2022-01-02' | +--------+-----------------------------------------------------+------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] } | -| Output | NullableColumn { column: [18994, 18994, 18994, 18994, 18994], validity: [0b___11111] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn[2022-01-02, 2022-01-02T03:25:02.868894-07:00, 2022-01-02 02:00:11, 2022-01-02T01:12:00-07:00, 2022-01-02T01] | +| Output | NullableColumn { column: [18994, 18994, 18994, 18994, 18994], validity: [0b___11111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------+ ast : TRY_CAST(TO_DATE(-354285) AS VARCHAR) @@ -3983,12 +3983,12 @@ evaluation: | Row 4 | '9999-12-31' | '9999-12-31' | +--------+---------------------+--------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | [-354285, -100, 0, 100, 2932896] | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1000-01-01, 1969-09-23, 1970-01-01, 1970-04-11, 9999-12-31] }, validity: [0b___11111] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------+ +| a | [-354285, -100, 0, 100, 2932896] | +| Output | NullableColumn { column: StringColumn[1000-01-01, 1969-09-23, 1970-01-01, 1970-04-11, 9999-12-31], validity: [0b___11111] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------+ error: @@ -4151,7 +4151,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| a | StringColumn[Abc, Dobrý den, ß😀山] | | Output | NullableColumn { column: BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____111] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4173,7 +4173,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | +| a | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4195,7 +4195,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | +| a | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x416263446f6272c3bd2064656ec39ff09f9880e5b1b1, offsets: [0, 3, 13, 22] }, validity: [0b_____011] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4259,12 +4259,12 @@ evaluation: | Row 2 | 'ß😀山' | 'ß😀山' | +--------+-------------------+-----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____111] } | -+--------+----------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| a | StringColumn[Abc, Dobrý den, ß😀山] | +| Output | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____111] } | ++--------+----------------------------------------------------------------------------------------+ ast : TRY_CAST(TRY_CAST(a AS BINARY) AS STRING) @@ -4281,12 +4281,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+----------------------------+-----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | -+--------+----------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | ++--------+----------------------------------------------------------------------------------------+ ast : TRY_CAST(TRY_CAST(a AS BINARY NULL) AS STRING NULL) @@ -4303,11 +4303,11 @@ evaluation: | Row 2 | NULL | NULL | +--------+----------------------------+-----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] }, validity: [0b_____011] } | -+--------+----------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn[Abc, Dobrý den, ß😀山], validity: [0b_____011] } | ++--------+----------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/comparison.txt b/src/query/functions/tests/it/scalars/testdata/comparison.txt index 7dd3ce500bd5..f68ccccfc9b8 100644 --- a/src/query/functions/tests/it/scalars/testdata/comparison.txt +++ b/src/query/functions/tests/it/scalars/testdata/comparison.txt @@ -223,13 +223,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | true | +--------+------------------------------------------+------------------------------------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | -| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] } | -| Output | Boolean([0b_1011111]) | -+--------+------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | +| rhs | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] | +| Output | Boolean([0b_1011111]) | ++--------+------------------------------------------------------------------------------------------------------------+ ast : lhs = rhs @@ -250,13 +250,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | true | +--------+------------------------------------------+------------------------------------------+---------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | -| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] } | -| Output | Boolean([0b_1011111]) | -+--------+------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | +| rhs | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] | +| Output | Boolean([0b_1011111]) | ++--------+------------------------------------------------------------------------------------------------------------+ ast : '1'!='2' @@ -405,13 +405,13 @@ evaluation: | Row 3 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | false | +--------+----------------------------------+----------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] } | -| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] } | -| Output | Boolean([0b____0000]) | -+--------+----------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| lhs | StringColumn[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] | +| rhs | StringColumn[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] | +| Output | Boolean([0b____0000]) | ++--------+----------------------------------------------------------------------+ ast : lhs != rhs @@ -429,13 +429,13 @@ evaluation: | Row 3 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | false | +--------+----------------------------------+----------------------------------+---------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] } | -| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] } | -| Output | Boolean([0b____0000]) | -+--------+----------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| lhs | StringColumn[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] | +| rhs | StringColumn[null, true, 9223372036854775807, [1,2,3,["a","b","c"]]] | +| Output | Boolean([0b____0000]) | ++--------+----------------------------------------------------------------------+ ast : '1'<'2' @@ -573,13 +573,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","c"]]' | '[0,2,3,["a","b","c"]]' | true | +--------+-------------------------+-------------------------+---------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, [1,2,3,["a","b","c"]]] } | -| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775800, -33768, 1234.5678, 1.912e2, [0,2,3,["a","b","c"]]] } | -| Output | Boolean([0b_1111111]) | -+--------+--------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| lhs | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, [1,2,3,["a","b","c"]]] | +| rhs | StringColumn[null, true, 9223372036854775800, -33768, 1234.5678, 1.912e2, [0,2,3,["a","b","c"]]] | +| Output | Boolean([0b_1111111]) | ++--------+--------------------------------------------------------------------------------------------------+ ast : lhs < rhs @@ -600,13 +600,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","c"]]' | '[0,2,3,["a","b","c"]]' | false | +--------+-------------------------+-------------------------+---------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, [1,2,3,["a","b","c"]]] } | -| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775800, -33768, 1234.5678, 1.912e2, [0,2,3,["a","b","c"]]] } | -| Output | Boolean([0b_0001000]) | -+--------+--------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------+ +| lhs | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, [1,2,3,["a","b","c"]]] | +| rhs | StringColumn[null, true, 9223372036854775800, -33768, 1234.5678, 1.912e2, [0,2,3,["a","b","c"]]] | +| Output | Boolean([0b_0001000]) | ++--------+--------------------------------------------------------------------------------------------------+ ast : '5'<='2' @@ -749,13 +749,13 @@ evaluation: | Row 2 | '[1,2,3,["a","b","c"]]' | '[0,2,3,["a","b","c"]]' | false | +--------+------------------------------------------------+------------------------------------------------+---------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray["databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | -| rhs | StringColumn { data: Utf8ViewArray["databend", {"k":"a","a":"d"}, [0,2,3,["a","b","c"]]] } | -| Output | Boolean([0b_____011]) | -+--------+--------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------+ +| lhs | StringColumn["databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | +| rhs | StringColumn["databend", {"k":"a","a":"d"}, [0,2,3,["a","b","c"]]] | +| Output | Boolean([0b_____011]) | ++--------+--------------------------------------------------------------------+ ast : lhs <= rhs @@ -772,13 +772,13 @@ evaluation: | Row 2 | '[1,2,3,["a","b","c"]]' | '[0,2,3,["a","b","c"]]' | false | +--------+------------------------------------------------+------------------------------------------------+---------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray["databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | -| rhs | StringColumn { data: Utf8ViewArray["databend", {"k":"a","a":"d"}, [0,2,3,["a","b","c"]]] } | -| Output | Boolean([0b_____001]) | -+--------+--------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------+ +| lhs | StringColumn["databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | +| rhs | StringColumn["databend", {"k":"a","a":"d"}, [0,2,3,["a","b","c"]]] | +| Output | Boolean([0b_____001]) | ++--------+--------------------------------------------------------------------+ ast : '3'>'2' @@ -923,13 +923,13 @@ evaluation: | Row 4 | '1234.5678' | '1234.5678' | false | +--------+-----------------------+-----------------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678] } | -| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775806, -32768, 1234.5678] } | -| Output | Boolean([0b___00100]) | -+--------+------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------+ +| lhs | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678] | +| rhs | StringColumn[null, true, 9223372036854775806, -32768, 1234.5678] | +| Output | Boolean([0b___00100]) | ++--------+------------------------------------------------------------------+ ast : lhs > rhs @@ -948,13 +948,13 @@ evaluation: | Row 4 | '1234.5678' | '1234.5678' | false | +--------+-----------------------+-----------------------+---------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678] } | -| rhs | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775806, -32768, 1234.5678] } | -| Output | Boolean([0b___00100]) | -+--------+------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------+ +| lhs | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678] | +| rhs | StringColumn[null, true, 9223372036854775806, -32768, 1234.5678] | +| Output | Boolean([0b___00100]) | ++--------+------------------------------------------------------------------+ ast : col > 'efg' @@ -971,12 +971,12 @@ evaluation: | Row 1 | 'efg' | false | +--------+-----------------+---------+ evaluation (internal): -+--------+------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------+ -| col | StringColumn { data: Utf8ViewArray[bcd, efg] } | -| Output | Boolean([0b______00]) | -+--------+------------------------------------------------+ ++--------+------------------------+ +| Column | Data | ++--------+------------------------+ +| col | StringColumn[bcd, efg] | +| Output | Boolean([0b______00]) | ++--------+------------------------+ ast : '2'>='1' @@ -1123,13 +1123,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","d"]]' | '[1,2,3,["a","b","c"]]' | true | +--------+-----------------------------------------------------------+-----------------------------------------------------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"b"}, [1,2,3,["a","b","d"]]] } | -| rhs | StringColumn { data: Utf8ViewArray[9223372036854775806, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] } | -| Output | Boolean([0b_1011111]) | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn[9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"b"}, [1,2,3,["a","b","d"]]] | +| rhs | StringColumn[9223372036854775806, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] | +| Output | Boolean([0b_1011111]) | ++--------+------------------------------------------------------------------------------------------------------------------------+ ast : lhs >= rhs @@ -1150,13 +1150,13 @@ evaluation: | Row 6 | '[1,2,3,["a","b","d"]]' | '[1,2,3,["a","b","c"]]' | true | +--------+-----------------------------------------------------------+-----------------------------------------------------------+---------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"b"}, [1,2,3,["a","b","d"]]] } | -| rhs | StringColumn { data: Utf8ViewArray[9223372036854775806, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] } | -| Output | Boolean([0b_1011111]) | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------+ +| lhs | StringColumn[9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"b"}, [1,2,3,["a","b","d"]]] | +| rhs | StringColumn[9223372036854775806, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", {"k":"v","a":"d"}, [1,2,3,["a","b","c"]]] | +| Output | Boolean([0b_1011111]) | ++--------+------------------------------------------------------------------------------------------------------------------------+ ast : '1' like '2' @@ -1223,12 +1223,12 @@ evaluation: | Row 3 | 'abf' | true | +--------+-----------------+---------------+ evaluation (internal): -+--------+----------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | -| Output | Boolean([0b____1111]) | -+--------+----------------------------------------------------------+ ++--------+----------------------------------+ +| Column | Data | ++--------+----------------------------------+ +| lhs | StringColumn[abc, abd, abe, abf] | +| Output | Boolean([0b____1111]) | ++--------+----------------------------------+ ast : lhs like 'b%' @@ -1247,12 +1247,12 @@ evaluation: | Row 3 | 'abf' | false | +--------+-----------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | -| Output | Boolean([0b____0000]) | -+--------+----------------------------------------------------------+ ++--------+----------------------------------+ +| Column | Data | ++--------+----------------------------------+ +| lhs | StringColumn[abc, abd, abe, abf] | +| Output | Boolean([0b____0000]) | ++--------+----------------------------------+ ast : lhs like 'ab%' @@ -1270,12 +1270,12 @@ evaluation: | Row 3 | 'abf' | true | +--------+-----------------+---------------+ evaluation (internal): -+--------+----------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | -| Output | Boolean([0b____1111]) | -+--------+----------------------------------------------------------+ ++--------+----------------------------------+ +| Column | Data | ++--------+----------------------------------+ +| lhs | StringColumn[abc, abd, abe, abf] | +| Output | Boolean([0b____1111]) | ++--------+----------------------------------+ ast : lhs like 'c' @@ -1294,12 +1294,12 @@ evaluation: | Row 3 | 'abf' | false | +--------+-----------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | -| Output | Boolean([0b____0000]) | -+--------+----------------------------------------------------------+ ++--------+----------------------------------+ +| Column | Data | ++--------+----------------------------------+ +| lhs | StringColumn[abc, abd, abe, abf] | +| Output | Boolean([0b____0000]) | ++--------+----------------------------------+ ast : lhs like rhs @@ -1317,13 +1317,13 @@ evaluation: | Row 3 | 'abf' | 'a' | false | +--------+-----------------+-----------------+---------------+ evaluation (internal): -+--------+----------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf] } | -| rhs | StringColumn { data: Utf8ViewArray[a%, _b_, abe, a] } | -| Output | Boolean([0b____0111]) | -+--------+----------------------------------------------------------+ ++--------+----------------------------------+ +| Column | Data | ++--------+----------------------------------+ +| lhs | StringColumn[abc, abd, abe, abf] | +| rhs | StringColumn[a%, _b_, abe, a] | +| Output | Boolean([0b____0111]) | ++--------+----------------------------------+ ast : parse_json('"hello"') like 'h%' @@ -1367,12 +1367,12 @@ evaluation: | Row 2 | '["abe","abf"]' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray["abc", {"abd":12}, ["abe","abf"]] } | -| Output | Boolean([0b_____001]) | -+--------+------------------------------------------------------------------------+ ++--------+------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------+ +| lhs | StringColumn["abc", {"abd":12}, ["abe","abf"]] | +| Output | Boolean([0b_____001]) | ++--------+------------------------------------------------+ ast : parse_json(lhs) like '%ab%' @@ -1389,12 +1389,12 @@ evaluation: | Row 2 | '["abe","abf"]' | true | +--------+------------------------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray["abc", {"abd":12}, ["abe","abf"]] } | -| Output | Boolean([0b_____111]) | -+--------+------------------------------------------------------------------------+ ++--------+------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------+ +| lhs | StringColumn["abc", {"abd":12}, ["abe","abf"]] | +| Output | Boolean([0b_____111]) | ++--------+------------------------------------------------+ ast : lhs regexp rhs @@ -1414,13 +1414,13 @@ evaluation: | Row 5 | '' | '' | true | +--------+--------------+--------------+---------------+ evaluation (internal): -+--------+-----------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf, abc, ] } | -| rhs | StringColumn { data: Utf8ViewArray[^a, ^b, abe, a, , ] } | -| Output | Boolean([0b__101101]) | -+--------+-----------------------------------------------------------------+ ++--------+-----------------------------------------+ +| Column | Data | ++--------+-----------------------------------------+ +| lhs | StringColumn[abc, abd, abe, abf, abc, ] | +| rhs | StringColumn[^a, ^b, abe, a, , ] | +| Output | Boolean([0b__101101]) | ++--------+-----------------------------------------+ ast : lhs rlike rhs @@ -1440,12 +1440,12 @@ evaluation: | Row 5 | '' | '' | true | +--------+--------------+--------------+---------------+ evaluation (internal): -+--------+-----------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------+ -| lhs | StringColumn { data: Utf8ViewArray[abc, abd, abe, abf, abc, ] } | -| rhs | StringColumn { data: Utf8ViewArray[^a, ^b, abe, a, , ] } | -| Output | Boolean([0b__101101]) | -+--------+-----------------------------------------------------------------+ ++--------+-----------------------------------------+ +| Column | Data | ++--------+-----------------------------------------+ +| lhs | StringColumn[abc, abd, abe, abf, abc, ] | +| rhs | StringColumn[^a, ^b, abe, a, , ] | +| Output | Boolean([0b__101101]) | ++--------+-----------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/geo_h3.txt b/src/query/functions/tests/it/scalars/testdata/geo_h3.txt index 3ee681496703..45fd05fa864a 100644 --- a/src/query/functions/tests/it/scalars/testdata/geo_h3.txt +++ b/src/query/functions/tests/it/scalars/testdata/geo_h3.txt @@ -763,12 +763,12 @@ evaluation: | Row 2 | 599686042433355775 | '85283473fffffff' | +--------+-------------------------------------------+-------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------+ -| h3 | UInt64([635318325446452991, 644325524701193897, 599686042433355775]) | -| Output | StringColumn { data: Utf8ViewArray[8d11aa6a38826ff, 8f11aa6a38826a9, 85283473fffffff] } | -+--------+-----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| h3 | UInt64([635318325446452991, 644325524701193897, 599686042433355775]) | +| Output | StringColumn[8d11aa6a38826ff, 8f11aa6a38826a9, 85283473fffffff] | ++--------+----------------------------------------------------------------------+ error: @@ -809,12 +809,12 @@ evaluation: | Row 2 | '85283473fffffff' | 599686042433355775 | +--------+-----------------------------------------+----------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------+ -| h3_str | StringColumn { data: Utf8ViewArray[8d11aa6a38826ff, 8f11aa6a38826a9, 85283473fffffff] } | -| Output | UInt64([635318325446452991, 644325524701193897, 599686042433355775]) | -+--------+-----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------+ +| h3_str | StringColumn[8d11aa6a38826ff, 8f11aa6a38826a9, 85283473fffffff] | +| Output | UInt64([635318325446452991, 644325524701193897, 599686042433355775]) | ++--------+----------------------------------------------------------------------+ error: diff --git a/src/query/functions/tests/it/scalars/testdata/geometry.txt b/src/query/functions/tests/it/scalars/testdata/geometry.txt index 44e7b265c3de..a69f920d3cfc 100644 --- a/src/query/functions/tests/it/scalars/testdata/geometry.txt +++ b/src/query/functions/tests/it/scalars/testdata/geometry.txt @@ -257,7 +257,7 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[9q60y60rhs, u4pruydqqvj0] } | +| a | StringColumn[9q60y60rhs, u4pruydqqvj0] | | Output | BinaryColumn { data: 0x0103000000010000000500000000000036632a5ec00000001470a6414000000036632a5ec00000004170a6414000000009632a5ec00000004170a6414000000009632a5ec00000001470a6414000000036632a5ec00000001470a6414001030000000100000005000000000000d99bd024400000000916d34c40000000d99bd024400000680a16d34c40000040e49bd024400000680a16d34c40000040e49bd024400000000916d34c40000000d99bd024400000000916d34c40, offsets: [0, 93, 186] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -406,7 +406,7 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[LINESTRING(0.0 0.0, 1.0 0.0, 1.0 2.0, 0.0 2.0, 0.0 0.0), LINESTRING(10.1 5.2, 15.2 7.3, 20.2 8.3, 10.9 7.7, 10.1 5.2)] } | +| a | StringColumn[LINESTRING(0.0 0.0, 1.0 0.0, 1.0 2.0, 0.0 2.0, 0.0 0.0), LINESTRING(10.1 5.2, 15.2 7.3, 20.2 8.3, 10.9 7.7, 10.1 5.2)] | | Output | BinaryColumn { data: 0x0103000000010000000500000000000000000000000000000000000000000000000000f03f0000000000000000000000000000f03f00000000000000400000000000000000000000000000004000000000000000000000000000000000010300000001000000050000003333333333332440cdcccccccccc14406666666666662e403333333333331d4033333333333334409a99999999992040cdcccccccccc2540cdcccccccccc1e403333333333332440cdcccccccccc1440, offsets: [0, 93, 186] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -704,13 +704,13 @@ evaluation: | Row 2 | 3 | 3 | 'POINT(3 3)' | +--------+---------+---------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------+ -| a | Float64([1, 2, 3]) | -| b | Float64([1, 2, 3]) | -| Output | StringColumn { data: Utf8ViewArray[POINT(1 1), POINT(2 2), POINT(3 3)] } | -+--------+--------------------------------------------------------------------------+ ++--------+--------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------+ +| a | Float64([1, 2, 3]) | +| b | Float64([1, 2, 3]) | +| Output | StringColumn[POINT(1 1), POINT(2 2), POINT(3 3)] | ++--------+--------------------------------------------------+ ast : try_to_geometry(NULL) @@ -938,7 +938,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[0101000020797f000066666666a9cb17411f85ebc19e325641, 0101000020797f000066666666a9cb17411f85ebc19e325641, 0101000020797f000066666666a9cb17411f85ebc19e325641] } | +| a | StringColumn[0101000020797f000066666666a9cb17411f85ebc19e325641, 0101000020797f000066666666a9cb17411f85ebc19e325641, 0101000020797f000066666666a9cb17411f85ebc19e325641] | | b | Int32([32633, 4326, 3857]) | | Output | BinaryColumn { data: 0x0101000020797f000066666666a9cb17411f85ebc19e3256410101000020e610000066666666a9cb17411f85ebc19e3256410101000020110f000066666666a9cb17411f85ebc19e325641, offsets: [0, 25, 50, 75] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -971,7 +971,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[POINT(389866.35 5819003.03), POINT(389866.35 5819003.03), POINT(389866.35 5819003.03)] } | +| a | StringColumn[POINT(389866.35 5819003.03), POINT(389866.35 5819003.03), POINT(389866.35 5819003.03)] | | Output | BinaryColumn { data: 0x010100000066666666a9cb17411f85ebc19e325641010100000066666666a9cb17411f85ebc19e325641010100000066666666a9cb17411f85ebc19e325641, offsets: [0, 21, 42, 63] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1003,7 +1003,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[POINT(389866.35 5819003.03), POINT(389866.35 5819003.03), POINT(389866.35 5819003.03)] } | +| a | StringColumn[POINT(389866.35 5819003.03), POINT(389866.35 5819003.03), POINT(389866.35 5819003.03)] | | b | Int32([32633, 4326, 3857]) | | Output | BinaryColumn { data: 0x0101000020797f000066666666a9cb17411f85ebc19e3256410101000020e610000066666666a9cb17411f85ebc19e3256410101000020110f000066666666a9cb17411f85ebc19e325641, offsets: [0, 25, 50, 75] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1214,7 +1214,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[POINT(389866.35 5819003.03)] } | +| a | StringColumn[POINT(389866.35 5819003.03)] | | b | Int32([32633]) | | c | Int32([3857]) | | Output | BinaryColumn { data: 0x0101000020110f00006f0c0118f4b83641522cb70c524b5a41, offsets: [0, 25] } | @@ -1246,7 +1246,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[POINT(4.500212 52.161170)] } | +| a | StringColumn[POINT(4.500212 52.161170)] | | b | Int32([4326]) | | c | Int32([28992]) | | Output | BinaryColumn { data: 0x0101000020407100005dfe43ba4a06f7402ffce0ac98521c41, offsets: [0, 25] } | diff --git a/src/query/functions/tests/it/scalars/testdata/hash.txt b/src/query/functions/tests/it/scalars/testdata/hash.txt index 971f377992c4..6c8cd0146af5 100644 --- a/src/query/functions/tests/it/scalars/testdata/hash.txt +++ b/src/query/functions/tests/it/scalars/testdata/hash.txt @@ -30,12 +30,12 @@ evaluation: | Row 2 | 'ß😀山' | 'b814c09d48b62faafc315df44a35863e' | +--------+-------------------+------------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | -| Output | StringColumn { data: Utf8ViewArray[35593b7ce5020eae3ca68fd5b6f3e031, 1ef0b4dab5588fe3f63cd84f6f1d6ab2, b814c09d48b62faafc315df44a35863e] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn[Abc, Dobrý den, ß😀山] | +| Output | StringColumn[35593b7ce5020eae3ca68fd5b6f3e031, 1ef0b4dab5588fe3f63cd84f6f1d6ab2, b814c09d48b62faafc315df44a35863e] | ++--------+--------------------------------------------------------------------------------------------------------------------+ ast : sha('Abc') @@ -70,12 +70,12 @@ evaluation: | Row 2 | 'ß😀山' | 'e978809ba007678383c2db3decbaf02eb0bf72a8' | +--------+-------------------+--------------------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | -| Output | StringColumn { data: Utf8ViewArray[915858afa2278f25527f192038108346164b47f2, c66d056c9fb2c9e6cd74b4555f64c0af4a7bc599, e978809ba007678383c2db3decbaf02eb0bf72a8] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn[Abc, Dobrý den, ß😀山] | +| Output | StringColumn[915858afa2278f25527f192038108346164b47f2, c66d056c9fb2c9e6cd74b4555f64c0af4a7bc599, e978809ba007678383c2db3decbaf02eb0bf72a8] | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------+ ast : blake3('Abc') @@ -110,12 +110,12 @@ evaluation: | Row 2 | 'ß😀山' | '56475d2e89dba36b511ddaa8e4e8e995c094f59f5fbfa0af5929f3f399d9a810' | +--------+-------------------+--------------------------------------------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | -| Output | StringColumn { data: Utf8ViewArray[1f7aa3978949c4275797ac6056c0108e852a5d07c49838a00799194957b82111, 0b449419834e5d285d3b95f892b5494db8d7798c11602ccde1f51c33a159677f, 56475d2e89dba36b511ddaa8e4e8e995c094f59f5fbfa0af5929f3f399d9a810] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn[Abc, Dobrý den, ß😀山] | +| Output | StringColumn[1f7aa3978949c4275797ac6056c0108e852a5d07c49838a00799194957b82111, 0b449419834e5d285d3b95f892b5494db8d7798c11602ccde1f51c33a159677f, 56475d2e89dba36b511ddaa8e4e8e995c094f59f5fbfa0af5929f3f399d9a810] | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : sha2('Abc',0) @@ -159,13 +159,13 @@ evaluation: | Row 2 | 'ß😀山' | 512 | '3bd4ca36a66c0675e695f3fc44af703cd6c110085adf105138ef56e6768a639f16a9c27b651a0c64f685b24be835e0a62485575477e06d530574865bf1670d30' | +--------+-------------------+-------------+------------------------------------------------------------------------------------------------------------------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | -| b | UInt16([224, 384, 512]) | -| Output | StringColumn { data: Utf8ViewArray[11d86770f501294c6b395942a39f60fe286a15e06282abcb2294cfa0, 1a5c66c918718c627e360e56833d2f6c638fd1a67086792606cfefe50089289ca34b52a72a833e6f2661b64417068846, 3bd4ca36a66c0675e695f3fc44af703cd6c110085adf105138ef56e6768a639f16a9c27b651a0c64f685b24be835e0a62485575477e06d530574865bf1670d30] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | StringColumn[Abc, Dobrý den, ß😀山] | +| b | UInt16([224, 384, 512]) | +| Output | StringColumn[11d86770f501294c6b395942a39f60fe286a15e06282abcb2294cfa0, 1a5c66c918718c627e360e56833d2f6c638fd1a67086792606cfefe50089289ca34b52a72a833e6f2661b64417068846, 3bd4ca36a66c0675e695f3fc44af703cd6c110085adf105138ef56e6768a639f16a9c27b651a0c64f685b24be835e0a62485575477e06d530574865bf1670d30] | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : city64withseed('Abc',0) @@ -275,7 +275,7 @@ evaluation (internal): +--------+----------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | +| a | StringColumn[Abc, Dobrý den, ß😀山] | | b | UInt16([10, 11, 12]) | | Output | UInt64([10385767944629066306, 12123249488783690377, 14631005279260459058]) | +--------+----------------------------------------------------------------------------+ @@ -366,12 +366,12 @@ evaluation: | Row 1 | 'ß😀山' | 1354619631122873228 | +--------+-------------------------+----------------------------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Dobrý den, ß😀山] } | -| Output | UInt64([5782510256878119795, 1354619631122873228]) | -+--------+--------------------------------------------------------+ ++--------+----------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------+ +| a | StringColumn[Dobrý den, ß😀山] | +| Output | UInt64([5782510256878119795, 1354619631122873228]) | ++--------+----------------------------------------------------+ ast : xxhash64('Abc') @@ -459,12 +459,12 @@ evaluation: | Row 1 | 'ß😀山' | 656695431091154575 | +--------+-------------------------+----------------------------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Dobrý den, ß😀山] } | -| Output | UInt64([314761032262035578, 656695431091154575]) | -+--------+--------------------------------------------------------+ ++--------+--------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------+ +| a | StringColumn[Dobrý den, ß😀山] | +| Output | UInt64([314761032262035578, 656695431091154575]) | ++--------+--------------------------------------------------+ ast : xxhash32('Abc') @@ -552,11 +552,11 @@ evaluation: | Row 1 | 'ß😀山' | 1401072642 | +--------+-------------------------+------------------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Dobrý den, ß😀山] } | -| Output | UInt32([19285785, 1401072642]) | -+--------+--------------------------------------------------------+ ++--------+--------------------------------+ +| Column | Data | ++--------+--------------------------------+ +| a | StringColumn[Dobrý den, ß😀山] | +| Output | UInt32([19285785, 1401072642]) | ++--------+--------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/map.txt b/src/query/functions/tests/it/scalars/testdata/map.txt index 63659a6561ea..e08900f0959e 100644 --- a/src/query/functions/tests/it/scalars/testdata/map.txt +++ b/src/query/functions/tests/it/scalars/testdata/map.txt @@ -69,17 +69,17 @@ evaluation: | Row 2 | 3 | 6 | 9 | 'c' | NULL | 'g' | {3:'c', 6:NULL, 9:'g'} | +--------+---------+---------+---------+-------------+---------------------+---------------------+------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | Int8([1, 2, 3]) | -| b_col | Int8([4, 5, 6]) | -| c_col | Int8([7, 8, 9]) | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[d, e, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[f, , g] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: Tuple([Int8([1, 4, 7, 2, 5, 8, 3, 6, 9]), NullableColumn { column: StringColumn { data: Utf8ViewArray[a, d, f, b, e, , c, , g] }, validity: [0b01011111, 0b_______1] }]), offsets: [0, 3, 6, 9] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | Int8([1, 2, 3]) | +| b_col | Int8([4, 5, 6]) | +| c_col | Int8([7, 8, 9]) | +| d_col | NullableColumn { column: StringColumn[a, b, c], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[d, e, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[f, , g], validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([Int8([1, 4, 7, 2, 5, 8, 3, 6, 9]), NullableColumn { column: StringColumn[a, d, f, b, e, , c, , g], validity: [0b01011111, 0b_______1] }]), offsets: [0, 3, 6, 9] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map(['k1', 'k2'], [a_col, b_col]) @@ -97,13 +97,13 @@ evaluation: | Row 2 | 3 | 6 | {'k1':3, 'k2':6} | +--------+---------+---------+-------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | Int8([1, 2, 3]) | -| b_col | Int8([4, 5, 6]) | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[k1, k2, k1, k2, k1, k2] }, Int8([1, 4, 2, 5, 3, 6])]), offsets: [0, 2, 4, 6] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------+ +| a_col | Int8([1, 2, 3]) | +| b_col | Int8([4, 5, 6]) | +| Output | ArrayColumn { values: Tuple([StringColumn[k1, k2, k1, k2, k1, k2], Int8([1, 4, 2, 5, 3, 6])]), offsets: [0, 2, 4, 6] } | ++--------+------------------------------------------------------------------------------------------------------------------------+ ast : map([],[])[1] @@ -201,15 +201,15 @@ evaluation: | Row 1 | 2 | 4 | 'v2' | 'v4' | NULL | +--------+---------+---------+---------------+---------------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------+ -| k1 | Int16([1, 2]) | -| k2 | Int16([3, 4]) | -| v1 | StringColumn { data: Utf8ViewArray[v1, v2] } | -| v2 | StringColumn { data: Utf8ViewArray[v3, v4] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, ] }, validity: [0b______01] } | -+--------+-----------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------+ +| k1 | Int16([1, 2]) | +| k2 | Int16([3, 4]) | +| v1 | StringColumn[v1, v2] | +| v2 | StringColumn[v3, v4] | +| Output | NullableColumn { column: StringColumn[v1, ], validity: [0b______01] } | ++--------+-----------------------------------------------------------------------+ ast : map_keys({}) @@ -262,17 +262,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | ['c', 'f', 'z'] | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+-----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | -| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | -| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: StringColumn { data: Utf8ViewArray[a, d, x, b, e, y, c, f, z] }, offsets: [0, 3, 6, 9] } | -+--------+----------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| a_col | StringColumn[a, b, c] | +| b_col | StringColumn[d, e, f] | +| c_col | StringColumn[x, y, z] | +| d_col | NullableColumn { column: StringColumn[v1, v2, v3], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[v4, v5, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[v6, , v7], validity: [0b_____101] } | +| Output | ArrayColumn { values: StringColumn[a, d, x, b, e, y, c, f, z], offsets: [0, 3, 6, 9] } | ++--------+----------------------------------------------------------------------------------------+ ast : map_values({}) @@ -334,17 +334,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | ['v3', NULL, 'v7'] | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | -| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | -| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v4, v6, v2, v5, , v3, , v7] }, validity: [0b01011111, 0b_______1] }, offsets: [0, 3, 6, 9] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn[a, b, c] | +| b_col | StringColumn[d, e, f] | +| c_col | StringColumn[x, y, z] | +| d_col | NullableColumn { column: StringColumn[v1, v2, v3], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[v4, v5, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[v6, , v7], validity: [0b_____101] } | +| Output | ArrayColumn { values: NullableColumn { column: StringColumn[v1, v4, v6, v2, v5, , v3, , v7], validity: [0b01011111, 0b_______1] }, offsets: [0, 3, 6, 9] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_size({}) @@ -388,17 +388,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | 3 | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | -| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | -| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | -| Output | UInt64([3, 3, 3]) | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| a_col | StringColumn[a, b, c] | +| b_col | StringColumn[d, e, f] | +| c_col | StringColumn[x, y, z] | +| d_col | NullableColumn { column: StringColumn[v1, v2, v3], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[v4, v5, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[v6, , v7], validity: [0b_____101] } | +| Output | UInt64([3, 3, 3]) | ++--------+-----------------------------------------------------------------------------+ ast : map_cat({}, {}) @@ -442,17 +442,17 @@ evaluation: | Row 2 | 'a_k3' | 'b_k3' | 'c_k3' | 'aaa3' | 'bbb3' | 'ccc3' | {'a_k3':'aaa3', 'b_k3':'bbb3', 'c_k3':'ccc3'} | +--------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-----------------------------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a_k1, a_k2, a_k3] } | -| b_col | StringColumn { data: Utf8ViewArray[b_k1, b_k2, b_k3] } | -| c_col | StringColumn { data: Utf8ViewArray[c_k1, c_k2, c_k3] } | -| d_col | StringColumn { data: Utf8ViewArray[aaa1, aaa2, aaa3] } | -| e_col | StringColumn { data: Utf8ViewArray[bbb1, bbb2, bbb3] } | -| f_col | StringColumn { data: Utf8ViewArray[ccc1, ccc2, ccc3] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a_k1, b_k1, c_k1, a_k2, b_k2, c_k2, a_k3, b_k3, c_k3] }, StringColumn { data: Utf8ViewArray[aaa1, bbb1, ccc1, aaa2, bbb2, ccc2, aaa3, bbb3, ccc3] }]), offsets: [0, 3, 6, 9] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn[a_k1, a_k2, a_k3] | +| b_col | StringColumn[b_k1, b_k2, b_k3] | +| c_col | StringColumn[c_k1, c_k2, c_k3] | +| d_col | StringColumn[aaa1, aaa2, aaa3] | +| e_col | StringColumn[bbb1, bbb2, bbb3] | +| f_col | StringColumn[ccc1, ccc2, ccc3] | +| Output | ArrayColumn { values: Tuple([StringColumn[a_k1, b_k1, c_k1, a_k2, b_k2, c_k2, a_k3, b_k3, c_k3], StringColumn[aaa1, bbb1, ccc1, aaa2, bbb2, ccc2, aaa3, bbb3, ccc3]]), offsets: [0, 3, 6, 9] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_cat({'k1':'v1','k2':'v2'}, {'k1':'abc'}) @@ -478,17 +478,17 @@ evaluation: | Row 2 | 'c_k3' | 'b_k3' | 'c_k3' | 'aaa3' | 'bbb3' | 'ccc3' | {'c_k3':'ccc3', 'b_k3':'bbb3'} | +--------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-----------------------------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a_k1, a_k2, c_k3] } | -| b_col | StringColumn { data: Utf8ViewArray[b_k1, c_k2, b_k3] } | -| c_col | StringColumn { data: Utf8ViewArray[c_k1, c_k2, c_k3] } | -| d_col | StringColumn { data: Utf8ViewArray[aaa1, aaa2, aaa3] } | -| e_col | StringColumn { data: Utf8ViewArray[bbb1, bbb2, bbb3] } | -| f_col | StringColumn { data: Utf8ViewArray[ccc1, ccc2, ccc3] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a_k1, b_k1, c_k1, a_k2, c_k2, c_k3, b_k3] }, StringColumn { data: Utf8ViewArray[aaa1, bbb1, ccc1, aaa2, ccc2, ccc3, bbb3] }]), offsets: [0, 3, 5, 7] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn[a_k1, a_k2, c_k3] | +| b_col | StringColumn[b_k1, c_k2, b_k3] | +| c_col | StringColumn[c_k1, c_k2, c_k3] | +| d_col | StringColumn[aaa1, aaa2, aaa3] | +| e_col | StringColumn[bbb1, bbb2, bbb3] | +| f_col | StringColumn[ccc1, ccc2, ccc3] | +| Output | ArrayColumn { values: Tuple([StringColumn[a_k1, b_k1, c_k1, a_k2, c_k2, c_k3, b_k3], StringColumn[aaa1, bbb1, ccc1, aaa2, ccc2, ccc3, bbb3]]), offsets: [0, 3, 5, 7] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_cat({'k1': 'v1', 'k2': 'v2'}, {'k3': 'v3'}) @@ -602,15 +602,15 @@ evaluation: | Row 2 | 'a_k3' | 'b_k3' | 'aaa3' | 'bbb3' | {'a_k3':'aaa3'} | +--------+-------------------+-------------------+-------------------+-------------------+--------------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a_k1, a_k2, a_k3] } | -| b_col | StringColumn { data: Utf8ViewArray[b_k1, b_k2, b_k3] } | -| d_col | StringColumn { data: Utf8ViewArray[aaa1, aaa2, aaa3] } | -| e_col | StringColumn { data: Utf8ViewArray[bbb1, bbb2, bbb3] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a_k1, b_k1, b_k2, a_k3] }, StringColumn { data: Utf8ViewArray[aaa1, bbb1, bbb2, aaa3] }]), offsets: [0, 2, 3, 4] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn[a_k1, a_k2, a_k3] | +| b_col | StringColumn[b_k1, b_k2, b_k3] | +| d_col | StringColumn[aaa1, aaa2, aaa3] | +| e_col | StringColumn[bbb1, bbb2, bbb3] | +| Output | ArrayColumn { values: Tuple([StringColumn[a_k1, b_k1, b_k2, a_k3], StringColumn[aaa1, bbb1, bbb2, aaa3]]), offsets: [0, 2, 3, 4] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ ast : map_delete({'k1': 'v1', 'k2': 'v2', 'k3': 'v3', 'k4': 'v4'}, string_key_col) @@ -627,12 +627,12 @@ evaluation: | Row 1 | 'k2' | {'k1':'v1', 'k3':'v3', 'k4':'v4'} | +--------+----------------+------------------------------------+ evaluation (internal): -+----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| string_key_col | StringColumn { data: Utf8ViewArray[k3, k2] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[k1, k2, k4, k1, k3, k4] }, StringColumn { data: Utf8ViewArray[v1, v2, v4, v1, v3, v4] }]), offsets: [0, 3, 6] } | -+----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++----------------+---------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++----------------+---------------------------------------------------------------------------------------------------------------------------------+ +| string_key_col | StringColumn[k3, k2] | +| Output | ArrayColumn { values: Tuple([StringColumn[k1, k2, k4, k1, k3, k4], StringColumn[v1, v2, v4, v1, v3, v4]]), offsets: [0, 3, 6] } | ++----------------+---------------------------------------------------------------------------------------------------------------------------------+ ast : map_delete({'k1': 'v1', 'k2': 'v2', 'k3': 'v3'}, 'k1', 'k2', 'k3') @@ -684,15 +684,15 @@ evaluation: | Row 2 | 'a_k3' | 559 | 'aaa3' | 662 | {'a_k3':559} | +--------+-------------------+-------------+-------------------+-------------+--------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a_k1, a_k2, a_k3] } | -| b_col | Int16([555, 557, 559]) | -| d_col | StringColumn { data: Utf8ViewArray[aaa1, aaa2, aaa3] } | -| e_col | Int16([666, 664, 662]) | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a_k1, aaa1, aaa2, a_k3] }, Int16([555, 666, 664, 559])]), offsets: [0, 2, 3, 4] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn[a_k1, a_k2, a_k3] | +| b_col | Int16([555, 557, 559]) | +| d_col | StringColumn[aaa1, aaa2, aaa3] | +| e_col | Int16([666, 664, 662]) | +| Output | ArrayColumn { values: Tuple([StringColumn[a_k1, aaa1, aaa2, a_k3], Int16([555, 666, 664, 559])]), offsets: [0, 2, 3, 4] } | ++--------+---------------------------------------------------------------------------------------------------------------------------+ error: @@ -753,17 +753,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | false | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | -| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | -| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | -| Output | Boolean([0b_____001]) | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| a_col | StringColumn[a, b, c] | +| b_col | StringColumn[d, e, f] | +| c_col | StringColumn[x, y, z] | +| d_col | NullableColumn { column: StringColumn[v1, v2, v3], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[v4, v5, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[v6, , v7], validity: [0b_____101] } | +| Output | Boolean([0b_____001]) | ++--------+-----------------------------------------------------------------------------+ ast : map_contains_key(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'd') @@ -780,17 +780,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | false | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | -| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | -| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | -| Output | Boolean([0b_____001]) | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| a_col | StringColumn[a, b, c] | +| b_col | StringColumn[d, e, f] | +| c_col | StringColumn[x, y, z] | +| d_col | NullableColumn { column: StringColumn[v1, v2, v3], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[v4, v5, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[v6, , v7], validity: [0b_____101] } | +| Output | Boolean([0b_____001]) | ++--------+-----------------------------------------------------------------------------+ ast : map_pick({'a':1,'b':2,'c':3}, 'a', 'b') @@ -861,17 +861,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | {} | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | -| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | -| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a, b] }, NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2] }, validity: [0b______11] }]), offsets: [0, 1, 2, 2] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn[a, b, c] | +| b_col | StringColumn[d, e, f] | +| c_col | StringColumn[x, y, z] | +| d_col | NullableColumn { column: StringColumn[v1, v2, v3], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[v4, v5, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[v6, , v7], validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([StringColumn[a, b], NullableColumn { column: StringColumn[v1, v2], validity: [0b______11] }]), offsets: [0, 1, 2, 2] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_insert({}, 'k1', 'v1') @@ -924,17 +924,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | {'c':'v3', 'f':NULL, 'z':'v7', 'k1':'v10'} | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+--------------------------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | -| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | -| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a, d, x, k1, b, e, y, k1, c, f, z, k1] }, NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v4, v6, v10, v2, v5, , v10, v3, , v7, v10] }, validity: [0b10111111, 0b____1101] }]), offsets: [0, 4, 8, 12] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn[a, b, c] | +| b_col | StringColumn[d, e, f] | +| c_col | StringColumn[x, y, z] | +| d_col | NullableColumn { column: StringColumn[v1, v2, v3], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[v4, v5, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[v6, , v7], validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([StringColumn[a, d, x, k1, b, e, y, k1, c, f, z, k1], NullableColumn { column: StringColumn[v1, v4, v6, v10, v2, v5, , v10, v3, , v7, v10], validity: [0b10111111, 0b____1101] }]), offsets: [0, 4, 8, 12] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : map_insert(map([a_col, b_col, c_col], [d_col, e_col, f_col]), 'a', 'v10', true) @@ -952,17 +952,17 @@ evaluation: | Row 2 | 'c' | 'f' | 'z' | 'v3' | NULL | 'v7' | {'c':'v3', 'f':NULL, 'z':'v7', 'a':'v10'} | +--------+-------------+-------------+-------------+---------------+----------------------+----------------------+-------------------------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| a_col | StringColumn { data: Utf8ViewArray[a, b, c] } | -| b_col | StringColumn { data: Utf8ViewArray[d, e, f] } | -| c_col | StringColumn { data: Utf8ViewArray[x, y, z] } | -| d_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v1, v2, v3] }, validity: [0b_____111] } | -| e_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v4, v5, ] }, validity: [0b_____011] } | -| f_col | NullableColumn { column: StringColumn { data: Utf8ViewArray[v6, , v7] }, validity: [0b_____101] } | -| Output | ArrayColumn { values: Tuple([StringColumn { data: Utf8ViewArray[a, d, x, b, e, y, a, c, f, z, a] }, NullableColumn { column: StringColumn { data: Utf8ViewArray[v10, v4, v6, v2, v5, , v10, v3, , v7, v10] }, validity: [0b11011111, 0b_____110] }]), offsets: [0, 3, 7, 11] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a_col | StringColumn[a, b, c] | +| b_col | StringColumn[d, e, f] | +| c_col | StringColumn[x, y, z] | +| d_col | NullableColumn { column: StringColumn[v1, v2, v3], validity: [0b_____111] } | +| e_col | NullableColumn { column: StringColumn[v4, v5, ], validity: [0b_____011] } | +| f_col | NullableColumn { column: StringColumn[v6, , v7], validity: [0b_____101] } | +| Output | ArrayColumn { values: Tuple([StringColumn[a, d, x, b, e, y, a, c, f, z, a], NullableColumn { column: StringColumn[v10, v4, v6, v2, v5, , v10, v3, , v7, v10], validity: [0b11011111, 0b_____110] }]), offsets: [0, 3, 7, 11] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ error: diff --git a/src/query/functions/tests/it/scalars/testdata/regexp.txt b/src/query/functions/tests/it/scalars/testdata/regexp.txt index 2481b476c7f5..be50dce9de50 100644 --- a/src/query/functions/tests/it/scalars/testdata/regexp.txt +++ b/src/query/functions/tests/it/scalars/testdata/regexp.txt @@ -56,13 +56,13 @@ evaluation: | Row 2 | '' | '' | 0 | +--------+---------------------------+--------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, ] } | -| pat | StringColumn { data: Utf8ViewArray[dog, a{2}, ] } | -| Output | UInt64([1, 1, 0]) | -+--------+------------------------------------------------------------------------------+ ++--------+------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------+ +| source | StringColumn[dog cat dog, aa aaa aaaa aa aaa aaaa, ] | +| pat | StringColumn[dog, a{2}, ] | +| Output | UInt64([1, 1, 0]) | ++--------+------------------------------------------------------+ ast : regexp_instr(source, pat, pos) @@ -79,14 +79,14 @@ evaluation: | Row 2 | '' | '' | 1 | 0 | +--------+---------------------------+--------------+---------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, ] } | -| pat | StringColumn { data: Utf8ViewArray[dog, a{2}, ] } | -| pos | Int64([1, 2, 1]) | -| Output | UInt64([1, 4, 0]) | -+--------+------------------------------------------------------------------------------+ ++--------+------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------+ +| source | StringColumn[dog cat dog, aa aaa aaaa aa aaa aaaa, ] | +| pat | StringColumn[dog, a{2}, ] | +| pos | Int64([1, 2, 1]) | +| Output | UInt64([1, 4, 0]) | ++--------+------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur) @@ -103,15 +103,15 @@ evaluation: | Row 2 | 'aa aa aa aaaa aaaa aaaa' | 'a{4}' | 9 | 2 | 15 | +--------+---------------------------------------------+------------------+---------+---------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] } | -| pat | StringColumn { data: Utf8ViewArray[dog, a{2}, a{4}] } | -| pos | Int64([1, 1, 9]) | -| occur | Int64([2, 3, 2]) | -| Output | UInt64([9, 8, 15]) | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| source | StringColumn[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] | +| pat | StringColumn[dog, a{2}, a{4}] | +| pos | Int64([1, 1, 9]) | +| occur | Int64([2, 3, 2]) | +| Output | UInt64([9, 8, 15]) | ++--------+-----------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro) @@ -129,16 +129,16 @@ evaluation: | Row 2 | 'aa aa aa aaaa aaaa aaaa' | 'a{4}' | 1 | 2 | 1 | 19 | +--------+---------------------------------------------+------------------+---------+---------+---------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] } | -| pat | StringColumn { data: Utf8ViewArray[dog, a{2}, a{4}] } | -| pos | Int64([1, 2, 1]) | -| occur | Int64([2, 2, 2]) | -| ro | Int64([0, 1, 1]) | -| Output | UInt64([9, 10, 19]) | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| source | StringColumn[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] | +| pat | StringColumn[dog, a{2}, a{4}] | +| pos | Int64([1, 2, 1]) | +| occur | Int64([2, 2, 2]) | +| ro | Int64([0, 1, 1]) | +| Output | UInt64([9, 10, 19]) | ++--------+-----------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro, mt) @@ -156,17 +156,17 @@ evaluation: | Row 2 | 'aa aa aa aaaa aaaa aaaa' | 'A{4}' | 1 | 2 | 1 | 'i' | 19 | +--------+---------------------------------------------+------------------+---------+---------+---------+-------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] } | -| pat | StringColumn { data: Utf8ViewArray[dog, A{2}, A{4}] } | -| pos | Int64([1, 2, 1]) | -| occur | Int64([2, 2, 2]) | -| ro | Int64([0, 1, 1]) | -| mt | StringColumn { data: Utf8ViewArray[i, c, i] } | -| Output | UInt64([9, 0, 19]) | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| source | StringColumn[dog cat dog, aa aaa aaaa aa aaa aaaa, aa aa aa aaaa aaaa aaaa] | +| pat | StringColumn[dog, A{2}, A{4}] | +| pos | Int64([1, 2, 1]) | +| occur | Int64([2, 2, 2]) | +| ro | Int64([0, 1, 1]) | +| mt | StringColumn[i, c, i] | +| Output | UInt64([9, 0, 19]) | ++--------+-----------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro) @@ -184,16 +184,16 @@ evaluation: | Row 3 | 'aa aa aa aaaa aaaa aaaa' | 'A{4}' | 1 | 1 | 1 | 14 | +--------+-------------------------------+-----------------------+---------+---------+---------+-------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, , aa aa aa aaaa aaaa aaaa] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[dog, , , A{4}] }, validity: [0b____1001] } | -| pos | Int64([1, 2, 1, 1]) | -| occur | Int64([2, 2, 2, 1]) | -| ro | Int64([0, 1, 1, 1]) | -| Output | NullableColumn { column: UInt64([9, 0, 0, 14]), validity: [0b____1001] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn[dog cat dog, aa aaa aaaa aa aaa aaaa, , aa aa aa aaaa aaaa aaaa], validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn[dog, , , A{4}], validity: [0b____1001] } | +| pos | Int64([1, 2, 1, 1]) | +| occur | Int64([2, 2, 2, 1]) | +| ro | Int64([0, 1, 1, 1]) | +| Output | NullableColumn { column: UInt64([9, 0, 0, 14]), validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro, mt) @@ -211,17 +211,17 @@ evaluation: | Row 3 | 'aa aa aa aaaa aaaa aaaa' | 'A{4}' | 1 | 1 | 1 | 'i' | 14 | +--------+-------------------------------+-----------------------+---------+---------+---------+-------------+-------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[dog cat dog, aa aaa aaaa aa aaa aaaa, , aa aa aa aaaa aaaa aaaa] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[dog, , , A{4}] }, validity: [0b____1001] } | -| pos | Int64([1, 2, 1, 1]) | -| occur | Int64([2, 2, 2, 1]) | -| ro | Int64([0, 1, 1, 1]) | -| mt | StringColumn { data: Utf8ViewArray[i, c, i, i] } | -| Output | NullableColumn { column: UInt64([9, 0, 0, 14]), validity: [0b____1001] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn[dog cat dog, aa aaa aaaa aa aaa aaaa, , aa aa aa aaaa aaaa aaaa], validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn[dog, , , A{4}], validity: [0b____1001] } | +| pos | Int64([1, 2, 1, 1]) | +| occur | Int64([2, 2, 2, 1]) | +| ro | Int64([0, 1, 1, 1]) | +| mt | StringColumn[i, c, i, i] | +| Output | NullableColumn { column: UInt64([9, 0, 0, 14]), validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------+ ast : regexp_instr(source, pat, pos, occur, ro) @@ -240,16 +240,16 @@ evaluation: | Row 3 | '周 周周 周周周 周周周周' | '周+' | 5 | 1 | 1 | 9 | +--------+---------------------------------------------------------+-----------------+---------+---------+---------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] } | -| pat | StringColumn { data: Utf8ViewArray[周+, 周+, 周+, 周+] } | -| pos | Int64([1, 2, 3, 5]) | -| occur | Int64([2, 2, 3, 1]) | -| ro | Int64([0, 1, 1, 1]) | -| Output | UInt64([3, 9, 14, 9]) | -+--------+------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------+ +| source | StringColumn[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] | +| pat | StringColumn[周+, 周+, 周+, 周+] | +| pos | Int64([1, 2, 3, 5]) | +| occur | Int64([2, 2, 3, 1]) | +| ro | Int64([0, 1, 1, 1]) | +| Output | UInt64([3, 9, 14, 9]) | ++--------+------------------------------------------------------------------------------------------------------------------+ error: @@ -349,15 +349,15 @@ evaluation: | Row 5 | '' | '' | true | +--------+----------------------+-------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[abc, abd, Abe, new* | -| | *line, fo | -| | fo, ] } | -| pat | StringColumn { data: Utf8ViewArray[^a, Ab, abe, new\*.\*line, ^fo$, ] } | -| Output | Boolean([0b__100111]) | -+--------+-------------------------------------------------------------------------+ ++--------+-------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------+ +| source | StringColumn[abc, abd, Abe, new* | +| | *line, fo | +| | fo, ] | +| pat | StringColumn[^a, Ab, abe, new\*.\*line, ^fo$, ] | +| Output | Boolean([0b__100111]) | ++--------+-------------------------------------------------+ ast : regexp_like(source, pat, mt) @@ -379,16 +379,16 @@ evaluation: | Row 5 | '' | '' | 'c' | true | +--------+----------------------+-------------------------+------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[abc, abd, Abe, new* | -| | *line, fo | -| | fo, ] } | -| pat | StringColumn { data: Utf8ViewArray[^a, Ab, abe, new\*.\*line, ^fo$, ] } | -| mt | StringColumn { data: Utf8ViewArray[, c, i, n, m, c] } | -| Output | Boolean([0b__111101]) | -+--------+-------------------------------------------------------------------------+ ++--------+-------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------+ +| source | StringColumn[abc, abd, Abe, new* | +| | *line, fo | +| | fo, ] | +| pat | StringColumn[^a, Ab, abe, new\*.\*line, ^fo$, ] | +| mt | StringColumn[, c, i, n, m, c] | +| Output | Boolean([0b__111101]) | ++--------+-------------------------------------------------+ ast : regexp_like(source, pat, mt) @@ -406,14 +406,14 @@ evaluation: | Row 3 | 'abc' | 'abc' | NULL | NULL | +--------+-----------------------+-----------------------+---------------------+--------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, abc, , abc] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, , , abc] }, validity: [0b____1001] } | -| mt | NullableColumn { column: StringColumn { data: Utf8ViewArray[, i, i, ] }, validity: [0b____0111] } | -| Output | NullableColumn { column: Boolean([0b____1101]), validity: [0b____0001] } | -+--------+----------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn[abc, abc, , abc], validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn[abc, , , abc], validity: [0b____1001] } | +| mt | NullableColumn { column: StringColumn[, i, i, ], validity: [0b____0111] } | +| Output | NullableColumn { column: Boolean([0b____1101]), validity: [0b____0001] } | ++--------+----------------------------------------------------------------------------------+ error: @@ -520,14 +520,14 @@ evaluation: | Row 3 | '' | 'b' | 'X' | '' | +--------+----------------+------------+-------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[a b c, a b c, a b c, ] } | -| pat | StringColumn { data: Utf8ViewArray[b, x, , b] } | -| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | -| Output | StringColumn { data: Utf8ViewArray[a X c, a b c, a b c, ] } | -+--------+-------------------------------------------------------------+ ++--------+-------------------------------------+ +| Column | Data | ++--------+-------------------------------------+ +| source | StringColumn[a b c, a b c, a b c, ] | +| pat | StringColumn[b, x, , b] | +| repl | StringColumn[X, X, X, X] | +| Output | StringColumn[a X c, a b c, a b c, ] | ++--------+-------------------------------------+ ast : regexp_replace(source, pat, repl, pos) @@ -546,15 +546,15 @@ evaluation: | Row 3 | 'abc def ghi' | '[a-z]+' | 'X' | 12 | 'abc def ghi' | +--------+---------------------------------+-----------------------+-------------+----------+---------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, abc def ghi, abc def ghi] } | -| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+, [a-z]+] } | -| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | -| pos | Int64([1, 4, 8, 12]) | -| Output | StringColumn { data: Utf8ViewArray[X X X, abc X X, abc def X, abc def ghi] } | -+--------+------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------+ +| source | StringColumn[abc def ghi, abc def ghi, abc def ghi, abc def ghi] | +| pat | StringColumn[[a-z]+, [a-z]+, [a-z]+, [a-z]+] | +| repl | StringColumn[X, X, X, X] | +| pos | Int64([1, 4, 8, 12]) | +| Output | StringColumn[X X X, abc X X, abc def X, abc def ghi] | ++--------+------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur) @@ -573,16 +573,16 @@ evaluation: | Row 3 | 'abc def ghi' | '[a-z]+' | 'X' | 4 | 3 | 'abc def ghi' | +--------+---------------------------------+-----------------------+-------------+---------+---------+---------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, abc def ghi, abc def ghi] } | -| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+, [a-z]+] } | -| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | -| pos | Int64([1, 1, 4, 4]) | -| occur | Int64([0, 1, 2, 3]) | -| Output | StringColumn { data: Utf8ViewArray[X X X, X def ghi, abc def X, abc def ghi] } | -+--------+------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------+ +| source | StringColumn[abc def ghi, abc def ghi, abc def ghi, abc def ghi] | +| pat | StringColumn[[a-z]+, [a-z]+, [a-z]+, [a-z]+] | +| repl | StringColumn[X, X, X, X] | +| pos | Int64([1, 1, 4, 4]) | +| occur | Int64([0, 1, 2, 3]) | +| Output | StringColumn[X X X, X def ghi, abc def X, abc def ghi] | ++--------+------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur, mt) @@ -600,17 +600,17 @@ evaluation: | Row 2 | 'abc DEF ghi' | '[a-z]+' | 'X' | 4 | 1 | 'i' | 'abc X ghi' | +--------+---------------------------------+-----------------------+-------------+---------+---------+------------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc DEF ghi, abc DEF ghi] } | -| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+] } | -| repl | StringColumn { data: Utf8ViewArray[X, X, X] } | -| pos | Int64([1, 1, 4]) | -| occur | Int64([0, 2, 1]) | -| mt | StringColumn { data: Utf8ViewArray[, c, i] } | -| Output | StringColumn { data: Utf8ViewArray[X X X, abc DEF X, abc X ghi] } | -+--------+-----------------------------------------------------------------------------+ ++--------+-----------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------+ +| source | StringColumn[abc def ghi, abc DEF ghi, abc DEF ghi] | +| pat | StringColumn[[a-z]+, [a-z]+, [a-z]+] | +| repl | StringColumn[X, X, X] | +| pos | Int64([1, 1, 4]) | +| occur | Int64([0, 2, 1]) | +| mt | StringColumn[, c, i] | +| Output | StringColumn[X X X, abc DEF X, abc X ghi] | ++--------+-----------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur) @@ -629,16 +629,16 @@ evaluation: | Row 3 | 'abc DEF ghi' | '[a-z]+' | 'X' | 4 | 1 | 'abc X ghi' | +--------+-------------------------------+--------------------------+-------------+---------+---------+-------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc def ghi, abc DEF ghi, , abc DEF ghi] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[[a-z]+, , , [a-z]+] }, validity: [0b____1001] } | -| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | -| pos | Int64([1, 1, 4, 4]) | -| occur | Int64([0, 2, 1, 1]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[X X X, abc DEF ghi, , abc X ghi] }, validity: [0b____1001] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn[abc def ghi, abc DEF ghi, , abc DEF ghi], validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn[[a-z]+, , , [a-z]+], validity: [0b____1001] } | +| repl | StringColumn[X, X, X, X] | +| pos | Int64([1, 1, 4, 4]) | +| occur | Int64([0, 2, 1, 1]) | +| Output | NullableColumn { column: StringColumn[X X X, abc DEF ghi, , abc X ghi], validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur, mt) @@ -657,17 +657,17 @@ evaluation: | Row 3 | 'abc DEF ghi' | '[a-z]+' | 'X' | 4 | 1 | 'i' | 'abc X ghi' | +--------+-------------------------------+--------------------------+-------------+---------+---------+------------+-------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc def ghi, abc DEF ghi, , abc DEF ghi] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[[a-z]+, , , [a-z]+] }, validity: [0b____1001] } | -| repl | StringColumn { data: Utf8ViewArray[X, X, X, X] } | -| pos | Int64([1, 1, 4, 4]) | -| occur | Int64([0, 2, 1, 1]) | -| mt | StringColumn { data: Utf8ViewArray[, c, i, i] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[X X X, abc DEF ghi, , abc X ghi] }, validity: [0b____1001] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn[abc def ghi, abc DEF ghi, , abc DEF ghi], validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn[[a-z]+, , , [a-z]+], validity: [0b____1001] } | +| repl | StringColumn[X, X, X, X] | +| pos | Int64([1, 1, 4, 4]) | +| occur | Int64([0, 2, 1, 1]) | +| mt | StringColumn[, c, i, i] | +| Output | NullableColumn { column: StringColumn[X X X, abc DEF ghi, , abc X ghi], validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------+ ast : regexp_replace(source, pat, repl, pos, occur) @@ -686,16 +686,16 @@ evaluation: | Row 3 | '周 周周 周周周 周周周周' | '周+' | '唐' | 5 | 1 | '周 周周 唐 周周周周' | +--------+---------------------------------------------------------+-----------------+---------------+---------+---------+-------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] } | -| pat | StringColumn { data: Utf8ViewArray[周+, 周+, 周+, 周+] } | -| repl | StringColumn { data: Utf8ViewArray[唐, 唐, 唐, 唐] } | -| pos | Int64([1, 2, 3, 5]) | -| occur | Int64([0, 1, 3, 1]) | -| Output | StringColumn { data: Utf8ViewArray[唐 唐 唐 唐, 周 唐 周周周 周周周周, 周 周周 周周周 唐, 周 周周 唐 周周周周] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------+ +| source | StringColumn[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] | +| pat | StringColumn[周+, 周+, 周+, 周+] | +| repl | StringColumn[唐, 唐, 唐, 唐] | +| pos | Int64([1, 2, 3, 5]) | +| occur | Int64([0, 1, 3, 1]) | +| Output | StringColumn[唐 唐 唐 唐, 周 唐 周周周 周周周周, 周 周周 周周周 唐, 周 周周 唐 周周周周] | ++--------+------------------------------------------------------------------------------------------------------------------+ error: @@ -790,13 +790,13 @@ evaluation: | Row 2 | '' | '' | NULL | +--------+----------------------+--------------+-------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, ] } | -| pat | StringColumn { data: Utf8ViewArray[[a-z]+, xxx, ] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, , ] }, validity: [0b_____001] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| source | StringColumn[abc def ghi, abc def ghi, ] | +| pat | StringColumn[[a-z]+, xxx, ] | +| Output | NullableColumn { column: StringColumn[abc, , ], validity: [0b_____001] } | ++--------+--------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos) @@ -814,14 +814,14 @@ evaluation: | Row 2 | 'abc def ghi' | '[a-z]+' | 12 | NULL | +--------+---------------------------------+-----------------------+----------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, abc def ghi] } | -| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+] } | -| pos | Int64([1, 4, 12]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, def, ] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| source | StringColumn[abc def ghi, abc def ghi, abc def ghi] | +| pat | StringColumn[[a-z]+, [a-z]+, [a-z]+] | +| pos | Int64([1, 4, 12]) | +| Output | NullableColumn { column: StringColumn[abc, def, ], validity: [0b_____011] } | ++--------+-----------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos, occur) @@ -839,15 +839,15 @@ evaluation: | Row 2 | 'abc def ghi' | '[a-z]+' | 12 | 3 | NULL | +--------+---------------------------------+-----------------------+----------+---------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[abc def ghi, abc def ghi, abc def ghi] } | -| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+] } | -| pos | Int64([1, 4, 12]) | -| occur | Int64([3, 2, 3]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[ghi, ghi, ] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| source | StringColumn[abc def ghi, abc def ghi, abc def ghi] | +| pat | StringColumn[[a-z]+, [a-z]+, [a-z]+] | +| pos | Int64([1, 4, 12]) | +| occur | Int64([3, 2, 3]) | +| Output | NullableColumn { column: StringColumn[ghi, ghi, ], validity: [0b_____011] } | ++--------+-----------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos, occur, mt) @@ -865,16 +865,16 @@ evaluation: | Row 2 | 'abc DEF ghi' | '[a-z]+' | 12 | 3 | 'i' | NULL | +--------+---------------------------------+-----------------------+----------+---------+-------------+-------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[ABC def ghi, abc def GHI, abc DEF ghi] } | -| pat | StringColumn { data: Utf8ViewArray[[a-z]+, [a-z]+, [a-z]+] } | -| pos | Int64([1, 4, 12]) | -| occur | Int64([3, 2, 3]) | -| mt | StringColumn { data: Utf8ViewArray[c, i, i] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, GHI, ] }, validity: [0b_____010] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| source | StringColumn[ABC def ghi, abc def GHI, abc DEF ghi] | +| pat | StringColumn[[a-z]+, [a-z]+, [a-z]+] | +| pos | Int64([1, 4, 12]) | +| occur | Int64([3, 2, 3]) | +| mt | StringColumn[c, i, i] | +| Output | NullableColumn { column: StringColumn[, GHI, ], validity: [0b_____010] } | ++--------+--------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos, occur, mt) @@ -892,16 +892,16 @@ evaluation: | Row 3 | 'abc DEF ghi' | '[a-z]+' | 4 | 1 | 'i' | 'DEF' | +--------+-------------------------------+--------------------------+---------+---------+------------+-------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| source | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc def ghi, abc DEF ghi, , abc DEF ghi] }, validity: [0b____1011] } | -| pat | NullableColumn { column: StringColumn { data: Utf8ViewArray[[a-z]+, , , [a-z]+] }, validity: [0b____1001] } | -| pos | Int64([1, 1, 4, 4]) | -| occur | Int64([1, 2, 1, 1]) | -| mt | StringColumn { data: Utf8ViewArray[, c, i, i] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, , , DEF] }, validity: [0b____1001] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------+ +| source | NullableColumn { column: StringColumn[abc def ghi, abc DEF ghi, , abc DEF ghi], validity: [0b____1011] } | +| pat | NullableColumn { column: StringColumn[[a-z]+, , , [a-z]+], validity: [0b____1001] } | +| pos | Int64([1, 1, 4, 4]) | +| occur | Int64([1, 2, 1, 1]) | +| mt | StringColumn[, c, i, i] | +| Output | NullableColumn { column: StringColumn[abc, , , DEF], validity: [0b____1001] } | ++--------+----------------------------------------------------------------------------------------------------------+ ast : regexp_substr(source, pat, pos, occur) @@ -919,15 +919,15 @@ evaluation: | Row 2 | '周 周周 周周周 周周周周' | '周+' | 14 | 1 | NULL | +--------+---------------------------------------------------------+-----------------+----------+---------+-------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------+ -| source | StringColumn { data: Utf8ViewArray[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] } | -| pat | StringColumn { data: Utf8ViewArray[周+, 周+, 周+] } | -| pos | Int64([1, 2, 14]) | -| occur | Int64([1, 2, 1]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[周, 周周周, ] }, validity: [0b_____011] } | -+--------+-----------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------+ +| source | StringColumn[周 周周 周周周 周周周周, 周 周周 周周周 周周周周, 周 周周 周周周 周周周周] | +| pat | StringColumn[周+, 周+, 周+] | +| pos | Int64([1, 2, 14]) | +| occur | Int64([1, 2, 1]) | +| Output | NullableColumn { column: StringColumn[周, 周周周, ], validity: [0b_____011] } | ++--------+-----------------------------------------------------------------------------------------+ error: diff --git a/src/query/functions/tests/it/scalars/testdata/string.txt b/src/query/functions/tests/it/scalars/testdata/string.txt index 63958666b0e4..6e83f4f09863 100644 --- a/src/query/functions/tests/it/scalars/testdata/string.txt +++ b/src/query/functions/tests/it/scalars/testdata/string.txt @@ -48,12 +48,12 @@ evaluation: | Row 2 | 'ß😀山' | 'SS😀山' | +--------+-------------------+-------------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, Dobrý den, ß😀山] } | -| Output | StringColumn { data: Utf8ViewArray[ABC, DOBRÝ DEN, SS😀山] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[Abc, Dobrý den, ß😀山] | +| Output | StringColumn[ABC, DOBRÝ DEN, SS😀山] | ++--------+--------------------------------------+ ast : lower('Abc') @@ -106,12 +106,12 @@ evaluation: | Row 2 | 'İ😀山' | 'i̇😀山' | +--------+-------------------+-------------+ evaluation (internal): -+--------+-------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[Abc, DOBRÝ DEN, İ😀山] } | -| Output | StringColumn { data: Utf8ViewArray[abc, dobrý den, i̇😀山] } | -+--------+-------------------------------------------------------------+ ++--------+-------------------------------------+ +| Column | Data | ++--------+-------------------------------------+ +| a | StringColumn[Abc, DOBRÝ DEN, İ😀山] | +| Output | StringColumn[abc, dobrý den, i̇😀山] | ++--------+-------------------------------------+ ast : bit_length('latin') @@ -182,12 +182,12 @@ evaluation: | Row 2 | 'кириллица and latin' | 224 | +--------+-----------------------------------+----------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[latin, кириллица, кириллица and latin] } | -| Output | UInt64([40, 144, 224]) | -+--------+-----------------------------------------------------------------------------+ ++--------+-----------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------+ +| a | StringColumn[latin, кириллица, кириллица and latin] | +| Output | UInt64([40, 144, 224]) | ++--------+-----------------------------------------------------+ ast : octet_length('latin') @@ -258,12 +258,12 @@ evaluation: | Row 2 | 'кириллица and latin' | 28 | +--------+-----------------------------------+----------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[latin, кириллица, кириллица and latin] } | -| Output | UInt64([5, 18, 28]) | -+--------+-----------------------------------------------------------------------------+ ++--------+-----------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------+ +| a | StringColumn[latin, кириллица, кириллица and latin] | +| Output | UInt64([5, 18, 28]) | ++--------+-----------------------------------------------------+ ast : char_length('latin') @@ -334,12 +334,12 @@ evaluation: | Row 2 | 'кириллица and latin' | 19 | +--------+-----------------------------------+----------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[latin, кириллица, кириллица and latin] } | -| Output | UInt64([5, 9, 19]) | -+--------+-----------------------------------------------------------------------------+ ++--------+-----------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------+ +| a | StringColumn[latin, кириллица, кириллица and latin] | +| Output | UInt64([5, 9, 19]) | ++--------+-----------------------------------------------------+ ast : quote('a\0b') @@ -465,12 +465,12 @@ evaluation: | Row 2 | 'a\nb' | 'a\\nb' | +--------+---------------------+----------+ evaluation (internal): -+--------+------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[a\0b, a\'b, a\nb] } | -| Output | StringColumn { data: Utf8ViewArray[a\\0b, a\\\'b, a\\nb] } | -+--------+------------------------------------------------------------+ ++--------+------------------------------------+ +| Column | Data | ++--------+------------------------------------+ +| a | StringColumn[a\0b, a\'b, a\nb] | +| Output | StringColumn[a\\0b, a\\\'b, a\\nb] | ++--------+------------------------------------+ ast : reverse('abc') @@ -550,12 +550,12 @@ evaluation: | Row 2 | '' | '' | +--------+--------------+--------+ evaluation (internal): -+--------+------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[abc, a, ] } | -| Output | StringColumn { data: Utf8ViewArray[cba, a, ] } | -+--------+------------------------------------------------+ ++--------+------------------------+ +| Column | Data | ++--------+------------------------+ +| a | StringColumn[abc, a, ] | +| Output | StringColumn[cba, a, ] | ++--------+------------------------+ ast : ascii('1') @@ -636,12 +636,12 @@ evaluation: | Row 3 | '你好' | 228 | +--------+-----------------+------------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[1, 123, -1, 你好] } | -| Output | UInt8([49, 49, 45, 228]) | -+--------+--------------------------------------------------------+ ++--------+--------------------------------+ +| Column | Data | ++--------+--------------------------------+ +| a | StringColumn[1, 123, -1, 你好] | +| Output | UInt8([49, 49, 45, 228]) | ++--------+--------------------------------+ ast : ascii(b) @@ -657,12 +657,12 @@ evaluation: | Row 0 | '' | 0 | +--------+-----------+---------+ evaluation (internal): -+--------+----------------------------------------+ -| Column | Data | -+--------+----------------------------------------+ -| b | StringColumn { data: Utf8ViewArray[] } | -| Output | UInt8([0]) | -+--------+----------------------------------------+ ++--------+----------------+ +| Column | Data | ++--------+----------------+ +| b | StringColumn[] | +| Output | UInt8([0]) | ++--------+----------------+ ast : ltrim(' abc ') @@ -734,12 +734,12 @@ evaluation: | Row 3 | 'abc ' | 'abc ' | +--------+-----------------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | -| Output | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[abc, abc, abc , abc ] | +| Output | StringColumn[abc, abc, abc , abc ] | ++--------+----------------------------------------------+ ast : rtrim(' abc ') @@ -811,12 +811,12 @@ evaluation: | Row 3 | 'abc ' | 'abc' | +--------+-----------------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | -| Output | StringColumn { data: Utf8ViewArray[abc, abc, abc, abc] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[abc, abc, abc , abc ] | +| Output | StringColumn[abc, abc, abc, abc] | ++--------+----------------------------------------------+ ast : trim_leading('aaabbaaa', 'a') @@ -915,12 +915,12 @@ evaluation: | Row 3 | 'aabbaa' | 'bbaa' | +--------+-----------------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | -| Output | StringColumn { data: Utf8ViewArray[bbaa, bbccbb, ccddcc, bbaa] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc, aabbaa] | +| Output | StringColumn[bbaa, bbccbb, ccddcc, bbaa] | ++--------+----------------------------------------------+ ast : trim_leading(a, b) @@ -938,13 +938,13 @@ evaluation: | Row 3 | 'aabbaa' | '' | 'aabbaa' | +--------+-----------------------+------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | -| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | -| Output | StringColumn { data: Utf8ViewArray[bbaa, ccbb, ddcc, aabbaa] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc, aabbaa] | +| b | StringColumn[a, b, c, ] | +| Output | StringColumn[bbaa, ccbb, ddcc, aabbaa] | ++--------+----------------------------------------------+ ast : trim_leading('aba', b) @@ -962,12 +962,12 @@ evaluation: | Row 3 | '' | 'aba' | +--------+------------+--------+ evaluation (internal): -+--------+---------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------+ -| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | -| Output | StringColumn { data: Utf8ViewArray[ba, aba, aba, aba] } | -+--------+---------------------------------------------------------+ ++--------+---------------------------------+ +| Column | Data | ++--------+---------------------------------+ +| b | StringColumn[a, b, c, ] | +| Output | StringColumn[ba, aba, aba, aba] | ++--------+---------------------------------+ ast : trim_trailing('aaabbaaa', 'a') @@ -1066,12 +1066,12 @@ evaluation: | Row 3 | 'aabbaa' | 'aabbaa' | +--------+-----------------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | -| Output | StringColumn { data: Utf8ViewArray[aabbaa, bbcc, ccddcc, aabbaa] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc, aabbaa] | +| Output | StringColumn[aabbaa, bbcc, ccddcc, aabbaa] | ++--------+----------------------------------------------+ ast : trim_trailing(a, b) @@ -1089,13 +1089,13 @@ evaluation: | Row 3 | 'aabbaa' | '' | 'aabbaa' | +--------+-----------------------+------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | -| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | -| Output | StringColumn { data: Utf8ViewArray[aabb, bbcc, ccdd, aabbaa] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc, aabbaa] | +| b | StringColumn[a, b, c, ] | +| Output | StringColumn[aabb, bbcc, ccdd, aabbaa] | ++--------+----------------------------------------------+ ast : trim_trailing('aba', b) @@ -1113,12 +1113,12 @@ evaluation: | Row 3 | '' | 'aba' | +--------+------------+--------+ evaluation (internal): -+--------+---------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------+ -| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | -| Output | StringColumn { data: Utf8ViewArray[ab, aba, aba, aba] } | -+--------+---------------------------------------------------------+ ++--------+---------------------------------+ +| Column | Data | ++--------+---------------------------------+ +| b | StringColumn[a, b, c, ] | +| Output | StringColumn[ab, aba, aba, aba] | ++--------+---------------------------------+ ast : trim_both('aaabbaaa', 'a') @@ -1226,12 +1226,12 @@ evaluation: | Row 3 | 'aabbaa' | 'bb' | +--------+-----------------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | -| Output | StringColumn { data: Utf8ViewArray[bb, bbccbb, ccddcc, bb] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc, aabbaa] | +| Output | StringColumn[bb, bbccbb, ccddcc, bb] | ++--------+----------------------------------------------+ ast : trim_both(a, b) @@ -1249,13 +1249,13 @@ evaluation: | Row 3 | 'aabbaa' | '' | 'aabbaa' | +--------+-----------------------+------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc, aabbaa] } | -| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | -| Output | StringColumn { data: Utf8ViewArray[bb, cc, dd, aabbaa] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc, aabbaa] | +| b | StringColumn[a, b, c, ] | +| Output | StringColumn[bb, cc, dd, aabbaa] | ++--------+----------------------------------------------+ ast : trim_both('aba', b) @@ -1273,12 +1273,12 @@ evaluation: | Row 3 | '' | 'aba' | +--------+------------+--------+ evaluation (internal): -+--------+--------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------+ -| b | StringColumn { data: Utf8ViewArray[a, b, c, ] } | -| Output | StringColumn { data: Utf8ViewArray[b, aba, aba, aba] } | -+--------+--------------------------------------------------------+ ++--------+--------------------------------+ +| Column | Data | ++--------+--------------------------------+ +| b | StringColumn[a, b, c, ] | +| Output | StringColumn[b, aba, aba, aba] | ++--------+--------------------------------+ ast : trim(' abc ') @@ -1350,12 +1350,12 @@ evaluation: | Row 3 | 'abc ' | 'abc' | +--------+-----------------------+--------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | -| Output | StringColumn { data: Utf8ViewArray[abc, abc, abc, abc] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[abc, abc, abc , abc ] | +| Output | StringColumn[abc, abc, abc, abc] | ++--------+----------------------------------------------+ ast : trim(both 'a' from 'aaabbaaa') @@ -1426,12 +1426,12 @@ evaluation: | Row 2 | 'ccddcc' | 'ccddcc' | +--------+-----------------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| Output | StringColumn { data: Utf8ViewArray[bb, bbccbb, ccddcc] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| Output | StringColumn[bb, bbccbb, ccddcc] | ++--------+--------------------------------------+ ast : trim(both b from a) @@ -1448,13 +1448,13 @@ evaluation: | Row 2 | 'ccddcc' | 'c' | 'dd' | +--------+-----------------------+-------------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| b | StringColumn { data: Utf8ViewArray[a, b, c] } | -| Output | StringColumn { data: Utf8ViewArray[bb, cc, dd] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| b | StringColumn[a, b, c] | +| Output | StringColumn[bb, cc, dd] | ++--------+--------------------------------------+ ast : trim(both a from a) @@ -1471,12 +1471,12 @@ evaluation: | Row 2 | 'ccddcc' | '' | +--------+-----------------------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| Output | StringColumn { data: Utf8ViewArray[, , ] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| Output | StringColumn[, , ] | ++--------+--------------------------------------+ ast : trim(both b from 'aba') @@ -1493,12 +1493,12 @@ evaluation: | Row 2 | 'c' | 'aba' | +--------+-------------+--------+ evaluation (internal): -+--------+---------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------+ -| b | StringColumn { data: Utf8ViewArray[a, b, c] } | -| Output | StringColumn { data: Utf8ViewArray[b, aba, aba] } | -+--------+---------------------------------------------------+ ++--------+---------------------------+ +| Column | Data | ++--------+---------------------------+ +| b | StringColumn[a, b, c] | +| Output | StringColumn[b, aba, aba] | ++--------+---------------------------+ ast : trim(leading 'a' from 'aaabbaaa') @@ -1569,12 +1569,12 @@ evaluation: | Row 2 | 'ccddcc' | 'ccddcc' | +--------+-----------------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| Output | StringColumn { data: Utf8ViewArray[bbaa, bbccbb, ccddcc] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| Output | StringColumn[bbaa, bbccbb, ccddcc] | ++--------+--------------------------------------+ ast : trim(leading b from a) @@ -1591,13 +1591,13 @@ evaluation: | Row 2 | 'ccddcc' | 'c' | 'ddcc' | +--------+-----------------------+-------------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| b | StringColumn { data: Utf8ViewArray[a, b, c] } | -| Output | StringColumn { data: Utf8ViewArray[bbaa, ccbb, ddcc] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| b | StringColumn[a, b, c] | +| Output | StringColumn[bbaa, ccbb, ddcc] | ++--------+--------------------------------------+ ast : trim(leading a from a) @@ -1614,12 +1614,12 @@ evaluation: | Row 2 | 'ccddcc' | '' | +--------+-----------------------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| Output | StringColumn { data: Utf8ViewArray[, , ] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| Output | StringColumn[, , ] | ++--------+--------------------------------------+ ast : trim(leading b from 'aba') @@ -1636,12 +1636,12 @@ evaluation: | Row 2 | 'c' | 'aba' | +--------+-------------+--------+ evaluation (internal): -+--------+----------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------+ -| b | StringColumn { data: Utf8ViewArray[a, b, c] } | -| Output | StringColumn { data: Utf8ViewArray[ba, aba, aba] } | -+--------+----------------------------------------------------+ ++--------+----------------------------+ +| Column | Data | ++--------+----------------------------+ +| b | StringColumn[a, b, c] | +| Output | StringColumn[ba, aba, aba] | ++--------+----------------------------+ ast : trim(trailing 'a' from 'aaabbaaa') @@ -1712,12 +1712,12 @@ evaluation: | Row 2 | 'ccddcc' | 'ccddcc' | +--------+-----------------------+----------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| Output | StringColumn { data: Utf8ViewArray[aabb, bbccbb, ccddcc] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| Output | StringColumn[aabb, bbccbb, ccddcc] | ++--------+--------------------------------------+ ast : trim(trailing b from a) @@ -1734,13 +1734,13 @@ evaluation: | Row 2 | 'ccddcc' | 'c' | 'ccdd' | +--------+-----------------------+-------------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| b | StringColumn { data: Utf8ViewArray[a, b, c] } | -| Output | StringColumn { data: Utf8ViewArray[aabb, bbcc, ccdd] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| b | StringColumn[a, b, c] | +| Output | StringColumn[aabb, bbcc, ccdd] | ++--------+--------------------------------------+ ast : trim(trailing a from a) @@ -1757,12 +1757,12 @@ evaluation: | Row 2 | 'ccddcc' | '' | +--------+-----------------------+--------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[aabbaa, bbccbb, ccddcc] } | -| Output | StringColumn { data: Utf8ViewArray[, , ] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | StringColumn[aabbaa, bbccbb, ccddcc] | +| Output | StringColumn[, , ] | ++--------+--------------------------------------+ ast : trim(trailing b from 'aba') @@ -1779,12 +1779,12 @@ evaluation: | Row 2 | 'c' | 'aba' | +--------+-------------+--------+ evaluation (internal): -+--------+----------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------+ -| b | StringColumn { data: Utf8ViewArray[a, b, c] } | -| Output | StringColumn { data: Utf8ViewArray[ab, aba, aba] } | -+--------+----------------------------------------------------+ ++--------+----------------------------+ +| Column | Data | ++--------+----------------------------+ +| b | StringColumn[a, b, c] | +| Output | StringColumn[ab, aba, aba] | ++--------+----------------------------+ ast : concat('5', '3', '4') @@ -1829,12 +1829,12 @@ evaluation: | Row 3 | 'abc ' | 'abc 345' | +--------+-----------------------+----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[abc, abc, abc , abc ] } | -| Output | StringColumn { data: Utf8ViewArray[abc345, abc345, abc 345, abc 345] } | -+--------+----------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------+ +| a | StringColumn[abc, abc, abc , abc ] | +| Output | StringColumn[abc345, abc345, abc 345, abc 345] | ++--------+----------------------------------------------------------+ ast : concat(a, '3') @@ -1853,12 +1853,12 @@ evaluation: | Row 3 | 'd' | 'd3' | +--------+----------------------+------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a3, b3, c3, d3] }, validity: [0b____1011] } | -+--------+---------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn[a3, b3, c3, d3], validity: [0b____1011] } | ++--------+---------------------------------------------------------------------------------+ ast : concat_ws('-', '3', null, '4', null, '5') @@ -1903,12 +1903,12 @@ evaluation: | Row 3 | '-' | '3-4-5' | +--------+-------------+---------+ evaluation (internal): -+--------+------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[,, -, ,, -] } | -| Output | StringColumn { data: Utf8ViewArray[3,4,5, 3-4-5, 3,4,5, 3-4-5] } | -+--------+------------------------------------------------------------------+ ++--------+------------------------------------------+ +| Column | Data | ++--------+------------------------------------------+ +| a | StringColumn[,, -, ,, -] | +| Output | StringColumn[3,4,5, 3-4-5, 3,4,5, 3-4-5] | ++--------+------------------------------------------+ ast : concat_ws(a, '3') @@ -1927,12 +1927,12 @@ evaluation: | Row 3 | 'd' | '3' | +--------+----------------------+-----------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[3, 3, , 3] }, validity: [0b____1011] } | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn[3, 3, , 3], validity: [0b____1011] } | ++--------+-----------------------------------------------------------------------------+ ast : concat_ws(a, '3', '4') @@ -1951,12 +1951,12 @@ evaluation: | Row 3 | 'd' | '3d4' | +--------+----------------------+-----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------+ -| a | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[3a4, 3b4, , 3d4] }, validity: [0b____1011] } | -+--------+----------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn[3a4, 3b4, , 3d4], validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------+ error: @@ -1985,12 +1985,12 @@ evaluation: | Row 2 | 3 | '11' | +--------+----------+--------------------------------------------------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------+ -| a | Int8([-1, 2, 3]) | -| Output | StringColumn { data: Utf8ViewArray[1111111111111111111111111111111111111111111111111111111111111111, 10, 11] } | -+--------+----------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------+ +| a | Int8([-1, 2, 3]) | +| Output | StringColumn[1111111111111111111111111111111111111111111111111111111111111111, 10, 11] | ++--------+----------------------------------------------------------------------------------------+ ast : bin(a2) @@ -2007,12 +2007,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+------------------+-----------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------+ -| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 10, 11] }, validity: [0b_____011] } | -+--------+----------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------+ +| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn[1, 10, 11], validity: [0b_____011] } | ++--------+----------------------------------------------------------------------------+ ast : bin(b) @@ -2029,12 +2029,12 @@ evaluation: | Row 2 | 6 | '110' | +--------+---------+--------+ evaluation (internal): -+--------+----------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------+ -| b | Int16([2, 4, 6]) | -| Output | StringColumn { data: Utf8ViewArray[10, 100, 110] } | -+--------+----------------------------------------------------+ ++--------+----------------------------+ +| Column | Data | ++--------+----------------------------+ +| b | Int16([2, 4, 6]) | +| Output | StringColumn[10, 100, 110] | ++--------+----------------------------+ ast : bin(c) @@ -2051,12 +2051,12 @@ evaluation: | Row 2 | 30 | '11110' | +--------+-----------+---------+ evaluation (internal): -+--------+----------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------+ -| c | UInt32([10, 20, 30]) | -| Output | StringColumn { data: Utf8ViewArray[1010, 10100, 11110] } | -+--------+----------------------------------------------------------+ ++--------+----------------------------------+ +| Column | Data | ++--------+----------------------------------+ +| c | UInt32([10, 20, 30]) | +| Output | StringColumn[1010, 10100, 11110] | ++--------+----------------------------------+ error: @@ -2093,12 +2093,12 @@ evaluation: | Row 2 | 3 | '3' | +--------+----------+--------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------+ -| a | Int8([-1, 2, 3]) | -| Output | StringColumn { data: Utf8ViewArray[1777777777777777777777, 2, 3] } | -+--------+--------------------------------------------------------------------+ ++--------+--------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------+ +| a | Int8([-1, 2, 3]) | +| Output | StringColumn[1777777777777777777777, 2, 3] | ++--------+--------------------------------------------+ ast : oct(a2) @@ -2115,12 +2115,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+------------------+-----------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 2, 3] }, validity: [0b_____011] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn[1, 2, 3], validity: [0b_____011] } | ++--------+--------------------------------------------------------------------------+ ast : oct(b) @@ -2137,12 +2137,12 @@ evaluation: | Row 2 | 6 | '6' | +--------+---------+--------+ evaluation (internal): -+--------+-----------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------+ -| b | Int16([2, 4, 6]) | -| Output | StringColumn { data: Utf8ViewArray[2, 4, 6] } | -+--------+-----------------------------------------------+ ++--------+-----------------------+ +| Column | Data | ++--------+-----------------------+ +| b | Int16([2, 4, 6]) | +| Output | StringColumn[2, 4, 6] | ++--------+-----------------------+ ast : oct(c) @@ -2159,12 +2159,12 @@ evaluation: | Row 2 | 30 | '36' | +--------+-----------+--------+ evaluation (internal): -+--------+--------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------+ -| c | UInt32([10, 20, 30]) | -| Output | StringColumn { data: Utf8ViewArray[12, 24, 36] } | -+--------+--------------------------------------------------+ ++--------+--------------------------+ +| Column | Data | ++--------+--------------------------+ +| c | UInt32([10, 20, 30]) | +| Output | StringColumn[12, 24, 36] | ++--------+--------------------------+ error: @@ -2201,12 +2201,12 @@ evaluation: | Row 2 | 3 | '3' | +--------+----------+--------------------+ evaluation (internal): -+--------+--------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------+ -| a | Int8([-1, 2, 3]) | -| Output | StringColumn { data: Utf8ViewArray[ffffffffffffffff, 2, 3] } | -+--------+--------------------------------------------------------------+ ++--------+--------------------------------------+ +| Column | Data | ++--------+--------------------------------------+ +| a | Int8([-1, 2, 3]) | +| Output | StringColumn[ffffffffffffffff, 2, 3] | ++--------+--------------------------------------+ ast : hex(a2) @@ -2223,12 +2223,12 @@ evaluation: | Row 2 | NULL | NULL | +--------+------------------+-----------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, 2, 3] }, validity: [0b_____011] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| a2 | NullableColumn { column: UInt8([1, 2, 3]), validity: [0b_____011] } | +| Output | NullableColumn { column: StringColumn[1, 2, 3], validity: [0b_____011] } | ++--------+--------------------------------------------------------------------------+ ast : hex(b) @@ -2245,12 +2245,12 @@ evaluation: | Row 2 | 6 | '6' | +--------+---------+--------+ evaluation (internal): -+--------+-----------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------+ -| b | Int16([2, 4, 6]) | -| Output | StringColumn { data: Utf8ViewArray[2, 4, 6] } | -+--------+-----------------------------------------------+ ++--------+-----------------------+ +| Column | Data | ++--------+-----------------------+ +| b | Int16([2, 4, 6]) | +| Output | StringColumn[2, 4, 6] | ++--------+-----------------------+ ast : hex(c) @@ -2267,12 +2267,12 @@ evaluation: | Row 2 | 30 | '1e' | +--------+-----------+--------+ evaluation (internal): -+--------+-------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------+ -| c | UInt32([10, 20, 30]) | -| Output | StringColumn { data: Utf8ViewArray[a, 14, 1e] } | -+--------+-------------------------------------------------+ ++--------+-------------------------+ +| Column | Data | ++--------+-------------------------+ +| c | UInt32([10, 20, 30]) | +| Output | StringColumn[a, 14, 1e] | ++--------+-------------------------+ error: @@ -2303,12 +2303,12 @@ evaluation: | Row 2 | 'databend' | '6461746162656e64' | +--------+-----------------+--------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------+ -| e | StringColumn { data: Utf8ViewArray[abc, def, databend] } | -| Output | StringColumn { data: Utf8ViewArray[616263, 646566, 6461746162656e64] } | -+--------+------------------------------------------------------------------------+ ++--------+------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------+ +| e | StringColumn[abc, def, databend] | +| Output | StringColumn[616263, 646566, 6461746162656e64] | ++--------+------------------------------------------------+ ast : lpad('hi', 2, '?') @@ -2403,14 +2403,14 @@ evaluation: | Row 2 | 'cc' | 5 | 'bb' | 'bbbcc' | +--------+-----------------+---------+-------------+---------+ evaluation (internal): -+--------+----------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[hi, test, cc] } | -| b | UInt8([0, 3, 5]) | -| c | StringColumn { data: Utf8ViewArray[?, x, bb] } | -| Output | StringColumn { data: Utf8ViewArray[, tes, bbbcc] } | -+--------+----------------------------------------------------+ ++--------+----------------------------+ +| Column | Data | ++--------+----------------------------+ +| a | StringColumn[hi, test, cc] | +| b | UInt8([0, 3, 5]) | +| c | StringColumn[?, x, bb] | +| Output | StringColumn[, tes, bbbcc] | ++--------+----------------------------+ error: @@ -2513,14 +2513,14 @@ evaluation: | Row 2 | 'cc' | 5 | 'bb' | 'ccbbb' | +--------+-----------------+---------+-------------+---------+ evaluation (internal): -+--------+----------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[hi, test, cc] } | -| b | UInt8([0, 3, 5]) | -| c | StringColumn { data: Utf8ViewArray[?, x, bb] } | -| Output | StringColumn { data: Utf8ViewArray[, tes, ccbbb] } | -+--------+----------------------------------------------------+ ++--------+----------------------------+ +| Column | Data | ++--------+----------------------------+ +| a | StringColumn[hi, test, cc] | +| b | UInt8([0, 3, 5]) | +| c | StringColumn[?, x, bb] | +| Output | StringColumn[, tes, ccbbb] | ++--------+----------------------------+ error: @@ -2582,14 +2582,14 @@ evaluation: | Row 3 | 'q' | '' | 'q' | 'q' | +--------+-----------------+-------------+-------------+--------+ evaluation (internal): -+--------+-------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[hi, test, cc, q] } | -| b | StringColumn { data: Utf8ViewArray[i, te, cc, ] } | -| c | StringColumn { data: Utf8ViewArray[?, x, bb, q] } | -| Output | StringColumn { data: Utf8ViewArray[h?, xst, bb, q] } | -+--------+-------------------------------------------------------+ ++--------+-------------------------------+ +| Column | Data | ++--------+-------------------------------+ +| a | StringColumn[hi, test, cc, q] | +| b | StringColumn[i, te, cc, ] | +| c | StringColumn[?, x, bb, q] | +| Output | StringColumn[h?, xst, bb, q] | ++--------+-------------------------------+ ast : translate('abcdefabcdef', 'dc', 'zy') @@ -2644,14 +2644,14 @@ evaluation: | Row 3 | 'abcdef' | 'dc' | 'dc' | 'abcdef' | +--------+-----------------------+-------------+-------------+----------+ evaluation (internal): -+--------+----------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[abcdef, abcdef, abcdef, abcdef] } | -| b | StringColumn { data: Utf8ViewArray[dc, , dc, dc] } | -| c | StringColumn { data: Utf8ViewArray[zy, zy, , dc] } | -| Output | StringColumn { data: Utf8ViewArray[abyzef, abcdef, abef, abcdef] } | -+--------+----------------------------------------------------------------------+ ++--------+----------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------+ +| a | StringColumn[abcdef, abcdef, abcdef, abcdef] | +| b | StringColumn[dc, , dc, dc] | +| c | StringColumn[zy, zy, , dc] | +| Output | StringColumn[abyzef, abcdef, abef, abcdef] | ++--------+----------------------------------------------+ ast : strcmp('text', 'text2') @@ -2696,13 +2696,13 @@ evaluation: | Row 3 | 'cc' | 'ccb' | -1 | +--------+-----------------+------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[i, h, test, cc] } | -| b | StringColumn { data: Utf8ViewArray[hi, hi, test, ccb] } | -| Output | Int8([1, -1, 0, -1]) | -+--------+---------------------------------------------------------+ ++--------+---------------------------------+ +| Column | Data | ++--------+---------------------------------+ +| a | StringColumn[i, h, test, cc] | +| b | StringColumn[hi, hi, test, ccb] | +| Output | Int8([1, -1, 0, -1]) | ++--------+---------------------------------+ ast : locate('bar', 'foobarbar') @@ -2837,14 +2837,14 @@ evaluation: | Row 3 | 'q' | '56' | 1 | 0 | +--------+---------------+---------------+---------+----------------------------+ evaluation (internal): -+--------+------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[bar, cc, cc, q] } | -| b | StringColumn { data: Utf8ViewArray[foobarbar, bdccacc, xx, 56] } | -| c | UInt8([1, 2, 0, 1]) | -| Output | UInt64([4, 3, 0, 0]) | -+--------+------------------------------------------------------------------+ ++--------+------------------------------------------+ +| Column | Data | ++--------+------------------------------------------+ +| a | StringColumn[bar, cc, cc, q] | +| b | StringColumn[foobarbar, bdccacc, xx, 56] | +| c | UInt8([1, 2, 0, 1]) | +| Output | UInt64([4, 3, 0, 0]) | ++--------+------------------------------------------+ ast : char(65,66,67) @@ -2974,12 +2974,12 @@ evaluation: | Row 3 | 'TEACHER' | 'T260' | +--------+------------------------------------+---------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[#🐑🐑he🐑llo🐑, 🐑he🐑llo🐑, teacher, TEACHER] } | -| Output | StringColumn { data: Utf8ViewArray[🐑400, 🐑400, T260, T260] } | -+--------+-------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------+ +| a | StringColumn[#🐑🐑he🐑llo🐑, 🐑he🐑llo🐑, teacher, TEACHER] | +| Output | StringColumn[🐑400, 🐑400, T260, T260] | ++--------+-------------------------------------------------------------+ ast : ord(NULL) @@ -3077,12 +3077,12 @@ evaluation: | Row 2 | 'c' | 'ccc' | +--------+-------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[a, b, c] } | -| Output | StringColumn { data: Utf8ViewArray[aaa, bbb, ccc] } | -+--------+-----------------------------------------------------+ ++--------+-----------------------------+ +| Column | Data | ++--------+-----------------------------+ +| a | StringColumn[a, b, c] | +| Output | StringColumn[aaa, bbb, ccc] | ++--------+-----------------------------+ error: @@ -3188,15 +3188,15 @@ evaluation: | Row 3 | 'q' | 1 | 1 | '56' | '56' | +--------+-----------------+---------+---------+---------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------+ -| a | StringColumn { data: Utf8ViewArray[hi, test, cc, q] } | -| b | UInt8([1, 4, 1, 1]) | -| c | UInt8([3, 5, 1, 1]) | -| d | StringColumn { data: Utf8ViewArray[xx, zc, 12, 56] } | -| Output | StringColumn { data: Utf8ViewArray[xx, teszc, 12c, 56] } | -+--------+----------------------------------------------------------+ ++--------+----------------------------------+ +| Column | Data | ++--------+----------------------------------+ +| a | StringColumn[hi, test, cc, q] | +| b | UInt8([1, 4, 1, 1]) | +| c | UInt8([3, 5, 1, 1]) | +| d | StringColumn[xx, zc, 12, 56] | +| Output | StringColumn[xx, teszc, 12c, 56] | ++--------+----------------------------------+ ast : insert(x, y, z, u) @@ -3214,15 +3214,15 @@ evaluation: | Row 3 | 'q' | 1 | 1 | '56' | '56' | +--------+--------------------------+------------------+------------------+------------------------+-----------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------+ -| x | NullableColumn { column: StringColumn { data: Utf8ViewArray[hi, test, cc, q] }, validity: [0b____1110] } | -| y | NullableColumn { column: UInt8([1, 4, 1, 1]), validity: [0b____1011] } | -| z | NullableColumn { column: UInt8([3, 5, 1, 1]), validity: [0b____1101] } | -| u | NullableColumn { column: StringColumn { data: Utf8ViewArray[xx, zc, 12, 56] }, validity: [0b____1110] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[xx, teszc, 12c, 56] }, validity: [0b____1000] } | -+--------+-------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------+ +| x | NullableColumn { column: StringColumn[hi, test, cc, q], validity: [0b____1110] } | +| y | NullableColumn { column: UInt8([1, 4, 1, 1]), validity: [0b____1011] } | +| z | NullableColumn { column: UInt8([3, 5, 1, 1]), validity: [0b____1101] } | +| u | NullableColumn { column: StringColumn[xx, zc, 12, 56], validity: [0b____1110] } | +| Output | NullableColumn { column: StringColumn[xx, teszc, 12c, 56], validity: [0b____1000] } | ++--------+-------------------------------------------------------------------------------------+ ast : space(0) @@ -3272,12 +3272,12 @@ evaluation: | Row 9 | 9 | ' ' | +--------+---------+-------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) | -| Output | StringColumn { data: Utf8ViewArray[, , , , , , , , , ] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------+ +| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) | +| Output | StringColumn[, , , , , , , , , ] | ++--------+-------------------------------------------------------------------------------+ ast : left('', 0) @@ -3320,12 +3320,12 @@ evaluation: | Row 10 | 10 | '123456789' | +--------+----------+--------------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------+ -| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) | -| Output | StringColumn { data: Utf8ViewArray[, 1, 12, 123, 1234, 12345, 123456, 1234567, 12345678, 123456789, 123456789] } | -+--------+------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------+ +| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) | +| Output | StringColumn[, 1, 12, 123, 1234, 12345, 123456, 1234567, 12345678, 123456789, 123456789] | ++--------+------------------------------------------------------------------------------------------+ ast : right('', 0) @@ -3368,12 +3368,12 @@ evaluation: | Row 10 | 10 | '123456789' | +--------+----------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------+ -| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) | -| Output | StringColumn { data: Utf8ViewArray[, 9, 89, 789, 6789, 56789, 456789, 3456789, 23456789, 123456789, 123456789] } | -+--------+------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------+ +| a | UInt8([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) | +| Output | StringColumn[, 9, 89, 789, 6789, 56789, 456789, 3456789, 23456789, 123456789, 123456789] | ++--------+------------------------------------------------------------------------------------------+ ast : mid('1234567890', -3, 3) @@ -3513,13 +3513,13 @@ evaluation: | Row 44 | -4 | 4 | '' | +--------+----------+---------+--------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| pos | Int8([0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -3, -3, -3, -3, -3, -4, -4, -4, -4, -4]) | -| len | UInt8([0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4]) | -| Output | StringColumn { data: Utf8ViewArray[, , , , , , a, ab, abc, abc, , b, bc, bc, bc, , c, c, c, c, , , , , , , c, c, c, c, , b, bc, bc, bc, , a, ab, abc, abc, , , , , ] } | -+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| pos | Int8([0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -3, -3, -3, -3, -3, -4, -4, -4, -4, -4]) | +| len | UInt8([0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4]) | +| Output | StringColumn[, , , , , , a, ab, abc, abc, , b, bc, bc, bc, , c, c, c, c, , , , , , , c, c, c, c, , b, bc, bc, bc, , a, ab, abc, abc, , , , , ] | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : split('Sakila', 'il') @@ -3582,12 +3582,12 @@ evaluation: | Row 3 | 'aeeceedeef' | 'ee' | ['a', 'c', 'd', 'f'] | +--------+-------------------------------+------------------------+---------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| str | NullableColumn { column: StringColumn { data: Utf8ViewArray[127.0.0.1, aaa--bbb-BBB--ccc, cc, aeeceedeef] }, validity: [0b____1110] } | -| sep | NullableColumn { column: StringColumn { data: Utf8ViewArray[., --, cc, ee] }, validity: [0b____1110] } | -| Output | NullableColumn { column: ArrayColumn { values: StringColumn { data: Utf8ViewArray[127, 0, 0, 1, aaa, bbb-BBB, ccc, , a, c, d, f] }, offsets: [0, 4, 7, 8, 12] }, validity: [0b____1110] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| str | NullableColumn { column: StringColumn[127.0.0.1, aaa--bbb-BBB--ccc, cc, aeeceedeef], validity: [0b____1110] } | +| sep | NullableColumn { column: StringColumn[., --, cc, ee], validity: [0b____1110] } | +| Output | NullableColumn { column: ArrayColumn { values: StringColumn[127, 0, 0, 1, aaa, bbb-BBB, ccc, , a, c, d, f], offsets: [0, 4, 7, 8, 12] }, validity: [0b____1110] } | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/tuple.txt b/src/query/functions/tests/it/scalars/testdata/tuple.txt index 7ab8f2c58ff8..0eacf5ebe1bc 100644 --- a/src/query/functions/tests/it/scalars/testdata/tuple.txt +++ b/src/query/functions/tests/it/scalars/testdata/tuple.txt @@ -49,12 +49,12 @@ evaluation: | Row 3 | 'd' | ('d', 'd') | +--------+----------------------+----------------------------------------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | -| Output | Tuple([NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] }, NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] }]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____1011] } | +| Output | Tuple([NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____1011] }, NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____1011] }]) | ++--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ error: @@ -141,12 +141,12 @@ evaluation: | Row 3 | 'd' | 'd' | +--------+----------------------+----------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____1011] } | -+--------+-----------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____1011] } | ++--------+-----------------------------------------------------------------------------+ ast : col.1 @@ -164,11 +164,11 @@ evaluation: | Row 3 | NULL | NULL | +--------+---------------------------------+----------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| col | NullableColumn { column: Tuple([NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____0011] }]), validity: [0b____0101] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[a, b, c, d] }, validity: [0b____0001] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------+ +| col | NullableColumn { column: Tuple([NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____0011] }]), validity: [0b____0101] } | +| Output | NullableColumn { column: StringColumn[a, b, c, d], validity: [0b____0001] } | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/variant.txt b/src/query/functions/tests/it/scalars/testdata/variant.txt index 77fcf04b20fa..4ec5b213363b 100644 --- a/src/query/functions/tests/it/scalars/testdata/variant.txt +++ b/src/query/functions/tests/it/scalars/testdata/variant.txt @@ -110,7 +110,7 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | +| s | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | | Output | BinaryColumn { data: 0x200000000000000020000000400000002000000020000009507fffffffffffffff200000002000000340800020000000200000096040934a456d5cfaad2000000020000009604067e6666666666620000000100000075c226162635c2220000000100000086461746162656e644000000210000001100000011000000110000001616b6276800000042000000220000002200000025000001350015002500380000003100000011000000110000001616263, offsets: [0, 8, 16, 33, 44, 61, 78, 93, 109, 133, 178] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -133,7 +133,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, false, , 1234] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[true, false, , 1234], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x2000000040000000200000003000000020000000200000035004d2, offsets: [0, 8, 16, 16, 27] }, validity: [0b____1011] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -243,7 +243,7 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] } | +| s | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | | Output | NullableColumn { column: BinaryColumn { data: 0x200000000000000020000000400000002000000020000009507fffffffffffffff200000002000000340800020000000200000096040934a456d5cfaad2000000020000009604067e6666666666620000000100000075c226162635c2220000000100000086461746162656e644000000210000001100000011000000110000001616b6276800000042000000220000002200000025000001350015002500380000003100000011000000110000001616263, offsets: [0, 8, 16, 33, 44, 61, 78, 93, 109, 133, 178] }, validity: [0b11111111, 0b______11] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -266,7 +266,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, ttt, , 1234] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[true, ttt, , 1234], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x200000004000000020000000200000035004d2, offsets: [0, 8, 8, 8, 19] }, validity: [0b____1001] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ @@ -312,12 +312,12 @@ evaluation: | Row 2 | 'true' | NULL | +--------+------------------+-------------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, abc, true] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, expected value, pos 1, ] }, validity: [0b_____010] } | -+--------+--------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------+ +| s | StringColumn[null, abc, true] | +| Output | NullableColumn { column: StringColumn[, expected value, pos 1, ], validity: [0b_____010] } | ++--------+--------------------------------------------------------------------------------------------+ ast : check_json(s) @@ -335,12 +335,12 @@ evaluation: | Row 3 | '1234' | NULL | +--------+-----------------------+-------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, ttt, , 1234] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, expected ident, pos 2, , ] }, validity: [0b____0010] } | -+--------+----------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, ttt, , 1234], validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn[, expected ident, pos 2, , ], validity: [0b____0010] } | ++--------+----------------------------------------------------------------------------------------------+ ast : length(parse_json('1234')) @@ -387,7 +387,7 @@ evaluation (internal): +--------+----------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, [1,2,3,4], ["a","b","c"]] } | +| s | StringColumn[true, [1,2,3,4], ["a","b","c"]] | | Output | NullableColumn { column: UInt32([0, 4, 3]), validity: [0b_____110] } | +--------+----------------------------------------------------------------------+ @@ -407,12 +407,12 @@ evaluation: | Row 3 | '["a","b","c"]' | 3 | +--------+------------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [1,2,3,4], , ["a","b","c"]] }, validity: [0b____1011] } | -| Output | NullableColumn { column: UInt32([0, 4, 0, 3]), validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, [1,2,3,4], , ["a","b","c"]], validity: [0b____1011] } | +| Output | NullableColumn { column: UInt32([0, 4, 0, 3]), validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------------+ ast : json_object_keys(parse_json('[1,2,3,4]')) @@ -450,7 +450,7 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[[1,2,3,4], {"a":"b","c":"d"}, {"k1":"v1","k2":"v2"}] } | +| s | StringColumn[[1,2,3,4], {"a":"b","c":"d"}, {"k1":"v1","k2":"v2"}] | | Output | NullableColumn { column: BinaryColumn { data: 0x80000002100000011000000161638000000210000002100000026b316b32, offsets: [0, 0, 14, 30] }, validity: [0b_____110] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -473,7 +473,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,4], {"a":"b","c":"d"}, , {"k1":"v1","k2":"v2"}] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[[1,2,3,4], {"a":"b","c":"d"}, , {"k1":"v1","k2":"v2"}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x80000002100000011000000161638000000210000002100000026b316b32, offsets: [0, 0, 14, 14, 30] }, validity: [0b____1010] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -558,7 +558,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, [1,2,3,4], ["a","b","c"]] } | +| s | StringColumn[true, [1,2,3,4], ["a","b","c"]] | | i | UInt64([0, 0, 1]) | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 19] }, validity: [0b_____110] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ @@ -582,7 +582,7 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [1,2,3,4], , ["a","b","c"]] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[true, [1,2,3,4], , ["a","b","c"]], validity: [0b____1011] } | | i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025003200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -605,8 +605,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, {"a":"b"}] } | -| k | StringColumn { data: Utf8ViewArray[k, k, x] } | +| s | StringColumn[true, {"k":1}, {"a":"b"}] | +| k | StringColumn[k, k, x] | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 0, 10, 10] }, validity: [0b_____010] } | +--------+---------------------------------------------------------------------------------------------------------------------------+ @@ -629,8 +629,8 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | -| k | StringColumn { data: Utf8ViewArray[, k, , a] } | +| s | NullableColumn { column: StringColumn[true, {"k":1}, , {"a":"b"}], validity: [0b____1011] } | +| k | StringColumn[, k, , a] | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -679,8 +679,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, {"a":"b"}] } | -| k | StringColumn { data: Utf8ViewArray[k, K, A] } | +| s | StringColumn[true, {"k":1}, {"a":"b"}] | +| k | StringColumn[k, K, A] | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 19] }, validity: [0b_____110] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ @@ -703,8 +703,8 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | -| k | StringColumn { data: Utf8ViewArray[, K, , A] } | +| s | NullableColumn { column: StringColumn[true, {"k":1}, , {"a":"b"}], validity: [0b____1011] } | +| k | StringColumn[, K, , A] | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -798,8 +798,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, ["a","b"]] } | -| k | StringColumn { data: Utf8ViewArray[k, ["k"], ["a"]] } | +| s | StringColumn[true, {"k":1}, ["a","b"]] | +| k | StringColumn[k, ["k"], ["a"]] | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 0, 10, 10] }, validity: [0b_____010] } | +--------+---------------------------------------------------------------------------------------------------------------------------+ @@ -822,8 +822,8 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , ["a","b"]] }, validity: [0b____1011] } | -| k | StringColumn { data: Utf8ViewArray[[0], ["k"], , [0]] } | +| s | NullableColumn { column: StringColumn[true, {"k":1}, , ["a","b"]], validity: [0b____1011] } | +| k | StringColumn[[0], ["k"], , [0]] | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000161, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -914,13 +914,13 @@ evaluation: | Row 2 | '["a","b"]' | '["a"]' | NULL | +--------+---------------------------------+-------------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, ["a","b"]] } | -| k | StringColumn { data: Utf8ViewArray[k, ["k"], ["a"]] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, ] }, validity: [0b_____010] } | -+--------+------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------+ +| s | StringColumn[true, {"k":1}, ["a","b"]] | +| k | StringColumn[k, ["k"], ["a"]] | +| Output | NullableColumn { column: StringColumn[, 1, ], validity: [0b_____010] } | ++--------+------------------------------------------------------------------------+ ast : json_extract_path_text(s, k) @@ -938,13 +938,13 @@ evaluation: | Row 3 | '["a","b"]' | '[0]' | 'a' | +--------+-----------------------------+--------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , ["a","b"]] }, validity: [0b____1011] } | -| k | StringColumn { data: Utf8ViewArray[[0], ["k"], , [0]] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, , a] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, {"k":1}, , ["a","b"]], validity: [0b____1011] } | +| k | StringColumn[[0], ["k"], , [0]] | +| Output | NullableColumn { column: StringColumn[, 1, , a], validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------+ ast : as_boolean(parse_json('true')) @@ -1073,12 +1073,12 @@ evaluation: | Row 6 | '{"a":"b"}' | NULL | +--------+------------------------------+--------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | NullableColumn { column: Boolean([0b_0000010]), validity: [0b_0000010] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Boolean([0b_0000010]), validity: [0b_0000010] } | ++--------+--------------------------------------------------------------------------+ ast : as_integer(parse_json(s)) @@ -1099,12 +1099,12 @@ evaluation: | Row 6 | '{"a":"b"}' | NULL | +--------+------------------------------+------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | NullableColumn { column: Int64([0, 0, 123, 0, 0, 0, 0]), validity: [0b_0000100] } | -+--------+----------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Int64([0, 0, 123, 0, 0, 0, 0]), validity: [0b_0000100] } | ++--------+-----------------------------------------------------------------------------------+ ast : as_float(parse_json(s)) @@ -1128,7 +1128,7 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | | Output | NullableColumn { column: Float64([0, 0, 123, 12.34, 0, 0, 0]), validity: [0b_0001100] } | +--------+-----------------------------------------------------------------------------------------+ @@ -1151,12 +1151,12 @@ evaluation: | Row 6 | '{"a":"b"}' | NULL | +--------+------------------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, , , , ab, , ] }, validity: [0b_0010000] } | -+--------+---------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: StringColumn[, , , , ab, , ], validity: [0b_0010000] } | ++--------+---------------------------------------------------------------------------------+ ast : as_array(parse_json(s)) @@ -1180,7 +1180,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | | Output | NullableColumn { column: BinaryColumn { data: 0x80000003200000022000000220000002500150025003, offsets: [0, 0, 0, 0, 0, 0, 22, 22] }, validity: [0b_0100000] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1206,7 +1206,7 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000110000001100000016162, offsets: [0, 0, 0, 0, 0, 0, 0, 14] }, validity: [0b_1000000] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------+ @@ -1355,12 +1355,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | Boolean([0b_0000001]) | -+--------+----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0000001]) | ++--------+----------------------------------------------------------------+ ast : is_boolean(parse_json(s)) @@ -1381,12 +1381,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | Boolean([0b_0000010]) | -+--------+----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0000010]) | ++--------+----------------------------------------------------------------+ ast : is_integer(parse_json(s)) @@ -1407,12 +1407,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | Boolean([0b_0000100]) | -+--------+----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0000100]) | ++--------+----------------------------------------------------------------+ ast : is_float(parse_json(s)) @@ -1433,12 +1433,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | Boolean([0b_0001100]) | -+--------+----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0001100]) | ++--------+----------------------------------------------------------------+ ast : is_string(parse_json(s)) @@ -1459,12 +1459,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | Boolean([0b_0010000]) | -+--------+----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0010000]) | ++--------+----------------------------------------------------------------+ ast : is_array(parse_json(s)) @@ -1485,12 +1485,12 @@ evaluation: | Row 6 | '{"a":"b"}' | false | +--------+------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | Boolean([0b_0100000]) | -+--------+----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0100000]) | ++--------+----------------------------------------------------------------+ ast : is_object(parse_json(s)) @@ -1511,12 +1511,12 @@ evaluation: | Row 6 | '{"a":"b"}' | true | +--------+------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] } | -| Output | Boolean([0b_1000000]) | -+--------+----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_1000000]) | ++--------+----------------------------------------------------------------+ ast : to_boolean(parse_json('true')) @@ -1678,12 +1678,12 @@ evaluation: | Row 2 | 'true' | true | +--------+------------------------+--------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, , true] }, validity: [0b_____101] } | -| Output | NullableColumn { column: Boolean([0b_____101]), validity: [0b_____101] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, , true], validity: [0b_____101] } | +| Output | NullableColumn { column: Boolean([0b_____101]), validity: [0b_____101] } | ++--------+-------------------------------------------------------------------------------+ ast : to_int64(parse_json(s)) @@ -1700,12 +1700,12 @@ evaluation: | Row 2 | '-10' | -10 | +--------+---------------------+------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, , -10] }, validity: [0b_____101] } | -| Output | NullableColumn { column: Int64([1, 0, -10]), validity: [0b_____101] } | -+--------+---------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[1, , -10], validity: [0b_____101] } | +| Output | NullableColumn { column: Int64([1, 0, -10]), validity: [0b_____101] } | ++--------+---------------------------------------------------------------------------+ ast : to_uint64(parse_json(s)) @@ -1722,12 +1722,12 @@ evaluation: | Row 2 | '20' | 20 | +--------+----------------------+-------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, , 20] }, validity: [0b_____101] } | -| Output | NullableColumn { column: UInt64([1, 0, 20]), validity: [0b_____101] } | -+--------+--------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[1, , 20], validity: [0b_____101] } | +| Output | NullableColumn { column: UInt64([1, 0, 20]), validity: [0b_____101] } | ++--------+--------------------------------------------------------------------------+ ast : to_float64(parse_json(s)) @@ -1744,12 +1744,12 @@ evaluation: | Row 2 | '100.2' | 100.2 | +--------+-------------------------+--------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[1.2, , 100.2] }, validity: [0b_____101] } | -| Output | NullableColumn { column: Float64([1.2, 0, 100.2]), validity: [0b_____101] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[1.2, , 100.2], validity: [0b_____101] } | +| Output | NullableColumn { column: Float64([1.2, 0, 100.2]), validity: [0b_____101] } | ++--------+-------------------------------------------------------------------------------+ ast : to_date(parse_json(s)) @@ -1766,12 +1766,12 @@ evaluation: | Row 2 | '"2023-10-01"' | '2023-10-01' | +--------+----------------------------------+--------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray["2020-01-01", , "2023-10-01"] }, validity: [0b_____101] } | -| Output | NullableColumn { column: [18262, 0, 19631], validity: [0b_____101] } | -+--------+-----------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn["2020-01-01", , "2023-10-01"], validity: [0b_____101] } | +| Output | NullableColumn { column: [18262, 0, 19631], validity: [0b_____101] } | ++--------+-----------------------------------------------------------------------------------------------+ ast : to_timestamp(parse_json(s)) @@ -1788,12 +1788,12 @@ evaluation: | Row 2 | '"2023-10-01 10:11:12"' | '2023-10-01 10:11:12.000000' | +--------+-------------------------------------------+------------------------------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray["2020-01-01 00:00:00", , "2023-10-01 10:11:12"] }, validity: [0b_____101] } | -| Output | NullableColumn { column: [1577836800000000, 0, 1696155072000000], validity: [0b_____101] } | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn["2020-01-01 00:00:00", , "2023-10-01 10:11:12"], validity: [0b_____101] } | +| Output | NullableColumn { column: [1577836800000000, 0, 1696155072000000], validity: [0b_____101] } | ++--------+-----------------------------------------------------------------------------------------------------------------+ ast : to_string(parse_json(s)) @@ -1810,12 +1810,12 @@ evaluation: | Row 2 | '123' | '123' | +--------+-----------------------+-------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray["abc", , 123] }, validity: [0b_____101] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[abc, , 123] }, validity: [0b_____101] } | -+--------+-------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn["abc", , 123], validity: [0b_____101] } | +| Output | NullableColumn { column: StringColumn[abc, , 123], validity: [0b_____101] } | ++--------+-------------------------------------------------------------------------------+ ast : try_to_boolean(parse_json('true')) @@ -1990,12 +1990,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | -| Output | NullableColumn { column: Boolean([0b00000001]), validity: [0b00000001] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"], validity: [0b11101111] } | +| Output | NullableColumn { column: Boolean([0b00000001]), validity: [0b00000001] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_int64(parse_json(s)) @@ -2017,12 +2017,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | -| Output | NullableColumn { column: Int64([1, 123, -100, 0, 0, 0, 0, 0]), validity: [0b00000111] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"], validity: [0b11101111] } | +| Output | NullableColumn { column: Int64([1, 123, -100, 0, 0, 0, 0, 0]), validity: [0b00000111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_uint64(parse_json(s)) @@ -2044,12 +2044,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | -| Output | NullableColumn { column: UInt64([1, 123, 0, 0, 0, 0, 0, 0]), validity: [0b00000011] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"], validity: [0b11101111] } | +| Output | NullableColumn { column: UInt64([1, 123, 0, 0, 0, 0, 0, 0]), validity: [0b00000011] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_float64(parse_json(s)) @@ -2071,12 +2071,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | -| Output | NullableColumn { column: Float64([1, 123, -100, 12.34, 0, 0, 0, 0]), validity: [0b00001111] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"], validity: [0b11101111] } | +| Output | NullableColumn { column: Float64([1, 123, -100, 12.34, 0, 0, 0, 0]), validity: [0b00001111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_date(parse_json(s)) @@ -2098,12 +2098,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+--------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | -| Output | NullableColumn { column: [0, 0, 0, 0, 0, 18262, 18628, 0], validity: [0b01100000] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"], validity: [0b11101111] } | +| Output | NullableColumn { column: [0, 0, 0, 0, 0, 18262, 18628, 0], validity: [0b01100000] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_timestamp(parse_json(s)) @@ -2125,12 +2125,12 @@ evaluation: | Row 7 | '"abc"' | NULL | +--------+-------------------------+------------------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | -| Output | NullableColumn { column: [0, 0, 0, 0, 0, 1577836800000000, 1609531200000000, 0], validity: [0b01100000] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"], validity: [0b11101111] } | +| Output | NullableColumn { column: [0, 0, 0, 0, 0, 1577836800000000, 1609531200000000, 0], validity: [0b01100000] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ ast : try_to_string(parse_json(s)) @@ -2152,12 +2152,12 @@ evaluation: | Row 7 | '"abc"' | 'abc' | +--------+-------------------------+-----------------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"] }, validity: [0b11101111] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, 123, -100, 12.34, , 2020-01-01, 2021-01-01 20:00:00, abc] }, validity: [0b11101111] } | -+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, 123, -100, 12.34, , "2020-01-01", "2021-01-01 20:00:00", "abc"], validity: [0b11101111] } | +| Output | NullableColumn { column: StringColumn[true, 123, -100, 12.34, , 2020-01-01, 2021-01-01 20:00:00, abc], validity: [0b11101111] } | ++--------+---------------------------------------------------------------------------------------------------------------------------------------+ ast : json_object() @@ -2229,10 +2229,10 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| k1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | -| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | -| k2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | -| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j2, k2, l2, m2] }, validity: [0b____1111] } | +| k1 | NullableColumn { column: StringColumn[a1, b1, , d1], validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn[j1, k1, l1, ], validity: [0b____0111] } | +| k2 | NullableColumn { column: StringColumn[a2, , c2, d2], validity: [0b____1101] } | +| v2 | NullableColumn { column: StringColumn[j2, k2, l2, m2], validity: [0b____1111] } | | Output | BinaryColumn { data: 0x4000000210000002100000021000000210000002613161326a316a3240000001100000021000000262316b3140000001100000021000000263326c3240000001100000021000000264326d32, offsets: [0, 28, 44, 60, 76] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2309,10 +2309,10 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| k1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | -| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | -| k2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | -| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j2, k2, l2, m2] }, validity: [0b____1111] } | +| k1 | NullableColumn { column: StringColumn[a1, b1, , d1], validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn[j1, k1, l1, ], validity: [0b____0111] } | +| k2 | NullableColumn { column: StringColumn[a2, , c2, d2], validity: [0b____1101] } | +| v2 | NullableColumn { column: StringColumn[j2, k2, l2, m2], validity: [0b____1111] } | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000210000002100000021000000210000002613161326a316a3240000001100000021000000262316b3140000001100000021000000263326c3240000001100000021000000264326d32, offsets: [0, 28, 44, 60, 76] }, validity: [0b____1111] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2386,10 +2386,10 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| k1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | -| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | -| k2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | -| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j2, k2, l2, m2] }, validity: [0b____1111] } | +| k1 | NullableColumn { column: StringColumn[a1, b1, , d1], validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn[j1, k1, l1, ], validity: [0b____0111] } | +| k2 | NullableColumn { column: StringColumn[a2, , c2, d2], validity: [0b____1101] } | +| v2 | NullableColumn { column: StringColumn[j2, k2, l2, m2], validity: [0b____1111] } | | Output | BinaryColumn { data: 0x4000000210000002100000021000000210000002613161326a316a3240000001100000021000000262316b3140000001100000021000000263326c324000000210000002100000020000000010000002643164326d32, offsets: [0, 28, 44, 60, 86] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2466,10 +2466,10 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| k1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | -| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | -| k2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | -| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j2, k2, l2, m2] }, validity: [0b____1111] } | +| k1 | NullableColumn { column: StringColumn[a1, b1, , d1], validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn[j1, k1, l1, ], validity: [0b____0111] } | +| k2 | NullableColumn { column: StringColumn[a2, , c2, d2], validity: [0b____1101] } | +| v2 | NullableColumn { column: StringColumn[j2, k2, l2, m2], validity: [0b____1111] } | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000210000002100000021000000210000002613161326a316a3240000001100000021000000262316b3140000001100000021000000263326c324000000210000002100000020000000010000002643164326d32, offsets: [0, 28, 44, 60, 86] }, validity: [0b____1111] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2546,8 +2546,8 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [{"k":1},{"k":2}], , [1,2,3,4]] }, validity: [0b____1011] } | -| p | StringColumn { data: Utf8ViewArray[$[0], $[*].k, $.a, $[0,2]] } | +| s | NullableColumn { column: StringColumn[true, [{"k":1},{"k":2}], , [1,2,3,4]], validity: [0b____1011] } | +| p | StringColumn[$[0], $[*].k, $.a, $[0,2]] | | Output | NullableColumn { column: BinaryColumn { data: 0x800000008000000220000002200000025001500280000002200000022000000250015003, offsets: [0, 4, 20, 20, 36] }, validity: [0b____1011] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2624,8 +2624,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [{"k":1},{"k":2}], , [1,2,3,4]] }, validity: [0b____1011] } | -| p | StringColumn { data: Utf8ViewArray[$[0], $[*].k, $.a, $[0,2]] } | +| s | NullableColumn { column: StringColumn[true, [{"k":1},{"k":2}], , [1,2,3,4]], validity: [0b____1011] } | +| p | StringColumn[$[0], $[*].k, $.a, $[0,2]] | | Output | NullableColumn { column: BinaryColumn { data: 0x2000000020000002500120000000200000025001, offsets: [0, 0, 10, 10, 20] }, validity: [0b____1010] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -2894,9 +2894,9 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a1, b1, , d1] }, validity: [0b____1011] } | -| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[j1, k1, l1, ] }, validity: [0b____0111] } | -| v3 | NullableColumn { column: StringColumn { data: Utf8ViewArray[a2, , c2, d2] }, validity: [0b____1101] } | +| v1 | NullableColumn { column: StringColumn[a1, b1, , d1], validity: [0b____1011] } | +| v2 | NullableColumn { column: StringColumn[j1, k1, l1, ], validity: [0b____0111] } | +| v3 | NullableColumn { column: StringColumn[a2, , c2, d2], validity: [0b____1101] } | | Output | BinaryColumn { data: 0x8000000310000002100000021000000261316a3161328000000310000002100000020000000062316b31800000030000000010000002100000026c3163328000000310000002000000001000000264316432, offsets: [0, 22, 42, 62, 82] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3062,7 +3062,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, [1,2,3,4], ["a","b","c"]] } | +| s | StringColumn[true, [1,2,3,4], ["a","b","c"]] | | i | UInt64([0, 0, 1]) | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 19] }, validity: [0b_____110] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3086,7 +3086,7 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [1,2,3,4], , ["a","b","c"]] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[true, [1,2,3,4], , ["a","b","c"]], validity: [0b____1011] } | | i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025003200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3109,8 +3109,8 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, {"a":"b"}] } | -| k | StringColumn { data: Utf8ViewArray[k, k, x] } | +| s | StringColumn[true, {"k":1}, {"a":"b"}] | +| k | StringColumn[k, k, x] | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 0, 10, 10] }, validity: [0b_____010] } | +--------+---------------------------------------------------------------------------------------------------------------------------+ @@ -3133,8 +3133,8 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | -| k | StringColumn { data: Utf8ViewArray[, k, , a] } | +| s | NullableColumn { column: StringColumn[true, {"k":1}, , {"a":"b"}], validity: [0b____1011] } | +| k | StringColumn[, k, , a] | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3216,13 +3216,13 @@ evaluation: | Row 2 | '["a","b","c"]' | 1 | 'b' | +--------+----------------------------------+---------+-------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, [1,2,3,4], ["a","b","c"]] } | -| i | UInt64([0, 0, 1]) | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, b] }, validity: [0b_____110] } | -+--------+-------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------+ +| s | StringColumn[true, [1,2,3,4], ["a","b","c"]] | +| i | UInt64([0, 0, 1]) | +| Output | NullableColumn { column: StringColumn[, 1, b], validity: [0b_____110] } | ++--------+-------------------------------------------------------------------------+ ast : parse_json(s)->>i @@ -3240,13 +3240,13 @@ evaluation: | Row 3 | '["a","b","c"]' | 1 | 'b' | +--------+------------------------+------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [1,2,3,4], , ["a","b","c"]] }, validity: [0b____1011] } | -| i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 3, , b] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, [1,2,3,4], , ["a","b","c"]], validity: [0b____1011] } | +| i | NullableColumn { column: UInt64([0, 2, 0, 1]), validity: [0b____1010] } | +| Output | NullableColumn { column: StringColumn[, 3, , b], validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------------+ ast : parse_json(s)->>k @@ -3263,13 +3263,13 @@ evaluation: | Row 2 | '{"a":"b"}' | 'x' | NULL | +--------+------------------------+-------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------+ -| s | StringColumn { data: Utf8ViewArray[true, {"k":1}, {"a":"b"}] } | -| k | StringColumn { data: Utf8ViewArray[k, k, x] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, ] }, validity: [0b_____010] } | -+--------+------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------+ +| s | StringColumn[true, {"k":1}, {"a":"b"}] | +| k | StringColumn[k, k, x] | +| Output | NullableColumn { column: StringColumn[, 1, ], validity: [0b_____010] } | ++--------+------------------------------------------------------------------------+ ast : parse_json(s)->>k @@ -3287,13 +3287,13 @@ evaluation: | Row 3 | '{"a":"b"}' | 'a' | 'b' | +--------+-----------------------------+------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | -| k | StringColumn { data: Utf8ViewArray[, k, , a] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, , b] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, {"k":1}, , {"a":"b"}], validity: [0b____1011] } | +| k | StringColumn[, k, , a] | +| Output | NullableColumn { column: StringColumn[, 1, , b], validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------+ error: @@ -3430,7 +3430,7 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3], {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[[1,2,3], {"k":1}, , {"a":"b"}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001, offsets: [0, 10, 10, 10, 10] }, validity: [0b____0001] } | +--------+--------------------------------------------------------------------------------------------------------------------------------+ @@ -3453,8 +3453,8 @@ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | -| k | NullableColumn { column: StringColumn { data: Utf8ViewArray[{1}, {k}, , {a}] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[true, {"k":1}, , {"a":"b"}], validity: [0b____1011] } | +| k | NullableColumn { column: StringColumn[{1}, {k}, , {a}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x20000000200000025001200000001000000162, offsets: [0, 0, 10, 10, 19] }, validity: [0b____1010] } | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -3573,12 +3573,12 @@ evaluation: | Row 3 | '{"a":"b"}' | NULL | +--------+-----------------------------+-------------+ evaluation (internal): -+--------+------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3], {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[1, , , ] }, validity: [0b____0001] } | -+--------+------------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[[1,2,3], {"k":1}, , {"a":"b"}], validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn[1, , , ], validity: [0b____0001] } | ++--------+------------------------------------------------------------------------------------------------+ ast : parse_json(s) #>> k @@ -3596,13 +3596,13 @@ evaluation: | Row 3 | '{"a":"b"}' | '{a}' | 'b' | +--------+-----------------------------+-----------------------+-------------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, {"k":1}, , {"a":"b"}] }, validity: [0b____1011] } | -| k | NullableColumn { column: StringColumn { data: Utf8ViewArray[{1}, {k}, , {a}] }, validity: [0b____1011] } | -| Output | NullableColumn { column: StringColumn { data: Utf8ViewArray[, 1, , b] }, validity: [0b____1010] } | -+--------+---------------------------------------------------------------------------------------------------------------------+ ++--------+---------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, {"k":1}, , {"a":"b"}], validity: [0b____1011] } | +| k | NullableColumn { column: StringColumn[{1}, {k}, , {a}], validity: [0b____1011] } | +| Output | NullableColumn { column: StringColumn[, 1, , b], validity: [0b____1010] } | ++--------+---------------------------------------------------------------------------------------------+ ast : parse_json('["1","2","3"]') ? NULL @@ -3675,12 +3675,12 @@ evaluation: | Row 3 | '{"b":1}' | false | +--------+-----------------------------+--------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3], {"a":1}, , {"b":1}] }, validity: [0b____1011] } | -| Output | NullableColumn { column: Boolean([0b____0010]), validity: [0b____1011] } | -+--------+----------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[[1,2,3], {"a":1}, , {"b":1}], validity: [0b____1011] } | +| Output | NullableColumn { column: Boolean([0b____0010]), validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------------------+ ast : parse_json('["1","2","3"]') ?| NULL @@ -3753,12 +3753,12 @@ evaluation: | Row 3 | '{"c":1}' | false | +--------+-----------------------------+--------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[["a","e","d"], {"a":1,"b":2}, , {"c":1}] }, validity: [0b____1011] } | -| Output | NullableColumn { column: Boolean([0b____0011]), validity: [0b____1011] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[["a","e","d"], {"a":1,"b":2}, , {"c":1}], validity: [0b____1011] } | +| Output | NullableColumn { column: Boolean([0b____0011]), validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------------------------------+ ast : parse_json('["1","2","3"]') ?& NULL @@ -3831,12 +3831,12 @@ evaluation: | Row 3 | '{"a":0,"c":1}' | false | +--------+-------------------------------------+--------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[["a","e","b"], {"a":1,"b":2}, , {"a":0,"c":1}] }, validity: [0b____1011] } | -| Output | NullableColumn { column: Boolean([0b____0011]), validity: [0b____1011] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[["a","e","b"], {"a":1,"b":2}, , {"a":0,"c":1}], validity: [0b____1011] } | +| Output | NullableColumn { column: Boolean([0b____0011]), validity: [0b____1011] } | ++--------+----------------------------------------------------------------------------------------------------------------+ ast : NULL @> NULL @@ -4295,13 +4295,13 @@ evaluation: | Row 3 | '[1,2,3,4]' | '$[*] > 2' | true | +--------+------------------------+-----------------------------+--------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [{"k":1},{"k":2}], , [1,2,3,4]] }, validity: [0b____1011] } | -| p | StringColumn { data: Utf8ViewArray[$.a > 0, $[*].k == 1, $[*] > 1, $[*] > 2] } | -| Output | NullableColumn { column: Boolean([0b____1010]), validity: [0b____1011] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, [{"k":1},{"k":2}], , [1,2,3,4]], validity: [0b____1011] } | +| p | StringColumn[$.a > 0, $[*].k == 1, $[*] > 1, $[*] > 2] | +| Output | NullableColumn { column: Boolean([0b____1010]), validity: [0b____1011] } | ++--------+-------------------------------------------------------------------------------------------------------+ ast : parse_json('{"a":1,"b":2}') @@ '$.a == 1' @@ -4400,13 +4400,13 @@ evaluation: | Row 3 | '[1,2,3,4]' | '$[*] > 2' | true | +--------+------------------------+-----------------------------+--------------+ evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[true, [{"k":1},{"k":2}], , [1,2,3,4]] }, validity: [0b____1011] } | -| p | StringColumn { data: Utf8ViewArray[$.a > 0, $[*].k == 1, $[*] > 1, $[*] > 2] } | -| Output | NullableColumn { column: Boolean([0b____1010]), validity: [0b____1011] } | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ ++--------+-------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, [{"k":1},{"k":2}], , [1,2,3,4]], validity: [0b____1011] } | +| p | StringColumn[$.a > 0, $[*].k == 1, $[*] > 1, $[*] > 2] | +| Output | NullableColumn { column: Boolean([0b____1010]), validity: [0b____1011] } | ++--------+-------------------------------------------------------------------------------------------------------+ ast : NULL @? '$.a' @@ -4866,7 +4866,7 @@ evaluation (internal): +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[{"a":1},2,3], [1,2,3], , {"a":"b"}] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[[{"a":1},2,3], [1,2,3], , {"a":"b"}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x800000035000000420000002200000024000000050025003800000032000000220000002200000025001500250034000000110000001100000016162, offsets: [0, 24, 46, 46, 60] }, validity: [0b____1011] } | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4889,8 +4889,8 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,{"a":2},3], {"k":[1,2,3]}, , {"a":"b"}] }, validity: [0b____1011] } | -| k | NullableColumn { column: StringColumn { data: Utf8ViewArray[{1,a}, {k,-1}, {k}, {c}] }, validity: [0b____1011] } | +| s | NullableColumn { column: StringColumn[[1,{"a":2},3], {"k":[1,2,3]}, , {"a":"b"}], validity: [0b____1011] } | +| k | NullableColumn { column: StringColumn[{1,a}, {k,-1}, {k}, {c}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x8000000320000002500000042000000250014000000050034000000110000001500000106b800000022000000220000002500150024000000110000001100000016162, offsets: [0, 24, 53, 53, 67] }, validity: [0b____1011] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -4968,8 +4968,8 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,null], ["A","B"], , {"a":"b"}] }, validity: [0b____1011] } | -| n | NullableColumn { column: StringColumn { data: Utf8ViewArray["hi", , true, [1,2,3]] }, validity: [0b____1101] } | +| v | NullableColumn { column: StringColumn[[1,2,3,null], ["A","B"], , {"a":"b"}], validity: [0b____1011] } | +| n | NullableColumn { column: StringColumn["hi", , true, [1,2,3]], validity: [0b____1101] } | | Output | NullableColumn { column: BinaryColumn { data: 0x8000000520000002200000021000000220000002000000005001500268695003800000025000000e50000016400000011000000110000001616280000003200000022000000220000002500150025003, offsets: [0, 32, 32, 32, 80] }, validity: [0b____1001] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5055,7 +5055,7 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,1,2,3,3,null,2,1,null], ["A","B","A","B","C"], , {"a":"b"}] }, validity: [0b____1011] } | +| v | NullableColumn { column: StringColumn[[1,1,2,3,3,null,2,1,null], ["A","B","A","B","C"], , {"a":"b"}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x800000042000000220000002200000020000000050015002500380000003100000011000000110000001414243800000015000000e4000000110000001100000016162, offsets: [0, 26, 45, 45, 67] }, validity: [0b____1011] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5168,8 +5168,8 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}] }, validity: [0b____1011] } | -| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}] }, validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}], validity: [0b____1011] } | +| v2 | NullableColumn { column: StringColumn[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x800000042000000220000002200000020000000050015002500380000000800000015000000e4000000110000001100000016162, offsets: [0, 26, 30, 30, 52] }, validity: [0b____1011] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5282,8 +5282,8 @@ evaluation (internal): +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}] }, validity: [0b____1011] } | -| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}] }, validity: [0b____1011] } | +| v1 | NullableColumn { column: StringColumn[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}], validity: [0b____1011] } | +| v2 | NullableColumn { column: StringColumn[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x8000000220000002000000005003800000051000000110000001100000011000000110000001414241424380000000, offsets: [0, 14, 43, 43, 47] }, validity: [0b____1011] } | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5393,13 +5393,13 @@ evaluation: | Row 3 | '{"a":"b"}' | '{"a":"b"}' | true | +--------+---------------------------------+---------------------------------+--------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ -| v1 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}] }, validity: [0b____1011] } | -| v2 | NullableColumn { column: StringColumn { data: Utf8ViewArray[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}] }, validity: [0b____1011] } | -| Output | NullableColumn { column: Boolean([0b____1001]), validity: [0b____1011] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------+ +| v1 | NullableColumn { column: StringColumn[[1,2,3,3,null,null], ["A","B","A","B","C"], , {"a":"b"}], validity: [0b____1011] } | +| v2 | NullableColumn { column: StringColumn[[1,1,2,3,4,5,null], ["X","Y","Z"], , {"a":"b"}], validity: [0b____1011] } | +| Output | NullableColumn { column: Boolean([0b____1001]), validity: [0b____1011] } | ++--------+--------------------------------------------------------------------------------------------------------------------------+ ast : json_object_insert('{"b":12,"d":34,"m":[1,2],"x":{"k":"v"}}'::variant, 'a', 'hello') @@ -5543,8 +5543,8 @@ evaluation (internal): +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}] }, validity: [0b____1011] } | -| n | NullableColumn { column: StringColumn { data: Utf8ViewArray["hi", , true, [1,2,3]] }, validity: [0b____1101] } | +| v | NullableColumn { column: StringColumn[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}], validity: [0b____1011] } | +| n | NullableColumn { column: StringColumn["hi", , true, [1,2,3]], validity: [0b____1101] } | | Output | NullableColumn { column: BinaryColumn { data: 0x40000002100000011000000110000001100000026b787668694000000110000001100000016d6e400000041000000110000001100000011000000110000001100000015000001610000001616378796264800000032000000220000002200000025001500250037a, offsets: [0, 25, 39, 39, 104] }, validity: [0b____1011] } | +--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5567,8 +5567,8 @@ evaluation (internal): +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}] }, validity: [0b____1011] } | -| n | NullableColumn { column: StringColumn { data: Utf8ViewArray["hi", , true, [1,2,3]] }, validity: [0b____1101] } | +| v | NullableColumn { column: StringColumn[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}], validity: [0b____1011] } | +| n | NullableColumn { column: StringColumn["hi", , true, [1,2,3]], validity: [0b____1101] } | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000210000001100000011000000210000001636b6869764000000110000001100000016d6e4000000310000001100000011000000110000001500000161000000161637962800000032000000220000002200000025001500250037a, offsets: [0, 25, 39, 39, 94] }, validity: [0b____1011] } | +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5635,7 +5635,7 @@ evaluation (internal): +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}] }, validity: [0b____1011] } | +| v | NullableColumn { column: StringColumn[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x4000000110000001100000016b764000000040000002100000011000000110000001100000016379647a, offsets: [0, 14, 18, 18, 42] }, validity: [0b____1011] } | +--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -5702,7 +5702,7 @@ evaluation (internal): +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| v | NullableColumn { column: StringColumn { data: Utf8ViewArray[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}] }, validity: [0b____1011] } | +| v | NullableColumn { column: StringColumn[{"k":"v"}, {"m":"n"}, , {"a":"b","c":"d","y":"z"}], validity: [0b____1011] } | | Output | NullableColumn { column: BinaryColumn { data: 0x400000004000000110000001100000016d6e4000000110000001100000016162, offsets: [0, 4, 18, 18, 32] }, validity: [0b____1011] } | +--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs index f554b6157f38..894cd2c2b89f 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs @@ -100,7 +100,7 @@ impl RowConverter for CommonRowConverter { let col = col.as_variant().unwrap(); let mut builder = BinaryColumnBuilder::with_capacity( col.len(), - col.current_buffer_len(), + col.total_bytes_len(), ); for (i, val) in col.iter().enumerate() { if let Some(validity) = validity { diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index b712b9ccbf4d..5106915297f2 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -13,7 +13,7 @@ test = true [features] default = ["simd"] -simd = ["databend-common-arrow/simd"] +simd = ["databend-common-column/simd"] python-udf = ["arrow-udf-python"] disable_initial_exec_tls = ["databend-common-base/disable_initial_exec_tls"] jemalloc = ["databend-common-storages-system/jemalloc"] @@ -51,12 +51,13 @@ chrono = { workspace = true } chrono-tz = { workspace = true } ctor = { workspace = true } dashmap = { workspace = true } -databend-common-arrow = { workspace = true } + databend-common-ast = { workspace = true } databend-common-base = { workspace = true } databend-common-cache = { workspace = true } databend-common-catalog = { workspace = true } databend-common-cloud-control = { workspace = true } +databend-common-column = { workspace = true } databend-common-config = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } diff --git a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_keys_builder.rs b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_keys_builder.rs index 3979aa44668a..41581af90896 100644 --- a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_keys_builder.rs +++ b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_keys_builder.rs @@ -16,7 +16,7 @@ use std::marker::PhantomData; use byteorder::BigEndian; use byteorder::WriteBytesExt; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::buffer::Buffer; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::decimal::Decimal; use databend_common_expression::types::number::Number; diff --git a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_keys_iter.rs b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_keys_iter.rs index fd2f13508cfd..efdaa0bd0d58 100644 --- a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_keys_iter.rs +++ b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_keys_iter.rs @@ -17,10 +17,10 @@ use std::slice::Iter; use byteorder::BigEndian; use byteorder::ReadBytesExt; -use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use databend_common_expression::types::binary::BinaryColumn; -use databend_common_expression::types::binary::BinaryIterator; +use databend_common_expression::types::binary::BinaryColumnIter; use databend_common_expression::types::number::Number; use databend_common_hashtable::DictionaryKeys; @@ -86,7 +86,7 @@ impl SerializedKeysColumnIter { } impl KeysColumnIter<[u8]> for SerializedKeysColumnIter { - type Iterator<'a> = BinaryIterator<'a> where Self: 'a; + type Iterator<'a> = BinaryColumnIter<'a> where Self: 'a; fn iter(&self) -> Self::Iterator<'_> { self.column.iter() diff --git a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_polymorphic_keys.rs b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_polymorphic_keys.rs index 1a7e3591deed..232a8990e1d1 100644 --- a/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_polymorphic_keys.rs +++ b/src/query/service/src/pipelines/processors/transforms/group_by/aggregator_polymorphic_keys.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use std::time::Instant; use bumpalo::Bump; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::number::*; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/common.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/common.rs index 47152a9087e3..71482ebfece9 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/common.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/common.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::Result; use databend_common_expression::arrow::or_validities; use databend_common_expression::types::nullable::NullableColumn; @@ -156,7 +156,7 @@ impl HashJoinState { match col { Column::Nullable(c) => { let bitmap = &c.validity; - if bitmap.unset_bits() == 0 { + if bitmap.null_count() == 0 { valids = Some(Bitmap::new_constant(true, num_rows)); break; } else { diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs index 54e6e73e5236..440178b6268b 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs @@ -20,11 +20,11 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_base::base::tokio::sync::Barrier; use databend_common_catalog::runtime_filter_info::RuntimeFilterInfo; use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::table_context::TableContext; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::arrow::and_validities; @@ -452,7 +452,7 @@ impl HashJoinBuildState { let build_keys_iter = $method.build_keys_iter(&keys_state)?; let valid_num = match &$valids { - Some(valids) => valids.len() - valids.unset_bits(), + Some(valids) => valids.len() - valids.null_count(), None => $chunk.num_rows(), }; let mut local_space: Vec = Vec::with_capacity(valid_num * entry_size); @@ -519,14 +519,14 @@ impl HashJoinBuildState { let space_size = match &keys_state { // safe to unwrap(): offset.len() >= 1. - KeysState::Column(Column::String(col)) => col.current_buffer_len(), + KeysState::Column(Column::String(col)) => col.total_bytes_len(), KeysState::Column( Column::Binary(col) | Column::Variant(col) | Column::Bitmap(col), ) => col.data().len(), _ => unreachable!(), }; let valid_num = match &$valids { - Some(valids) => valids.len() - valids.unset_bits(), + Some(valids) => valids.len() - valids.null_count(), None => $chunk.num_rows(), }; let mut entry_local_space: Vec = Vec::with_capacity(valid_num * entry_size); @@ -692,10 +692,10 @@ impl HashJoinBuildState { }); match valids { ControlFlow::Continue(Some(valids)) | ControlFlow::Break(Some(valids)) => { - if valids.unset_bits() == valids.len() { + if valids.null_count() == valids.len() { return Ok(()); } - if valids.unset_bits() != 0 { + if valids.null_count() != 0 { Some(valids) } else { None @@ -717,7 +717,7 @@ impl HashJoinBuildState { JoinType::RightMark => { if !_has_null && !keys_columns.is_empty() { if let Some(validity) = keys_columns[0].validity().1 { - if validity.unset_bits() > 0 { + if validity.null_count() > 0 { _has_null = true; let mut has_null_ref = self .hash_join_state diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs index 5e27be5eb94a..5cb8b8e835de 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs @@ -18,9 +18,9 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_base::base::tokio::sync::Barrier; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::arrow::and_validities; @@ -278,7 +278,7 @@ impl HashJoinProbeState { // Thanks to the **adaptive** execution strategy of early filtering, we don't experience a performance decrease // when all keys have matches. This allows us to achieve the same performance as before. probe_state.num_keys += if let Some(valids) = &valids { - (valids.len() - valids.unset_bits()) as u64 + (valids.len() - valids.null_count()) as u64 } else { input_num_rows as u64 }; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs index 0292ec9eb554..50ac70f659a1 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs @@ -55,7 +55,7 @@ impl HashJoinProbeState { .get_by_offset(0) .to_column(process_state.input.num_rows()); // Check if there is any null in the probe column. - if matches!(probe_column.validity().1, Some(x) if x.unset_bits() > 0) { + if matches!(probe_column.validity().1, Some(x) if x.null_count() > 0) { let mut has_null = self .hash_join_state .hash_join_desc @@ -153,7 +153,7 @@ impl HashJoinProbeState { .get_by_offset(0) .to_column(process_state.input.num_rows()); // Check if there is any null in the probe column. - if matches!(probe_column.validity().1, Some(x) if x.unset_bits() > 0) { + if matches!(probe_column.validity().1, Some(x) if x.null_count() > 0) { let mut has_null = self .hash_join_state .hash_join_desc diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs index 87ac376e47da..af4cb2a22ed0 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_expression::filter::FilterExecutor; use databend_common_expression::DataBlock; use databend_common_expression::Expr; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/result_blocks.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/result_blocks.rs index cfe04ef351e7..5533499a39f2 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/result_blocks.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/result_blocks.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockEntry; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/spill_common.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/spill_common.rs index 3c1837044b1e..c67678a683b8 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/spill_common.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/spill_common.rs @@ -14,7 +14,7 @@ // Define some methods that are used by both the build and probe spilling of the hash join. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; diff --git a/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_state.rs b/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_state.rs index 2034b0e277ff..02f369b66ed7 100644 --- a/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_state.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_catalog::table_context::TableContext; +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberColumnBuilder; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_srf.rs b/src/query/service/src/pipelines/processors/transforms/transform_srf.rs index 315cca051ba7..63e6b94b3bab 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_srf.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_srf.rs @@ -451,12 +451,12 @@ pub fn push_string_column( if let Column::Nullable(box nullable_column) = column { if let Column::String(string_column) = nullable_column.column { let validity = nullable_column.validity; - if validity.unset_bits() == 0 { + if validity.null_count() == 0 { for idx in 0..repeat_times { builder.push(unsafe { string_column.index_unchecked(idx) }); } builder.push_repeat_null(num_rows - repeat_times); - } else if validity.unset_bits() == validity.len() { + } else if validity.null_count() == validity.len() { builder.push_repeat_null(num_rows); } else { for idx in 0..repeat_times { @@ -485,12 +485,12 @@ fn push_variant_column( if let Column::Nullable(box nullable_column) = column { if let Column::Variant(variant_column) = nullable_column.column { let validity = nullable_column.validity; - if validity.unset_bits() == 0 { + if validity.null_count() == 0 { for idx in 0..repeat_times { builder.push(unsafe { variant_column.index_unchecked(idx) }); } builder.push_repeat_null(num_rows - repeat_times); - } else if validity.unset_bits() == validity.len() { + } else if validity.null_count() == validity.len() { builder.push_repeat_null(num_rows); } else { for idx in 0..repeat_times { @@ -519,12 +519,12 @@ fn push_number_column( if let Column::Nullable(box nullable_column) = column { if let Column::Number(NumberColumn::UInt64(number_column)) = nullable_column.column { let validity = nullable_column.validity; - if validity.unset_bits() == 0 { + if validity.null_count() == 0 { for idx in 0..repeat_times { builder.push(unsafe { *number_column.get_unchecked(idx) }); } builder.push_repeat_null(num_rows - repeat_times); - } else if validity.unset_bits() == validity.len() { + } else if validity.null_count() == validity.len() { builder.push_repeat_null(num_rows); } else { for idx in 0..repeat_times { diff --git a/src/query/service/src/pipelines/processors/transforms/transform_udf_script.rs b/src/query/service/src/pipelines/processors/transforms/transform_udf_script.rs index f5001fa8917d..caf34b7c5ac2 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_udf_script.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_udf_script.rs @@ -22,8 +22,8 @@ use arrow_array::RecordBatch; use arrow_schema::Schema; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::converts::arrow::ARROW_EXT_TYPE_VARIANT; use databend_common_expression::converts::arrow::EXTENSION_KEY; -use databend_common_expression::converts::arrow2::ARROW_EXT_TYPE_VARIANT; use databend_common_expression::variant_transform::contains_variant; use databend_common_expression::variant_transform::transform_variant; use databend_common_expression::BlockEntry; diff --git a/src/query/service/src/servers/flight/v1/scatter/flight_scatter_hash.rs b/src/query/service/src/servers/flight/v1/scatter/flight_scatter_hash.rs index ff1d07a2ab9e..cc82de51e9dc 100644 --- a/src/query/service/src/servers/flight/v1/scatter/flight_scatter_hash.rs +++ b/src/query/service/src/servers/flight/v1/scatter/flight_scatter_hash.rs @@ -15,12 +15,12 @@ use std::collections::hash_map::DefaultHasher; use std::hash::Hasher; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::check_function; use databend_common_expression::types::number::NumberScalar; use databend_common_expression::types::AnyType; +use databend_common_expression::types::Buffer; use databend_common_expression::types::DataType; use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberDataType; @@ -245,9 +245,9 @@ fn get_hash_values( NullableType::>::try_downcast_column(&c) { let null_map = column.validity; - if null_map.unset_bits() == 0 { + if null_map.null_count() == 0 { Ok(column.column) - } else if null_map.unset_bits() == null_map.len() { + } else if null_map.null_count() == null_map.len() { Ok(vec![default_scatter_index; rows].into()) } else { let mut need_new_vec = true; diff --git a/src/query/service/src/spillers/serialize.rs b/src/query/service/src/spillers/serialize.rs index 1854cf019d4d..6eec57747d3b 100644 --- a/src/query/service/src/spillers/serialize.rs +++ b/src/query/service/src/spillers/serialize.rs @@ -15,6 +15,7 @@ use std::io::Write; use std::sync::Arc; +use arrow_schema::Schema; use buf_list::BufList; use buf_list::Cursor; use bytes::Buf; @@ -23,7 +24,6 @@ use databend_common_base::base::DmaWriteBuf; use databend_common_exception::Result; use databend_common_expression::arrow::read_column; use databend_common_expression::arrow::write_column; -use databend_common_expression::converts::arrow::table_schema_to_arrow_schema; use databend_common_expression::infer_table_schema; use databend_common_expression::DataBlock; use databend_common_expression::DataField; @@ -166,7 +166,7 @@ fn bare_blocks_to_parquet( .into_iter() .map(|block| block.to_record_batch(&table_schema)) .collect::>>()?; - let arrow_schema = Arc::new(table_schema_to_arrow_schema(&table_schema)); + let arrow_schema = Arc::new(Schema::from(table_schema.as_ref())); let mut writer = ArrowWriter::try_new(write_buffer, arrow_schema, Some(props))?; for batch in batches { writer.write(&batch)?; diff --git a/src/query/service/tests/it/servers/http/json_block.rs b/src/query/service/tests/it/servers/http/json_block.rs index 0ca921a6ed66..b29fa79611d7 100644 --- a/src/query/service/tests/it/servers/http/json_block.rs +++ b/src/query/service/tests/it/servers/http/json_block.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::number::Float64Type; diff --git a/src/query/service/tests/it/storages/fuse/operations/read_plan.rs b/src/query/service/tests/it/storages/fuse/operations/read_plan.rs index ae769f50dc25..62018708d4e0 100644 --- a/src/query/service/tests/it/storages/fuse/operations/read_plan.rs +++ b/src/query/service/tests/it/storages/fuse/operations/read_plan.rs @@ -16,9 +16,9 @@ use std::collections::HashMap; use std::iter::Iterator; use std::sync::Arc; +use arrow_schema::DataType as ArrowType; +use arrow_schema::Field; use chrono::Utc; -use databend_common_arrow::arrow::datatypes::DataType as ArrowType; -use databend_common_arrow::arrow::datatypes::Field as ArrowField; use databend_common_base::base::tokio; use databend_common_catalog::plan::Projection; use databend_common_catalog::plan::PushDownInfo; @@ -61,13 +61,16 @@ fn test_to_partitions() -> Result<()> { }) }; - let col_nodes_gen = |field_index| ColumnNode { - field: ArrowField::new("".to_string(), ArrowType::Int64, false), - is_nested: false, - init: vec![], - leaf_indices: vec![field_index], - leaf_column_ids: vec![field_index as ColumnId], - children: None, + let col_nodes_gen = |field_index| { + let mut n = ColumnNode::new( + Field::new("".to_string(), ArrowType::Int64, false), + false, + vec![], + vec![field_index], + None, + ); + n.leaf_column_ids = vec![field_index as ColumnId]; + n }; // generates fake data. diff --git a/src/query/sql/src/planner/plans/constant_table_scan.rs b/src/query/sql/src/planner/plans/constant_table_scan.rs index dae453f79e8e..fe09d7afa1e6 100644 --- a/src/query/sql/src/planner/plans/constant_table_scan.rs +++ b/src/query/sql/src/planner/plans/constant_table_scan.rs @@ -190,7 +190,7 @@ impl Operator for ConstantTableScan { let (is_all_null, bitmap) = value.validity(); let null_count = match (is_all_null, bitmap) { (true, _) => self.num_rows as u64, - (false, Some(bitmap)) => bitmap.unset_bits() as u64, + (false, Some(bitmap)) => bitmap.null_count() as u64, (false, None) => 0, }; diff --git a/src/query/storages/common/blocks/src/parquet_rs.rs b/src/query/storages/common/blocks/src/parquet_rs.rs index c002896a74df..53c32579a2ea 100644 --- a/src/query/storages/common/blocks/src/parquet_rs.rs +++ b/src/query/storages/common/blocks/src/parquet_rs.rs @@ -15,7 +15,6 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::converts::arrow::table_schema_to_arrow_schema; use databend_common_expression::DataBlock; use databend_common_expression::TableSchema; use databend_storages_common_table_meta::table::TableCompression; @@ -46,7 +45,7 @@ pub fn blocks_to_parquet( .into_iter() .map(|block| block.to_record_batch(table_schema)) .collect::>>()?; - let arrow_schema = Arc::new(table_schema_to_arrow_schema(table_schema)); + let arrow_schema = Arc::new(table_schema.into()); let mut writer = ArrowWriter::try_new(write_buffer, arrow_schema, Some(props))?; for batch in batches { writer.write(&batch)?; diff --git a/src/query/storages/common/cache/Cargo.toml b/src/query/storages/common/cache/Cargo.toml index 4973c6eec93c..750cd984cf48 100644 --- a/src/query/storages/common/cache/Cargo.toml +++ b/src/query/storages/common/cache/Cargo.toml @@ -11,7 +11,8 @@ doctest = false test = true [dependencies] -databend-common-arrow = { workspace = true } +arrow = { workspace = true } + databend-common-base = { workspace = true } databend-common-cache = { workspace = true } databend-common-catalog = { workspace = true } diff --git a/src/query/storages/common/cache/src/caches.rs b/src/query/storages/common/cache/src/caches.rs index 250574e5fc7d..8f79174e23d2 100644 --- a/src/query/storages/common/cache/src/caches.rs +++ b/src/query/storages/common/cache/src/caches.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use arrow::array::ArrayRef; use databend_common_cache::MemSized; use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::Partitions; @@ -58,10 +59,7 @@ pub type PrunePartitionsCache = InMemoryLruCache<(PartStatistics, Partitions)>; /// In memory object cache of table column array pub type ColumnArrayCache = InMemoryLruCache; pub type ArrayRawDataUncompressedSize = usize; -pub type SizedColumnArray = ( - Box, - ArrayRawDataUncompressedSize, -); +pub type SizedColumnArray = (ArrayRef, ArrayRawDataUncompressedSize); // Bind Type of cached objects to Caches // diff --git a/src/query/storages/common/index/Cargo.toml b/src/query/storages/common/index/Cargo.toml index 40d42533df4e..c6cb3a3930b3 100644 --- a/src/query/storages/common/index/Cargo.toml +++ b/src/query/storages/common/index/Cargo.toml @@ -16,7 +16,7 @@ ignored = ["xorfilter-rs", "match-template"] [dependencies] anyerror = { workspace = true } cbordata = { workspace = true } -databend-common-arrow = { workspace = true } + databend-common-ast = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } @@ -38,7 +38,7 @@ xorfilter-rs = { workspace = true, features = ["cbordata"] } [dev-dependencies] criterion = { workspace = true } -databend-common-arrow = { workspace = true } + rand = { workspace = true } [[bench]] diff --git a/src/query/storages/common/index/src/bloom_index.rs b/src/query/storages/common/index/src/bloom_index.rs index 8763f35ae9a5..9cf69e2488d9 100644 --- a/src/query/storages/common/index/src/bloom_index.rs +++ b/src/query/storages/common/index/src/bloom_index.rs @@ -17,8 +17,6 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -27,6 +25,8 @@ use databend_common_expression::eval_function; use databend_common_expression::types::boolean::BooleanDomain; use databend_common_expression::types::nullable::NullableDomain; use databend_common_expression::types::AnyType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; use databend_common_expression::types::DataType; use databend_common_expression::types::MapType; use databend_common_expression::types::NullableType; @@ -272,7 +272,7 @@ impl BloomIndex { // create filter per column let mut filter_builder = Xor8Builder::create(); - if validity.as_ref().map(|v| v.unset_bits()).unwrap_or(0) > 0 { + if validity.as_ref().map(|v| v.null_count()).unwrap_or(0) > 0 { let validity = validity.unwrap(); let it = column.deref().iter().zip(validity.iter()).map( |(v, b)| { @@ -545,7 +545,7 @@ impl BloomIndex { /// If it does, the bloom index for the column will not be established. fn check_large_string(column: &Column) -> bool { if let Column::String(v) = &column { - let bytes_per_row = v.current_buffer_len() / v.len().max(1); + let bytes_per_row = v.total_bytes_len() / v.len().max(1); if bytes_per_row > 256 { return true; } diff --git a/src/query/storages/common/index/tests/it/filters/bloom_filter.rs b/src/query/storages/common/index/tests/it/filters/bloom_filter.rs index 32ceb5d15d08..ce83bfe70fe3 100644 --- a/src/query/storages/common/index/tests/it/filters/bloom_filter.rs +++ b/src/query/storages/common/index/tests/it/filters/bloom_filter.rs @@ -16,7 +16,6 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::sync::Arc; -use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::Result; use databend_common_expression::type_check::check_function; use databend_common_expression::types::array::ArrayColumn; @@ -25,6 +24,7 @@ use databend_common_expression::types::map::KvPair; use databend_common_expression::types::number::NumberScalar; use databend_common_expression::types::number::UInt8Type; use databend_common_expression::types::AnyType; +use databend_common_expression::types::Buffer; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; diff --git a/src/query/storages/common/table_meta/Cargo.toml b/src/query/storages/common/table_meta/Cargo.toml index abccd45c9db7..baea9374e977 100644 --- a/src/query/storages/common/table_meta/Cargo.toml +++ b/src/query/storages/common/table_meta/Cargo.toml @@ -10,12 +10,13 @@ edition = { workspace = true } dev = ["snap"] [dependencies] -databend-common-arrow = { workspace = true } + databend-common-base = { workspace = true } databend-common-datavalues = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } databend-common-io = { workspace = true } +databend-common-native = { workspace = true } databend-common-storage = { workspace = true } bincode_v1 = { workspace = true } diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 226dc74de269..021df36be06a 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -18,11 +18,11 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; -use databend_common_arrow::native::ColumnMeta as NativeColumnMeta; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ColumnId; use databend_common_expression::TableField; +use databend_common_native::ColumnMeta as NativeColumnMeta; use enum_as_inner::EnumAsInner; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index aebed5d56ba4..378ceb884658 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -105,7 +105,7 @@ pub struct NativeColumnMeta { pub pages: Vec, } -impl From for databend_common_arrow::native::ColumnMeta { +impl From for databend_common_native::ColumnMeta { fn from(value: NativeColumnMeta) -> Self { Self { offset: value.offset, @@ -114,7 +114,7 @@ impl From for databend_common_arrow::native::ColumnMeta { } } -impl From for databend_common_arrow::native::PageMeta { +impl From for databend_common_native::PageMeta { fn from(value: PageMeta) -> Self { Self { length: value.length, diff --git a/src/query/storages/common/table_meta/src/table/table_compression.rs b/src/query/storages/common/table_meta/src/table/table_compression.rs index b542555fc8d8..a33cd09a3387 100644 --- a/src/query/storages/common/table_meta/src/table/table_compression.rs +++ b/src/query/storages/common/table_meta/src/table/table_compression.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_arrow::native; use databend_common_exception::ErrorCode; +use databend_common_native; use parquet::basic::Compression as ParquetCompression; use parquet::basic::GzipLevel; use parquet::basic::ZstdLevel; @@ -49,13 +49,13 @@ impl TryFrom<&str> for TableCompression { } /// Convert to native Compression. -impl From for native::CommonCompression { +impl From for databend_common_native::CommonCompression { fn from(value: TableCompression) -> Self { match value { - TableCompression::None => native::CommonCompression::None, - TableCompression::LZ4 => native::CommonCompression::Lz4, - TableCompression::Snappy => native::CommonCompression::Snappy, - TableCompression::Zstd => native::CommonCompression::Zstd, + TableCompression::None => databend_common_native::CommonCompression::None, + TableCompression::LZ4 => databend_common_native::CommonCompression::Lz4, + TableCompression::Snappy => databend_common_native::CommonCompression::Snappy, + TableCompression::Zstd => databend_common_native::CommonCompression::Zstd, } } } diff --git a/src/query/storages/fuse/Cargo.toml b/src/query/storages/fuse/Cargo.toml index e788fe21365c..921e0b488154 100644 --- a/src/query/storages/fuse/Cargo.toml +++ b/src/query/storages/fuse/Cargo.toml @@ -11,7 +11,7 @@ doctest = false test = true [dependencies] -databend-common-arrow = { workspace = true } + databend-common-base = { workspace = true } databend-common-catalog = { workspace = true } databend-common-exception = { workspace = true } @@ -23,6 +23,7 @@ databend-common-license = { workspace = true } databend-common-meta-app = { workspace = true } databend-common-meta-types = { workspace = true } databend-common-metrics = { workspace = true } +databend-common-native = { workspace = true } databend-common-pipeline-core = { workspace = true } databend-common-pipeline-sinks = { workspace = true } databend-common-pipeline-sources = { workspace = true } diff --git a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_native.rs b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_native.rs index 8a64ac3cf8a7..6b20a43b4d2c 100644 --- a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_native.rs +++ b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_native.rs @@ -14,10 +14,9 @@ use std::sync::Arc; -use databend_common_arrow::arrow::chunk::Chunk; -use databend_common_arrow::native::read as nread; use databend_common_exception::Result; use databend_common_expression::DataBlock; +use databend_common_native::read as nread; use databend_storages_common_table_meta::meta::ColumnMeta; use log::debug; @@ -142,7 +141,7 @@ impl AggIndexReader { for (index, column_node) in self.reader.project_column_nodes.iter().enumerate() { let readers = data.remove(&index).unwrap(); - let array_iter = self.reader.build_array_iter(column_node, readers)?; + let array_iter = self.reader.build_column_iter(column_node, readers)?; let arrays = array_iter.map(|a| Ok(a?)).collect::>>()?; all_columns_arrays.push(arrays); } @@ -160,12 +159,11 @@ impl AggIndexReader { let mut blocks = Vec::with_capacity(page_num); for i in 0..page_num { - let mut arrays = Vec::with_capacity(all_columns_arrays.len()); - for array in all_columns_arrays.iter() { - arrays.push(array[i].clone()); + let mut columns = Vec::with_capacity(all_columns_arrays.len()); + for cs in all_columns_arrays.iter() { + columns.push(cs[i].clone()); } - let chunk = Chunk::new(arrays); - let block = DataBlock::from_arrow_chunk(&chunk, &self.reader.data_schema())?; + let block = DataBlock::new_from_columns(columns); blocks.push(block); } let block = DataBlock::concat(&blocks)?; diff --git a/src/query/storages/fuse/src/io/read/block/block_reader.rs b/src/query/storages/fuse/src/io/read/block/block_reader.rs index 975bcd1e8d77..8c339e4f7c9e 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader.rs @@ -15,9 +15,9 @@ use std::collections::BTreeMap; use std::sync::Arc; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::arrow::datatypes::Schema; -use databend_common_arrow::native::read::NativeColumnsReader; +use arrow_schema::Field; +use arrow_schema::Schema; +use arrow_schema::SchemaRef; use databend_common_catalog::plan::Projection; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; @@ -29,6 +29,7 @@ use databend_common_expression::DataSchema; use databend_common_expression::FieldIndex; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; +use databend_common_native::read::NativeColumnsReader; use databend_common_sql::field_default_value; use databend_common_storage::ColumnNode; use databend_common_storage::ColumnNodes; @@ -43,6 +44,7 @@ pub struct BlockReader { pub(crate) operator: Operator, pub(crate) projection: Projection, pub(crate) projected_schema: TableSchemaRef, + pub(crate) arrow_schema: SchemaRef, pub(crate) project_indices: BTreeMap, pub(crate) project_column_nodes: Vec, pub(crate) default_vals: Vec, @@ -127,7 +129,7 @@ impl BlockReader { }; let arrow_schema: Schema = schema.as_ref().into(); - let native_columns_reader = NativeColumnsReader::new(arrow_schema.clone())?; + let native_columns_reader = NativeColumnsReader::new()?; let column_nodes = ColumnNodes::new_from_schema(&arrow_schema, Some(&schema)); let project_column_nodes: Vec = projection @@ -143,6 +145,7 @@ impl BlockReader { operator, projection, projected_schema, + arrow_schema: arrow_schema.into(), project_indices, project_column_nodes, default_vals, @@ -191,6 +194,10 @@ impl BlockReader { self.projected_schema.clone() } + pub fn arrow_schema(&self) -> SchemaRef { + self.arrow_schema.clone() + } + pub fn data_fields(&self) -> Vec { self.schema().fields().iter().map(DataField::from).collect() } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs index 8e29a6d8d52e..9f229ffc0f49 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; -use databend_common_arrow::arrow::array::Array; +use arrow_array::ArrayRef; use databend_common_catalog::plan::PartInfoPtr; use databend_common_exception::Result; use databend_common_expression::ColumnId; @@ -34,8 +34,8 @@ use crate::FuseStorageFormat; pub enum DeserializedArray<'a> { Cached(&'a Arc), - Deserialized((ColumnId, Box, usize)), - NoNeedToCache(Box), + Deserialized((ColumnId, ArrayRef, usize)), + NoNeedToCache(ArrayRef), } pub struct FieldDeserializationContext<'a> { diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_native.rs b/src/query/storages/fuse/src/io/read/block/block_reader_native.rs index 8f2058b8ceef..802321a2562b 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_native.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_native.rs @@ -19,10 +19,6 @@ use std::ops::Range; use std::sync::Arc; use arrow::datatypes::Schema as ArrowSchema; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::native::read::reader::read_meta_async; -use databend_common_arrow::native::read::reader::NativeReader; -use databend_common_arrow::native::read::NativeReadBuf; use databend_common_catalog::plan::PartInfoPtr; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; @@ -34,6 +30,9 @@ use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::Value; use databend_common_metrics::storage::*; +use databend_common_native::read::reader::read_meta_async; +use databend_common_native::read::reader::NativeReader; +use databend_common_native::read::NativeReadBuf; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::ColumnMeta; use opendal::Operator; @@ -217,23 +216,23 @@ impl BlockReader { pub fn build_block( &self, - chunks: &[(usize, Box)], + columns: &[(usize, Column)], default_val_indices: Option>, ) -> Result { let mut nums_rows: Option = None; let mut entries = Vec::with_capacity(self.project_column_nodes.len()); for (index, _) in self.project_column_nodes.iter().enumerate() { - if let Some(array) = chunks.iter().find(|c| c.0 == index).map(|c| c.1.clone()) { + if let Some(column) = columns.iter().find(|c| c.0 == index).map(|c| c.1.clone()) { let data_type: DataType = self.projected_schema.field(index).data_type().into(); entries.push(BlockEntry::new( data_type.clone(), - Value::Column(Column::from_arrow(array.as_ref(), &data_type)?), + Value::Column(column.clone()), )); match nums_rows { Some(rows) => { - debug_assert_eq!(rows, array.len(), "Column array lengths are not equal") + debug_assert_eq!(rows, column.len(), "Column lengths are not equal") } - None => nums_rows = Some(array.len()), + None => nums_rows = Some(column.len()), } } else if let Some(ref default_val_indices) = default_val_indices { if default_val_indices.contains(&index) { @@ -265,7 +264,7 @@ impl BlockReader { .into_iter() .map(ColumnMeta::Native) .collect::>(); - let schema = DataSchema::try_from(&schema).ok()?; + let schema = DataSchema::from(&schema); Some((metas, ArrowSchema::from(&schema))) } } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs index d656758174bb..5ac0643be8f0 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs @@ -15,19 +15,17 @@ use std::collections::HashMap; use std::time::Instant; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::chunk::Chunk; -use databend_common_arrow::arrow::datatypes::DataType as ArrowType; -use databend_common_arrow::arrow::datatypes::Field as ArrowField; -use databend_common_arrow::arrow::datatypes::Schema as ArrowSchema; -use databend_common_arrow::native::read::reader::NativeReader; -use databend_common_arrow::native::read::ArrayIter; -use databend_common_arrow::native::read::NativeColumnsReader; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; use databend_common_metrics::storage::*; +use databend_common_native::read::reader::NativeReader; +use databend_common_native::read::ColumnIter; +use databend_common_native::read::NativeColumnsReader; use databend_common_storage::ColumnNode; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; @@ -117,23 +115,19 @@ impl BlockReader { for array in &deserialized_column_arrays { match array { DeserializedArray::Deserialized((_, array, ..)) => { - chunk_arrays.push(array); + chunk_arrays.push(array.clone()); } DeserializedArray::NoNeedToCache(array) => { - chunk_arrays.push(array); + chunk_arrays.push(array.clone()); } DeserializedArray::Cached(sized_column) => { - chunk_arrays.push(&sized_column.0); + chunk_arrays.push(sized_column.0.clone()); } } } // build data block - let chunk = Chunk::try_new(chunk_arrays)?; - let data_block = if !need_to_fill_default_val { - DataBlock::from_arrow_chunk(&chunk, &self.data_schema()) - } else { - let data_schema = self.data_schema(); + let data_block = if need_to_fill_default_val { let mut default_vals = Vec::with_capacity(need_default_vals.len()); for (i, need_default_val) in need_default_vals.iter().enumerate() { if !need_default_val { @@ -142,12 +136,21 @@ impl BlockReader { default_vals.push(Some(self.default_vals[i].clone())); } } - DataBlock::create_with_default_value_and_chunk( - &data_schema, - &chunk, + DataBlock::create_with_opt_default_value( + chunk_arrays, + &self.data_schema(), &default_vals, num_rows, - ) + )? + } else { + debug_assert!(chunk_arrays.len() == self.projected_schema.num_fields()); + let cols = chunk_arrays + .into_iter() + .zip(self.data_schema().fields()) + .map(|(arr, f)| Column::from_arrow_rs(arr, f.data_type())) + .collect::>>()?; + + DataBlock::new_from_columns(cols) }; // populate cache if necessary @@ -164,17 +167,15 @@ impl BlockReader { } } } - data_block + Ok(data_block) } - fn chunks_to_native_array( + fn chunks_to_native_column( &self, - _column_node: &ColumnNode, metas: Vec<&ColumnMeta>, chunks: Vec<&[u8]>, - _leaf_ids: Vec, - field: ArrowField, - ) -> Result> { + field: TableField, + ) -> Result { let mut page_metas = Vec::with_capacity(chunks.len()); let mut readers = Vec::with_capacity(chunks.len()); for (chunk, meta) in chunks.into_iter().zip(metas.into_iter()) { @@ -184,10 +185,11 @@ impl BlockReader { page_metas.push(meta.pages.clone()); } - match self - .native_columns_reader - .batch_read_array(readers, field, page_metas) - { + match self.native_columns_reader.batch_read_column( + readers, + field.data_type().clone(), + page_metas, + ) { Ok(array) => Ok(array), Err(err) => Err(err.into()), } @@ -196,13 +198,13 @@ impl BlockReader { fn deserialize_native_field<'a>( &self, deserialization_context: &'a FieldDeserializationContext, - column: &ColumnNode, + column_node: &ColumnNode, ) -> Result>> { - let indices = &column.leaf_indices; + let indices = &column_node.leaf_indices; let column_chunks = deserialization_context.column_chunks; // column passed in may be a compound field (with sub leaves), // or a leaf column of compound field - let is_nested = column.has_children(); + let is_nested = column_node.has_children(); let estimated_cap = indices.len(); let mut field_column_metas = Vec::with_capacity(estimated_cap); let mut field_column_data = Vec::with_capacity(estimated_cap); @@ -210,7 +212,7 @@ impl BlockReader { let mut field_uncompressed_size = 0; for (i, leaf_index) in indices.iter().enumerate() { - let column_id = column.leaf_column_ids[i]; + let column_id = column_node.leaf_column_ids[i]; if let Some(column_meta) = deserialization_context.column_metas.get(&column_id) { if let Some(chunk) = column_chunks.get(&column_id) { match chunk { @@ -243,13 +245,12 @@ impl BlockReader { } if !field_column_metas.is_empty() { - let array = self.chunks_to_native_array( - column, + let column = self.chunks_to_native_column( field_column_metas, field_column_data, - field_leaf_ids, - column.field.clone(), + column_node.table_field.clone(), )?; + let array = column.clone().into_arrow_rs(); // mark the array if is_nested { // the array is not intended to be cached @@ -257,7 +258,7 @@ impl BlockReader { Ok(Some(DeserializedArray::NoNeedToCache(array))) } else { // the array is deserialized from raw bytes, should be cached - let column_id = column.leaf_column_ids[0]; + let column_id = column_node.leaf_column_ids[0]; Ok(Some(DeserializedArray::Deserialized(( column_id, array, @@ -269,40 +270,30 @@ impl BlockReader { } } - pub(crate) fn build_array_iter( + pub(crate) fn build_column_iter( &self, column_node: &ColumnNode, readers: Vec>>, - ) -> Result> { - let field = column_node.field.clone(); - - match self.native_columns_reader.column_iter_to_arrays( + ) -> Result> { + match self.native_columns_reader.column_iters( readers, - field, + column_node.table_field.clone(), column_node.init.clone(), ) { - Ok(array_iter) => Ok(array_iter), + Ok(column_iter) => Ok(column_iter), Err(err) => Err(err.into()), } } - pub(crate) fn build_virtual_array_iter( + pub(crate) fn build_virtual_column_iter( name: String, readers: Vec>>, - ) -> Result> { - let field = ArrowField::new( - name, - ArrowType::Extension( - "Variant".to_string(), - Box::new(ArrowType::LargeBinary), - None, - ), - true, - ); - let schema = ArrowSchema::from(vec![field.clone()]); - let native_column_reader = NativeColumnsReader::new(schema)?; - match native_column_reader.column_iter_to_arrays(readers, field, vec![]) { - Ok(array_iter) => Ok(array_iter), + ) -> Result> { + let field = TableField::new(&name, TableDataType::Variant.wrap_nullable()); + + let native_column_reader = NativeColumnsReader::new()?; + match native_column_reader.column_iters(readers, field, vec![]) { + Ok(iter) => Ok(iter), Err(err) => Err(err.into()), } } diff --git a/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs b/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs index 8f4495402f39..6eea7ae2517a 100644 --- a/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use arrow_array::RecordBatch; -use databend_common_expression::converts::arrow::table_schema_to_arrow_schema; +use arrow_schema::Schema; use databend_common_expression::ColumnId; use databend_common_expression::TableSchema; use databend_storages_common_table_meta::meta::Compression; @@ -35,7 +35,7 @@ pub fn column_chunks_to_record_batch( column_chunks: &HashMap, compression: &Compression, ) -> databend_common_exception::Result { - let arrow_schema = table_schema_to_arrow_schema(original_schema); + let arrow_schema = Schema::from(original_schema); let parquet_schema = arrow_to_parquet_schema(&arrow_schema)?; let column_id_to_dfs_id = original_schema .to_leaf_column_ids() diff --git a/src/query/storages/fuse/src/io/read/block/parquet/mod.rs b/src/query/storages/fuse/src/io/read/block/parquet/mod.rs index c4b212b4e0f4..2212bf9c5d56 100644 --- a/src/query/storages/fuse/src/io/read/block/parquet/mod.rs +++ b/src/query/storages/fuse/src/io/read/block/parquet/mod.rs @@ -95,18 +95,16 @@ impl BlockReader { Some(DataItem::RawData(data)) => { // get the deserialized arrow array, which may be a nested array let arrow_array = column_by_name(&record_batch, &name_paths[i]); - let arrow2_array: Box = - arrow_array.into(); if !column_node.is_nested { if let Some(cache) = &array_cache { let meta = column_metas.get(&field.column_id).unwrap(); let (offset, len) = meta.offset_length(); let key = TableDataCacheKey::new(block_path, field.column_id, offset, len); - cache.insert(key.into(), (arrow2_array.clone(), data.len())); + cache.insert(key.into(), (arrow_array.clone(), data.len())); } } - Value::Column(Column::from_arrow(arrow2_array.as_ref(), &data_type)?) + Value::Column(Column::from_arrow_rs(arrow_array, &data_type)?) } Some(DataItem::ColumnArray(cached)) => { if column_node.is_nested { @@ -115,7 +113,7 @@ impl BlockReader { "unexpected nested field: nested leaf field hits cached", )); } - Value::Column(Column::from_arrow(cached.0.as_ref(), &data_type)?) + Value::Column(Column::from_arrow_rs(cached.0.clone(), &data_type)?) } None => Value::Scalar(self.default_vals[i].clone()), }; diff --git a/src/query/storages/fuse/src/io/read/virtual_column/virtual_column_reader_parquet.rs b/src/query/storages/fuse/src/io/read/virtual_column/virtual_column_reader_parquet.rs index fd5eb9274639..742213e32027 100644 --- a/src/query/storages/fuse/src/io/read/virtual_column/virtual_column_reader_parquet.rs +++ b/src/query/storages/fuse/src/io/read/virtual_column/virtual_column_reader_parquet.rs @@ -153,10 +153,8 @@ impl VirtualColumnReader { .as_ref() .and_then(|r| r.column_by_name(&virtual_column_field.name).cloned()) { - let arrow2_array: Box = - arrow_array.into(); let data_type: DataType = virtual_column_field.data_type.as_ref().into(); - let value = Value::Column(Column::from_arrow(arrow2_array.as_ref(), &data_type)?); + let value = Value::Column(Column::from_arrow_rs(arrow_array, &data_type)?); data_block.add_column(BlockEntry::new(data_type, value)); continue; } diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index f803a4b58201..29276b1563ba 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -18,11 +18,10 @@ use std::sync::Arc; use std::time::Instant; use chrono::Utc; -use databend_common_arrow::arrow::chunk::Chunk as ArrowChunk; -use databend_common_arrow::native::write::NativeWriter; use databend_common_catalog::plan::Projection; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::DataField; @@ -41,6 +40,7 @@ use databend_common_metrics::storage::metrics_inc_block_inverted_index_write_mil use databend_common_metrics::storage::metrics_inc_block_inverted_index_write_nums; use databend_common_metrics::storage::metrics_inc_block_write_milliseconds; use databend_common_metrics::storage::metrics_inc_block_write_nums; +use databend_common_native::write::NativeWriter; use databend_storages_common_blocks::blocks_to_parquet; use databend_storages_common_index::BloomIndex; use databend_storages_common_io::ReadSettings; @@ -76,7 +76,6 @@ pub fn serialize_block( Ok(meta) } FuseStorageFormat::Native => { - let arrow_schema = schema.as_ref().into(); let leaf_column_ids = schema.to_leaf_column_ids(); let mut default_compress_ratio = Some(2.10f64); @@ -86,8 +85,8 @@ pub fn serialize_block( let mut writer = NativeWriter::new( buf, - arrow_schema, - databend_common_arrow::native::write::WriteOptions { + schema.as_ref().clone(), + databend_common_native::write::WriteOptions { default_compression: write_settings.table_compression.into(), max_page_size: Some(write_settings.max_page_size), default_compress_ratio, @@ -95,7 +94,12 @@ pub fn serialize_block( }, )?; - let batch = ArrowChunk::try_from(block)?; + let block = block.consume_convert_to_full(); + let batch: Vec = block + .columns() + .iter() + .map(|x| x.value.as_column().unwrap().clone()) + .collect(); writer.start()?; writer.write(&batch)?; diff --git a/src/query/storages/fuse/src/io/write/inverted_index_writer.rs b/src/query/storages/fuse/src/io/write/inverted_index_writer.rs index d0b288251ba0..daf6952e13e1 100644 --- a/src/query/storages/fuse/src/io/write/inverted_index_writer.rs +++ b/src/query/storages/fuse/src/io/write/inverted_index_writer.rs @@ -19,9 +19,9 @@ use std::sync::Arc; use arrow_ipc::writer::write_message; use arrow_ipc::writer::IpcDataGenerator; use arrow_ipc::writer::IpcWriteOptions; +use arrow_schema::Schema as ArrowSchema; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::converts::arrow::table_schema_to_arrow_schema; use databend_common_expression::types::BinaryType; use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; @@ -201,7 +201,7 @@ pub(crate) fn block_to_inverted_index( } // footer: schema + offsets + schema_len + meta_len - let arrow_schema = Arc::new(table_schema_to_arrow_schema(table_schema)); + let arrow_schema = Arc::new(ArrowSchema::from(table_schema)); let generator = IpcDataGenerator {}; let write_options = IpcWriteOptions::default(); #[allow(deprecated)] diff --git a/src/query/storages/fuse/src/operations/merge_into/mutator/matched_mutator.rs b/src/query/storages/fuse/src/operations/merge_into/mutator/matched_mutator.rs index d89504a9ebe8..9e1c939d3621 100644 --- a/src/query/storages/fuse/src/operations/merge_into/mutator/matched_mutator.rs +++ b/src/query/storages/fuse/src/operations/merge_into/mutator/matched_mutator.rs @@ -19,7 +19,6 @@ use std::sync::Arc; use std::time::Instant; use ahash::AHashMap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_base::base::tokio::sync::Semaphore; use databend_common_base::base::ProgressValues; use databend_common_base::runtime::GlobalIORuntime; @@ -34,6 +33,7 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberDataType; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; diff --git a/src/query/storages/fuse/src/operations/merge_into/mutator/merge_into_split_mutator.rs b/src/query/storages/fuse/src/operations/merge_into/mutator/merge_into_split_mutator.rs index ca570669e130..2a72a61972c5 100644 --- a/src/query/storages/fuse/src/operations/merge_into/mutator/merge_into_split_mutator.rs +++ b/src/query/storages/fuse/src/operations/merge_into/mutator/merge_into_split_mutator.rs @@ -14,9 +14,9 @@ use std::ops::Not; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::DataBlock; diff --git a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs index 21a27c8f495d..3e834f1c4159 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs @@ -200,7 +200,7 @@ impl Processor for MutationSource { 0 } } - Value::Column(bitmap) => bitmap.len() - bitmap.unset_bits(), + Value::Column(bitmap) => bitmap.len() - bitmap.null_count(), }; if affect_rows != 0 { diff --git a/src/query/storages/fuse/src/operations/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation_source.rs index 6a96f1681246..bd2ee76b0b19 100644 --- a/src/query/storages/fuse/src/operations/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation_source.rs @@ -88,7 +88,7 @@ impl FuseTable { Ok(match &predicates { Value::Scalar(v) => *v, - Value::Column(bitmap) => bitmap.unset_bits() == 0, + Value::Column(bitmap) => bitmap.null_count() == 0, }) } diff --git a/src/query/storages/fuse/src/operations/read/fuse_rows_fetcher.rs b/src/query/storages/fuse/src/operations/read/fuse_rows_fetcher.rs index a68f9d49d375..97b32725f528 100644 --- a/src/query/storages/fuse/src/operations/read/fuse_rows_fetcher.rs +++ b/src/query/storages/fuse/src/operations/read/fuse_rows_fetcher.rs @@ -14,13 +14,13 @@ use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Projection; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::BlockEntry; @@ -175,7 +175,7 @@ where F: RowsFetcher + Send + Sync + 'static } else { // From merge into matched data, the row id column is nullable but has no null value. let value = *value.into_nullable().unwrap(); - debug_assert!(value.validity.unset_bits() == 0); + debug_assert!(value.validity.null_count() == 0); value.column.into_number().unwrap().into_u_int64().unwrap() }; diff --git a/src/query/storages/fuse/src/operations/read/native_data_source_deserializer.rs b/src/query/storages/fuse/src/operations/read/native_data_source_deserializer.rs index db5387aa92ad..6b3c0945d0e4 100644 --- a/src/query/storages/fuse/src/operations/read/native_data_source_deserializer.rs +++ b/src/query/storages/fuse/src/operations/read/native_data_source_deserializer.rs @@ -19,9 +19,6 @@ use std::collections::VecDeque; use std::ops::BitAnd; use std::sync::Arc; -use databend_common_arrow::arrow::array::Array; -use databend_common_arrow::arrow::bitmap::MutableBitmap; -use databend_common_arrow::native::read::ArrayIter; use databend_common_base::base::Progress; use databend_common_base::base::ProgressValues; use databend_common_base::runtime::profile::Profile; @@ -37,6 +34,7 @@ use databend_common_expression::eval_function; use databend_common_expression::filter_helper::FilterHelpers; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::BlockEntry; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::Column; @@ -54,6 +52,7 @@ use databend_common_expression::TopKSorter; use databend_common_expression::Value; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_metrics::storage::*; +use databend_common_native::read::ColumnIter; use databend_common_pipeline_core::processors::Event; use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::OutputPort; @@ -77,10 +76,10 @@ use crate::DEFAULT_ROW_PER_PAGE; #[derive(Default)] struct ReadPartState { // Structures for reading a partition: - /// The [`ArrayIter`] of each columns to read native pages in order. - array_iters: BTreeMap>, - /// The number of pages need to be skipped for each iter in `array_iters`. - array_skip_pages: BTreeMap, + /// The [`columnIter`] of each columns to read native pages in order. + column_iters: BTreeMap>, + /// The number of pages need to be skipped for each iter in `column_iters`. + column_skip_pages: BTreeMap, /// `read_column_ids` is the columns that are in the block to read. /// /// The not read columns may have two cases: @@ -103,7 +102,7 @@ struct ReadPartState { /// It's used to mark the prefethed columns such as top-k and prewhere columns. read_columns: HashSet, /// Columns are already read into memory. - arrays: Vec<(usize, Box)>, + columns: Vec<(usize, Column)>, /// The number of rows that are filtered while reading current set of pages. /// It's used for the filter executor. filtered_count: Option, @@ -112,22 +111,22 @@ struct ReadPartState { impl ReadPartState { fn new() -> Self { Self { - array_iters: BTreeMap::new(), - array_skip_pages: BTreeMap::new(), + column_iters: BTreeMap::new(), + column_skip_pages: BTreeMap::new(), read_column_ids: HashSet::new(), if_need_fill_defaults: false, is_finished: true, // new state should be finished. offset: 0, read_columns: HashSet::new(), - arrays: Vec::new(), + columns: Vec::new(), filtered_count: None, } } /// Reset all the state. Mark the state as finished. fn finish(&mut self) { - self.array_iters.clear(); - self.array_skip_pages.clear(); + self.column_iters.clear(); + self.column_skip_pages.clear(); self.read_column_ids.clear(); self.if_need_fill_defaults = false; self.offset = 0; @@ -139,21 +138,21 @@ impl ReadPartState { /// Reset the state for reading a new set of pages (prepare to produce a new block). fn new_pages(&mut self) { self.read_columns.clear(); - self.arrays.clear(); + self.columns.clear(); self.filtered_count = None; } /// Skip one page for each unread column. fn skip_pages(&mut self) { - for (i, s) in self.array_skip_pages.iter_mut() { + for (i, s) in self.column_skip_pages.iter_mut() { if self.read_columns.contains(i) { continue; } *s += 1; } - if let Some((_, array)) = self.arrays.first() { + if let Some((_, column)) = self.columns.first() { // Advance the offset. - self.offset += array.len(); + self.offset += column.len(); } } @@ -166,14 +165,14 @@ impl ReadPartState { return Ok(true); } - if let Some(array_iter) = self.array_iters.get_mut(&index) { - let skipped_pages = self.array_skip_pages.get(&index).unwrap(); - match array_iter.nth(*skipped_pages) { - Some(array) => { + if let Some(column_iter) = self.column_iters.get_mut(&index) { + let skipped_pages = self.column_skip_pages.get(&index).unwrap(); + match column_iter.nth(*skipped_pages) { + Some(column) => { self.read_columns.insert(index); - self.arrays.push((index, array?)); + self.columns.push((index, column?)); // reset the skipped pages for next reading. - self.array_skip_pages.insert(index, 0); + self.column_skip_pages.insert(index, 0); } None => { self.finish(); @@ -197,7 +196,7 @@ pub struct NativeDeserializeDataTransform { output: Arc, output_data: Option, parts: VecDeque, - chunks: VecDeque, + columns: VecDeque, scan_progress: Arc, // Structures for table scan information: @@ -372,7 +371,7 @@ impl NativeDeserializeDataTransform { output, output_data: None, parts: VecDeque::new(), - chunks: VecDeque::new(), + columns: VecDeque::new(), prewhere_columns, prewhere_schema, remain_columns, @@ -425,7 +424,7 @@ impl NativeDeserializeDataTransform { /// otherwise extract it from the source column fn add_virtual_columns( &self, - chunks: &[(usize, Box)], + columns: &[(usize, Column)], schema: &DataSchema, virtual_column_fields: &Option>, block: &mut DataBlock, @@ -436,20 +435,17 @@ impl NativeDeserializeDataTransform { .src_schema .index_of(&virtual_column_field.name) .unwrap(); - if let Some(array) = chunks + if let Some(column) = columns .iter() .find(|c| c.0 == src_index) .map(|c| c.1.clone()) { let data_type: DataType = (*self.src_schema.field(src_index).data_type()).clone(); - let column = BlockEntry::new( - data_type.clone(), - Value::Column(Column::from_arrow(array.as_ref(), &data_type)?), - ); + let num_rows = column.len(); + let column = BlockEntry::new(data_type.clone(), Value::Column(column.clone())); // If the source column is the default value, num_rows may be zero if block.num_columns() > 0 && block.num_rows() == 0 { - let num_rows = array.len(); let mut columns = block.columns().to_vec(); columns.push(column); *block = DataBlock::new(columns, num_rows); @@ -489,7 +485,7 @@ impl NativeDeserializeDataTransform { fn check_default_values(&mut self) -> Result { if self.prewhere_columns.len() > 1 { if let Some((_, sorter, index)) = self.top_k.as_mut() { - if !self.read_state.array_iters.contains_key(index) { + if !self.read_state.column_iters.contains_key(index) { let default_val = self.block_reader.default_vals[*index].clone(); if sorter.never_match_value(&default_val) { return Ok(true); @@ -501,13 +497,13 @@ impl NativeDeserializeDataTransform { let all_defaults = &self .prewhere_columns .iter() - .all(|index| !self.read_state.array_iters.contains_key(index)); + .all(|index| !self.read_state.column_iters.contains_key(index)); let all_virtual_defaults = match &self.prewhere_virtual_column_fields { Some(ref prewhere_virtual_column_fields) => { prewhere_virtual_column_fields.iter().all(|c| { let src_index = self.src_schema.index_of(&c.source_name).unwrap(); - !self.read_state.array_iters.contains_key(&src_index) + !self.read_state.column_iters.contains_key(&src_index) }) } None => true, @@ -551,7 +547,7 @@ impl NativeDeserializeDataTransform { // Default value satisfies the filter, update the value of top-k column. if let Some((_, sorter, index)) = self.top_k.as_mut() { - if !self.read_state.array_iters.contains_key(index) { + if !self.read_state.column_iters.contains_key(index) { let part = FuseBlockPartInfo::from_part(&self.parts[0])?; let num_rows = part.nums_rows; @@ -571,7 +567,7 @@ impl NativeDeserializeDataTransform { /// Finish the processing of current partition. fn finish_partition(&mut self) { self.read_state.finish(); - self.chunks.pop_front(); + self.columns.pop_front(); self.parts.pop_front(); } @@ -607,10 +603,10 @@ impl NativeDeserializeDataTransform { /// Initialize the read state for a new partition. fn new_read_state(&mut self) -> Result<()> { debug_assert!(self.read_state.is_finished()); - debug_assert!(!self.chunks.is_empty()); + debug_assert!(!self.columns.is_empty()); debug_assert!(!self.parts.is_empty()); - if let NativeDataSource::Normal(chunks) = self.chunks.front_mut().unwrap() { + if let NativeDataSource::Normal(columns) = self.columns.front_mut().unwrap() { let part = self.parts.front().unwrap(); let part = FuseBlockPartInfo::from_part(part)?; @@ -619,11 +615,11 @@ impl NativeDeserializeDataTransform { } for (index, column_node) in self.block_reader.project_column_nodes.iter().enumerate() { - let readers = chunks.remove(&index).unwrap_or_default(); + let readers = columns.remove(&index).unwrap_or_default(); if !readers.is_empty() { - let array_iter = self.block_reader.build_array_iter(column_node, readers)?; - self.read_state.array_iters.insert(index, array_iter); - self.read_state.array_skip_pages.insert(index, 0); + let column_iter = self.block_reader.build_column_iter(column_node, readers)?; + self.read_state.column_iters.insert(index, column_iter); + self.read_state.column_skip_pages.insert(index, 0); for column_id in &column_node.leaf_column_ids { self.read_state.read_column_ids.insert(*column_id); @@ -633,7 +629,7 @@ impl NativeDeserializeDataTransform { } } - // Add optional virtual columns' array_iters. + // Add optional virtual columns' column_iters. if let Some(virtual_reader) = self.virtual_reader.as_ref() { for (index, virtual_column_info) in virtual_reader .virtual_column_info @@ -642,14 +638,14 @@ impl NativeDeserializeDataTransform { .enumerate() { let virtual_index = index + self.block_reader.project_column_nodes.len(); - if let Some(readers) = chunks.remove(&virtual_index) { - let array_iter = BlockReader::build_virtual_array_iter( + if let Some(readers) = columns.remove(&virtual_index) { + let column_iter = BlockReader::build_virtual_column_iter( virtual_column_info.name.clone(), readers, )?; let index = self.src_schema.index_of(&virtual_column_info.name)?; - self.read_state.array_iters.insert(index, array_iter); - self.read_state.array_skip_pages.insert(index, 0); + self.read_state.column_iters.insert(index, column_iter); + self.read_state.column_skip_pages.insert(index, 0); } } } @@ -718,7 +714,7 @@ impl NativeDeserializeDataTransform { } let mut block = self .block_reader - .build_block(&self.read_state.arrays, None)?; + .build_block(&self.read_state.columns, None)?; // 6. fill missing fields with default values. if self.read_state.if_need_fill_defaults { @@ -729,7 +725,7 @@ impl NativeDeserializeDataTransform { // 7. add optional virtual columns. self.add_virtual_columns( - &self.read_state.arrays, + &self.read_state.columns, &self.src_schema, &self.virtual_column_fields, &mut block, @@ -746,18 +742,16 @@ impl NativeDeserializeDataTransform { /// /// Returns false if skip the current page or the partition is finished. fn read_and_check_topk(&mut self) -> Result { - if let Some((top_k, sorter, index)) = self.top_k.as_mut() { + if let Some((_top_k, sorter, index)) = self.top_k.as_mut() { if !self.read_state.read_page(*index)? { debug_assert!(self.read_state.is_finished()); return Ok(false); } // TopK should always be the first read column. - debug_assert_eq!(self.read_state.arrays.len(), 1); - let (i, array) = self.read_state.arrays.last().unwrap(); + debug_assert_eq!(self.read_state.columns.len(), 1); + let (i, column) = self.read_state.columns.last().unwrap(); debug_assert_eq!(i, index); - let data_type = top_k.field.data_type().into(); - let col = Column::from_arrow(array.as_ref(), &data_type)?; - if sorter.never_match_any(&col) { + if sorter.never_match_any(column) { // skip current page. return Ok(false); } @@ -776,34 +770,35 @@ impl NativeDeserializeDataTransform { continue; } - let num_columns = self.read_state.arrays.len(); + let num_columns = self.read_state.columns.len(); if !self.read_state.read_page(*index)? { debug_assert!(self.read_state.is_finished()); return Ok(false); } - if num_columns == self.read_state.arrays.len() { + if num_columns == self.read_state.columns.len() { // It means the column is not read and it's a default value. prewhere_default_val_indices.insert(*index); } } // Evaluate the filter. - // If `self.read_state.arrays.is_empty()`, + // If `self.read_state.columns.is_empty()`, // it means there are only default columns in prewhere columns. (all prewhere columns are newly added by `alter table`) // In this case, we don't need to evaluate the filter, because the unsatisfied blocks are already filtered in `read_partitions`. - if self.prewhere_filter.is_some() && !self.read_state.arrays.is_empty() { + if self.prewhere_filter.is_some() && !self.read_state.columns.is_empty() { debug_assert!(self.filter_executor.is_some()); - let mut prewhere_block = if self.read_state.arrays.len() < self.prewhere_columns.len() { + let mut prewhere_block = if self.read_state.columns.len() < self.prewhere_columns.len() + { self.block_reader - .build_block(&self.read_state.arrays, Some(prewhere_default_val_indices))? + .build_block(&self.read_state.columns, Some(prewhere_default_val_indices))? } else { self.block_reader - .build_block(&self.read_state.arrays, None)? + .build_block(&self.read_state.columns, None)? }; // Add optional virtual columns for prewhere self.add_virtual_columns( - &self.read_state.arrays, + &self.read_state.columns, &self.prewhere_schema, &self.prewhere_virtual_column_fields, &mut prewhere_block, @@ -832,25 +827,25 @@ impl NativeDeserializeDataTransform { if let Some(bloom_runtime_filter) = self.bloom_runtime_filter.as_ref() { let mut bitmaps = Vec::with_capacity(bloom_runtime_filter.len()); for (idx, filter) in bloom_runtime_filter.iter() { - let array = if let Some((_, array)) = - self.read_state.arrays.iter().find(|(i, _)| i == idx) + let column = if let Some((_, column)) = + self.read_state.columns.iter().find(|(i, _)| i == idx) { - (*idx, array.clone()) + (*idx, column.clone()) } else if !self.read_state.read_page(*idx)? { debug_assert!(self.read_state.is_finished()); return Ok(false); } else { // The runtime filter column must be the last column to read. - let (i, array) = self.read_state.arrays.last().unwrap(); + let (i, column) = self.read_state.columns.last().unwrap(); debug_assert_eq!(i, idx); - (*idx, array.clone()) + (*idx, column.clone()) }; - let probe_block = self.block_reader.build_block(&[array], None)?; + let probe_block = self.block_reader.build_block(&[column], None)?; let mut bitmap = MutableBitmap::from_len_zeroed(probe_block.num_rows()); let probe_column = probe_block.get_last_column().clone(); update_bitmap_with_bloom_filter(probe_column, filter, &mut bitmap)?; - let unset_bits = bitmap.unset_bits(); + let unset_bits = bitmap.null_count(); if unset_bits == bitmap.len() { // skip current page. return Ok(false); @@ -882,24 +877,21 @@ impl NativeDeserializeDataTransform { /// /// Returns false if skip the current page. fn update_topk_heap(&mut self) -> Result { - if let Some((top_k, sorter, index)) = &mut self.top_k { + if let Some((_top_k, sorter, index)) = &mut self.top_k { // Topk column should always be the first column read. - let (i, array) = self.read_state.arrays.first().unwrap(); + let (i, col) = self.read_state.columns.first().unwrap(); debug_assert_eq!(i, index); - let data_type = top_k.field.data_type().into(); - let col = Column::from_arrow(array.as_ref(), &data_type)?; - let filter_executor = self.filter_executor.as_mut().unwrap(); let count = if let Some(count) = self.read_state.filtered_count { sorter.push_column_with_selection::( - &col, + col, filter_executor.mutable_true_selection(), count, ) } else { // If there is no prewhere filter, initialize the true selection. sorter.push_column_with_selection::( - &col, + col, filter_executor.mutable_true_selection(), col.len(), ) @@ -953,7 +945,7 @@ impl NativeDeserializeDataTransform { /// Pre-process the partition before reading it. fn pre_process_partition(&mut self) -> Result<()> { - debug_assert!(!self.chunks.is_empty()); + debug_assert!(!self.columns.is_empty()); debug_assert!(!self.parts.is_empty()); // Create a new read state. @@ -966,7 +958,7 @@ impl NativeDeserializeDataTransform { return Ok(()); } - if self.read_state.array_iters.is_empty() { + if self.read_state.column_iters.is_empty() { // All columns are default values, not need to read. let part = self.parts.front().unwrap(); let fuse_part = FuseBlockPartInfo::from_part(part)?; @@ -1050,7 +1042,7 @@ impl Processor for NativeDeserializeDataTransform { return Ok(Event::NeedConsume); } - if !self.chunks.is_empty() { + if !self.columns.is_empty() { if !self.input.has_data() { self.input.set_need_data(); } @@ -1062,7 +1054,7 @@ impl Processor for NativeDeserializeDataTransform { if let Some(block_meta) = data_block.take_meta() { if let Some(source_meta) = DataSourceWithMeta::downcast_from(block_meta) { self.parts = VecDeque::from(source_meta.meta); - self.chunks = VecDeque::from(source_meta.data); + self.columns = VecDeque::from(source_meta.data); return Ok(Event::Sync); } } @@ -1086,8 +1078,8 @@ impl Processor for NativeDeserializeDataTransform { // Only if current read state is finished can we start to read a new partition. if self.read_state.is_finished() { - if let Some(chunks) = self.chunks.front_mut() { - let chunks = match chunks { + if let Some(columns) = self.columns.front_mut() { + let columns = match columns { NativeDataSource::AggIndex(data) => { let agg_index_reader = self.index_reader.as_ref().as_ref().unwrap(); let block = agg_index_reader.deserialize_native_data(data)?; @@ -1098,7 +1090,7 @@ impl Processor for NativeDeserializeDataTransform { NativeDataSource::Normal(data) => data, }; - if chunks.is_empty() { + if columns.is_empty() { // This means it's an empty projection let part = self.parts.front().unwrap(); let fuse_part = FuseBlockPartInfo::from_part(part)?; diff --git a/src/query/storages/fuse/src/operations/read/native_rows_fetcher.rs b/src/query/storages/fuse/src/operations/read/native_rows_fetcher.rs index 791c6544ceea..b7e1b16a0e3b 100644 --- a/src/query/storages/fuse/src/operations/read/native_rows_fetcher.rs +++ b/src/query/storages/fuse/src/operations/read/native_rows_fetcher.rs @@ -238,7 +238,7 @@ impl NativeRowsFetcher { for (index, column_node) in reader.project_column_nodes.iter().enumerate() { let readers = chunks.remove(&index).unwrap(); if !readers.is_empty() { - let array_iter = reader.build_array_iter(column_node, readers)?; + let array_iter = reader.build_column_iter(column_node, readers)?; array_iters.insert(index, array_iter); } } @@ -253,13 +253,13 @@ impl NativeRowsFetcher { // discarded, and also that calling `nth(0)` multiple times on the same iterator // will return different elements. let pos = *page - offset; - let mut arrays = Vec::with_capacity(array_iters.len()); + let mut columns = Vec::with_capacity(array_iters.len()); for (index, array_iter) in array_iters.iter_mut() { let array = array_iter.nth(pos as usize).unwrap()?; - arrays.push((*index, array)); + columns.push((*index, array)); } offset = *page + 1; - let block = reader.build_block(&arrays, None)?; + let block = reader.build_block(&columns, None)?; blocks.push(block); } diff --git a/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs b/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs index d58dffde4a7a..b08b81e86c99 100644 --- a/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs +++ b/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs @@ -17,8 +17,6 @@ use std::ops::BitAnd; use std::sync::Arc; use std::time::Instant; -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_base::base::Progress; use databend_common_base::base::ProgressValues; use databend_common_base::runtime::profile::Profile; @@ -29,7 +27,9 @@ use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_expression::DataField; diff --git a/src/query/storages/fuse/src/operations/read/runtime_filter_prunner.rs b/src/query/storages/fuse/src/operations/read/runtime_filter_prunner.rs index 066a418b626d..ba0253ea4754 100644 --- a/src/query/storages/fuse/src/operations/read/runtime_filter_prunner.rs +++ b/src/query/storages/fuse/src/operations/read/runtime_filter_prunner.rs @@ -15,11 +15,11 @@ use std::collections::HashMap; use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::plan::PartInfoPtr; use databend_common_exception::Result; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberColumn; use databend_common_expression::Column; use databend_common_expression::ConstantFolder; diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/merge_into_mutator.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/merge_into_mutator.rs index 17e6b15218a7..640f0853078a 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/merge_into_mutator.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/merge_into_mutator.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use std::time::Instant; use ahash::AHashMap; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_base::base::tokio::sync::Semaphore; use databend_common_base::base::ProgressValues; use databend_common_base::runtime::GlobalIORuntime; @@ -29,6 +28,7 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockEntry; @@ -451,7 +451,7 @@ impl AggregationContext { } } - let delete_nums = bitmap.unset_bits(); + let delete_nums = bitmap.null_count(); info!("number of row deleted: {}", delete_nums); // shortcut: nothing to be deleted diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/mutator_replace_into.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/mutator_replace_into.rs index 1196fc5d63f5..b50ee261bd1d 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/mutator_replace_into.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/mutator_replace_into.rs @@ -19,12 +19,12 @@ use std::iter::once; use ahash::HashSet; use ahash::HashSetExt; -use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::AnyType; use databend_common_expression::types::DataType; +use databend_common_expression::types::MutableBitmap; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; @@ -435,7 +435,7 @@ impl Partitioner { let row_bloom_hashes: Vec> = column_bloom_hashes .iter() .filter_map(|(hashes, validity)| match validity { - Some(v) if v.unset_bits() != 0 => v + Some(v) if v.null_count() != 0 => v .get(row_idx) .map(|v| if v { hashes.get(row_idx) } else { None }), _ => Some(hashes.get(row_idx)), diff --git a/src/query/storages/fuse/src/statistics/column_statistic.rs b/src/query/storages/fuse/src/statistics/column_statistic.rs index 0c171dedd507..ff2374129b36 100644 --- a/src/query/storages/fuse/src/statistics/column_statistic.rs +++ b/src/query/storages/fuse/src/statistics/column_statistic.rs @@ -140,7 +140,7 @@ pub fn gen_columns_statistics( let (is_all_null, bitmap) = col.validity(); let unset_bits = match (is_all_null, bitmap) { (true, _) => rows, - (false, Some(bitmap)) => bitmap.unset_bits(), + (false, Some(bitmap)) => bitmap.null_count(), (false, None) => 0, }; diff --git a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs index f6ab657277c2..f2879c81637b 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs @@ -14,11 +14,6 @@ use std::sync::Arc; -use databend_common_arrow::arrow::datatypes::Field; -use databend_common_arrow::native::read::reader::NativeReader; -use databend_common_arrow::native::stat::stat_simple; -use databend_common_arrow::native::stat::ColumnInfo; -use databend_common_arrow::native::stat::PageBody; use databend_common_catalog::catalog_kind::CATALOG_DEFAULT; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Filters; @@ -49,6 +44,10 @@ use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; use databend_common_expression::Value; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_native::read::reader::NativeReader; +use databend_common_native::stat::stat_simple; +use databend_common_native::stat::ColumnInfo; +use databend_common_native::stat::PageBody; use databend_storages_common_io::MergeIOReader; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::SegmentInfo; @@ -175,7 +174,6 @@ impl<'a> FuseEncodingImpl<'a> { continue; } let column_id = field.column_id; - let arrow_field: Field = field.into(); let column_meta = block.col_metas.get(&column_id).unwrap(); let (offset, len) = column_meta.offset_length(); let ranges = vec![(column_id, offset..(offset + len))]; @@ -200,7 +198,7 @@ impl<'a> FuseEncodingImpl<'a> { let pages = std::io::Cursor::new(pages); let page_metas = column_meta.as_native().unwrap().pages.clone(); let reader = NativeReader::new(pages, page_metas, vec![]); - let this_column_info = stat_simple(reader, arrow_field.clone())?; + let this_column_info = stat_simple(reader, field.clone())?; columns_info.push((field.data_type.sql_name(), this_column_info)); } } diff --git a/src/query/storages/parquet/Cargo.toml b/src/query/storages/parquet/Cargo.toml index 6cd32a9da075..f93a0483dd6c 100644 --- a/src/query/storages/parquet/Cargo.toml +++ b/src/query/storages/parquet/Cargo.toml @@ -18,7 +18,7 @@ async-backtrace = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } -databend-common-arrow = { workspace = true } + databend-common-base = { workspace = true } databend-common-catalog = { workspace = true } databend-common-exception = { workspace = true } diff --git a/src/query/storages/parquet/src/parquet_rs/parquet_reader/predicate.rs b/src/query/storages/parquet/src/parquet_rs/parquet_reader/predicate.rs index e5203ef3f28c..2b60725769ba 100644 --- a/src/query/storages/parquet/src/parquet_rs/parquet_reader/predicate.rs +++ b/src/query/storages/parquet/src/parquet_rs/parquet_reader/predicate.rs @@ -16,10 +16,10 @@ use std::sync::Arc; use arrow_array::BooleanArray; use arrow_array::RecordBatch; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_catalog::plan::PrewhereInfo; use databend_common_catalog::plan::Projection; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; diff --git a/src/query/storages/parquet/src/parquet_rs/parquet_reader/read_policy/predicate_and_topk.rs b/src/query/storages/parquet/src/parquet_rs/parquet_reader/read_policy/predicate_and_topk.rs index c459159a898a..012e149e8e96 100644 --- a/src/query/storages/parquet/src/parquet_rs/parquet_reader/read_policy/predicate_and_topk.rs +++ b/src/query/storages/parquet/src/parquet_rs/parquet_reader/read_policy/predicate_and_topk.rs @@ -204,7 +204,7 @@ impl ReadPolicyBuilder for PredicateAndTopkPolicyBuilder { num_rows, )?; let filter = self.predicate.evaluate_block(&block)?; - if filter.unset_bits() == num_rows { + if filter.null_count() == num_rows { // All rows in current row group are filtered out. return Ok(None); } diff --git a/src/query/storages/parquet/src/parquet_rs/parquet_reader/read_policy/utils.rs b/src/query/storages/parquet/src/parquet_rs/parquet_reader/read_policy/utils.rs index 3ca6fa891370..7615afd36edd 100644 --- a/src/query/storages/parquet/src/parquet_rs/parquet_reader/read_policy/utils.rs +++ b/src/query/storages/parquet/src/parquet_rs/parquet_reader/read_policy/utils.rs @@ -53,7 +53,7 @@ pub fn evaluate_topk( let topk_col = block.columns()[0].value.as_column().unwrap(); let num_rows = topk_col.len(); let filter = topk.evaluate_column(topk_col, sorter); - if filter.unset_bits() == num_rows { + if filter.null_count() == num_rows { // All rows are filtered out. return Ok(None); } diff --git a/src/query/storages/parquet/src/parquet_rs/parquet_reader/topk.rs b/src/query/storages/parquet/src/parquet_rs/parquet_reader/topk.rs index 4fa1620c0411..d53b9a72a375 100644 --- a/src/query/storages/parquet/src/parquet_rs/parquet_reader/topk.rs +++ b/src/query/storages/parquet/src/parquet_rs/parquet_reader/topk.rs @@ -14,9 +14,9 @@ use std::sync::Arc; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_catalog::plan::TopK; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::Column; use databend_common_expression::TableField; use databend_common_expression::TableSchema; diff --git a/src/query/storages/parquet/src/parquet_rs/parquet_reader/utils.rs b/src/query/storages/parquet/src/parquet_rs/parquet_reader/utils.rs index 96aff368bee9..4bae1ff6dde3 100644 --- a/src/query/storages/parquet/src/parquet_rs/parquet_reader/utils.rs +++ b/src/query/storages/parquet/src/parquet_rs/parquet_reader/utils.rs @@ -15,10 +15,9 @@ use arrow_array::BooleanArray; use arrow_array::RecordBatch; use arrow_array::StructArray; -use databend_common_arrow::arrow::array::Arrow2Arrow; -use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::types::Bitmap; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataField; @@ -95,15 +94,7 @@ pub fn transform_record_batch_by_field_paths( } pub fn bitmap_to_boolean_array(bitmap: Bitmap) -> BooleanArray { - let res = Box::new( - databend_common_arrow::arrow::array::BooleanArray::try_new( - databend_common_arrow::arrow::datatypes::DataType::Boolean, - bitmap, - None, - ) - .unwrap(), - ); - BooleanArray::from(res.to_data()) + BooleanArray::from(bitmap.into_array_data()) } /// FieldPaths is used to traverse nested columns in [`RecordBatch`]. diff --git a/src/query/storages/stage/src/append/parquet_file/writer_processor.rs b/src/query/storages/stage/src/append/parquet_file/writer_processor.rs index e1d7a3669e16..ff8aad1d6fdd 100644 --- a/src/query/storages/stage/src/append/parquet_file/writer_processor.rs +++ b/src/query/storages/stage/src/append/parquet_file/writer_processor.rs @@ -17,12 +17,11 @@ use std::collections::VecDeque; use std::mem; use std::sync::Arc; -use arrow_schema::Schema as ArrowSchema; +use arrow_schema::Schema; use async_trait::async_trait; use databend_common_catalog::plan::StageTableInfo; use databend_common_config::DATABEND_SEMVER; use databend_common_exception::Result; -use databend_common_expression::converts::arrow::table_schema_to_arrow_schema; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_pipeline_core::processors::Event; @@ -47,7 +46,7 @@ pub struct ParquetFileWriter { output: Arc, table_info: StageTableInfo, - arrow_schema: Arc, + arrow_schema: Arc, input_data: Vec, @@ -74,7 +73,7 @@ const MAX_BUFFER_SIZE: usize = 64 * 1024 * 1024; const MAX_ROW_GROUP_SIZE: usize = 1024 * 1024; fn create_writer( - arrow_schema: Arc, + arrow_schema: Arc, targe_file_size: Option, ) -> Result>> { let props = WriterProperties::builder() @@ -107,7 +106,7 @@ impl ParquetFileWriter { let unload_output = UnloadOutput::create(table_info.copy_into_location_options.detailed_output); - let arrow_schema = Arc::new(table_schema_to_arrow_schema(&table_info.schema)); + let arrow_schema = Arc::new(Schema::from(table_info.schema.as_ref())); let writer = create_writer(arrow_schema.clone(), targe_file_size)?; Ok(ProcessorPtr::create(Box::new(ParquetFileWriter { diff --git a/tests/sqllogictests/suites/mode/cluster/memo/aggregate_property.test b/tests/sqllogictests/suites/mode/cluster/memo/aggregate_property.test index 32405dc66997..205b7ebb4bf0 100644 --- a/tests/sqllogictests/suites/mode/cluster/memo/aggregate_property.test +++ b/tests/sqllogictests/suites/mode/cluster/memo/aggregate_property.test @@ -26,7 +26,7 @@ where t_10.a = t_1000.a and t_100.a = t_1000.a ---- Memo ├── root group: #8 -├── estimated memory: 14.91 KiB +├── estimated memory: 12.09 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -89,7 +89,7 @@ group by t_10.a, t_100.a ---- Memo ├── root group: #8 -├── estimated memory: 37.27 KiB +├── estimated memory: 30.23 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] diff --git a/tests/sqllogictests/suites/mode/cluster/memo/join_property.test b/tests/sqllogictests/suites/mode/cluster/memo/join_property.test index c6d9effd4cd4..2bab888b4a88 100644 --- a/tests/sqllogictests/suites/mode/cluster/memo/join_property.test +++ b/tests/sqllogictests/suites/mode/cluster/memo/join_property.test @@ -25,7 +25,7 @@ select * from t_10, t_100, t_1000 where t_10.a = t_1000.a and t_100.a = t_1000.a ---- Memo ├── root group: #5 -├── estimated memory: 11.59 KiB +├── estimated memory: 9.41 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -73,7 +73,7 @@ select * from t_1000 left join t_10 on t_1000.a = t_10.a left join t_100 on t_10 ---- Memo ├── root group: #5 -├── estimated memory: 10.77 KiB +├── estimated memory: 8.73 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -119,7 +119,7 @@ select * from t_1000 right join t_10 on t_1000.a = t_10.a right join t_100 on t_ ---- Memo ├── root group: #5 -├── estimated memory: 9.11 KiB +├── estimated memory: 7.39 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -161,7 +161,7 @@ select * from t_1000 full join t_10 on t_1000.a = t_10.a full join t_100 on t_10 ---- Memo ├── root group: #5 -├── estimated memory: 9.11 KiB +├── estimated memory: 7.39 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] @@ -203,7 +203,7 @@ select * from t_10, t_100, t_1000 ---- Memo ├── root group: #5 -├── estimated memory: 7.45 KiB +├── estimated memory: 6.05 KiB ├── Group #0 │ ├── Best properties │ │ └── { dist: Any }: expr: #0, cost: 10.000, children: [] diff --git a/tests/sqllogictests/suites/mode/cluster/memo/mix_property.test b/tests/sqllogictests/suites/mode/cluster/memo/mix_property.test index cb8c9ad7328f..5643c1172676 100644 --- a/tests/sqllogictests/suites/mode/cluster/memo/mix_property.test +++ b/tests/sqllogictests/suites/mode/cluster/memo/mix_property.test @@ -29,7 +29,7 @@ limit 10 ---- Memo ├── root group: #10 -├── estimated memory: 38.92 KiB +├── estimated memory: 31.58 KiB ├── Group #0 │ ├── Best properties │ │ ├── { dist: Any }: expr: #0, cost: 1000.000, children: [] From 4af19cb807c5f1d54634dbcdc360eab540801752 Mon Sep 17 00:00:00 2001 From: TCeason <33082201+TCeason@users.noreply.github.com> Date: Tue, 19 Nov 2024 23:53:18 +0800 Subject: [PATCH 59/92] refactor: optimize query system.tables when query single table (#16869) optimize: optimize query system.tables when query single table --- Cargo.lock | 1 + src/meta/app/src/principal/mod.rs | 1 + src/meta/app/src/principal/user_grant.rs | 27 ++ src/query/catalog/src/table_context.rs | 5 +- .../interpreters/access/privilege_access.rs | 28 +- src/query/service/src/sessions/query_ctx.rs | 10 +- src/query/service/src/sessions/session.rs | 9 +- .../src/sessions/session_privilege_mgr.rs | 41 +- .../table_functions/infer_schema/parquet.rs | 2 +- .../inspect_parquet/inspect_parquet_table.rs | 2 +- .../list_stage/list_stage_table.rs | 2 +- .../show_grants/show_grants_table.rs | 2 +- .../tests/it/sql/exec/get_table_bind_test.rs | 5 +- .../it/storages/fuse/operations/commit.rs | 5 +- src/query/storages/system/Cargo.toml | 1 + .../storages/system/src/columns_table.rs | 2 +- .../storages/system/src/databases_table.rs | 2 +- .../storages/system/src/indexes_table.rs | 2 +- src/query/storages/system/src/stages_table.rs | 2 +- .../storages/system/src/streams_table.rs | 2 +- src/query/storages/system/src/tables_table.rs | 395 +++++++++++------- .../system/src/user_functions_table.rs | 2 +- 22 files changed, 327 insertions(+), 221 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1deb8a980af..56128851893b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4568,6 +4568,7 @@ dependencies = [ "databend-common-exception", "databend-common-expression", "databend-common-functions", + "databend-common-management", "databend-common-meta-api", "databend-common-meta-app", "databend-common-meta-types", diff --git a/src/meta/app/src/principal/mod.rs b/src/meta/app/src/principal/mod.rs index 80fc40dcf4ca..568253041164 100644 --- a/src/meta/app/src/principal/mod.rs +++ b/src/meta/app/src/principal/mod.rs @@ -102,6 +102,7 @@ pub use user_defined_function::UserDefinedFunction; pub use user_grant::GrantEntry; pub use user_grant::GrantObject; pub use user_grant::UserGrantSet; +pub use user_grant::SYSTEM_TABLES_ALLOW_LIST; pub use user_identity::UserIdentity; pub use user_info::UserInfo; pub use user_info::UserOption; diff --git a/src/meta/app/src/principal/user_grant.rs b/src/meta/app/src/principal/user_grant.rs index 7ba3df94a084..e08d86495473 100644 --- a/src/meta/app/src/principal/user_grant.rs +++ b/src/meta/app/src/principal/user_grant.rs @@ -21,6 +21,33 @@ use enumflags2::BitFlags; use crate::principal::UserPrivilegeSet; use crate::principal::UserPrivilegeType; +// some statements like `SELECT 1`, `SHOW USERS`, `SHOW ROLES`, `SHOW TABLES` will be +// rewritten to the queries on the system tables, we need to skip the privilege check on +// these tables. +pub const SYSTEM_TABLES_ALLOW_LIST: [&str; 21] = [ + "catalogs", + "columns", + "databases", + "databases_with_history", + "dictionaries", + "tables", + "views", + "tables_with_history", + "views_with_history", + "password_policies", + "streams", + "streams_terse", + "virtual_columns", + "users", + "roles", + "stages", + "one", + "processes", + "user_functions", + "functions", + "indexes", +]; + #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq, Hash)] pub enum GrantObject { Global, diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index 9ff28e69347e..5b3fec728981 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -224,7 +224,10 @@ pub trait TableContext: Send + Sync { check_current_role_only: bool, ) -> Result<()>; async fn get_available_roles(&self) -> Result>; - async fn get_visibility_checker(&self) -> Result; + async fn get_visibility_checker( + &self, + ignore_ownership: bool, + ) -> Result; fn get_fuse_version(&self) -> String; fn get_format_settings(&self) -> Result; fn get_tenant(&self) -> Tenant; diff --git a/src/query/service/src/interpreters/access/privilege_access.rs b/src/query/service/src/interpreters/access/privilege_access.rs index 95d538266d3e..3281a19ef742 100644 --- a/src/query/service/src/interpreters/access/privilege_access.rs +++ b/src/query/service/src/interpreters/access/privilege_access.rs @@ -29,6 +29,7 @@ use databend_common_meta_app::principal::StageType; use databend_common_meta_app::principal::UserGrantSet; use databend_common_meta_app::principal::UserPrivilegeSet; use databend_common_meta_app::principal::UserPrivilegeType; +use databend_common_meta_app::principal::SYSTEM_TABLES_ALLOW_LIST; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::seq_value::SeqV; use databend_common_sql::binder::MutationType; @@ -58,33 +59,6 @@ enum ObjectId { Table(u64, u64), } -// some statements like `SELECT 1`, `SHOW USERS`, `SHOW ROLES`, `SHOW TABLES` will be -// rewritten to the queries on the system tables, we need to skip the privilege check on -// these tables. -const SYSTEM_TABLES_ALLOW_LIST: [&str; 21] = [ - "catalogs", - "columns", - "databases", - "databases_with_history", - "dictionaries", - "tables", - "views", - "tables_with_history", - "views_with_history", - "password_policies", - "streams", - "streams_terse", - "virtual_columns", - "users", - "roles", - "stages", - "one", - "processes", - "user_functions", - "functions", - "indexes", -]; - // table functions that need `Super` privilege const SYSTEM_TABLE_FUNCTIONS: [&str; 1] = ["fuse_amend"]; diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index c5822b6da492..a8dd5e987cf5 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -692,8 +692,14 @@ impl TableContext for QueryContext { self.get_current_session().get_id() } - async fn get_visibility_checker(&self) -> Result { - self.shared.session.get_visibility_checker().await + async fn get_visibility_checker( + &self, + ignore_ownership: bool, + ) -> Result { + self.shared + .session + .get_visibility_checker(ignore_ownership) + .await } fn get_fuse_version(&self) -> String { diff --git a/src/query/service/src/sessions/session.rs b/src/query/service/src/sessions/session.rs index 7f1bedc15782..878224401f24 100644 --- a/src/query/service/src/sessions/session.rs +++ b/src/query/service/src/sessions/session.rs @@ -310,8 +310,13 @@ impl Session { } #[async_backtrace::framed] - pub async fn get_visibility_checker(&self) -> Result { - self.privilege_mgr().get_visibility_checker().await + pub async fn get_visibility_checker( + &self, + ignore_ownership: bool, + ) -> Result { + self.privilege_mgr() + .get_visibility_checker(ignore_ownership) + .await } pub fn get_settings(&self) -> Arc { diff --git a/src/query/service/src/sessions/session_privilege_mgr.rs b/src/query/service/src/sessions/session_privilege_mgr.rs index 8d772eb1c867..914629cc3cd2 100644 --- a/src/query/service/src/sessions/session_privilege_mgr.rs +++ b/src/query/service/src/sessions/session_privilege_mgr.rs @@ -75,7 +75,10 @@ pub trait SessionPrivilegeManager { async fn validate_available_role(&self, role_name: &str) -> Result; - async fn get_visibility_checker(&self) -> Result; + async fn get_visibility_checker( + &self, + ignore_ownership: bool, + ) -> Result; // fn show_grants(&self); } @@ -336,27 +339,31 @@ impl<'a> SessionPrivilegeManager for SessionPrivilegeManagerImpl<'a> { } #[async_backtrace::framed] - async fn get_visibility_checker(&self) -> Result { + async fn get_visibility_checker( + &self, + ignore_ownership: bool, + ) -> Result { // TODO(liyz): is it check the visibility according onwerships? - let user_api = UserApiProvider::instance(); - let ownerships = user_api - .role_api(&self.session_ctx.get_current_tenant()) - .get_ownerships() - .await?; let roles = self.get_all_effective_roles().await?; let roles_name: Vec = roles.iter().map(|role| role.name.to_string()).collect(); - let ownership_objects = if roles_name.contains(&"account_admin".to_string()) { - vec![] - } else { - let mut ownership_objects = vec![]; - for ownership in ownerships { - if roles_name.contains(&ownership.data.role) { - ownership_objects.push(ownership.data.object); + let ownership_objects = + if roles_name.contains(&"account_admin".to_string()) || ignore_ownership { + vec![] + } else { + let user_api = UserApiProvider::instance(); + let ownerships = user_api + .role_api(&self.session_ctx.get_current_tenant()) + .get_ownerships() + .await?; + let mut ownership_objects = vec![]; + for ownership in ownerships { + if roles_name.contains(&ownership.data.role) { + ownership_objects.push(ownership.data.object); + } } - } - ownership_objects - }; + ownership_objects + }; Ok(GrantObjectVisibilityChecker::new( &self.get_current_user()?, diff --git a/src/query/service/src/table_functions/infer_schema/parquet.rs b/src/query/service/src/table_functions/infer_schema/parquet.rs index de814c75c395..5bcbd8eaf8c1 100644 --- a/src/query/service/src/table_functions/infer_schema/parquet.rs +++ b/src/query/service/src/table_functions/infer_schema/parquet.rs @@ -99,7 +99,7 @@ impl AsyncSource for ParquetInferSchemaSource { .get_settings() .get_enable_experimental_rbac_check()?; if enable_experimental_rbac_check { - let visibility_checker = self.ctx.get_visibility_checker().await?; + let visibility_checker = self.ctx.get_visibility_checker(false).await?; if !(stage_info.is_temporary || visibility_checker.check_stage_read_visibility(&stage_info.stage_name) || stage_info.stage_type == StageType::User diff --git a/src/query/service/src/table_functions/inspect_parquet/inspect_parquet_table.rs b/src/query/service/src/table_functions/inspect_parquet/inspect_parquet_table.rs index a0e5e13f2baa..e483df46c8a7 100644 --- a/src/query/service/src/table_functions/inspect_parquet/inspect_parquet_table.rs +++ b/src/query/service/src/table_functions/inspect_parquet/inspect_parquet_table.rs @@ -213,7 +213,7 @@ impl AsyncSource for InspectParquetSource { .get_settings() .get_enable_experimental_rbac_check()?; if enable_experimental_rbac_check { - let visibility_checker = self.ctx.get_visibility_checker().await?; + let visibility_checker = self.ctx.get_visibility_checker(false).await?; if !(stage_info.is_temporary || visibility_checker.check_stage_read_visibility(&stage_info.stage_name) || stage_info.stage_type == StageType::User diff --git a/src/query/service/src/table_functions/list_stage/list_stage_table.rs b/src/query/service/src/table_functions/list_stage/list_stage_table.rs index 89c3e8e45a0f..d8f51289553e 100644 --- a/src/query/service/src/table_functions/list_stage/list_stage_table.rs +++ b/src/query/service/src/table_functions/list_stage/list_stage_table.rs @@ -187,7 +187,7 @@ impl ListStagesSource { .get_settings() .get_enable_experimental_rbac_check()?; if enable_experimental_rbac_check { - let visibility_checker = self.ctx.get_visibility_checker().await?; + let visibility_checker = self.ctx.get_visibility_checker(false).await?; if !(stage_info.is_temporary || visibility_checker.check_stage_read_visibility(&stage_info.stage_name) || stage_info.stage_type == StageType::User diff --git a/src/query/service/src/table_functions/show_grants/show_grants_table.rs b/src/query/service/src/table_functions/show_grants/show_grants_table.rs index d6357c0d1a5d..8976ce72dafe 100644 --- a/src/query/service/src/table_functions/show_grants/show_grants_table.rs +++ b/src/query/service/src/table_functions/show_grants/show_grants_table.rs @@ -566,7 +566,7 @@ async fn show_object_grant( let tenant = ctx.get_tenant(); let user_api = UserApiProvider::instance(); let roles = user_api.get_roles(&tenant).await?; - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker(false).await?; let current_user = ctx.get_current_user()?.identity().username; let (object, owner_object, object_id, object_name) = match grant_type { "table" => { diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 16a8824240a8..ea8d1476bc25 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -690,7 +690,10 @@ impl TableContext for CtxDelegation { todo!() } - async fn get_visibility_checker(&self) -> Result { + async fn get_visibility_checker( + &self, + _ignore_ownership: bool, + ) -> Result { todo!() } diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 94b3f39457c9..02db1fb9ffbf 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -594,7 +594,10 @@ impl TableContext for CtxDelegation { todo!() } - async fn get_visibility_checker(&self) -> Result { + async fn get_visibility_checker( + &self, + _ignore_ownership: bool, + ) -> Result { todo!() } diff --git a/src/query/storages/system/Cargo.toml b/src/query/storages/system/Cargo.toml index 31b4e329688c..79790a6233d0 100644 --- a/src/query/storages/system/Cargo.toml +++ b/src/query/storages/system/Cargo.toml @@ -26,6 +26,7 @@ databend-common-config = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } databend-common-functions = { workspace = true } +databend-common-management = { workspace = true } databend-common-meta-api = { workspace = true } databend-common-meta-app = { workspace = true } databend-common-meta-types = { workspace = true } diff --git a/src/query/storages/system/src/columns_table.rs b/src/query/storages/system/src/columns_table.rs index 094a01613f16..5d9b2cf296df 100644 --- a/src/query/storages/system/src/columns_table.rs +++ b/src/query/storages/system/src/columns_table.rs @@ -274,7 +274,7 @@ pub(crate) async fn dump_tables( } } - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker(false).await?; let mut final_dbs: Vec<(String, u64)> = Vec::new(); diff --git a/src/query/storages/system/src/databases_table.rs b/src/query/storages/system/src/databases_table.rs index 4adbbb880d73..b2284a96790a 100644 --- a/src/query/storages/system/src/databases_table.rs +++ b/src/query/storages/system/src/databases_table.rs @@ -117,7 +117,7 @@ where DatabasesTable: HistoryAware let mut owners: Vec> = vec![]; let mut dropped_on: Vec> = vec![]; - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker(false).await?; let catalog_dbs = visibility_checker.get_visibility_database(); // None means has global level privileges if let Some(catalog_dbs) = catalog_dbs { diff --git a/src/query/storages/system/src/indexes_table.rs b/src/query/storages/system/src/indexes_table.rs index 47daad8deebf..d133aa931e8b 100644 --- a/src/query/storages/system/src/indexes_table.rs +++ b/src/query/storages/system/src/indexes_table.rs @@ -154,7 +154,7 @@ impl IndexesTable { ctx: Arc, ) -> Result> { let tenant = ctx.get_tenant(); - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker(false).await?; let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; let ctl_name = catalog.name(); diff --git a/src/query/storages/system/src/stages_table.rs b/src/query/storages/system/src/stages_table.rs index 91a09bfaacbb..13a03d1ed9fa 100644 --- a/src/query/storages/system/src/stages_table.rs +++ b/src/query/storages/system/src/stages_table.rs @@ -62,7 +62,7 @@ impl AsyncSystemTable for StagesTable { let enable_experimental_rbac_check = ctx.get_settings().get_enable_experimental_rbac_check()?; let stages = if enable_experimental_rbac_check { - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker(false).await?; stages .into_iter() .filter(|stage| { diff --git a/src/query/storages/system/src/streams_table.rs b/src/query/storages/system/src/streams_table.rs index 6bf7013a1652..614839112377 100644 --- a/src/query/storages/system/src/streams_table.rs +++ b/src/query/storages/system/src/streams_table.rs @@ -80,7 +80,7 @@ impl AsyncSystemTable for StreamsTable { .iter() .map(|e| (e.name(), e.clone())) .collect::>(); - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker(false).await?; let user_api = UserApiProvider::instance(); let mut catalogs = vec![]; diff --git a/src/query/storages/system/src/tables_table.rs b/src/query/storages/system/src/tables_table.rs index 0817f756e5f1..f092b4c22b62 100644 --- a/src/query/storages/system/src/tables_table.rs +++ b/src/query/storages/system/src/tables_table.rs @@ -38,6 +38,7 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_management::RoleApi; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::TableIdent; @@ -46,7 +47,6 @@ use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; use databend_common_storages_fuse::FuseTable; use databend_common_storages_view::view_table::QUERY; -use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::UserApiProvider; use log::warn; @@ -129,9 +129,7 @@ where TablesTable: HistoryAware .into_iter() .map(|cat| cat.disable_table_info_refresh()) .collect::>>()?; - let visibility_checker = ctx.get_visibility_checker().await?; - - self.get_full_data_from_catalogs(ctx, push_downs, catalogs, visibility_checker) + self.get_full_data_from_catalogs(ctx, push_downs, catalogs) .await } } @@ -237,7 +235,6 @@ where TablesTable: HistoryAware ctx: Arc, push_downs: Option, catalogs: Vec>, - visibility_checker: GrantObjectVisibilityChecker, ) -> Result { let tenant = ctx.get_tenant(); @@ -328,199 +325,277 @@ where TablesTable: HistoryAware ); } } - let catalog_dbs = visibility_checker.get_visibility_database(); - - for (ctl_name, ctl) in ctls.iter() { - if let Some(push_downs) = &push_downs { - if push_downs.filters.as_ref().map(|f| &f.filter).is_some() { - for db in &db_name { - match ctl.get_database(&tenant, db.as_str()).await { - Ok(database) => dbs.push(database), - Err(err) => { - let msg = format!("Failed to get database: {}, {}", db, err); - warn!("{}", msg); - } + + // from system.tables where database = 'db' and name = 'name' + // from system.tables where database = 'db' and table_id = 123 + if db_name.len() == 1 + && !invalid_optimize + && tables_names.len() + tables_ids.len() == 1 + && !invalid_tables_ids + && !WITH_HISTORY + { + let visibility_checker = ctx.get_visibility_checker(true).await?; + for (ctl_name, ctl) in ctls.iter() { + for db in &db_name { + match ctl.get_database(&tenant, db.as_str()).await { + Ok(database) => dbs.push(database), + Err(err) => { + let msg = format!("Failed to get database: {}, {}", db, err); + warn!("{}", msg); } } + } + if let Err(err) = ctl.mget_table_names_by_ids(&tenant, &tables_ids).await { + warn!("Failed to get tables: {}, {}", ctl.name(), err); + } else { + let new_tables_names = ctl + .mget_table_names_by_ids(&tenant, &tables_ids) + .await? + .into_iter() + .flatten() + .filter(|table| !tables_names.contains(table)) + .collect::>(); + tables_names.extend(new_tables_names); + } - if !WITH_HISTORY { - match ctl.mget_table_names_by_ids(&tenant, &tables_ids).await { - Ok(tables) => { - for table in tables.into_iter().flatten() { - if !tables_names.contains(&table) { - tables_names.push(table.clone()); + for table_name in &tables_names { + for db in &dbs { + match ctl.get_table(&tenant, db.name(), table_name).await { + Ok(t) => { + let db_id = db.get_db_info().database_id.db_id; + let table_id = t.get_id(); + let role = user_api + .role_api(&tenant) + .get_ownership(&OwnershipObject::Table { + catalog_name: ctl_name.to_string(), + db_id, + table_id, + }) + .await? + .map(|o| o.role); + if visibility_checker.check_table_visibility( + ctl_name, + db.name(), + table_name, + db_id, + t.get_id(), + ) { + catalogs.push(ctl_name.as_str()); + databases.push(db.name().to_owned()); + database_tables.push(t); + owner.push(role); + } else if let Some(role) = role { + let roles = ctx.get_all_effective_roles().await?; + if roles.iter().any(|r| r.name == role) { + catalogs.push(ctl_name.as_str()); + databases.push(db.name().to_owned()); + database_tables.push(t); + owner.push(Some(role)); } } } Err(err) => { - let msg = format!("Failed to get tables: {}, {}", ctl.name(), err); + let msg = format!( + "Failed to get table in database: {}, {}", + db.name(), + err + ); + // warn no need to pad in ctx warn!("{}", msg); + continue; } } } } } - - if dbs.is_empty() || invalid_optimize { - // None means has global level privileges - dbs = if let Some(catalog_dbs) = &catalog_dbs { - let mut final_dbs = vec![]; - for (catalog_name, dbs) in catalog_dbs { - if ctl.name() == catalog_name.to_string() { - let mut catalog_db_ids = vec![]; - let mut catalog_db_names = vec![]; - catalog_db_names.extend( - dbs.iter() - .filter_map(|(db_name, _)| *db_name) - .map(|db_name| db_name.to_string()), - ); - catalog_db_ids.extend(dbs.iter().filter_map(|(_, db_id)| *db_id)); - if let Ok(databases) = ctl - .mget_database_names_by_ids(&tenant, &catalog_db_ids) - .await - { - catalog_db_names.extend(databases.into_iter().flatten()); - } else { - let msg = - format!("Failed to get database name by id: {}", ctl.name()); - warn!("{}", msg); + } else { + let visibility_checker = ctx.get_visibility_checker(false).await?; + let catalog_dbs = visibility_checker.get_visibility_database(); + + for (ctl_name, ctl) in ctls.iter() { + if let Some(push_downs) = &push_downs { + if push_downs.filters.as_ref().map(|f| &f.filter).is_some() { + for db in &db_name { + match ctl.get_database(&tenant, db.as_str()).await { + Ok(database) => dbs.push(database), + Err(err) => { + let msg = format!("Failed to get database: {}, {}", db, err); + warn!("{}", msg); + } } - let db_idents = catalog_db_names - .iter() - .map(|name| DatabaseNameIdent::new(&tenant, name)) - .collect::>(); - let dbs = ctl.mget_databases(&tenant, &db_idents).await?; - final_dbs.extend(dbs); } - } - final_dbs - } else { - match ctl.list_databases(&tenant).await { - Ok(dbs) => dbs, - Err(err) => { - let msg = - format!("List databases failed on catalog {}: {}", ctl.name(), err); - warn!("{}", msg); - ctx.push_warning(msg); - vec![] + if !WITH_HISTORY { + match ctl.mget_table_names_by_ids(&tenant, &tables_ids).await { + Ok(tables) => { + for table in tables.into_iter().flatten() { + if !tables_names.contains(&table) { + tables_names.push(table.clone()); + } + } + } + Err(err) => { + let msg = + format!("Failed to get tables: {}, {}", ctl.name(), err); + warn!("{}", msg); + } + } } } } - } - let final_dbs = dbs - .clone() - .into_iter() - .filter(|db| { - visibility_checker.check_database_visibility( - ctl_name, - db.name(), - db.get_db_info().database_id.db_id, - ) - }) - .collect::>(); - - let ownership = if get_ownership { - user_api.get_ownerships(&tenant).await.unwrap_or_default() - } else { - HashMap::new() - }; - for db in final_dbs { - let db_id = db.get_db_info().database_id.db_id; - let db_name = db.name(); - let tables = if tables_names.is_empty() - || tables_names.len() > 10 - || invalid_tables_ids - || invalid_optimize - { - match Self::list_tables(ctl, &tenant, db_name, WITH_HISTORY, WITHOUT_VIEW).await - { - Ok(tables) => tables, - Err(err) => { - // swallow the errors related with remote database or tables, avoid ANY of bad table config corrupt ALL of the results. - // these databases might be: - // - sharing database - // - hive database - // - iceberg database - // - others - // TODO(liyz): return the warnings in the HTTP query protocol. - let msg = - format!("Failed to list tables in database: {}, {}", db_name, err); - warn!("{}", msg); - ctx.push_warning(msg); - - continue; + if dbs.is_empty() || invalid_optimize { + // None means has global level privileges + dbs = if let Some(catalog_dbs) = &catalog_dbs { + let mut final_dbs = vec![]; + for (catalog_name, dbs) in catalog_dbs { + if ctl.name() == catalog_name.to_string() { + let mut catalog_db_ids = vec![]; + let mut catalog_db_names = vec![]; + catalog_db_names.extend( + dbs.iter() + .filter_map(|(db_name, _)| *db_name) + .map(|db_name| db_name.to_string()), + ); + catalog_db_ids.extend(dbs.iter().filter_map(|(_, db_id)| *db_id)); + if let Ok(databases) = ctl + .mget_database_names_by_ids(&tenant, &catalog_db_ids) + .await + { + catalog_db_names.extend(databases.into_iter().flatten()); + } else { + let msg = format!( + "Failed to get database name by id: {}", + ctl.name() + ); + warn!("{}", msg); + } + let db_idents = catalog_db_names + .iter() + .map(|name| DatabaseNameIdent::new(&tenant, name)) + .collect::>(); + let dbs = ctl.mget_databases(&tenant, &db_idents).await?; + final_dbs.extend(dbs); + } } - } - } else if WITH_HISTORY { - // Only can call get_table - let mut tables = Vec::new(); - for table_name in &tables_names { - match ctl.get_table_history(&tenant, db_name, table_name).await { - Ok(t) => tables.extend(t), + final_dbs + } else { + match ctl.list_databases(&tenant).await { + Ok(dbs) => dbs, Err(err) => { let msg = format!( - "Failed to get_table_history tables in database: {}, {}", - db_name, err + "List databases failed on catalog {}: {}", + ctl.name(), + err ); - // warn no need to pad in ctx warn!("{}", msg); - continue; + ctx.push_warning(msg); + + vec![] } } } - tables + } + + let final_dbs = dbs + .clone() + .into_iter() + .filter(|db| { + visibility_checker.check_database_visibility( + ctl_name, + db.name(), + db.get_db_info().database_id.db_id, + ) + }) + .collect::>(); + + let ownership = if get_ownership { + user_api.get_ownerships(&tenant).await.unwrap_or_default() } else { - // Only can call get_table - let mut tables = Vec::new(); - for table_name in &tables_names { - match ctl.get_table(&tenant, db_name, table_name).await { - Ok(t) => tables.push(t), + HashMap::new() + }; + for db in final_dbs { + let db_id = db.get_db_info().database_id.db_id; + let db_name = db.name(); + let tables = if tables_names.is_empty() + || tables_names.len() > 10 + || invalid_tables_ids + || invalid_optimize + { + match Self::list_tables(ctl, &tenant, db_name, WITH_HISTORY, WITHOUT_VIEW) + .await + { + Ok(tables) => tables, Err(err) => { + // swallow the errors related with remote database or tables, avoid ANY of bad table config corrupt ALL of the results. + // these databases might be: + // - sharing database + // - hive database + // - iceberg database + // - others + // TODO(liyz): return the warnings in the HTTP query protocol. let msg = format!( - "Failed to get table in database: {}, {}", + "Failed to list tables in database: {}, {}", db_name, err ); - // warn no need to pad in ctx warn!("{}", msg); + ctx.push_warning(msg); + continue; } } - } - tables - }; - - for table in tables { - let table_id = table.get_id(); - // If db1 is visible, do not mean db1.table1 is visible. A user may have a grant about db1.table2, so db1 is visible - // for her, but db1.table1 may be not visible. So we need an extra check about table here after db visibility check. - if visibility_checker.check_table_visibility( - ctl_name, - db_name, - table.name(), - db_id, - table_id, - ) && !table.is_stream() - { - if !WITHOUT_VIEW && table.get_table_info().engine() == "VIEW" { - catalogs.push(ctl_name.as_str()); - databases.push(db_name.to_owned()); - database_tables.push(table); - if ownership.is_empty() { - owner.push(None); - } else { - owner.push( - ownership - .get(&OwnershipObject::Table { - catalog_name: ctl_name.to_string(), - db_id, - table_id, - }) - .map(|role| role.to_string()), - ); + } else if WITH_HISTORY { + // Only can call get_table + let mut tables = Vec::new(); + for table_name in &tables_names { + match ctl.get_table_history(&tenant, db_name, table_name).await { + Ok(t) => tables.extend(t), + Err(err) => { + let msg = format!( + "Failed to get_table_history tables in database: {}, {}", + db_name, err + ); + // warn no need to pad in ctx + warn!("{}", msg); + continue; + } } - } else if WITHOUT_VIEW { + } + tables + } else { + // Only can call get_table + let mut tables = Vec::new(); + for table_name in &tables_names { + match ctl.get_table(&tenant, db_name, table_name).await { + Ok(t) => tables.push(t), + Err(err) => { + let msg = format!( + "Failed to get table in database: {}, {}", + db_name, err + ); + // warn no need to pad in ctx + warn!("{}", msg); + continue; + } + } + } + tables + }; + + for table in tables { + let table_id = table.get_id(); + // If db1 is visible, do not mean db1.table1 is visible. A user may have a grant about db1.table2, so db1 is visible + // for her, but db1.table1 may be not visible. So we need an extra check about table here after db visibility check. + if (table.get_table_info().engine() == "VIEW" || WITHOUT_VIEW) + && !table.is_stream() + && visibility_checker.check_table_visibility( + ctl_name, + db_name, + table.name(), + db_id, + table_id, + ) + { // system.tables store view name but not store view query // decrease information_schema.tables union. catalogs.push(ctl_name.as_str()); diff --git a/src/query/storages/system/src/user_functions_table.rs b/src/query/storages/system/src/user_functions_table.rs index b1580f15d48a..dc6ca994ed05 100644 --- a/src/query/storages/system/src/user_functions_table.rs +++ b/src/query/storages/system/src/user_functions_table.rs @@ -60,7 +60,7 @@ impl AsyncSystemTable for UserFunctionsTable { let enable_experimental_rbac_check = ctx.get_settings().get_enable_experimental_rbac_check()?; let user_functions = if enable_experimental_rbac_check { - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker(false).await?; let udfs = UserFunctionsTable::get_udfs(&ctx.get_tenant()).await?; udfs.into_iter() .filter(|udf| visibility_checker.check_udf_visibility(&udf.name)) From 918b9c2c0f4348666cf23972a4c3944fc09aad02 Mon Sep 17 00:00:00 2001 From: baishen Date: Wed, 20 Nov 2024 10:43:22 +0800 Subject: [PATCH 60/92] refactor(query): refactor geometry functions (#16870) * refactor(query): refactor geometry functions * fix * fix --- Cargo.lock | 126 +- Cargo.toml | 13 +- src/common/exception/Cargo.toml | 4 - src/common/io/Cargo.toml | 5 +- src/common/io/src/geography.rs | 26 +- src/common/io/src/geometry.rs | 308 +++- src/common/io/src/lib.rs | 11 +- src/query/ast/Cargo.toml | 2 +- src/query/ast/src/parser/error.rs | 4 +- src/query/ast/src/parser/expr.rs | 6 +- src/query/expression/Cargo.toml | 4 - src/query/expression/src/utils/display.rs | 37 +- src/query/formats/Cargo.toml | 3 +- .../formats/src/field_decoder/json_ast.rs | 4 +- src/query/formats/src/field_encoder/values.rs | 68 +- src/query/functions/Cargo.toml | 2 - src/query/functions/src/scalars/geometry.rs | 1440 ++++++++--------- .../it/scalars/testdata/function_list.txt | 6 +- .../tests/it/scalars/testdata/geometry.txt | 14 +- .../functions/02_0060_function_geometry.test | 4 +- 20 files changed, 1039 insertions(+), 1048 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56128851893b..edadfe03ad7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e0966165eaf052580bd70eb1b32cb3d6245774c0104d1b2793e9650bf83b52a" +dependencies = [ + "equator", +] + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -1771,12 +1780,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "c_vec" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd7a427adc0135366d99db65b36dae9237130997e560ed61118041fb72be6e8" - [[package]] name = "camino" version = "1.1.7" @@ -3034,7 +3037,7 @@ dependencies = [ "derive-visitor", "enum-as-inner 0.5.1", "ethnum", - "fast-float", + "fast-float2", "fastrace", "goldenfile", "indent", @@ -3042,7 +3045,7 @@ dependencies = [ "logos", "nom", "nom-rule", - "ordered-float 4.2.2", + "ordered-float 4.5.0", "pratt", "pretty", "pretty_assertions", @@ -3281,7 +3284,6 @@ dependencies = [ "backtrace", "bincode 2.0.0-rc.3", "databend-common-ast", - "geos", "geozero 0.14.0", "gimli 0.31.1", "http 1.1.0", @@ -3338,7 +3340,6 @@ dependencies = [ "ethnum", "futures", "geo", - "geos", "geozero 0.14.0", "goldenfile", "hex", @@ -3384,7 +3385,6 @@ dependencies = [ "databend-common-settings", "databend-storages-common-blocks", "databend-storages-common-table-meta", - "geos", "geozero 0.14.0", "hex", "jsonb", @@ -3425,9 +3425,7 @@ dependencies = [ "dtparse", "ethnum", "geo", - "geo-types", "geohash", - "geos", "geozero 0.14.0", "goldenfile", "h3o", @@ -3525,8 +3523,8 @@ dependencies = [ "enumflags2", "ethnum", "geo", - "geos", "geozero 0.14.0", + "hex", "lexical-core", "micromarshal", "rand", @@ -4319,7 +4317,7 @@ dependencies = [ "iceberg-catalog-hms", "iceberg-catalog-rest", "match-template", - "ordered-float 4.2.2", + "ordered-float 4.5.0", "serde", "tokio", "typetag", @@ -5149,7 +5147,7 @@ dependencies = [ "opensrv-mysql", "opentelemetry", "opentelemetry_sdk", - "ordered-float 4.2.2", + "ordered-float 4.5.0", "p256 0.13.2", "parking_lot 0.12.3", "parquet", @@ -6113,6 +6111,26 @@ dependencies = [ "log", ] +[[package]] +name = "equator" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c35da53b5a021d2484a7cc49b2ac7f2d840f8236a286f84202369bd338d761ea" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -6221,10 +6239,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] -name = "fast-float" -version = "0.2.0" +name = "fast-float2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" [[package]] name = "fastdivide" @@ -6875,42 +6893,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "geos" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "468b7ac7233c0f417b2c161d28b6d1e88c025eaf79303b69e6e1aa40a2ac1367" -dependencies = [ - "c_vec", - "geo-types", - "geos-sys", - "libc", - "num", - "wkt 0.10.3", -] - -[[package]] -name = "geos-src" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8f5247861370ecfcd91a239376c572021cf355cddf52a6602ebbf403d9bb99e" -dependencies = [ - "cmake", -] - -[[package]] -name = "geos-sys" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc873d24aefc72aa94c3c1c251afb82beb7be5926002746c0e1f585fef9854c" -dependencies = [ - "geos-src", - "libc", - "link-cplusplus", - "pkg-config", - "semver", -] - [[package]] name = "geozero" version = "0.13.0" @@ -6934,7 +6916,6 @@ checksum = "e5f28f34864745eb2f123c990c6ffd92c1584bd39439b3f27ff2a0f4ea5b309b" dependencies = [ "geo-types", "geojson", - "geos", "log", "scroll 0.11.0", "serde_json", @@ -8502,7 +8483,7 @@ dependencies = [ "num-bigint", "once_cell", "opendal", - "ordered-float 4.2.2", + "ordered-float 4.5.0", "parquet", "paste", "rand", @@ -9042,14 +9023,15 @@ dependencies = [ [[package]] name = "jsonb" -version = "0.4.3" -source = "git+https://github.com/databendlabs/jsonb?rev=ada713c#ada713c16369cd0c0b85f3bbae22183111325ee9" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd7dc2490b13d09367f5dc4bf202a5d70958dd5b9b2758e2708ee062752a824" dependencies = [ "byteorder", - "fast-float", + "fast-float2", "itoa", "nom", - "ordered-float 4.2.2", + "ordered-float 4.5.0", "rand", "ryu", "serde_json", @@ -9398,15 +9380,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "link-cplusplus" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" -dependencies = [ - "cc", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -9742,7 +9715,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "863693c933db83464c52adf30809f583aae924f30fa114f303aa5360b4a399ec" dependencies = [ "ethnum", - "ordered-float 4.2.2", + "ordered-float 4.5.0", ] [[package]] @@ -10759,9 +10732,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.2" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" +checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e" dependencies = [ "num-traits", "rand", @@ -11073,7 +11046,7 @@ dependencies = [ "integer-encoding 4.0.2", "lazy_static", "linkedbytes", - "ordered-float 4.2.2", + "ordered-float 4.5.0", "paste", "serde", "smallvec", @@ -11310,10 +11283,11 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "pprof" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" +checksum = "ebbe2f8898beba44815fdc9e5a4ae9c929e21c5dc29b0c774a15555f7f58d6d0" dependencies = [ + "aligned-vec", "backtrace", "cfg-if", "findshlibs", diff --git a/Cargo.toml b/Cargo.toml index 80de5122a3e8..802833cf39c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -288,10 +288,8 @@ futures = "0.3.24" futures-async-stream = { version = "0.2.7" } futures-util = "0.3.24" geo = { version = "0.28.0", features = ["use-serde"] } -geo-types = "0.7.13" geohash = "0.13.0" -geos = { version = "9.0.0", features = ["static", "geo", "geo-types"] } -geozero = { version = "0.14.0", features = ["default", "with-wkb", "with-geos", "with-geojson"] } +geozero = { version = "0.14.0", features = ["with-geo", "with-geojson", "with-wkb", "with-wkt"] } gimli = "0.31.0" globiter = "0.1" goldenfile = "1.4" @@ -320,7 +318,7 @@ jaq-core = "1.5.1" jaq-interpret = "1.5.0" jaq-parse = "1.0.3" jaq-std = "1.6.0" -jsonb = "0.4.3" +jsonb = "0.4.4" jwt-simple = { version = "0.12.10", default-features = false, features = ["pure-rust"] } lenient_semver = "0.4.2" levenshtein_automata = "0.2.1" @@ -379,7 +377,7 @@ openraft = { version = "0.10.0", features = [ ] } opensrv-mysql = { version = "0.7.0", features = ["tls"] } orc-rust = "0.5.0" -ordered-float = { version = "4.1.0", default-features = false } +ordered-float = { version = "4.5.0", default-features = false } ordq = "0.2.0" p256 = "0.13" parking_lot = "0.12.1" @@ -393,7 +391,7 @@ pin-project = "1" pin-project-lite = "0.2.9" poem = { version = "3.0", features = ["openssl-tls", "multipart", "compression", "cookie"] } pot = "2.0.0" -pprof = { version = "0.13.0", features = [ +pprof = { version = "0.14.0", features = [ "flamegraph", "protobuf-codec", "protobuf", @@ -501,7 +499,7 @@ zstd = "0.12.3" # AST needed cargo-license = "0.6.1" cargo_metadata = "0.18" -fast-float = "0.2.0" +fast-float2 = "0.2.3" gix = "0.63.0" indent = "0.1.1" logos = "0.12.1" @@ -620,7 +618,6 @@ backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "7226 color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" } deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "3038c145" } ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" } -jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "ada713c" } openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" } openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.7" } orc-rust = { git = "https://github.com/datafusion-contrib/orc-rust", rev = "dfb1ede" } diff --git a/src/common/exception/Cargo.toml b/src/common/exception/Cargo.toml index f7d33a1a0277..b1950b19735c 100644 --- a/src/common/exception/Cargo.toml +++ b/src/common/exception/Cargo.toml @@ -18,7 +18,6 @@ arrow-flight = { workspace = true } arrow-schema = { workspace = true } backtrace = { workspace = true, features = ["std", "serialize-serde"] } bincode = { workspace = true } -geos = { workspace = true } geozero = { workspace = true } gimli = { workspace = true } http = { workspace = true } @@ -38,8 +37,5 @@ tantivy = { workspace = true } thiserror = { workspace = true } tonic = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["geos"] - [lints] workspace = true diff --git a/src/common/io/Cargo.toml b/src/common/io/Cargo.toml index 1be97f6de668..f1c60ff26e5d 100644 --- a/src/common/io/Cargo.toml +++ b/src/common/io/Cargo.toml @@ -22,8 +22,8 @@ enquote = { workspace = true } enumflags2 = { workspace = true } ethnum = { workspace = true } geo = { workspace = true } -geos = { workspace = true } geozero = { workspace = true } +hex = { workspace = true } lexical-core = { workspace = true } micromarshal = { workspace = true } roaring = { workspace = true, features = ["serde"] } @@ -36,8 +36,5 @@ aho-corasick = { workspace = true } rand = { workspace = true } rmp-serde = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["geos"] - [lints] workspace = true diff --git a/src/common/io/src/geography.rs b/src/common/io/src/geography.rs index 6eaf0acfc2a6..ef436ad44b92 100644 --- a/src/common/io/src/geography.rs +++ b/src/common/io/src/geography.rs @@ -15,8 +15,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use geo::CoordsIter; -use geo::Geometry; -use geos::wkt::TryFromWkt; use geozero::ToWkb; pub const LONGITUDE_MIN: f64 = -180.0; @@ -24,20 +22,28 @@ pub const LONGITUDE_MAX: f64 = 180.0; pub const LATITUDE_MIN: f64 = -90.0; pub const LATITUDE_MAX: f64 = 90.0; -use super::geometry::cut_srid; +use super::geometry::ewkt_str_to_geo; pub fn geography_from_ewkt_bytes(ewkt: &[u8]) -> Result> { let s = std::str::from_utf8(ewkt).map_err(|e| ErrorCode::GeometryError(e.to_string()))?; geography_from_ewkt(s) } -pub fn geography_from_ewkt(ewkt: &str) -> Result> { - let (srid, wkt) = cut_srid(ewkt)?; - let geog: Geometry = - Geometry::try_from_wkt_str(wkt).map_err(|e| ErrorCode::GeometryError(e.to_string()))?; - geog.coords_iter().try_for_each(|c| check_point(c.x, c.y))?; - geog.to_ewkb(geozero::CoordDimensions::xy(), srid) - .map_err(ErrorCode::from) +/// Parses an EWKT input and returns a value of EWKB Geography. +pub fn geography_from_ewkt(input: &str) -> Result> { + let input = input.trim(); + let (geo, parsed_srid) = ewkt_str_to_geo(input)?; + if let Some(parsed_srid) = parsed_srid { + if parsed_srid != 4326 { + return Err(ErrorCode::GeometryError(format!( + "SRIDs other than 4326 are not supported. Got SRID: {}", + parsed_srid + ))); + } + } + geo.coords_iter().try_for_each(|c| check_point(c.x, c.y))?; + geo.to_ewkb(geozero::CoordDimensions::xy(), parsed_srid) + .map_err(|e| ErrorCode::GeometryError(e.to_string())) } pub fn check_point(lon: f64, lat: f64) -> Result<()> { diff --git a/src/common/io/src/geometry.rs b/src/common/io/src/geometry.rs index f186c7291bbf..c22747386b47 100644 --- a/src/common/io/src/geometry.rs +++ b/src/common/io/src/geometry.rs @@ -18,9 +18,13 @@ use std::str::FromStr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use geo::Geometry; +use geozero::geo_types::GeoWriter; +use geozero::geojson::GeoJson; use geozero::wkb::Ewkb; use geozero::CoordDimensions; +use geozero::GeomProcessor; use geozero::GeozeroGeometry; +use geozero::ToGeo; use geozero::ToJson; use geozero::ToWkb; use geozero::ToWkt; @@ -82,39 +86,12 @@ impl Display for GeometryDataType { pub fn parse_bytes_to_ewkb(buf: &[u8], srid: Option) -> Result> { let s = std::str::from_utf8(buf).map_err(|e| ErrorCode::GeometryError(e.to_string()))?; - parse_to_ewkb(s, srid) + geometry_from_str(s, srid) } -pub fn parse_to_ewkb(buf: &str, srid: Option) -> Result> { - let (parsed_srid, geo_part) = cut_srid(buf)?; - let srid = srid.or(parsed_srid); - let geom: Geometry = Geometry::try_from_wkt_str(geo_part) - .map_err(|e| ErrorCode::GeometryError(e.to_string()))?; - - geom.to_ewkb(CoordDimensions::xy(), srid) - .map_err(ErrorCode::from) -} - -pub fn cut_srid(ewkt: &str) -> Result<(Option, &str)> { - match ewkt.find(';') { - None => Ok((None, ewkt)), - Some(idx) => { - let prefix = ewkt[..idx].trim(); - match prefix - .strip_prefix("SRID=") - .or_else(|| prefix.strip_prefix("srid=")) - .and_then(|srid| srid.trim().parse::().ok()) - { - Some(srid) => Ok((Some(srid), &ewkt[idx + 1..])), - None => Err(ErrorCode::GeometryError(format!( - "invalid EWKT with prefix {prefix}" - ))), - } - } - } -} - -/// An enum representing any possible geometry subtype. +/// Parses an input and returns a value of ewkb geometry. +/// +/// Support any possible geometry format. /// /// WKB/EWKB: start with 01/00(1bit) /// @@ -142,49 +119,69 @@ pub fn cut_srid(ewkt: &str) -> Result<(Option, &str)> { /// let wkb: &[u8] = "0101000020797f000066666666a9cb17411f85ebc19e325641".as_bytes(); /// println!( /// "wkt:{ } wkb:{ } json: { }", -/// parse_to_subtype(wkt).unwrap(), -/// parse_to_subtype(wkb).unwrap(), -/// parse_to_subtype(geo_json.as_bytes()).unwrap() +/// geometry_from_str(wkt).unwrap(), +/// geometry_from_str(wkb).unwrap(), +/// geometry_from_str(geo_json.as_bytes()).unwrap() /// ); /// ``` -pub fn parse_to_subtype(buf: &[u8]) -> Result { - let bit_0_1: u8 = buf[0] & 1 | (buf[0] >> 1) & 1; - match std::str::from_utf8(buf) { - Ok(str) => { - let text: &String = &str.replace([' ', '\n'], ""); - let prefixes = [ - "SRID", - "POINT", - "LINESTRING", - "POLYGON", - "MULTIPOINT", - "MULTILINESTRING", - "MULTIPOLYGON", - "GEOMETRYCOLLECTION", - ]; - if prefixes.iter().any(|&prefix| text.starts_with(prefix)) { - Ok(GeometryDataType::EWKT) - } else if (text.starts_with('{')) && (text.ends_with('}')) { - Ok(GeometryDataType::GEOJSON) - } else if bit_0_1 == 0b00 || bit_0_1 == 0b01 { - Ok(GeometryDataType::EWKB) - } else { - Err(ErrorCode::GeometryError( - "Invalid geometry type format".to_string(), - )) - } - } - Err(_) => { - if bit_0_1 == 0b00 || bit_0_1 == 0b01 { - Ok(GeometryDataType::EWKB) - } else { - Err(ErrorCode::GeometryError( - "Invalid geometry type format".to_string(), - )) +pub fn geometry_from_str(input: &str, srid: Option) -> Result> { + let input = input.trim(); + let (geo, parsed_srid) = str_to_geo(input)?; + let srid = srid.or(parsed_srid); + + geo.to_ewkb(CoordDimensions::xy(), srid) + .map_err(|e| ErrorCode::GeometryError(e.to_string())) +} + +/// Parses an EWKT input and returns a value of EWKB geometry. +pub fn geometry_from_ewkt(input: &str, srid: Option) -> Result> { + let input = input.trim(); + let (geo, parsed_srid) = ewkt_str_to_geo(input)?; + let srid = srid.or(parsed_srid); + geo.to_ewkb(CoordDimensions::xy(), srid) + .map_err(|e| ErrorCode::GeometryError(e.to_string())) +} + +/// Parses a GEOJSON/EWKB/WKB/EWKT/WKT input and returns a Geometry object. +pub(crate) fn str_to_geo(input: &str) -> Result<(Geometry, Option)> { + if input.starts_with(['{']) { + let geo = GeoJson(input) + .to_geo() + .map_err(|e| ErrorCode::GeometryError(e.to_string()))?; + Ok((geo, None)) + } else if input.starts_with(['0', '0']) || input.starts_with(['0', '1']) { + let binary = match hex::decode(input) { + Ok(binary) => binary, + Err(e) => return Err(ErrorCode::GeometryError(e.to_string())), + }; + ewkb_to_geo(&mut Ewkb(&binary)) + } else { + ewkt_str_to_geo(input) + } +} + +/// Parses an EWKT input and returns Geometry object and SRID. +pub(crate) fn ewkt_str_to_geo(input: &str) -> Result<(Geometry, Option)> { + if input.starts_with(['s']) || input.starts_with(['S']) { + if let Some((srid_input, wkt_input)) = input.split_once(';') { + let srid_input = srid_input.to_uppercase(); + if let Some(srid_str) = srid_input.strip_prefix("SRID") { + if let Some(srid_str) = srid_str.trim().strip_prefix("=") { + let parsed_srid = srid_str.trim().parse::()?; + let geo = Geometry::try_from_wkt_str(wkt_input) + .map_err(|e| ErrorCode::GeometryError(e.to_string()))?; + return Ok((geo, Some(parsed_srid))); + } } } + Err(ErrorCode::GeometryError("invalid srid")) + } else { + let geo = Geometry::try_from_wkt_str(input) + .map_err(|e| ErrorCode::GeometryError(e.to_string()))?; + Ok((geo, None)) } } + pub trait GeometryFormatOutput { fn format(self, data_type: GeometryDataType) -> Result; } @@ -227,3 +224,178 @@ pub fn geometry_format( ) -> Result { geometry.format(format_type) } + +/// Convert Geometry object to GEOJSON format. +pub fn geo_to_json(geo: Geometry) -> Result { + geo.to_json() + .map_err(|e| ErrorCode::GeometryError(e.to_string())) +} + +/// Convert Geometry object to WKB format. +pub fn geo_to_wkb(geo: Geometry) -> Result> { + geo.to_wkb(geo.dims()) + .map_err(|e| ErrorCode::GeometryError(e.to_string())) +} + +/// Convert Geometry object to EWKB format. +pub fn geo_to_ewkb(geo: Geometry, srid: Option) -> Result> { + geo.to_ewkb(geo.dims(), srid) + .map_err(|e| ErrorCode::GeometryError(e.to_string())) +} + +/// Convert Geometry object to WKT format. +pub fn geo_to_wkt(geo: Geometry) -> Result { + geo.to_wkt() + .map_err(|e| ErrorCode::GeometryError(e.to_string())) +} + +/// Convert Geometry object to EWKT format. +pub fn geo_to_ewkt(geo: Geometry, srid: Option) -> Result { + geo.to_ewkt(srid) + .map_err(|e| ErrorCode::GeometryError(e.to_string())) +} + +/// Process EWKB input and return SRID. +pub fn read_srid>(ewkb: &mut Ewkb) -> Option { + let mut srid_processor = SridProcessor::new(); + ewkb.process_geom(&mut srid_processor).ok()?; + + srid_processor.srid +} + +/// Process EWKB input and return Geometry object and SRID. +pub fn ewkb_to_geo>(ewkb: &mut Ewkb) -> Result<(Geometry, Option)> { + let mut ewkb_processor = EwkbProcessor::new(); + ewkb.process_geom(&mut ewkb_processor)?; + + let geo = ewkb_processor + .geo_writer + .take_geometry() + .ok_or_else(|| ErrorCode::GeometryError("Invalid ewkb format"))?; + let srid = ewkb_processor.srid; + Ok((geo, srid)) +} + +struct SridProcessor { + srid: Option, +} + +impl SridProcessor { + fn new() -> Self { + Self { srid: None } + } +} + +impl GeomProcessor for SridProcessor { + fn srid(&mut self, srid: Option) -> geozero::error::Result<()> { + self.srid = srid; + Ok(()) + } +} + +struct EwkbProcessor { + geo_writer: GeoWriter, + srid: Option, +} + +impl EwkbProcessor { + fn new() -> Self { + Self { + geo_writer: GeoWriter::new(), + srid: None, + } + } +} + +impl GeomProcessor for EwkbProcessor { + fn srid(&mut self, srid: Option) -> geozero::error::Result<()> { + self.srid = srid; + Ok(()) + } + + fn xy(&mut self, x: f64, y: f64, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.xy(x, y, idx) + } + + fn point_begin(&mut self, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.point_begin(idx) + } + + fn point_end(&mut self, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.point_end(idx) + } + + fn multipoint_begin(&mut self, size: usize, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.multipoint_begin(size, idx) + } + + fn multipoint_end(&mut self, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.multipoint_end(idx) + } + + fn linestring_begin( + &mut self, + tagged: bool, + size: usize, + idx: usize, + ) -> geozero::error::Result<()> { + self.geo_writer.linestring_begin(tagged, size, idx) + } + + fn linestring_end(&mut self, tagged: bool, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.linestring_end(tagged, idx) + } + + fn multilinestring_begin(&mut self, size: usize, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.multilinestring_begin(size, idx) + } + + fn multilinestring_end(&mut self, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.multilinestring_end(idx) + } + + fn polygon_begin( + &mut self, + tagged: bool, + size: usize, + idx: usize, + ) -> geozero::error::Result<()> { + self.geo_writer.polygon_begin(tagged, size, idx) + } + + fn polygon_end(&mut self, tagged: bool, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.polygon_end(tagged, idx) + } + + fn multipolygon_begin(&mut self, size: usize, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.multipolygon_begin(size, idx) + } + + fn multipolygon_end(&mut self, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.multipolygon_end(idx) + } + + fn geometrycollection_begin(&mut self, size: usize, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.geometrycollection_begin(size, idx) + } + + fn geometrycollection_end(&mut self, idx: usize) -> geozero::error::Result<()> { + self.geo_writer.geometrycollection_end(idx) + } +} + +/// Return Geometry type name. +pub fn geometry_type_name(geo: &Geometry) -> &'static str { + match geo { + Geometry::Point(_) => "Point", + Geometry::Line(_) => "Line", + Geometry::LineString(_) => "LineString", + Geometry::Polygon(_) => "Polygon", + Geometry::MultiPoint(_) => "MultiPoint", + Geometry::MultiLineString(_) => "MultiLineString", + Geometry::MultiPolygon(_) => "MultiPolygon", + Geometry::GeometryCollection(_) => "GeometryCollection", + Geometry::Rect(_) => "Rect", + Geometry::Triangle(_) => "Triangle", + } +} diff --git a/src/common/io/src/lib.rs b/src/common/io/src/lib.rs index 522c52fedd01..d03b1e05c530 100644 --- a/src/common/io/src/lib.rs +++ b/src/common/io/src/lib.rs @@ -52,10 +52,17 @@ pub use decimal::display_decimal_128; pub use decimal::display_decimal_256; pub use escape::escape_string; pub use escape::escape_string_with_quote; +pub use geometry::ewkb_to_geo; +pub use geometry::geo_to_ewkb; +pub use geometry::geo_to_ewkt; +pub use geometry::geo_to_json; +pub use geometry::geo_to_wkb; +pub use geometry::geo_to_wkt; pub use geometry::geometry_format; +pub use geometry::geometry_from_ewkt; +pub use geometry::geometry_type_name; pub use geometry::parse_bytes_to_ewkb; -pub use geometry::parse_to_ewkb; -pub use geometry::parse_to_subtype; +pub use geometry::read_srid; pub use geometry::Axis; pub use geometry::Extremum; pub use geometry::GeometryDataType; diff --git a/src/query/ast/Cargo.toml b/src/query/ast/Cargo.toml index dfbe0f88e181..3d33cd406ba6 100644 --- a/src/query/ast/Cargo.toml +++ b/src/query/ast/Cargo.toml @@ -15,7 +15,7 @@ doctest = false derive-visitor = { workspace = true } enum-as-inner = { workspace = true } ethnum = { workspace = true } -fast-float = { workspace = true } +fast-float2 = { workspace = true } fastrace = { workspace = true } indent = { workspace = true } itertools = { workspace = true } diff --git a/src/query/ast/src/parser/error.rs b/src/query/ast/src/parser/error.rs index 7c5db1d936ae..3049f26685d4 100644 --- a/src/query/ast/src/parser/error.rs +++ b/src/query/ast/src/parser/error.rs @@ -163,8 +163,8 @@ impl<'a> Error<'a> { } } -impl From for ErrorKind { - fn from(_: fast_float::Error) -> Self { +impl From for ErrorKind { + fn from(_: fast_float2::Error) -> Self { ErrorKind::Other("unable to parse float number") } } diff --git a/src/query/ast/src/parser/expr.rs b/src/query/ast/src/parser/expr.rs index 58fdb9cc7780..a1523d4d66ab 100644 --- a/src/query/ast/src/parser/expr.rs +++ b/src/query/ast/src/parser/expr.rs @@ -1913,12 +1913,12 @@ pub fn parse_float(text: &str) -> Result { let exp = match e_part { Some(s) => match s.parse::() { Ok(i) => i, - Err(_) => return Ok(Literal::Float64(fast_float::parse(text)?)), + Err(_) => return Ok(Literal::Float64(fast_float2::parse(text)?)), }, None => 0, }; if i_part.len() as i32 + exp > 76 { - Ok(Literal::Float64(fast_float::parse(text)?)) + Ok(Literal::Float64(fast_float2::parse(text)?)) } else { let mut digits = String::with_capacity(76); digits.push_str(i_part); @@ -1960,7 +1960,7 @@ pub fn parse_uint(text: &str, radix: u32) -> Result { if text.is_empty() { return Ok(Literal::UInt64(0)); } else if text.len() > 76 { - return Ok(Literal::Float64(fast_float::parse(text)?)); + return Ok(Literal::Float64(fast_float2::parse(text)?)); } let value = i256::from_str_radix(text, radix)?; diff --git a/src/query/expression/Cargo.toml b/src/query/expression/Cargo.toml index bc49814fe8c2..7ecfb7cb3813 100644 --- a/src/query/expression/Cargo.toml +++ b/src/query/expression/Cargo.toml @@ -40,7 +40,6 @@ enum-as-inner = { workspace = true } ethnum = { workspace = true, features = ["serde", "macros", "borsh"] } futures = { workspace = true } geo = { workspace = true } -geos = { workspace = true } geozero = { workspace = true } hex = { workspace = true } hyper-util = { workspace = true } @@ -79,6 +78,3 @@ harness = false [lints] workspace = true - -[package.metadata.cargo-machete] -ignored = ["geos"] diff --git a/src/query/expression/src/utils/display.rs b/src/query/expression/src/utils/display.rs index 94d898fdc2f7..b89834ed3362 100755 --- a/src/query/expression/src/utils/display.rs +++ b/src/query/expression/src/utils/display.rs @@ -24,10 +24,9 @@ use databend_common_ast::parser::Dialect; use databend_common_io::deserialize_bitmap; use databend_common_io::display_decimal_128; use databend_common_io::display_decimal_256; +use databend_common_io::ewkb_to_geo; +use databend_common_io::geo_to_ewkt; use geozero::wkb::Ewkb; -use geozero::GeozeroGeometry; -use geozero::ToGeos; -use geozero::ToWkt; use itertools::Itertools; use num_traits::FromPrimitive; use rust_decimal::Decimal; @@ -167,16 +166,16 @@ impl<'a> Debug for ScalarRef<'a> { Ok(()) } ScalarRef::Geometry(s) => { - let ewkb = Ewkb(s); - let geos = ewkb.to_geos().unwrap(); - let geom = geos - .to_ewkt(geos.srid()) - .unwrap_or_else(|x| format!("GeozeroError: {:?}", x)); + let geom = ewkb_to_geo(&mut Ewkb(s)) + .and_then(|(geo, srid)| geo_to_ewkt(geo, srid)) + .unwrap_or_else(|e| format!("GeozeroError: {:?}", e)); write!(f, "{geom:?}") } ScalarRef::Geography(v) => { - let ewkt = v.to_ewkt().unwrap_or_else(|e| format!("Invalid data: {e}")); - write!(f, "{ewkt:?}") + let geog = ewkb_to_geo(&mut Ewkb(v.0)) + .and_then(|(geo, srid)| geo_to_ewkt(geo, srid)) + .unwrap_or_else(|e| format!("GeozeroError: {:?}", e)); + write!(f, "{geog:?}") } } } @@ -261,20 +260,16 @@ impl<'a> Display for ScalarRef<'a> { write!(f, "'{value}'") } ScalarRef::Geometry(s) => { - let ewkb = Ewkb(s); - let geos = ewkb.to_geos().unwrap(); - let geom = geos - .to_ewkt(geos.srid()) - .unwrap_or_else(|x| format!("GeozeroError: {:?}", x)); + let geom = ewkb_to_geo(&mut Ewkb(s)) + .and_then(|(geo, srid)| geo_to_ewkt(geo, srid)) + .unwrap_or_else(|e| format!("GeozeroError: {:?}", e)); write!(f, "'{geom}'") } ScalarRef::Geography(v) => { - let ewkb = Ewkb(v.0); - let geos = ewkb.to_geos().unwrap(); - let geom = geos - .to_ewkt(geos.srid()) - .unwrap_or_else(|x| format!("GeozeroError: {:?}", x)); - write!(f, "'{geom}'") + let geog = ewkb_to_geo(&mut Ewkb(v.0)) + .and_then(|(geo, srid)| geo_to_ewkt(geo, srid)) + .unwrap_or_else(|e| format!("GeozeroError: {:?}", e)); + write!(f, "'{geog}'") } } } diff --git a/src/query/formats/Cargo.toml b/src/query/formats/Cargo.toml index 6f2de788af28..8f11e6661940 100644 --- a/src/query/formats/Cargo.toml +++ b/src/query/formats/Cargo.toml @@ -26,7 +26,6 @@ async-trait = { workspace = true } base64 = { workspace = true } bstr = { workspace = true } chrono-tz = { workspace = true } -geos = { workspace = true } geozero = { workspace = true } hex = { workspace = true } jsonb = { workspace = true } @@ -47,4 +46,4 @@ tokio = { workspace = true } workspace = true [package.metadata.cargo-machete] -ignored = ["geos", "match-template"] +ignored = ["match-template"] diff --git a/src/query/formats/src/field_decoder/json_ast.rs b/src/query/formats/src/field_decoder/json_ast.rs index ef3dc809f091..c258207f70e3 100644 --- a/src/query/formats/src/field_decoder/json_ast.rs +++ b/src/query/formats/src/field_decoder/json_ast.rs @@ -39,8 +39,8 @@ use databend_common_expression::ColumnBuilder; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::DateTimeResType; use databend_common_io::geography::geography_from_ewkt; +use databend_common_io::geometry_from_ewkt; use databend_common_io::parse_bitmap; -use databend_common_io::parse_to_ewkb; use lexical_core::FromLexical; use num::cast::AsPrimitive; use num_traits::NumCast; @@ -346,7 +346,7 @@ impl FieldJsonAstDecoder { fn read_geometry(&self, column: &mut BinaryColumnBuilder, value: &Value) -> Result<()> { match value { Value::String(v) => { - let geom = parse_to_ewkb(v, None)?; + let geom = geometry_from_ewkt(v, None)?; column.put_slice(&geom); column.commit_row(); Ok(()) diff --git a/src/query/formats/src/field_encoder/values.rs b/src/query/formats/src/field_encoder/values.rs index 6c7a13d2e09b..1c3fa889da92 100644 --- a/src/query/formats/src/field_encoder/values.rs +++ b/src/query/formats/src/field_encoder/values.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use bstr::ByteSlice; use chrono_tz::Tz; use databend_common_base::base::OrderedFloat; use databend_common_expression::types::array::ArrayColumn; @@ -35,14 +34,14 @@ use databend_common_io::constants::NAN_BYTES_LOWER; use databend_common_io::constants::NAN_BYTES_SNAKE; use databend_common_io::constants::NULL_BYTES_UPPER; use databend_common_io::constants::TRUE_BYTES_NUM; +use databend_common_io::ewkb_to_geo; +use databend_common_io::geo_to_ewkb; +use databend_common_io::geo_to_ewkt; +use databend_common_io::geo_to_json; +use databend_common_io::geo_to_wkb; +use databend_common_io::geo_to_wkt; use databend_common_io::GeometryDataType; use geozero::wkb::Ewkb; -use geozero::CoordDimensions; -use geozero::GeozeroGeometry; -use geozero::ToGeos; -use geozero::ToJson; -use geozero::ToWkb; -use geozero::ToWkt; use lexical_core::ToLexical; use micromarshal::Marshal; use micromarshal::Unmarshal; @@ -308,24 +307,17 @@ impl FieldEncoderValues { in_nested: bool, ) { let v = unsafe { column.index_unchecked(row_index) }; - let s = match self.common_settings().geometry_format { - GeometryDataType::WKB => hex::encode_upper( - Ewkb(v.to_vec()) - .to_wkb(CoordDimensions::xy()) - .unwrap() - .as_bytes(), - ) - .as_bytes() - .to_vec(), - GeometryDataType::WKT => Ewkb(v.to_vec()).to_wkt().unwrap().as_bytes().to_vec(), - GeometryDataType::EWKB => hex::encode_upper(v).as_bytes().to_vec(), - GeometryDataType::EWKT => { - let ewkb = Ewkb(v.to_vec()); - let geos = ewkb.to_geos().unwrap(); - geos.to_ewkt(geos.srid()).unwrap().as_bytes().to_vec() - } - GeometryDataType::GEOJSON => Ewkb(v.to_vec()).to_json().unwrap().as_bytes().to_vec(), - }; + let s = ewkb_to_geo(&mut Ewkb(v)) + .and_then(|(geo, srid)| match self.common_settings().geometry_format { + GeometryDataType::WKB => geo_to_wkb(geo).map(|v| hex::encode_upper(v).into_bytes()), + GeometryDataType::WKT => geo_to_wkt(geo).map(|v| v.as_bytes().to_vec()), + GeometryDataType::EWKB => { + geo_to_ewkb(geo, srid).map(|v| hex::encode_upper(v).into_bytes()) + } + GeometryDataType::EWKT => geo_to_ewkt(geo, srid).map(|v| v.as_bytes().to_vec()), + GeometryDataType::GEOJSON => geo_to_json(geo).map(|v| v.as_bytes().to_vec()), + }) + .unwrap_or_else(|_| v.to_vec()); self.write_string_inner(&s, out_buf, in_nested); } @@ -338,21 +330,17 @@ impl FieldEncoderValues { in_nested: bool, ) { let v = unsafe { column.index_unchecked(row_index) }; - let s = match self.common_settings().geometry_format { - GeometryDataType::WKB => { - hex::encode_upper(Ewkb(v.0).to_wkb(CoordDimensions::xy()).unwrap().as_bytes()) - .as_bytes() - .to_vec() - } - GeometryDataType::WKT => Ewkb(v.0).to_wkt().unwrap().as_bytes().to_vec(), - GeometryDataType::EWKB => hex::encode_upper(v.0).as_bytes().to_vec(), - GeometryDataType::EWKT => { - let ewkb = Ewkb(v.0); - let geos = ewkb.to_geos().unwrap(); - geos.to_ewkt(geos.srid()).unwrap().as_bytes().to_vec() - } - GeometryDataType::GEOJSON => Ewkb(v.0).to_json().unwrap().as_bytes().to_vec(), - }; + let s = ewkb_to_geo(&mut Ewkb(v.0)) + .and_then(|(geo, srid)| match self.common_settings().geometry_format { + GeometryDataType::WKB => geo_to_wkb(geo).map(|v| hex::encode_upper(v).into_bytes()), + GeometryDataType::WKT => geo_to_wkt(geo).map(|v| v.as_bytes().to_vec()), + GeometryDataType::EWKB => { + geo_to_ewkb(geo, srid).map(|v| hex::encode_upper(v).into_bytes()) + } + GeometryDataType::EWKT => geo_to_ewkt(geo, srid).map(|v| v.as_bytes().to_vec()), + GeometryDataType::GEOJSON => geo_to_json(geo).map(|v| v.as_bytes().to_vec()), + }) + .unwrap_or_else(|_| v.0.to_vec()); self.write_string_inner(&s, out_buf, in_nested); } diff --git a/src/query/functions/Cargo.toml b/src/query/functions/Cargo.toml index dfac09efd4e8..652fe5bb59f5 100644 --- a/src/query/functions/Cargo.toml +++ b/src/query/functions/Cargo.toml @@ -30,9 +30,7 @@ databend-common-vector = { workspace = true } dtparse = { workspace = true } ethnum = { workspace = true } geo = { workspace = true } -geo-types = { workspace = true } geohash = { workspace = true } -geos = { workspace = true } geozero = { workspace = true } h3o = { workspace = true } hex = { workspace = true } diff --git a/src/query/functions/src/scalars/geometry.rs b/src/query/functions/src/scalars/geometry.rs index 6ba0d6419a67..c35cc7c59e5e 100644 --- a/src/query/functions/src/scalars/geometry.rs +++ b/src/query/functions/src/scalars/geometry.rs @@ -13,6 +13,7 @@ // limitations under the License. use databend_common_exception::ErrorCode; +use databend_common_exception::Result; use databend_common_expression::types::geometry::GeometryType; use databend_common_expression::types::BinaryType; use databend_common_expression::types::BooleanType; @@ -29,40 +30,47 @@ use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::vectorize_with_builder_4_arg; use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionRegistry; +use databend_common_io::ewkb_to_geo; +use databend_common_io::geo_to_ewkb; +use databend_common_io::geo_to_ewkt; +use databend_common_io::geo_to_json; +use databend_common_io::geo_to_wkb; +use databend_common_io::geo_to_wkt; +use databend_common_io::geometry::geometry_from_str; use databend_common_io::geometry_format; -use databend_common_io::parse_to_ewkb; -use databend_common_io::parse_to_subtype; +use databend_common_io::geometry_from_ewkt; +use databend_common_io::geometry_type_name; +use databend_common_io::read_srid; use databend_common_io::Axis; use databend_common_io::Extremum; -use databend_common_io::GeometryDataType; +use geo::coord; use geo::dimensions::Dimensions; use geo::BoundingRect; use geo::Contains; +use geo::Coord; use geo::EuclideanDistance; use geo::EuclideanLength; +use geo::Geometry; use geo::HasDimensions; use geo::HaversineDistance; +use geo::Line; +use geo::LineString; +use geo::MultiLineString; +use geo::MultiPolygon; use geo::Point; +use geo::Polygon; +use geo::Rect; use geo::ToDegrees; use geo::ToRadians; -use geo_types::coord; -use geo_types::Polygon; +use geo::Triangle; use geohash::decode_bbox; use geohash::encode; -use geos::geo_types; -use geos::geo_types::Coord; -use geos::geo_types::LineString; -use geos::Geometry; use geozero::geojson::GeoJson; use geozero::wkb::Ewkb; -use geozero::wkb::Wkb; use geozero::CoordDimensions; use geozero::GeozeroGeometry; use geozero::ToGeo; -use geozero::ToGeos; -use geozero::ToJson; use geozero::ToWkb; -use geozero::ToWkt; use jsonb::parse_value; use jsonb::to_string; use num_traits::AsPrimitive; @@ -94,18 +102,26 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_4_arg::, NumberType, NumberType, NumberType, NumberType, _, _>( "haversine", |_, _, _, _, _| FunctionDomain::Full, - vectorize_with_builder_4_arg::, NumberType, NumberType, NumberType, NumberType,>(|lat1, lon1, lat2, lon2, builder, _| { + vectorize_with_builder_4_arg::, NumberType, NumberType, NumberType, NumberType,>(|lat1, lon1, lat2, lon2, builder, ctx| { + if let Some(validity) = &ctx.validity { + if !validity.get_bit(builder.len()) { + builder.push(0_f64.into()); + return; + } + } let p1 = Point::new(lon1, lat1); let p2 = Point::new(lon2, lat2); let distance = p1.haversine_distance(&p2) * 0.001; - builder.push(format!("{:.9}",distance.into_inner()).parse().unwrap()); + + let distance = (distance * 1_000_000_000_f64).round() / 1_000_000_000_f64; + builder.push(distance.into()); }), ); registry.register_passthrough_nullable_1_arg::( "st_asgeojson", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -113,18 +129,19 @@ pub fn register(registry: &mut FunctionRegistry) { } } - let json = ewkb_to_json(geometry); - match parse_value(json.unwrap().as_bytes()) { - Ok(json) => { - json.write_to_vec(&mut builder.data); - } + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, _)| geo_to_json(geo)) { + Ok(json) => match parse_value(json.as_bytes()) { + Ok(json_val) => { + json_val.write_to_vec(&mut builder.data); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + } + }, Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } - }; + } builder.commit_row(); }), ); @@ -132,24 +149,22 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "st_asewkb", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); return; } } - let ewkb = Ewkb(geometry); - let srid = ewkb.to_geos().unwrap().srid(); - match Ewkb(geometry).to_ewkb(CoordDimensions::xy(), srid) { - Ok(wkb) => builder.put_slice(wkb.as_slice()), + + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, srid)| geo_to_ewkb(geo, srid)) { + Ok(ewkb) => { + builder.put_slice(ewkb.as_slice()); + } Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } - }; + } builder.commit_row(); }), ); @@ -157,7 +172,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "st_aswkb", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -165,15 +180,14 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match Ewkb(geometry).to_wkb(CoordDimensions::xy()) { - Ok(wkb) => builder.put_slice(wkb.as_slice()), + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, _)| geo_to_wkb(geo)) { + Ok(wkb) => { + builder.put_slice(wkb.as_slice()); + } Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } - }; + } builder.commit_row(); }), ); @@ -181,7 +195,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "st_asewkt", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -189,15 +203,14 @@ pub fn register(registry: &mut FunctionRegistry) { } } - let srid = Ewkb(geometry).to_geo().unwrap().srid(); - - match Ewkb(geometry).to_ewkt(srid) { - Ok(ewkt) => builder.put_str(&ewkt), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, srid)| geo_to_ewkt(geo, srid)) { + Ok(ewkt) => { + builder.put_str(&ewkt); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + } + } builder.commit_row(); }), ); @@ -205,7 +218,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "st_aswkt", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -213,13 +226,14 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match Ewkb(geometry).to_wkt() { - Ok(wkt) => builder.put_str(&wkt), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, _)| geo_to_wkt(geo)) { + Ok(wkt) => { + builder.put_str(&wkt); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + } + } builder.commit_row(); }), ); @@ -228,41 +242,46 @@ pub fn register(registry: &mut FunctionRegistry) { "st_contains", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |l_geometry, r_geometry, builder, ctx| { + |l_ewkb, r_ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push(false); return; } } - let l_ewkb = Ewkb(l_geometry); - let r_ewkb = Ewkb(r_geometry); - let l_geos: Geometry = l_ewkb.to_geos().unwrap(); - let r_geos: Geometry = r_ewkb.to_geos().unwrap(); - let l_srid = l_geos.srid(); - let r_srid = r_geos.srid(); - if l_srid != r_srid { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError("Srid does not match!").to_string(), - ); - builder.push(false); - } else { - let l_geo: geo::Geometry = l_geos.to_geo().unwrap(); - let r_geo: geo::Geometry = r_geos.to_geo().unwrap(); - if matches!(l_geo, geo::Geometry::GeometryCollection(_)) - || matches!(r_geo, geo::Geometry::GeometryCollection(_)) - { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError( - "A GEOMETRY object that is a GeometryCollection", - ) - .to_string(), - ); + + match ( + ewkb_to_geo(&mut Ewkb(l_ewkb)), + ewkb_to_geo(&mut Ewkb(r_ewkb)), + ) { + (Ok((l_geo, l_srid)), Ok((r_geo, r_srid))) => { + if !l_srid.eq(&r_srid) { + ctx.set_error( + builder.len(), + format!( + "Incompatible SRID: {} and {}", + l_srid.unwrap_or_default(), + r_srid.unwrap_or_default() + ), + ); + builder.push(false); + return; + } + if matches!(l_geo, Geometry::GeometryCollection(_)) + || matches!(r_geo, Geometry::GeometryCollection(_)) + { + ctx.set_error( + builder.len(), + "A GEOMETRY object that is a GeometryCollection".to_string(), + ); + builder.push(false); + } else { + builder.push(l_geo.contains(&r_geo)); + } + } + (Err(e), _) | (_, Err(e)) => { + ctx.set_error(builder.len(), e.to_string()); builder.push(false); - } else { - builder.push(l_geo.contains(&r_geo)); } } }, @@ -274,7 +293,7 @@ pub fn register(registry: &mut FunctionRegistry) { "st_distance", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::>( - |l_geometry, r_geometry, builder, ctx| { + |l_ewkb, r_ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push(F64::from(0_f64)); @@ -282,35 +301,43 @@ pub fn register(registry: &mut FunctionRegistry) { } } - let left_geo = Ewkb(l_geometry); - let right_geo = Ewkb(r_geometry); - let geos = &vec![left_geo.to_geos().unwrap(), right_geo.to_geos().unwrap()]; - if let Err(e) = get_shared_srid(geos) { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.push(F64::from(0_f64)); - return; - } - match ( - >::try_into( - left_geo.to_geo().unwrap(), - ), - >::try_into( - right_geo.to_geo().unwrap(), - ), + ewkb_to_geo(&mut Ewkb(l_ewkb)), + ewkb_to_geo(&mut Ewkb(r_ewkb)), ) { - (Ok(l_point), Ok(r_point)) => { - let distance = l_point.euclidean_distance(&r_point); - builder.push(format!("{:.9}", distance).parse().unwrap()); + (Ok((l_geo, l_srid)), Ok((r_geo, r_srid))) => { + if !l_srid.eq(&r_srid) { + ctx.set_error( + builder.len(), + format!( + "Incompatible SRID: {} and {}", + l_srid.unwrap_or_default(), + r_srid.unwrap_or_default() + ), + ); + builder.push(F64::from(0_f64)); + return; + } + + let distance = match l_geo { + Geometry::Point(l) => l.euclidean_distance(&r_geo), + Geometry::Line(l) => l.euclidean_distance(&r_geo), + Geometry::LineString(l) => l.euclidean_distance(&r_geo), + Geometry::Polygon(l) => l.euclidean_distance(&r_geo), + Geometry::MultiPoint(l) => l.euclidean_distance(&r_geo), + Geometry::MultiLineString(l) => l.euclidean_distance(&r_geo), + Geometry::MultiPolygon(l) => l.euclidean_distance(&r_geo), + Geometry::GeometryCollection(l) => l.euclidean_distance(&r_geo), + Geometry::Rect(l) => l.euclidean_distance(&r_geo), + Geometry::Triangle(l) => l.euclidean_distance(&r_geo), + }; + + let distance = + (distance * 1_000_000_000_f64).round() / 1_000_000_000_f64; + builder.push(distance.into()); } (Err(e), _) | (_, Err(e)) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); builder.push(F64::from(0_f64)); } } @@ -318,119 +345,163 @@ pub fn register(registry: &mut FunctionRegistry) { ), ); - registry.register_passthrough_nullable_1_arg::( + registry.register_combine_nullable_1_arg::( "st_endpoint", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { - if let Some(validity) = &ctx.validity { - if !validity.get_bit(builder.len()) { - builder.commit_row(); - return; + vectorize_with_builder_1_arg::>( + |ewkb, builder, ctx| { + if let Some(validity) = &ctx.validity { + if !validity.get_bit(builder.len()) { + builder.push_null(); + return; + } } - } - let point = match >::try_into( - Ewkb(geometry).to_geo().unwrap(), - ) { - Ok(line_string) => line_string.points().last().unwrap(), - Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, srid)| match geo { + Geometry::LineString(line_string) => { + let points = line_string.points(); + if let Some(point) = points.last() { + geo_to_ewkb(Geometry::from(point), srid).map(Some) + } else { + Ok(None) + } + } + _ => Err(ErrorCode::GeometryError(format!( + "Type {} is not supported as argument to ST_ENDPOINT", + geometry_type_name(&geo) + ))), + }) { + Ok(Some(binary)) => { + builder.push(binary.as_slice()); + } + Ok(None) => { + builder.push_null(); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + builder.push_null(); + } } - }; + }, + ), + ); - match geo_types::Geometry::from(point).to_wkb(CoordDimensions::xy()) { - Ok(binary) => builder.put_slice(binary.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; + registry.register_combine_nullable_1_arg::( + "st_startpoint", + |_, _| FunctionDomain::MayThrow, + vectorize_with_builder_1_arg::>( + |ewkb, builder, ctx| { + if let Some(validity) = &ctx.validity { + if !validity.get_bit(builder.len()) { + builder.push_null(); + return; + } + } - builder.commit_row(); - }), + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, srid)| match geo { + Geometry::LineString(line_string) => { + let mut points = line_string.points(); + if let Some(point) = points.next() { + geo_to_ewkb(Geometry::from(point), srid).map(Some) + } else { + Ok(None) + } + } + _ => Err(ErrorCode::GeometryError(format!( + "Type {} is not supported as argument to ST_STARTPOINT", + geometry_type_name(&geo) + ))), + }) { + Ok(Some(binary)) => { + builder.push(binary.as_slice()); + } + Ok(None) => { + builder.push_null(); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + builder.push_null(); + } + } + }, + ), ); - registry.register_passthrough_nullable_2_arg::( + registry.register_combine_nullable_2_arg::( "st_pointn", |_, _, _| FunctionDomain::MayThrow, - vectorize_with_builder_2_arg::( - |geometry, index, builder, ctx| { + vectorize_with_builder_2_arg::>( + |ewkb, index, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { - builder.commit_row(); + builder.push_null(); return; } } - let point = match >::try_into( - Ewkb(geometry).to_geo().unwrap(), - ) { - Ok(line_string) => { - let len = line_string.0.len() as i32; - if index >= -len && index < len && index != 0 { - Point( - line_string.0 - [if index < 0 { len + index } else { index - 1 } as usize], - ) + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, srid)| match geo { + Geometry::LineString(line_string) => { + let mut points = line_string.points(); + let len = points.len() as i32; + let idx = if index < 0 { len + index } else { index - 1 }; + if let Some(point) = points.nth(idx as usize) { + geo_to_ewkb(Geometry::from(point), srid).map(Some) } else { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(format!( - "Index {} is out of bounds", - index - )) - .to_string(), - ); - builder.commit_row(); - return; + Ok(None) } } + _ => Err(ErrorCode::GeometryError(format!( + "Type {} is not supported as argument to ST_POINTN", + geometry_type_name(&geo) + ))), + }) { + Ok(Some(binary)) => { + builder.push(binary.as_slice()); + } + Ok(None) => { + builder.push_null(); + } Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; + ctx.set_error(builder.len(), e.to_string()); + builder.push_null(); } - }; - - match geo_types::Geometry::from(point).to_wkb(CoordDimensions::xy()) { - Ok(binary) => builder.put_slice(binary.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; - - builder.commit_row(); + } }, ), ); registry.register_combine_nullable_1_arg::( "st_dimension", - |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::>(|ewkb, output, _| { - let geo: geo_types::Geometry = Ewkb(ewkb).to_geo().unwrap(); - - let dimension: Option = match geo.dimensions() { - Dimensions::Empty => None, - Dimensions::ZeroDimensional => Some(0), - Dimensions::OneDimensional => Some(1), - Dimensions::TwoDimensional => Some(2), - }; + |_, _| FunctionDomain::MayThrow, + vectorize_with_builder_1_arg::>( + |ewkb, output, ctx| { + if let Some(validity) = &ctx.validity { + if !validity.get_bit(output.len()) { + output.push_null(); + return; + } + } - match dimension { - Some(dimension) => output.push(dimension), - None => output.push_null(), - } - }), + match Ewkb(ewkb).to_geo() { + Ok(geo) => { + let dimension: Option = match geo.dimensions() { + Dimensions::Empty => None, + Dimensions::ZeroDimensional => Some(0), + Dimensions::OneDimensional => Some(1), + Dimensions::TwoDimensional => Some(2), + }; + match dimension { + Some(dimension) => output.push(dimension), + None => output.push_null(), + } + } + Err(e) => { + ctx.set_error(output.len(), e.to_string()); + output.push_null(); + } + } + }, + ), ); registry.register_passthrough_nullable_1_arg::( @@ -447,31 +518,24 @@ pub fn register(registry: &mut FunctionRegistry) { if geohash.len() > 12 { ctx.set_error( builder.len(), - "Currently the precision only implement within 12 digits!", + "Currently the precision only implement within 12 digits!".to_string(), ); builder.commit_row(); return; } - let geo: geo_types::Geometry = match decode_bbox(geohash) { - Ok(rect) => rect.into(), + match decode_bbox(geohash) { + Ok(rect) => { + let geo: Geometry = rect.into(); + match geo_to_ewkb(geo, None) { + Ok(binary) => builder.put_slice(binary.as_slice()), + Err(e) => ctx.set_error(builder.len(), e.to_string()), + } + } Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; + ctx.set_error(builder.len(), e.to_string()); } }; - - match geo.to_wkb(CoordDimensions::xy()) { - Ok(binary) => builder.put_slice(binary.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; builder.commit_row(); }), ); @@ -489,41 +553,31 @@ pub fn register(registry: &mut FunctionRegistry) { if geohash.len() > 12 { ctx.set_error( builder.len(), - ErrorCode::GeometryError( - "Currently the precision only implement within 12 digits!".to_string(), - ) - .to_string(), + "Currently the precision only implement within 12 digits!".to_string(), ); builder.commit_row(); return; } - let geo: geo_types::Geometry = match decode_bbox(geohash) { - Ok(rect) => Point::from(rect.center()).into(), + match decode_bbox(geohash) { + Ok(rect) => { + let geo: Geometry = Point::from(rect.center()).into(); + match geo_to_ewkb(geo, None) { + Ok(binary) => builder.put_slice(binary.as_slice()), + Err(e) => ctx.set_error(builder.len(), e.to_string()), + } + } Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; + ctx.set_error(builder.len(), e.to_string()); } }; - - match geo.to_wkb(CoordDimensions::xy()) { - Ok(binary) => builder.put_slice(binary.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; builder.commit_row(); }), ); registry.register_passthrough_nullable_2_arg::, NumberType, GeometryType, _, _>( "st_makegeompoint", - |_,_, _| FunctionDomain::Full, + |_,_, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::, NumberType, GeometryType>(|longitude, latitude, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { @@ -531,8 +585,14 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - let geom = geo::Geometry::from(Point::new(longitude.0, latitude.0)); - builder.put_slice(geom.to_wkb(CoordDimensions::xy()).unwrap().as_slice()); + let geo = Geometry::from(Point::new(longitude.0, latitude.0)); + match geo_to_ewkb(geo, None) { + Ok(binary) => builder.put_slice(binary.as_slice()), + Err(e) => ctx.set_error( + builder.len(), + e.to_string(), + ), + } builder.commit_row(); }) ); @@ -540,7 +600,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "st_makepolygon", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|wkb, builder, ctx| { + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -548,35 +608,36 @@ pub fn register(registry: &mut FunctionRegistry) { } } - let polygon = Wkb(wkb) - .to_geo() - .unwrap() - .try_into() - .map_err(|e: geo_types::Error| ErrorCode::GeometryError(e.to_string())) - .and_then(|line_string: LineString| { + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, srid)| match geo { + Geometry::LineString(line_string) => { let points = line_string.into_points(); if points.len() < 4 { - Err(ErrorCode::GeometryError( - "Input lines must have at least 4 points!", - )) + Err(ErrorCode::GeometryError(format!( + "Input lines must have at least 4 points, but got {}", + points.len() + ))) } else if points.last() != points.first() { Err(ErrorCode::GeometryError( - "The first and last elements are not equal.", + "The first point and last point are not equal".to_string(), )) } else { - geo_types::Geometry::from(Polygon::new(LineString::from(points), vec![])) - .to_wkb(CoordDimensions::xy()) - .map_err(|e| ErrorCode::GeometryError(e.to_string())) + let polygon = Polygon::new(LineString::from(points), vec![]); + let geo: Geometry = polygon.into(); + geo_to_ewkb(geo, srid) } - }); - - match polygon { - Ok(p) => builder.put_slice(p.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; + } + _ => Err(ErrorCode::GeometryError(format!( + "Type {} is not supported as argument to ST_MAKEPOLYGON", + geometry_type_name(&geo) + ))), + }) { + Ok(binary) => { + builder.put_slice(binary.as_slice()); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + } + } builder.commit_row(); }), ); @@ -585,55 +646,64 @@ pub fn register(registry: &mut FunctionRegistry) { "st_makeline", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |left_ewkb, right_ewkb, builder, ctx| { + |l_ewkb, r_ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); return; } } - let left_geo = Ewkb(left_ewkb); - let right_geo = Ewkb(right_ewkb); - let geos = &vec![left_geo.to_geos().unwrap(), right_geo.to_geos().unwrap()]; - // check srid - let srid = match get_shared_srid(geos) { - Ok(srid) => srid, - Err(e) => { - ctx.set_error(builder.len(), ErrorCode::GeometryError(e).to_string()); - builder.commit_row(); - return; - } - }; - let mut coords: Vec = vec![]; - for geometry in geos.iter() { - let g : geo_types::Geometry = geometry.try_into().unwrap(); - match g { - geo_types::Geometry::Point(point) => { - coords.push(point.0); - }, - geo_types::Geometry::LineString(line)=> { - coords.append(&mut line.clone().into_inner()); - }, - geo_types::Geometry::MultiPoint(multi_point)=> { - for point in multi_point.into_iter() { - coords.push(point.0); - } - }, - _ => { + match (ewkb_to_geo(&mut Ewkb(l_ewkb)), ewkb_to_geo(&mut Ewkb(r_ewkb))) { + (Ok((l_geo, l_srid)), Ok((r_geo, r_srid))) => { + if !l_srid.eq(&r_srid) { ctx.set_error( builder.len(), - ErrorCode::GeometryError("Geometry expression must be a Point, MultiPoint, or LineString.").to_string(), + format!("Incompatible SRID: {} and {}", l_srid.unwrap_or_default(), r_srid.unwrap_or_default()) ); builder.commit_row(); return; } + let geos = [l_geo, r_geo]; + let mut coords: Vec = vec![]; + for geo in geos.iter() { + match geo { + Geometry::Point(point) => { + coords.push(point.0); + }, + Geometry::LineString(line)=> { + coords.append(&mut line.clone().into_inner()); + }, + Geometry::MultiPoint(multi_point)=> { + for point in multi_point.into_iter() { + coords.push(point.0); + } + }, + _ => { + ctx.set_error( + builder.len(), + format!("Geometry expression must be a Point, MultiPoint, or LineString, but got {}", + geometry_type_name(geo) + ) + ); + builder.commit_row(); + return; + } + } + } + + let geo = Geometry::from(LineString::new(coords)); + match geo_to_ewkb(geo, l_srid) { + Ok(data) => builder.put_slice(data.as_slice()), + Err(e) => ctx.set_error(builder.len(), e.to_string()), + } + } + (Err(e), _) | (_, Err(e)) => { + ctx.set_error( + builder.len(), + e.to_string(), + ); } - } - let geom = geo::Geometry::from(LineString::new(coords)); - match geom.to_ewkb(CoordDimensions::xy(), srid) { - Ok(data) => builder.put_slice(data.as_slice()), - Err(e) => ctx.set_error(builder.len(), e.to_string()), } builder.commit_row(); }, @@ -643,7 +713,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "st_geohash", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -651,13 +721,10 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match point_to_geohash(geometry, None) { + match point_to_geohash(ewkb, None) { Ok(hash) => builder.put_str(&hash), Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } }; builder.commit_row(); @@ -688,10 +755,7 @@ pub fn register(registry: &mut FunctionRegistry) { match point_to_geohash(geometry, Some(precision)) { Ok(hash) => builder.put_str(&hash), Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } }; builder.commit_row(); @@ -702,7 +766,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "st_geometryfromwkb", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|str, builder, ctx| { + vectorize_with_builder_1_arg::(|s, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -710,38 +774,19 @@ pub fn register(registry: &mut FunctionRegistry) { } } - let ewkb = match hex::decode(str) { - Ok(binary) => Ewkb(binary), + let binary = match hex::decode(s) { + Ok(binary) => binary, Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; - } - }; - let geos = match ewkb.to_geos() { - Ok(geos) => geos, - Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); builder.commit_row(); return; } }; - match geos.to_ewkb(CoordDimensions::xy(), geos.srid()) { - Ok(ewkb) => { - builder.put_slice(ewkb.as_slice()); - } + match ewkb_to_geo(&mut Ewkb(binary)).and_then(|(geo, srid)| geo_to_ewkb(geo, srid)) { + Ok(ewkb) => builder.put_slice(ewkb.as_slice()), Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } } builder.commit_row(); @@ -758,28 +803,11 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - let ewkb = Ewkb(binary); - let geos = match ewkb.to_geos() { - Ok(geos) => geos, - Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; - } - }; - match geos.to_ewkb(CoordDimensions::xy(), geos.srid()) { - Ok(ewkb) => { - builder.put_slice(ewkb.as_slice()); - } + match ewkb_to_geo(&mut Ewkb(binary)).and_then(|(geo, srid)| geo_to_ewkb(geo, srid)) { + Ok(ewkb) => builder.put_slice(ewkb.as_slice()), Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } } builder.commit_row(); @@ -790,7 +818,7 @@ pub fn register(registry: &mut FunctionRegistry) { "st_geometryfromwkb", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |str, srid, builder, ctx| { + |s, srid, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -798,27 +826,21 @@ pub fn register(registry: &mut FunctionRegistry) { } } - let binary = match hex::decode(str) { + let binary = match hex::decode(s) { Ok(binary) => binary, Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); builder.commit_row(); return; } }; - match Ewkb(binary).to_ewkb(CoordDimensions::xy(), Some(srid)) { - Ok(ewkb) => { - builder.put_slice(ewkb.as_slice()); - } + match ewkb_to_geo(&mut Ewkb(binary)) + .and_then(|(geo, _)| geo_to_ewkb(geo, Some(srid))) + { + Ok(ewkb) => builder.put_slice(ewkb.as_slice()), Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } } builder.commit_row(); @@ -831,34 +853,19 @@ pub fn register(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |binary, srid, builder, ctx| { - if let Some(validity) = &ctx.validity { - if !validity.get_bit(builder.len()) { - builder.commit_row(); - return; - } - } - let ewkb = Ewkb(binary); - let geos = match ewkb.to_geos() { - Ok(geos) => geos, - Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + if let Some(validity) = &ctx.validity { + if !validity.get_bit(builder.len()) { builder.commit_row(); return; } - }; + } - match geos.to_ewkb(CoordDimensions::xy(), Some(srid)) { - Ok(ewkb) => { - builder.put_slice(ewkb.as_slice()); - } + match ewkb_to_geo(&mut Ewkb(binary)) + .and_then(|(geo, _)| geo_to_ewkb(geo, Some(srid))) + { + Ok(ewkb) => builder.put_slice(ewkb.as_slice()), Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } } builder.commit_row(); @@ -877,12 +884,9 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match parse_to_ewkb(wkt, None) { + match geometry_from_ewkt(wkt, None) { Ok(data) => builder.put_slice(data.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), + Err(e) => ctx.set_error(builder.len(), e.to_string()), } builder.commit_row(); }), @@ -899,58 +903,19 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - match parse_to_ewkb(wkt, Some(srid)) { + match geometry_from_ewkt(wkt, Some(srid)) { Ok(data) => builder.put_slice(data.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), + Err(e) => ctx.set_error(builder.len(), e.to_string()), } builder.commit_row(); }, ), ); - registry.register_passthrough_nullable_1_arg::( - "st_startpoint", - |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { - if let Some(validity) = &ctx.validity { - if !validity.get_bit(builder.len()) { - builder.commit_row(); - return; - } - } - - let point = match >::try_into( - Ewkb(geometry).to_geo().unwrap(), - ) { - Ok(line_string) => line_string.points().next().unwrap(), - Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; - } - }; - - match geo_types::Geometry::from(point).to_wkb(CoordDimensions::xy()) { - Ok(binary) => builder.put_slice(binary.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; - builder.commit_row(); - }), - ); - registry.register_passthrough_nullable_1_arg::, _, _>( "st_length", - |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::>(|geometry, builder, ctx| { + |_, _| FunctionDomain::MayThrow, + vectorize_with_builder_1_arg::>(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push(F64::from(0_f64)); @@ -958,41 +923,48 @@ pub fn register(registry: &mut FunctionRegistry) { } } - let g: geo_types::Geometry = Ewkb(geometry).to_geos().unwrap().try_into().unwrap(); - let mut distance = 0f64; - match g { - geo_types::Geometry::LineString(lines) => { - for line in lines.lines() { - distance += line.euclidean_length(); - } - } - geo_types::Geometry::MultiLineString(multi_lines) => { - for line_string in multi_lines.0 { - for line in line_string.lines() { - distance += line.euclidean_length(); - } - } - } - geo_types::Geometry::GeometryCollection(geom_c) => { - for geometry in geom_c.0 { - if let geo::Geometry::LineString(line_string) = geometry { - for line in line_string.lines() { + match Ewkb(ewkb).to_geo() { + Ok(geo) => { + let mut distance = 0f64; + match geo { + Geometry::LineString(lines) => { + for line in lines.lines() { distance += line.euclidean_length(); } } + Geometry::MultiLineString(multi_lines) => { + for line_string in multi_lines.0 { + for line in line_string.lines() { + distance += line.euclidean_length(); + } + } + } + Geometry::GeometryCollection(geom_c) => { + for geometry in geom_c.0 { + if let Geometry::LineString(line_string) = geometry { + for line in line_string.lines() { + distance += line.euclidean_length(); + } + } + } + } + _ => {} } + let distance = (distance * 1_000_000_000_f64).round() / 1_000_000_000_f64; + builder.push(distance.into()); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + builder.push(F64::from(0_f64)); } - _ => {} } - - builder.push(format!("{:.9}", distance).parse().unwrap()); }), ); registry.register_passthrough_nullable_1_arg::, _, _>( "st_x", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::>(|geometry, builder, ctx| { + vectorize_with_builder_1_arg::>(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push(F64::from(0_f64)); @@ -1000,25 +972,28 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match >::try_into( - Ewkb(geometry).to_geo().unwrap(), - ) { - Ok(point) => builder.push(F64::from(AsPrimitive::::as_(point.x()))), + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, _)| match geo { + Geometry::Point(point) => Ok(F64::from(AsPrimitive::::as_(point.x()))), + _ => Err(ErrorCode::GeometryError(format!( + "Type {} is not supported as argument to ST_X", + geometry_type_name(&geo) + ))), + }) { + Ok(x) => { + builder.push(x); + } Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); builder.push(F64::from(0_f64)); } - }; + } }), ); registry.register_passthrough_nullable_1_arg::, _, _>( "st_y", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::>(|geometry, builder, ctx| { + vectorize_with_builder_1_arg::>(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push(F64::from(0_f64)); @@ -1026,38 +1001,46 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match >::try_into( - Ewkb(geometry).to_geo().unwrap(), - ) { - Ok(point) => builder.push(F64::from(AsPrimitive::::as_(point.y()))), + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, _)| match geo { + Geometry::Point(point) => Ok(F64::from(AsPrimitive::::as_(point.y()))), + _ => Err(ErrorCode::GeometryError(format!( + "Type {} is not supported as argument to ST_Y", + geometry_type_name(&geo) + ))), + }) { + Ok(x) => { + builder.push(x); + } Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); builder.push(F64::from(0_f64)); } - }; + } }), ); registry.register_passthrough_nullable_2_arg::( "st_setsrid", - |_, _, _| FunctionDomain::Full, + |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |geometry, srid, output, ctx| { + |ewkb, srid, builder, ctx| { if let Some(validity) = &ctx.validity { - if !validity.get_bit(output.len()) { - output.commit_row(); + if !validity.get_bit(builder.len()) { + builder.commit_row(); return; } } - let ewkb = Ewkb(geometry); - let mut ggeom = ewkb.to_geos().unwrap(); - ggeom.set_srid(srid as usize); - let geo = ggeom.to_ewkb(ggeom.dims(), ggeom.srid()).unwrap(); - output.put_slice(&geo); - output.commit_row(); + + match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, _)| geo_to_ewkb(geo, Some(srid))) + { + Ok(ewkb) => { + builder.put_slice(ewkb.as_slice()); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + } + } + builder.commit_row(); }, ), ); @@ -1065,7 +1048,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "st_srid", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|geometry, output, ctx| { + vectorize_with_builder_1_arg::(|ewkb, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { output.push(0); @@ -1073,54 +1056,66 @@ pub fn register(registry: &mut FunctionRegistry) { } } - output.push(Ewkb(geometry).to_geos().unwrap().srid().unwrap_or(4326)); + let srid = read_srid(&mut Ewkb(ewkb)).unwrap_or(4326); + output.push(srid); }), ); registry.register_combine_nullable_1_arg::, _, _>( "st_xmax", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>>( - |geometry, builder, ctx| { + |ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push_null(); return; } } - - match st_extreme(&Ewkb(geometry).to_geo().unwrap(), Axis::X, Extremum::Max) { - None => builder.push_null(), - Some(x_max) => builder.push(F64::from(AsPrimitive::::as_(x_max))), - }; + match Ewkb(ewkb).to_geo() { + Ok(geo) => match st_extreme(&geo, Axis::X, Extremum::Max) { + Some(x_max) => builder.push(F64::from(AsPrimitive::::as_(x_max))), + None => builder.push_null(), + }, + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + builder.push_null(); + } + } }, ), ); registry.register_combine_nullable_1_arg::, _, _>( "st_xmin", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>>( - |geometry, builder, ctx| { + |ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push_null(); return; } } - match st_extreme(&Ewkb(geometry).to_geo().unwrap(), Axis::X, Extremum::Min) { - None => builder.push_null(), - Some(x_min) => builder.push(F64::from(AsPrimitive::::as_(x_min))), - }; + match Ewkb(ewkb).to_geo() { + Ok(geo) => match st_extreme(&geo, Axis::X, Extremum::Min) { + Some(x_max) => builder.push(F64::from(AsPrimitive::::as_(x_max))), + None => builder.push_null(), + }, + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + builder.push_null(); + } + } }, ), ); registry.register_combine_nullable_1_arg::, _, _>( "st_ymax", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>>( - |geometry, builder, ctx| { + |ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push_null(); @@ -1128,19 +1123,25 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match st_extreme(&Ewkb(geometry).to_geo().unwrap(), Axis::Y, Extremum::Max) { - None => builder.push_null(), - Some(y_max) => builder.push(F64::from(AsPrimitive::::as_(y_max))), - }; + match Ewkb(ewkb).to_geo() { + Ok(geo) => match st_extreme(&geo, Axis::Y, Extremum::Max) { + Some(x_max) => builder.push(F64::from(AsPrimitive::::as_(x_max))), + None => builder.push_null(), + }, + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + builder.push_null(); + } + } }, ), ); registry.register_combine_nullable_1_arg::, _, _>( "st_ymin", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>>( - |geometry, builder, ctx| { + |ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push_null(); @@ -1148,32 +1149,48 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match st_extreme(&Ewkb(geometry).to_geo().unwrap(), Axis::Y, Extremum::Min) { - None => builder.push_null(), - Some(y_min) => builder.push(F64::from(AsPrimitive::::as_(y_min))), - }; + match Ewkb(ewkb).to_geo() { + Ok(geo) => match st_extreme(&geo, Axis::Y, Extremum::Min) { + Some(x_max) => builder.push(F64::from(AsPrimitive::::as_(x_max))), + None => builder.push_null(), + }, + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + builder.push_null(); + } + } }, ), ); registry.register_passthrough_nullable_1_arg::( "st_npoints", - |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|geometry, builder, ctx| { + |_, _| FunctionDomain::MayThrow, + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.push(0); return; } } - builder.push(count_points(&Ewkb(geometry).to_geo().unwrap()) as u32); + + match Ewkb(ewkb).to_geo() { + Ok(geo) => { + let npoints = count_points(&geo); + builder.push(npoints as u32); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + builder.push(0); + } + } }), ); registry.register_passthrough_nullable_1_arg::( "to_string", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|b, builder, ctx| { + vectorize_with_builder_1_arg::(|ewkb, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); @@ -1181,13 +1198,10 @@ pub fn register(registry: &mut FunctionRegistry) { } } - match geometry_format(Ewkb(b), ctx.func_ctx.geometry_output_format) { + match geometry_format(Ewkb(ewkb), ctx.func_ctx.geometry_output_format) { Ok(data) => builder.put_str(&data), Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); + ctx.set_error(builder.len(), e.to_string()); } } builder.commit_row(); @@ -1197,19 +1211,16 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "to_geometry", |_, _| FunctionDomain::MayThrow, - vectorize_with_builder_1_arg::(|str, builder, ctx| { + vectorize_with_builder_1_arg::(|s, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); return; } } - match str_to_geometry_impl(str, None) { + match geometry_from_str(s, None) { Ok(data) => builder.put_slice(data.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), + Err(e) => ctx.set_error(builder.len(), e.to_string()), } builder.commit_row(); }), @@ -1219,19 +1230,16 @@ pub fn register(registry: &mut FunctionRegistry) { "to_geometry", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |str, srid, builder, ctx| { + |s, srid, builder, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(builder.len()) { builder.commit_row(); return; } } - match str_to_geometry_impl(str, Some(srid)) { + match geometry_from_str(s, Some(srid)) { Ok(data) => builder.put_slice(data.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), + Err(e) => ctx.set_error(builder.len(), e.to_string()), } builder.commit_row(); }, @@ -1248,25 +1256,12 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - let ewkb = Ewkb(binary); - let geos = match ewkb.to_geos() { - Ok(geos) => geos, - Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; - } - }; - match geos.to_ewkb(CoordDimensions::xy(), geos.srid()) { + match ewkb_to_geo(&mut Ewkb(binary)).and_then(|(geo, srid)| geo_to_ewkb(geo, srid)) { Ok(ewkb) => builder.put_slice(ewkb.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + } } builder.commit_row(); }), @@ -1284,25 +1279,14 @@ pub fn register(registry: &mut FunctionRegistry) { } } - let geo = match Ewkb(binary).to_geo() { - Ok(geo) => geo, + match ewkb_to_geo(&mut Ewkb(binary)) + .and_then(|(geo, _)| geo_to_ewkb(geo, Some(srid))) + { + Ok(ewkb) => builder.put_slice(ewkb.as_slice()), Err(e) => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ); - builder.commit_row(); - return; + ctx.set_error(builder.len(), e.to_string()); } - }; - - match geo.to_ewkb(CoordDimensions::xy(), Some(srid)) { - Ok(ewkb) => builder.put_slice(ewkb.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), - }; + } builder.commit_row(); }, ), @@ -1320,10 +1304,7 @@ pub fn register(registry: &mut FunctionRegistry) { } match json_to_geometry_impl(json, None) { Ok(data) => builder.put_slice(data.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), + Err(e) => ctx.set_error(builder.len(), e.to_string()), } builder.commit_row(); }), @@ -1342,10 +1323,7 @@ pub fn register(registry: &mut FunctionRegistry) { } match json_to_geometry_impl(json, Some(srid)) { Ok(data) => builder.put_slice(data.as_slice()), - Err(e) => ctx.set_error( - builder.len(), - ErrorCode::GeometryError(e.to_string()).to_string(), - ), + Err(e) => ctx.set_error(builder.len(), e.to_string()), } builder.commit_row(); }, @@ -1365,9 +1343,7 @@ pub fn register(registry: &mut FunctionRegistry) { } match json_to_geometry_impl(json, None) { Ok(data) => { - output.validity.push(true); - output.builder.put_slice(data.as_slice()); - output.builder.commit_row(); + output.push(data.as_slice()); } Err(_) => output.push_null(), } @@ -1388,9 +1364,7 @@ pub fn register(registry: &mut FunctionRegistry) { } match json_to_geometry_impl(json, Some(srid)) { Ok(data) => { - output.validity.push(true); - output.builder.put_slice(data.as_slice()); - output.builder.commit_row(); + output.push(data.as_slice()); } Err(_) => output.push_null(), } @@ -1401,42 +1375,36 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_1_arg::( "try_to_geometry", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::>( - |str, output, ctx| { - if let Some(validity) = &ctx.validity { - if !validity.get_bit(output.len()) { - output.push_null(); - return; - } + vectorize_with_builder_1_arg::>(|s, output, ctx| { + if let Some(validity) = &ctx.validity { + if !validity.get_bit(output.len()) { + output.push_null(); + return; } - match str_to_geometry_impl(str, None) { - Ok(data) => { - output.validity.push(true); - output.builder.put_slice(data.as_slice()); - output.builder.commit_row(); - } - Err(_) => output.push_null(), + } + match geometry_from_str(s, None) { + Ok(data) => { + output.push(data.as_slice()); } - }, - ), + Err(_) => output.push_null(), + } + }), ); registry.register_combine_nullable_2_arg::( "try_to_geometry", |_, _, _| FunctionDomain::Full, vectorize_with_builder_2_arg::>( - |str, srid, output, ctx| { + |s, srid, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { output.push_null(); return; } } - match str_to_geometry_impl(str, Some(srid)) { + match geometry_from_str(s, Some(srid)) { Ok(data) => { - output.validity.push(true); - output.builder.put_slice(data.as_slice()); - output.builder.commit_row(); + output.push(data.as_slice()); } Err(_) => output.push_null(), } @@ -1455,21 +1423,14 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - let ewkb = Ewkb(binary); - let geos = match ewkb.to_geos() { - Ok(geos) => geos, - Err(_) => { - output.push_null(); - return; - } - }; - match geos.to_ewkb(CoordDimensions::xy(), geos.srid()) { - Ok(ewkb) => { - output.validity.push(true); - output.builder.put_slice(ewkb.as_slice()); - output.builder.commit_row(); - } + match ewkb_to_geo(&mut Ewkb(binary)) { + Ok((geo, srid)) => match geo.to_ewkb(CoordDimensions::xy(), srid) { + Ok(ewkb) => { + output.push(ewkb.as_slice()); + } + Err(_) => output.push_null(), + }, Err(_) => output.push_null(), } }, @@ -1487,20 +1448,14 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - let geo = match Ewkb(binary).to_geo() { - Ok(geo) => geo, - Err(_) => { - output.push_null(); - return; - } - }; - match geo.to_ewkb(CoordDimensions::xy(), Some(srid)) { - Ok(ewkb) => { - output.validity.push(true); - output.builder.put_slice(ewkb.as_slice()); - output.builder.commit_row(); - } + match Ewkb(binary).to_geo() { + Ok(geo) => match geo.to_ewkb(CoordDimensions::xy(), Some(srid)) { + Ok(ewkb) => { + output.push(ewkb.as_slice()); + } + Err(_) => output.push_null(), + }, Err(_) => output.push_null(), } }, @@ -1521,19 +1476,16 @@ pub fn register(registry: &mut FunctionRegistry) { // All representations of the geo types supported by crates under the GeoRust organization, have not implemented srid(). // Currently, the srid() of all types returns the default value `None`, so we need to parse it manually here. - let from_srid = match Ewkb(original).to_geos().unwrap().srid() { - Some(srid) => srid, - _ => { - ctx.set_error( - builder.len(), - ErrorCode::GeometryError("input geometry must has the correct SRID") - .to_string(), - ); - builder.commit_row(); - return; - } - }; - + let from_srid = read_srid(&mut Ewkb(original)); + if from_srid.is_none() { + ctx.set_error( + builder.len(), + "input geometry must has the correct SRID".to_string(), + ); + builder.commit_row(); + return; + } + let from_srid = from_srid.unwrap(); match st_transform_impl(original, from_srid, to_srid) { Ok(data) => { builder.put_slice(data.as_slice()); @@ -1542,7 +1494,6 @@ pub fn register(registry: &mut FunctionRegistry) { ctx.set_error(builder.len(), e.to_string()); } } - builder.commit_row(); }, ), @@ -1568,7 +1519,6 @@ pub fn register(registry: &mut FunctionRegistry) { ctx.set_error(builder.len(), e.to_string()); } } - builder.commit_row(); }, ), @@ -1614,41 +1564,6 @@ fn st_transform_impl( }) } -#[inline] -fn get_shared_srid(geometries: &Vec) -> Result, String> { - let mut srid: Option = None; - let mut error_srid: String = String::new(); - let check_srid = geometries.windows(2).all(|w| { - let v1 = w[0].srid(); - let v2 = w[1].srid(); - match v1.eq(&v2) { - true => { - srid = v1; - true - } - false => { - error_srid = "Srid does not match!".to_string(); - false - } - } - }); - match check_srid { - true => Ok(srid), - false => Err(error_srid.clone()), - } -} - -pub fn ewkb_to_json(buf: &[u8]) -> databend_common_exception::Result { - Ewkb(buf) - .to_geos() - .map_err(|e| ErrorCode::GeometryError(e.to_string())) - .and_then(|geos| { - geos.to_json() - .map_err(|e| ErrorCode::GeometryError(e.to_string())) - .map(|json: String| json) - }) -} - /// The argument str must be a string expression that represents a valid geometric object in one of the following formats: /// /// WKT (well-known text). @@ -1656,45 +1571,8 @@ pub fn ewkb_to_json(buf: &[u8]) -> databend_common_exception::Result { /// EWKT (extended well-known text). /// EWKB (extended well-known binary) in hexadecimal format (without a leading 0x). /// GEOJSON -fn str_to_geometry_impl( - str: &str, - srid: Option, -) -> databend_common_exception::Result> { - let geo_type = match parse_to_subtype(str.as_bytes()) { - Ok(geo_types) => geo_types, - Err(e) => return Err(ErrorCode::GeometryError(e.to_string())), - }; - let ewkb = match geo_type { - GeometryDataType::WKT | GeometryDataType::EWKT => parse_to_ewkb(str, srid), - GeometryDataType::GEOJSON => GeoJson(str) - .to_ewkb(CoordDimensions::xy(), srid) - .map_err(|e| ErrorCode::GeometryError(e.to_string())), - GeometryDataType::WKB | GeometryDataType::EWKB => { - let ewkb = match hex::decode(str) { - Ok(binary) => Ewkb(binary), - Err(e) => return Err(ErrorCode::GeometryError(e.to_string())), - }; - - let geos = match ewkb.to_geos() { - Ok(geos) => geos, - Err(e) => return Err(ErrorCode::GeometryError(e.to_string())), - }; - - geos.to_ewkb(CoordDimensions::xy(), srid.or(geos.srid())) - .map_err(|e| ErrorCode::GeometryError(e.to_string())) - } - }; - - match ewkb { - Ok(data) => Ok(data), - Err(e) => Err(ErrorCode::GeometryError(e.to_string())), - } -} -fn json_to_geometry_impl( - binary: &[u8], - srid: Option, -) -> databend_common_exception::Result> { +fn json_to_geometry_impl(binary: &[u8], srid: Option) -> Result> { let s = to_string(binary); let json = GeoJson(s.as_str()); match json.to_ewkb(CoordDimensions::xy(), srid) { @@ -1703,10 +1581,7 @@ fn json_to_geometry_impl( } } -fn point_to_geohash( - geometry: &[u8], - precision: Option, -) -> databend_common_exception::Result { +fn point_to_geohash(geometry: &[u8], precision: Option) -> Result { let point = match Ewkb(geometry).to_geo() { Ok(geo) => Point::try_from(geo), Err(e) => return Err(ErrorCode::GeometryError(e.to_string())), @@ -1722,20 +1597,19 @@ fn point_to_geohash( } } -fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremum) -> Option { +fn st_extreme(geometry: &Geometry, axis: Axis, extremum: Extremum) -> Option { match geometry { - geo_types::Geometry::Point(point) => { + Geometry::Point(point) => { let coord = match axis { Axis::X => point.x(), Axis::Y => point.y(), }; Some(coord) } - geo_types::Geometry::MultiPoint(multi_point) => { + Geometry::MultiPoint(multi_point) => { let mut extreme_coord: Option = None; for point in multi_point { - if let Some(coord) = st_extreme(&geo_types::Geometry::Point(*point), axis, extremum) - { + if let Some(coord) = st_extreme(&Geometry::Point(*point), axis, extremum) { extreme_coord = match extreme_coord { Some(existing) => match extremum { Extremum::Max => Some(existing.max(coord)), @@ -1747,7 +1621,7 @@ fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremu } extreme_coord } - geo_types::Geometry::Line(line) => { + Geometry::Line(line) => { let bounding_rect = line.bounding_rect(); let coord = match axis { Axis::X => match extremum { @@ -1761,14 +1635,11 @@ fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremu }; Some(coord) } - geo_types::Geometry::MultiLineString(multi_line) => { + Geometry::MultiLineString(multi_line) => { let mut extreme_coord: Option = None; for line in multi_line { - if let Some(coord) = st_extreme( - &geo_types::Geometry::LineString(line.clone()), - axis, - extremum, - ) { + if let Some(coord) = st_extreme(&Geometry::LineString(line.clone()), axis, extremum) + { extreme_coord = match extreme_coord { Some(existing) => match extremum { Extremum::Max => Some(existing.max(coord)), @@ -1780,7 +1651,7 @@ fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremu } extreme_coord } - geo_types::Geometry::Polygon(polygon) => { + Geometry::Polygon(polygon) => { let bounding_rect = polygon.bounding_rect().unwrap(); let coord = match axis { Axis::X => match extremum { @@ -1794,14 +1665,11 @@ fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremu }; Some(coord) } - geo_types::Geometry::MultiPolygon(multi_polygon) => { + Geometry::MultiPolygon(multi_polygon) => { let mut extreme_coord: Option = None; for polygon in multi_polygon { - if let Some(coord) = st_extreme( - &geo_types::Geometry::Polygon(polygon.clone()), - axis, - extremum, - ) { + if let Some(coord) = st_extreme(&Geometry::Polygon(polygon.clone()), axis, extremum) + { extreme_coord = match extreme_coord { Some(existing) => match extremum { Extremum::Max => Some(existing.max(coord)), @@ -1813,7 +1681,7 @@ fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremu } extreme_coord } - geo_types::Geometry::GeometryCollection(geometry_collection) => { + Geometry::GeometryCollection(geometry_collection) => { let mut extreme_coord: Option = None; for geometry in geometry_collection { if let Some(coord) = st_extreme(geometry, axis, extremum) { @@ -1828,19 +1696,17 @@ fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremu } extreme_coord } - geo_types::Geometry::LineString(line_string) => { - line_string.bounding_rect().map(|rect| match axis { - Axis::X => match extremum { - Extremum::Max => rect.max().x, - Extremum::Min => rect.min().x, - }, - Axis::Y => match extremum { - Extremum::Max => rect.max().y, - Extremum::Min => rect.min().y, - }, - }) - } - geo_types::Geometry::Rect(rect) => { + Geometry::LineString(line_string) => line_string.bounding_rect().map(|rect| match axis { + Axis::X => match extremum { + Extremum::Max => rect.max().x, + Extremum::Min => rect.min().x, + }, + Axis::Y => match extremum { + Extremum::Max => rect.max().y, + Extremum::Min => rect.min().y, + }, + }), + Geometry::Rect(rect) => { let coord = match axis { Axis::X => match extremum { Extremum::Max => rect.max().x, @@ -1853,7 +1719,7 @@ fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremu }; Some(coord) } - geo_types::Geometry::Triangle(triangle) => { + Geometry::Triangle(triangle) => { let bounding_rect = triangle.bounding_rect(); let coord = match axis { Axis::X => match extremum { @@ -1870,12 +1736,12 @@ fn st_extreme(geometry: &geo_types::Geometry, axis: Axis, extremum: Extremu } } -fn count_points(geom: &geo_types::Geometry) -> usize { +fn count_points(geom: &Geometry) -> usize { match geom { - geo_types::Geometry::Point(_) => 1, - geo_types::Geometry::Line(_) => 2, - geo_types::Geometry::LineString(line_string) => line_string.0.len(), - geo_types::Geometry::Polygon(polygon) => { + Geometry::Point(_) => 1, + Geometry::Line(_) => 2, + Geometry::LineString(line_string) => line_string.0.len(), + Geometry::Polygon(polygon) => { polygon.exterior().0.len() + polygon .interiors() @@ -1883,46 +1749,46 @@ fn count_points(geom: &geo_types::Geometry) -> usize { .map(|line_string| line_string.0.len()) .sum::() } - geo_types::Geometry::MultiPoint(multi_point) => multi_point.0.len(), - geo_types::Geometry::MultiLineString(multi_line_string) => multi_line_string + Geometry::MultiPoint(multi_point) => multi_point.0.len(), + Geometry::MultiLineString(multi_line_string) => multi_line_string .0 .iter() .map(|line_string| line_string.0.len()) .sum::(), - geo_types::Geometry::MultiPolygon(multi_polygon) => multi_polygon + Geometry::MultiPolygon(multi_polygon) => multi_polygon .0 .iter() - .map(|polygon| count_points(&geo_types::Geometry::Polygon(polygon.clone()))) + .map(|polygon| count_points(&Geometry::Polygon(polygon.clone()))) .sum::(), - geo_types::Geometry::GeometryCollection(geometry_collection) => geometry_collection + Geometry::GeometryCollection(geometry_collection) => geometry_collection .0 .iter() .map(count_points) .sum::(), - geo_types::Geometry::Rect(_) => 5, - geo_types::Geometry::Triangle(_) => 4, + Geometry::Rect(_) => 5, + Geometry::Triangle(_) => 4, } } /// The last three decimal places of the f64 type are inconsistent between aarch64 and x86 platforms, /// causing unit test results to fail. We will only retain six decimal places. -fn round_geometry_coordinates(geom: geo::Geometry) -> geo::Geometry { +fn round_geometry_coordinates(geom: Geometry) -> Geometry { fn round_coordinate(coord: f64) -> f64 { (coord * 1_000_000.0).round() / 1_000_000.0 } match geom { - geo::Geometry::Point(point) => geo::Geometry::Point(Point::new( + Geometry::Point(point) => Geometry::Point(Point::new( round_coordinate(point.x()), round_coordinate(point.y()), )), - geo::Geometry::LineString(linestring) => geo::Geometry::LineString( + Geometry::LineString(linestring) => Geometry::LineString( linestring .into_iter() .map(|coord| coord!(x:round_coordinate(coord.x), y:round_coordinate(coord.y))) .collect(), ), - geo::Geometry::Polygon(polygon) => { + Geometry::Polygon(polygon) => { let outer_ring = polygon.exterior(); let mut rounded_inner_rings = Vec::new(); @@ -1946,15 +1812,15 @@ fn round_geometry_coordinates(geom: geo::Geometry) -> geo::Geometry { rounded_inner_rings, ); - geo::Geometry::Polygon(rounded_polygon) + Geometry::Polygon(rounded_polygon) } - geo::Geometry::MultiPoint(multipoint) => geo::Geometry::MultiPoint( + Geometry::MultiPoint(multipoint) => Geometry::MultiPoint( multipoint .into_iter() .map(|point| Point::new(round_coordinate(point.x()), round_coordinate(point.y()))) .collect(), ), - geo::Geometry::MultiLineString(multilinestring) => { + Geometry::MultiLineString(multilinestring) => { let rounded_lines: Vec> = multilinestring .into_iter() .map(|linestring| { @@ -1967,9 +1833,9 @@ fn round_geometry_coordinates(geom: geo::Geometry) -> geo::Geometry { }) .collect(); - geo::Geometry::MultiLineString(geo::MultiLineString::new(rounded_lines)) + Geometry::MultiLineString(MultiLineString::new(rounded_lines)) } - geo::Geometry::MultiPolygon(multipolygon) => { + Geometry::MultiPolygon(multipolygon) => { let rounded_polygons: Vec> = multipolygon .into_iter() .map(|polygon| { @@ -1989,22 +1855,22 @@ fn round_geometry_coordinates(geom: geo::Geometry) -> geo::Geometry { Polygon::new(LineString(outer_ring), rounded_inner_rings) }) .collect(); - geo::Geometry::MultiPolygon(geo::MultiPolygon::new(rounded_polygons)) + Geometry::MultiPolygon(MultiPolygon::new(rounded_polygons)) } - geo::Geometry::GeometryCollection(geometrycollection) => geo::Geometry::GeometryCollection( + Geometry::GeometryCollection(geometrycollection) => Geometry::GeometryCollection( geometrycollection .into_iter() .map(round_geometry_coordinates) .collect(), ), - geo::Geometry::Line(line) => geo::Geometry::Line(geo::Line::new( + Geometry::Line(line) => Geometry::Line(Line::new( Point::new( round_coordinate(line.start.x), round_coordinate(line.start.y), ), Point::new(round_coordinate(line.end.x), round_coordinate(line.end.y)), )), - geo::Geometry::Rect(rect) => geo::Geometry::Rect(geo::Rect::new( + Geometry::Rect(rect) => Geometry::Rect(Rect::new( Point::new( round_coordinate(rect.min().x), round_coordinate(rect.min().y), @@ -2014,7 +1880,7 @@ fn round_geometry_coordinates(geom: geo::Geometry) -> geo::Geometry { round_coordinate(rect.max().y), ), )), - geo::Geometry::Triangle(triangle) => geo::Geometry::Triangle(geo::Triangle::new( + Geometry::Triangle(triangle) => Geometry::Triangle(Triangle::new( coord!(x: round_coordinate(triangle.0.x), y: round_coordinate(triangle.0.y)), coord!(x: round_coordinate(triangle.1.x), y: round_coordinate(triangle.1.y)), coord!(x: round_coordinate(triangle.2.x), y: round_coordinate(triangle.2.y)), diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index ec521332ed50..bb706257a921 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -3641,7 +3641,7 @@ Functions overloads: 1 st_dimension(Geometry NULL) :: Int32 NULL 0 st_distance(Geometry, Geometry) :: Float64 1 st_distance(Geometry NULL, Geometry NULL) :: Float64 NULL -0 st_endpoint(Geometry) :: Geometry +0 st_endpoint(Geometry) :: Geometry NULL 1 st_endpoint(Geometry NULL) :: Geometry NULL 0 st_geographyfromewkt(String) :: Geography 1 st_geographyfromewkt(String NULL) :: Geography NULL @@ -3677,13 +3677,13 @@ Functions overloads: 1 st_makepolygon(Geometry NULL) :: Geometry NULL 0 st_npoints(Geometry) :: UInt32 1 st_npoints(Geometry NULL) :: UInt32 NULL -0 st_pointn(Geometry, Int32) :: Geometry +0 st_pointn(Geometry, Int32) :: Geometry NULL 1 st_pointn(Geometry NULL, Int32 NULL) :: Geometry NULL 0 st_setsrid(Geometry, Int32) :: Geometry 1 st_setsrid(Geometry NULL, Int32 NULL) :: Geometry NULL 0 st_srid(Geometry) :: Int32 1 st_srid(Geometry NULL) :: Int32 NULL -0 st_startpoint(Geometry) :: Geometry +0 st_startpoint(Geometry) :: Geometry NULL 1 st_startpoint(Geometry NULL) :: Geometry NULL 0 st_transform(Geometry, Int32) :: Geometry 1 st_transform(Geometry NULL, Int32 NULL) :: Geometry NULL diff --git a/src/query/functions/tests/it/scalars/testdata/geometry.txt b/src/query/functions/tests/it/scalars/testdata/geometry.txt index a69f920d3cfc..89cb96d0c9f0 100644 --- a/src/query/functions/tests/it/scalars/testdata/geometry.txt +++ b/src/query/functions/tests/it/scalars/testdata/geometry.txt @@ -19,10 +19,10 @@ output : 0101000020E61000006666666666965EC06666666666C64240 ast : st_asewkb(to_geometry('SRID=0;LINESTRING(0.75 0.75, -10 20)')) raw expr : st_asewkb(to_geometry('SRID=0;LINESTRING(0.75 0.75, -10 20)')) checked expr : st_asewkb(to_geometry("SRID=0;LINESTRING(0.75 0.75, -10 20)")) -optimized expr : 010200000002000000000000000000E83F000000000000E83F00000000000024C00000000000003440 +optimized expr : 01020000200000000002000000000000000000E83F000000000000E83F00000000000024C00000000000003440 output type : Binary output domain : Undefined -output : 010200000002000000000000000000E83F000000000000E83F00000000000024C00000000000003440 +output : 01020000200000000002000000000000000000E83F000000000000E83F00000000000024C00000000000003440 ast : st_aswkb(to_geometry('POINT(-122.35 37.55)')) @@ -119,7 +119,7 @@ ast : st_endpoint(to_geometry('LINESTRING(1 1, 2 2, 3 3, 4 4)')) raw expr : st_endpoint(to_geometry('LINESTRING(1 1, 2 2, 3 3, 4 4)')) checked expr : st_endpoint(to_geometry("LINESTRING(1 1, 2 2, 3 3, 4 4)")) optimized expr : "POINT(4 4)" -output type : Geometry +output type : Geometry NULL output domain : Undefined output : 'POINT(4 4)' @@ -357,7 +357,7 @@ evaluation: | | a | b | Output | +--------+---------+---------+--------------+ | Type | Float64 | Float64 | Geometry | -| Domain | {1..=3} | {1..=3} | Undefined | +| Domain | {1..=3} | {1..=3} | Unknown | | Row 0 | 1 | 1 | 'POINT(1 1)' | | Row 1 | 2 | 2 | 'POINT(2 2)' | | Row 2 | 3 | 3 | 'POINT(3 3)' | @@ -478,7 +478,7 @@ ast : ST_POINTN(TO_GEOMETRY('LINESTRING(1 1, 2 2, 3 3, 4 4)'), 1) raw expr : ST_POINTN(TO_GEOMETRY('LINESTRING(1 1, 2 2, 3 3, 4 4)'), 1) checked expr : st_pointn(to_geometry("LINESTRING(1 1, 2 2, 3 3, 4 4)"), to_int32(1_u8)) optimized expr : "POINT(1 1)" -output type : Geometry +output type : Geometry NULL output domain : Undefined output : 'POINT(1 1)' @@ -487,7 +487,7 @@ ast : ST_POINTN(TO_GEOMETRY('LINESTRING(1 1, 2 2, 3 3, 4 4)'), -2) raw expr : ST_POINTN(TO_GEOMETRY('LINESTRING(1 1, 2 2, 3 3, 4 4)'), minus(2)) checked expr : st_pointn(to_geometry("LINESTRING(1 1, 2 2, 3 3, 4 4)"), to_int32(minus(2_u8))) optimized expr : "POINT(3 3)" -output type : Geometry +output type : Geometry NULL output domain : Undefined output : 'POINT(3 3)' @@ -541,7 +541,7 @@ ast : st_startpoint(to_geometry('LINESTRING(1 1, 2 2, 3 3, 4 4)')) raw expr : st_startpoint(to_geometry('LINESTRING(1 1, 2 2, 3 3, 4 4)')) checked expr : st_startpoint(to_geometry("LINESTRING(1 1, 2 2, 3 3, 4 4)")) optimized expr : "POINT(1 1)" -output type : Geometry +output type : Geometry NULL output domain : Undefined output : 'POINT(1 1)' diff --git a/tests/sqllogictests/suites/query/functions/02_0060_function_geometry.test b/tests/sqllogictests/suites/query/functions/02_0060_function_geometry.test index a31c0c522cce..136d80fc21a0 100644 --- a/tests/sqllogictests/suites/query/functions/02_0060_function_geometry.test +++ b/tests/sqllogictests/suites/query/functions/02_0060_function_geometry.test @@ -414,7 +414,7 @@ query T SELECT UPPER(TO_HEX(ST_ASEWKB(g))) FROM t1 ---- 0101000020E61000006666666666965EC06666666666C64240 -010200000002000000000000000000E83F000000000000E83F00000000000024C00000000000003440 +01020000200000000002000000000000000000E83F000000000000E83F00000000000024C00000000000003440 # srid=0 has't supported by geozero crate.See also: https://github.com/georust/geozero/pull/210/files query T @@ -642,4 +642,4 @@ statement ok SET enable_geo_create_table=0 statement ok -DROP TABLE IF EXISTS t1; \ No newline at end of file +DROP TABLE IF EXISTS t1; From 9b1e4cbf610066e7ba1ca4ab02da1804b0fbe176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Wed, 20 Nov 2024 11:38:43 +0800 Subject: [PATCH 61/92] fix: raft-log wont write when exceeding 256M (#16876) - Fixed in: https://github.com/drmingdrmer/raft-log/commit/2693db500efe3862d775ed986f46824a7c5660e9 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edadfe03ad7b..817428ea8e17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11992,9 +11992,9 @@ dependencies = [ [[package]] name = "raft-log" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e7f06d77e694fc984ba36494c091169d9c6695ce2af174e8f1e167a05f588" +checksum = "3d5edc50b7d30e6b04683575129a996cf210951fc36c6e5856eb8dc746fef77a" dependencies = [ "byteorder", "clap", diff --git a/Cargo.toml b/Cargo.toml index 802833cf39c3..47f7abd796dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -404,7 +404,7 @@ prost = { version = "0.13" } prost-build = { version = "0.13" } prqlc = "0.11.3" quanta = "0.11.1" -raft-log = { version = "0.2.2" } +raft-log = { version = "0.2.3" } rand = { version = "0.8.5", features = ["small_rng"] } rayon = "1.9.0" recursive = "0.1.1" From 170b4b2b905f71ec73f6b26c437beea251a4c698 Mon Sep 17 00:00:00 2001 From: TCeason <33082201+TCeason@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:27:05 +0800 Subject: [PATCH 62/92] refactor: use jiff crate replace chrono in date/timestamp func (#16787) * chore: use jiff crate replace some chrono Done: register_diff_functions(registry); register_to_number_functions(registry); register_add_functions(registry); register_sub_functions(registry); register_timestamp_to_date register_date_to_timestamp register_to_string register_string_to_date register_string_to_timestamp TODO: register_real_time_functions(registry); register_rounder_functions(registry); * add feature tzdb-bundle-always * register_real_time_functions done * register_rounder_functions done * clamp max/min second * rm TzLut * cargo machete * support year < 1000 * resolve conflicts --- Cargo.lock | 14 +- Cargo.toml | 1 + src/common/io/Cargo.toml | 1 + .../cursor_ext/cursor_read_datetime_ext.rs | 424 +++++------ src/common/io/src/format_settings.rs | 3 + .../tests/it/cursor_ext/read_datetime_ext.rs | 96 ++- src/query/expression/Cargo.toml | 2 +- src/query/expression/src/filter/selector.rs | 2 - src/query/expression/src/function.rs | 16 +- src/query/expression/src/types/date.rs | 17 +- src/query/expression/src/types/timestamp.rs | 17 +- src/query/expression/src/types/variant.rs | 15 +- src/query/expression/src/utils/date_helper.rs | 699 ++++++++---------- src/query/expression/src/utils/display.rs | 6 +- src/query/expression/src/utils/serialize.rs | 12 +- src/query/expression/tests/it/types.rs | 6 +- src/query/formats/Cargo.toml | 1 + src/query/formats/src/common_settings.rs | 3 + .../formats/src/field_decoder/fast_values.rs | 14 +- .../formats/src/field_decoder/json_ast.rs | 15 +- src/query/formats/src/field_decoder/nested.rs | 14 +- .../src/field_decoder/separated_text.rs | 15 +- src/query/formats/src/field_encoder/csv.rs | 2 + src/query/formats/src/field_encoder/json.rs | 1 + src/query/formats/src/field_encoder/values.rs | 20 +- src/query/formats/src/file_format_type.rs | 16 + src/query/formats/src/output_format/json.rs | 10 +- src/query/functions/Cargo.toml | 2 +- .../aggregates/aggregate_json_array_agg.rs | 6 +- .../aggregates/aggregate_json_object_agg.rs | 6 +- src/query/functions/src/scalars/datetime.rs | 466 ++++++------ src/query/functions/src/scalars/variant.rs | 72 +- .../tests/it/scalars/testdata/cast.txt | 12 +- .../tests/it/scalars/testdata/datetime.txt | 78 +- src/query/service/Cargo.toml | 1 + .../transforms/transform_dictionary.rs | 8 +- .../src/servers/http/v1/query/string_block.rs | 7 +- .../mysql/writers/query_result_writer.rs | 1 + src/query/service/src/sessions/query_ctx.rs | 25 +- .../src/table_functions/cloud/task_history.rs | 12 +- src/query/storages/common/cache/Cargo.toml | 1 - src/query/storages/system/Cargo.toml | 2 +- .../storages/system/src/task_history_table.rs | 8 +- src/tests/sqlsmith/Cargo.toml | 1 + src/tests/sqlsmith/src/sql_gen/dml.rs | 2 + .../11_0001_data_type_date_time.test | 2 +- .../11_data_type/11_0005_data_type_date.test | 4 +- .../functions/02_0012_function_datetimes.test | 67 +- .../02_0012_function_datetimes_tz.test | 86 ++- .../02_0075_function_datetimes_tz.test | 30 +- 50 files changed, 1140 insertions(+), 1201 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 817428ea8e17..835bbf246e22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3345,6 +3345,7 @@ dependencies = [ "hex", "hyper-util", "itertools 0.13.0", + "jiff", "jsonb", "lexical-core", "log", @@ -3387,6 +3388,7 @@ dependencies = [ "databend-storages-common-table-meta", "geozero 0.14.0", "hex", + "jiff", "jsonb", "lexical-core", "match-template", @@ -3409,7 +3411,6 @@ dependencies = [ "bstr", "bumpalo", "chrono", - "chrono-tz 0.8.6", "comfy-table", "crc32fast", "criterion", @@ -3435,6 +3436,7 @@ dependencies = [ "jaq-interpret", "jaq-parse", "jaq-std", + "jiff", "jsonb", "lexical-core", "libm", @@ -3525,6 +3527,7 @@ dependencies = [ "geo", "geozero 0.14.0", "hex", + "jiff", "lexical-core", "micromarshal", "rand", @@ -4556,7 +4559,6 @@ dependencies = [ "async-backtrace", "async-trait", "chrono", - "chrono-tz 0.8.6", "databend-common-ast", "databend-common-base", "databend-common-building", @@ -4583,6 +4585,7 @@ dependencies = [ "databend-storages-common-cache", "futures", "itertools 0.13.0", + "jiff", "jsonb", "log", "once_cell", @@ -5132,6 +5135,7 @@ dependencies = [ "hyper-util", "indicatif", "itertools 0.13.0", + "jiff", "jsonb", "jwt-simple", "log", @@ -5234,6 +5238,7 @@ dependencies = [ "databend-driver-core", "ethnum", "itertools 0.13.0", + "jiff", "jsonb", "rand", "tokio", @@ -8979,10 +8984,11 @@ dependencies = [ [[package]] name = "jiff" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a45489186a6123c128fdf6016183fcfab7113e1820eb813127e036e287233fb" +checksum = "b9d9d414fc817d3e3d62b2598616733f76c4cc74fbac96069674739b881295c8" dependencies = [ + "jiff-tzdb", "jiff-tzdb-platform", "serde", "windows-sys 0.59.0", diff --git a/Cargo.toml b/Cargo.toml index 47f7abd796dd..af99aca4b21c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -318,6 +318,7 @@ jaq-core = "1.5.1" jaq-interpret = "1.5.0" jaq-parse = "1.0.3" jaq-std = "1.6.0" +jiff = { version = "0.1.14", features = ["serde", "tzdb-bundle-always"] } jsonb = "0.4.4" jwt-simple = { version = "0.12.10", default-features = false, features = ["pure-rust"] } lenient_semver = "0.4.2" diff --git a/src/common/io/Cargo.toml b/src/common/io/Cargo.toml index f1c60ff26e5d..e56851991ddf 100644 --- a/src/common/io/Cargo.toml +++ b/src/common/io/Cargo.toml @@ -24,6 +24,7 @@ ethnum = { workspace = true } geo = { workspace = true } geozero = { workspace = true } hex = { workspace = true } +jiff = { workspace = true } lexical-core = { workspace = true } micromarshal = { workspace = true } roaring = { workspace = true, features = ["serde"] } diff --git a/src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs b/src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs index 0a7927b219ca..5d85fc38451b 100644 --- a/src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs +++ b/src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs @@ -14,45 +14,48 @@ use std::io::Cursor; use std::io::Read; +use std::time::Duration as SysDuration; use chrono::DateTime; -use chrono::Datelike; use chrono::Duration; -use chrono::FixedOffset; use chrono::LocalResult; use chrono::MappedLocalTime; -use chrono::NaiveDate; use chrono::NaiveDateTime; -use chrono::Offset; -use chrono::TimeZone; +use chrono::TimeZone as ChronoTimeZone; use chrono_tz::Tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; +use jiff::civil::date; +use jiff::civil::Date; +use jiff::tz::Offset; +use jiff::tz::TimeZone; +use jiff::Timestamp; +use jiff::Zoned; use crate::cursor_ext::cursor_read_bytes_ext::ReadBytesExt; pub enum DateTimeResType { - Datetime(DateTime), - Date(NaiveDate), + Datetime(Zoned), + Date(Date), } pub trait BufferReadDateTimeExt { - fn read_date_text(&mut self, tz: &Tz, enable_dst_hour_fix: bool) -> Result; - fn read_timestamp_text( - &mut self, - tz: &Tz, - only_date_text: bool, - enable_dst_hour_fix: bool, - ) -> Result; + fn read_date_text(&mut self, jiff_tz: &TimeZone) -> Result; + fn read_timestamp_text(&mut self, jiff_tz: &TimeZone) -> Result; fn parse_time_offset( &mut self, - tz: &Tz, + tz: &TimeZone, buf: &mut Vec, - dt: &DateTime, + dt: &Zoned, west_tz: bool, - calc_offset: impl Fn(i64, i64, &DateTime) -> Result>, - ) -> Result>; + calc_offset: impl Fn(i64, i64, &Zoned) -> Result, + ) -> Result; + fn read_text_to_datetime( + &mut self, + jiff_tz: &TimeZone, + need_date: bool, + ) -> Result; } const DATE_LEN: usize = 10; @@ -72,21 +75,134 @@ fn parse_time_part(buf: &[u8], size: usize) -> Result { impl BufferReadDateTimeExt for Cursor where T: AsRef<[u8]> { - fn read_date_text(&mut self, tz: &Tz, enable_dst_hour_fix: bool) -> Result { + fn read_date_text(&mut self, jiff_tz: &TimeZone) -> Result { // TODO support YYYYMMDD format - // Also need to consider enable_dst_hour_fix, to_date('1947-04-15 00:00:00') - self.read_timestamp_text(tz, true, enable_dst_hour_fix) + self.read_text_to_datetime(jiff_tz, true) .map(|dt| match dt { - DateTimeResType::Datetime(dt) => dt.naive_local().date(), DateTimeResType::Date(nd) => nd, + DateTimeResType::Datetime(dt) => dt.date(), }) } - fn read_timestamp_text( + fn read_timestamp_text(&mut self, jiff_tz: &TimeZone) -> Result { + self.read_text_to_datetime(jiff_tz, false) + } + + // Only support HH:mm format + fn parse_time_offset( + &mut self, + tz: &TimeZone, + buf: &mut Vec, + dt: &Zoned, + west_tz: bool, + calc_offset: impl Fn(i64, i64, &Zoned) -> Result, + ) -> Result { + fn get_hour_minute_offset( + tz: &TimeZone, + dt: &Zoned, + west_tz: bool, + calc_offset: &impl Fn(i64, i64, &Zoned) -> Result, + hour_offset: i32, + minute_offset: i32, + ) -> Result { + if (hour_offset == 14 && minute_offset == 0) + || ((0..60).contains(&minute_offset) && hour_offset < 14) + { + if dt.year() < 1970 { + Ok(date(1970, 1, 1) + .at(0, 0, 0, 0) + .to_zoned(tz.clone()) + .map_err_to_code(ErrorCode::BadBytes, || format!("dt parse error"))?) + } else { + let current_tz_sec = dt.offset().seconds(); + let mut val_tz_sec = + Offset::from_seconds(hour_offset * 3600 + minute_offset * 60) + .map_err_to_code(ErrorCode::BadBytes, || { + "calc offset failed.".to_string() + })? + .seconds(); + if west_tz { + val_tz_sec = -val_tz_sec; + } + calc_offset(current_tz_sec.into(), val_tz_sec.into(), dt) + } + } else { + Err(ErrorCode::BadBytes(format!( + "Invalid Timezone Offset: The minute offset '{}' is outside the valid range. Expected range is [00-59] within a timezone gap of [-14:00, +14:00]", + minute_offset + ))) + } + } + let n = self.keep_read(buf, |f| f.is_ascii_digit()); + match n { + 2 => { + let hour_offset: i32 = + lexical_core::FromLexical::from_lexical(buf.as_slice()).unwrap(); + if (0..15).contains(&hour_offset) { + buf.clear(); + if self.ignore_byte(b':') { + if self.keep_read(buf, |f| f.is_ascii_digit()) != 2 { + // +08[other byte]00 will err in there, e.g. +08-00 + return Err(ErrorCode::BadBytes( + "Timezone Parsing Error: Incorrect format in hour part. The time zone format must conform to the ISO 8601 standard", + )); + } + let minute_offset: i32 = + lexical_core::FromLexical::from_lexical(buf.as_slice()).unwrap(); + // max utc: 14:00, min utc: 00:00 + get_hour_minute_offset( + tz, + dt, + west_tz, + &calc_offset, + hour_offset, + minute_offset, + ) + } else { + get_hour_minute_offset(tz, dt, west_tz, &calc_offset, hour_offset, 0) + } + } else { + Err(ErrorCode::BadBytes(format!( + "Invalid Timezone Offset: The hour offset '{}' is outside the valid range. Expected range is [00-14] within a timezone gap of [-14:00, +14:00]", + hour_offset + ))) + } + } + 4 => { + let hour_offset = &buf.as_slice()[..2]; + let hour_offset: i32 = + lexical_core::FromLexical::from_lexical(hour_offset).unwrap(); + let minute_offset = &buf.as_slice()[2..]; + let minute_offset: i32 = + lexical_core::FromLexical::from_lexical(minute_offset).unwrap(); + buf.clear(); + // max utc: 14:00, min utc: 00:00 + if (0..15).contains(&hour_offset) { + get_hour_minute_offset( + tz, + dt, + west_tz, + &calc_offset, + hour_offset, + minute_offset, + ) + } else { + Err(ErrorCode::BadBytes(format!( + "Invalid Timezone Offset: The hour offset '{}' is outside the valid range. Expected range is [00-14] within a timezone gap of [-14:00, +14:00]", + hour_offset + ))) + } + } + _ => Err(ErrorCode::BadBytes( + "Timezone Parsing Error: Incorrect format. The time zone format must conform to the ISO 8601 standard", + )), + } + } + + fn read_text_to_datetime( &mut self, - tz: &Tz, - only_date_text: bool, - enable_dst_hour_fix: bool, + jiff_tz: &TimeZone, + need_date: bool, ) -> Result { // Date Part YYYY-MM-DD let mut buf = vec![0; DATE_LEN]; @@ -103,28 +219,13 @@ where T: AsRef<[u8]> if v == "0000-00-00" { v = "1970-01-01"; } - let d = v - .parse::() - .map_err_to_code(ErrorCode::BadBytes, || { - format!( - "Date Parsing Error: The value '{}' could not be parsed into a valid Date", - v - ) - })?; - let less_1000 = |dt: DateTime| { - // convert timestamp less than `1000-01-01 00:00:00` to `1000-01-01 00:00:00` - if dt.year() < 1000 { - tz.from_utc_datetime( - &NaiveDate::from_ymd_opt(1000, 1, 1) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap(), - ) - } else { - dt - } - }; + let d = v.parse::().map_err_to_code(ErrorCode::BadBytes, || { + format!( + "Date Parsing Error: The value '{}' could not be parsed into a valid Date", + v + ) + })?; // Time Part buf.clear(); @@ -150,11 +251,11 @@ where T: AsRef<[u8]> // Examples: '2022-02-02T', '2022-02-02 ', '2022-02-02T02', '2022-02-02T3:', '2022-02-03T03:13', '2022-02-03T03:13:' if times.len() < 3 { times.resize(3, 0); - let dt = get_local_time(tz, &d, &mut times, enable_dst_hour_fix)?; - return Ok(DateTimeResType::Datetime(less_1000(dt))); + let dt = get_local_time(jiff_tz, &d, &mut times)?; + return Ok(DateTimeResType::Datetime(dt)); } - let dt = get_local_time(tz, &d, &mut times, enable_dst_hour_fix)?; + let dt = get_local_time(jiff_tz, &d, &mut times)?; // ms .microseconds let dt = if self.ignore_byte(b'.') { @@ -165,56 +266,49 @@ where T: AsRef<[u8]> "Microsecond Parsing Error: Expecting a format like [.123456] for microseconds part", )); } - let mut scales: i64 = + let mut scales: u64 = lexical_core::FromLexical::from_lexical(buf.as_slice()).unwrap(); if size <= 9 { - scales *= 10_i64.pow(9 - size as u32) + scales *= 10_u64.pow(9 - size as u32) } else { - scales /= (size as i64 - 9) * 10 + scales /= (size as u64 - 9) * 10 } - dt.checked_add_signed(Duration::nanoseconds(scales)) - .unwrap() + dt.checked_add(SysDuration::from_nanos(scales)) + .map_err_to_code(ErrorCode::BadBytes, || { + format!("Datetime {} add nanos {} with error", dt, scales) + })? } else { dt }; // Timezone 2022-02-02T03:00:03.123[z/Z[+/-08:00]] buf.clear(); - let calc_offset = |current_tz_sec: i64, val_tz_sec: i64, dt: &DateTime| { + let calc_offset = |current_tz_sec: i64, val_tz_sec: i64, dt: &Zoned| { let offset = (current_tz_sec - val_tz_sec) * 1000 * 1000; - let mut ts = dt.timestamp_micros(); + let mut ts = dt.timestamp().as_microsecond(); ts += offset; - // TODO: need support timestamp_micros in chrono-0.4.22/src/offset/mod.rs - // use like tz.timestamp_nanos() let (mut secs, mut micros) = (ts / 1_000_000, ts % 1_000_000); if ts < 0 { secs -= 1; micros += 1_000_000; } - Ok(tz.timestamp_opt(secs, (micros as u32) * 1000).unwrap()) + Ok(Timestamp::new(secs, (micros as i32) * 1000) + .map_err_to_code(ErrorCode::BadBytes, || { + format!("Datetime {} add offset {} with error", dt, offset) + })? + .to_zoned(jiff_tz.clone())) }; if self.ignore(|b| b == b'z' || b == b'Z') { // ISO 8601 The Z on the end means UTC (that is, an offset-from-UTC of zero hours-minutes-seconds). - if dt.year() < 1000 { - Ok(DateTimeResType::Datetime( - tz.from_utc_datetime( - &NaiveDate::from_ymd_opt(1000, 1, 1) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap(), - ), - )) - } else { - let current_tz = dt.offset().fix().local_minus_utc(); - Ok(DateTimeResType::Datetime(calc_offset( - current_tz.into(), - 0, - &dt, - )?)) - } + let current_tz = dt.offset().seconds(); + Ok(DateTimeResType::Datetime(calc_offset( + current_tz.into(), + 0, + &dt, + )?)) } else if self.ignore_byte(b'+') { Ok(DateTimeResType::Datetime(self.parse_time_offset( - tz, + jiff_tz, &mut buf, &dt, false, @@ -222,7 +316,7 @@ where T: AsRef<[u8]> )?)) } else if self.ignore_byte(b'-') { Ok(DateTimeResType::Datetime(self.parse_time_offset( - tz, + jiff_tz, &mut buf, &dt, true, @@ -230,162 +324,22 @@ where T: AsRef<[u8]> )?)) } else { // only datetime part - Ok(DateTimeResType::Datetime(less_1000(dt))) + Ok(DateTimeResType::Datetime(dt)) } } else { // only date part - if d.year() < 1000 { + if need_date { + Ok(DateTimeResType::Date(d)) + } else { Ok(DateTimeResType::Datetime( - tz.from_utc_datetime( - &NaiveDate::from_ymd_opt(1000, 1, 1) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap(), - ), + d.to_zoned(jiff_tz.clone()) + .map_err_to_code(ErrorCode::BadBytes, || { + format!("Failed to parse date {} as timestamp.", d) + })?, )) - } else { - let res = tz.from_local_datetime(&d.and_hms_opt(0, 0, 0).unwrap()); - match res { - LocalResult::Single(t) => Ok(DateTimeResType::Datetime(t)), - LocalResult::Ambiguous(t1, t2) => Err(ErrorCode::BadBytes(format!( - "Ambiguous Local Time: The local time is ambiguous, with possible times ranging from {:?} to {:?}", - t1, t2 - ))), - LocalResult::None => { - // like to_date('1941-03-15') => 1941-03-15 00:00:00 in Asia/Shanghai is not exists - // but if just convert to Date, it should can return NaiveDate - if only_date_text { - Ok(DateTimeResType::Date(d)) - } else { - // Now add a setting enable_dst_hour_fix to control this behavior. If true, try to add a hour. - if let Some(naive_datetime) = &d.and_hms_opt(1, 0, 0) { - return Ok(DateTimeResType::Datetime(unwrap_local_time( - tz, - enable_dst_hour_fix, - naive_datetime, - )?)); - } - Err(ErrorCode::BadBytes(format!( - "Local Time Error: No valid local time found for timezone '{:?}' with date '{}'", - tz, d - ))) - } - } - } } } } - - // Only support HH:mm format - fn parse_time_offset( - &mut self, - tz: &Tz, - buf: &mut Vec, - dt: &DateTime, - west_tz: bool, - calc_offset: impl Fn(i64, i64, &DateTime) -> Result>, - ) -> Result> { - fn get_hour_minute_offset( - tz: &Tz, - dt: &DateTime, - west_tz: bool, - calc_offset: &impl Fn(i64, i64, &DateTime) -> Result>, - hour_offset: i32, - minute_offset: i32, - ) -> Result> { - if (hour_offset == 14 && minute_offset == 0) - || ((0..60).contains(&minute_offset) && hour_offset < 14) - { - if dt.year() < 1970 { - Ok(tz.from_utc_datetime( - &NaiveDate::from_ymd_opt(1970, 1, 1) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap(), - )) - } else { - let current_tz_sec = dt.offset().fix().local_minus_utc(); - let mut val_tz_sec = - FixedOffset::east_opt(hour_offset * 3600 + minute_offset * 60) - .unwrap() - .local_minus_utc(); - if west_tz { - val_tz_sec = -val_tz_sec; - } - calc_offset(current_tz_sec.into(), val_tz_sec.into(), dt) - } - } else { - Err(ErrorCode::BadBytes(format!( - "Invalid Timezone Offset: The minute offset '{}' is outside the valid range. Expected range is [00-59] within a timezone gap of [-14:00, +14:00]", - minute_offset - ))) - } - } - let n = self.keep_read(buf, |f| f.is_ascii_digit()); - match n { - 2 => { - let hour_offset: i32 = - lexical_core::FromLexical::from_lexical(buf.as_slice()).unwrap(); - if (0..15).contains(&hour_offset) { - buf.clear(); - if self.ignore_byte(b':') { - if self.keep_read(buf, |f| f.is_ascii_digit()) != 2 { - // +08[other byte]00 will err in there, e.g. +08-00 - return Err(ErrorCode::BadBytes( - "Timezone Parsing Error: Incorrect format in hour part. The time zone format must conform to the ISO 8601 standard", - )); - } - let minute_offset: i32 = - lexical_core::FromLexical::from_lexical(buf.as_slice()).unwrap(); - // max utc: 14:00, min utc: 00:00 - get_hour_minute_offset( - tz, - dt, - west_tz, - &calc_offset, - hour_offset, - minute_offset, - ) - } else { - get_hour_minute_offset(tz, dt, west_tz, &calc_offset, hour_offset, 0) - } - } else { - Err(ErrorCode::BadBytes(format!( - "Invalid Timezone Offset: The hour offset '{}' is outside the valid range. Expected range is [00-14] within a timezone gap of [-14:00, +14:00]", - hour_offset - ))) - } - } - 4 => { - let hour_offset = &buf.as_slice()[..2]; - let hour_offset: i32 = - lexical_core::FromLexical::from_lexical(hour_offset).unwrap(); - let minute_offset = &buf.as_slice()[2..]; - let minute_offset: i32 = - lexical_core::FromLexical::from_lexical(minute_offset).unwrap(); - buf.clear(); - // max utc: 14:00, min utc: 00:00 - if (0..15).contains(&hour_offset) { - get_hour_minute_offset( - tz, - dt, - west_tz, - &calc_offset, - hour_offset, - minute_offset, - ) - } else { - Err(ErrorCode::BadBytes(format!( - "Invalid Timezone Offset: The hour offset '{}' is outside the valid range. Expected range is [00-14] within a timezone gap of [-14:00, +14:00]", - hour_offset - ))) - } - } - _ => Err(ErrorCode::BadBytes( - "Timezone Parsing Error: Incorrect format. The time zone format must conform to the ISO 8601 standard", - )), - } - } } // Can not directly unwrap, because of DST. @@ -395,16 +349,12 @@ where T: AsRef<[u8]> // -- https://github.com/chronotope/chrono/blob/v0.4.24/src/offset/mod.rs#L186 // select to_date(to_timestamp('2021-03-28 01:00:00')); // Now add a setting enable_dst_hour_fix to control this behavior. If true, try to add a hour. -fn get_local_time( - tz: &Tz, - d: &NaiveDate, - times: &mut Vec, - enable_dst_hour_fix: bool, -) -> Result> { - d.and_hms_opt(times[0], times[1], times[2]) - .map(|naive_datetime| unwrap_local_time(tz, enable_dst_hour_fix, &naive_datetime)) - .transpose()? - .ok_or_else(|| ErrorCode::BadBytes(format!("Invalid time provided in times: {:?}", times))) +fn get_local_time(tz: &TimeZone, d: &Date, times: &mut Vec) -> Result { + d.at(times[0] as i8, times[1] as i8, times[2] as i8, 0) + .to_zoned(tz.clone()) + .map_err_to_code(ErrorCode::BadBytes, || { + format!("Invalid time provided in times: {:?}", times) + }) } #[inline] diff --git a/src/common/io/src/format_settings.rs b/src/common/io/src/format_settings.rs index a203266a98f7..a8975daf1d81 100644 --- a/src/common/io/src/format_settings.rs +++ b/src/common/io/src/format_settings.rs @@ -13,12 +13,14 @@ // limitations under the License. use chrono_tz::Tz; +use jiff::tz::TimeZone; use crate::GeometryDataType; #[derive(Debug, Clone, PartialEq, Eq)] pub struct FormatSettings { pub timezone: Tz, + pub jiff_timezone: TimeZone, pub geometry_format: GeometryDataType, pub enable_dst_hour_fix: bool, pub format_null_as_str: bool, @@ -29,6 +31,7 @@ impl Default for FormatSettings { fn default() -> Self { Self { timezone: "UTC".parse::().unwrap(), + jiff_timezone: TimeZone::UTC, geometry_format: GeometryDataType::default(), enable_dst_hour_fix: false, format_null_as_str: true, diff --git a/src/common/io/tests/it/cursor_ext/read_datetime_ext.rs b/src/common/io/tests/it/cursor_ext/read_datetime_ext.rs index 7ccf8363876c..4ed4f87e8ff7 100644 --- a/src/common/io/tests/it/cursor_ext/read_datetime_ext.rs +++ b/src/common/io/tests/it/cursor_ext/read_datetime_ext.rs @@ -14,51 +14,50 @@ use std::io::Cursor; -use chrono::Offset; -use chrono_tz::Tz; use databend_common_exception::Result; use databend_common_io::cursor_ext::*; +use jiff::tz::TimeZone; #[test] fn test_read_timestamp_text() -> Result<()> { let mut reader = Cursor::new( "2023-12-25T02:31:07.485281+0545,2023-12-25T02:31:07.485281-0545,2023-12-25T02:31:07.485281+05,2023-12-25T02:31:07.485281-05,2009-01-01 00:00:00.12,2009-01-01 00:00:00.1234,2009-01-01 00:00:00.1234567891,2022-02-02T,2022-02-02 12,2022-02-02T13:4:,2022-02-02 12:03,2023-03-03,2022-02-02,2009-01-01 3:2:1.123,2009-01-01 0:00:00,2009-01-01 00:00:00.123,2009-01-01 00:00:00.123456,0002-03-03T00:01:02,2022-03-04T00:01:02+08:00,2022-03-04T00:01:02-08:00,0000-00-00,0000-00-00 00:00:00,0001-01-01 00:00:00,2020-01-01T11:11:11Z,2009-01-03 00:00:00,2020-01-01T11:11:11.123Z,2055-02-03 10:00:20.234+08:00,2055-02-03 10:00:20.234-08:00,1022-05-16T03:25:02.000000+08:00".as_bytes(), ); - let tz = Tz::UTC; - let expected = vec![ - "2023-12-24T20:46:07.485281UTC", - "2023-12-25T08:16:07.485281UTC", - "2023-12-24T21:31:07.485281UTC", - "2023-12-25T07:31:07.485281UTC", - "2009-01-01T00:00:00.120UTC", - "2009-01-01T00:00:00.123400UTC", - "2009-01-01T00:00:00.123456789UTC", - "2022-02-02T00:00:00UTC", - "2022-02-02T12:00:00UTC", - "2022-02-02T13:04:00UTC", - "2022-02-02T12:03:00UTC", - "2023-03-03T00:00:00UTC", - "2022-02-02T00:00:00UTC", - "2009-01-01T03:02:01.123UTC", - "2009-01-01T00:00:00UTC", - "2009-01-01T00:00:00.123UTC", - "2009-01-01T00:00:00.123456UTC", - "1000-01-01T00:00:00UTC", - "2022-03-03T16:01:02UTC", - "2022-03-04T08:01:02UTC", - "1970-01-01T00:00:00UTC", - "1970-01-01T00:00:00UTC", - "1000-01-01T00:00:00UTC", - "2020-01-01T11:11:11UTC", - "2009-01-03T00:00:00UTC", - "2020-01-01T11:11:11.123UTC", - "2055-02-03T02:00:20.234UTC", - "2055-02-03T18:00:20.234UTC", - "1970-01-01T00:00:00UTC", + let jiff_tz = TimeZone::UTC; + let expected = [ + "2023-12-24T20:46:07.485281+00:00[UTC]", + "2023-12-25T08:16:07.485281+00:00[UTC]", + "2023-12-24T21:31:07.485281+00:00[UTC]", + "2023-12-25T07:31:07.485281+00:00[UTC]", + "2009-01-01T00:00:00.12+00:00[UTC]", + "2009-01-01T00:00:00.1234+00:00[UTC]", + "2009-01-01T00:00:00.123456789+00:00[UTC]", + "2022-02-02T00:00:00+00:00[UTC]", + "2022-02-02T12:00:00+00:00[UTC]", + "2022-02-02T13:04:00+00:00[UTC]", + "2022-02-02T12:03:00+00:00[UTC]", + "2023-03-03T00:00:00+00:00[UTC]", + "2022-02-02T00:00:00+00:00[UTC]", + "2009-01-01T03:02:01.123+00:00[UTC]", + "2009-01-01T00:00:00+00:00[UTC]", + "2009-01-01T00:00:00.123+00:00[UTC]", + "2009-01-01T00:00:00.123456+00:00[UTC]", + "0002-03-03T00:01:02+00:00[UTC]", + "2022-03-03T16:01:02+00:00[UTC]", + "2022-03-04T08:01:02+00:00[UTC]", + "1970-01-01T00:00:00+00:00[UTC]", + "1970-01-01T00:00:00+00:00[UTC]", + "0001-01-01T00:00:00+00:00[UTC]", + "2020-01-01T11:11:11+00:00[UTC]", + "2009-01-03T00:00:00+00:00[UTC]", + "2020-01-01T11:11:11.123+00:00[UTC]", + "2055-02-03T02:00:20.234+00:00[UTC]", + "2055-02-03T18:00:20.234+00:00[UTC]", + "1970-01-01T00:00:00+00:00[UTC]", ]; let mut res = vec![]; for _ in 0..expected.len() { - let time = reader.read_timestamp_text(&tz, false, false)?; + let time = reader.read_timestamp_text(&jiff_tz)?; if let DateTimeResType::Datetime(time) = time { res.push(format!("{:?}", time)); reader.ignore_byte(b','); @@ -71,28 +70,28 @@ fn test_read_timestamp_text() -> Result<()> { #[test] fn test_read_dst_timestamp_text() -> Result<()> { let mut reader = Cursor::new("1947-04-15 01:00:00,1990-09-16 01:00:00".as_bytes()); - let tz = Tz::Asia__Shanghai; - let expected = vec![ - "1947-04-15T01:00:00CDT,+09:00", - "1990-09-16T01:00:00CDT,+09:00", + let jiff_tz = TimeZone::get("Asia/Shanghai").unwrap(); + let expected = [ + "\"1947-04-15T01:00:00+09:00[Asia/Shanghai]\"", + "\"1990-09-16T01:00:00+09:00[Asia/Shanghai]\"", ]; let mut res = vec![]; for _ in 0..expected.len() { - let time = reader.read_timestamp_text(&tz, false, true)?; + let time = reader.read_timestamp_text(&jiff_tz)?; if let DateTimeResType::Datetime(time) = time { - res.push(format!("{:?},{}", time, time.offset().fix())); + res.push(format!("{:?}", time.to_string())); reader.ignore_byte(b','); } } assert_eq!(res, expected); reader = Cursor::new("1990-09-16 01:00:00".as_bytes()); - let expected2 = vec!["1990-09-16T01:00:00CST,+08:00"]; + let expected2 = ["\"1990-09-16T01:00:00+09:00[Asia/Shanghai]\""]; let mut res = vec![]; for _ in 0..expected2.len() { - let time = reader.read_timestamp_text(&tz, false, false)?; + let time = reader.read_timestamp_text(&jiff_tz)?; if let DateTimeResType::Datetime(time) = time { - res.push(format!("{:?},{}", time, time.offset().fix())); + res.push(format!("{:?}", time.to_string())); reader.ignore_byte(b','); } } @@ -103,9 +102,8 @@ fn test_read_dst_timestamp_text() -> Result<()> { #[test] fn test_read_date_text() -> Result<()> { let mut reader = Cursor::new("2009-01-01,1000-01-01,2023-03-03,2022-02-02,2009-01-01 3:2:1.123,2009-01-01 0:00:00,2009-01-01 00:00:00.123,2009-01-01 00:00:00.123456,0002-03-03T00:01:02,2022-03-04T00:01:02+08:00,2022-03-04T00:01:02-08:00,0000-00-00,0000-00-00 00:00:00,0001-01-01 00:00:00,2020-01-01T11:11:11Z,2009-01-03 00:00:00,2020-01-01T11:11:11.123Z,2055-02-03 10:00:20.234+08:00,2055-02-03 10:00:20.234-08:00,1022-05-16T03:25:02.000000+08:00,2055-01-01".as_bytes()); - let tz = Tz::UTC; - - let expected = vec![ + let jiff_tz = TimeZone::get("UTC").unwrap(); + let expected = [ "2009-01-01", "1000-01-01", "2023-03-03", @@ -114,12 +112,12 @@ fn test_read_date_text() -> Result<()> { "2009-01-01", "2009-01-01", "2009-01-01", - "1000-01-01", + "0002-03-03", "2022-03-03", "2022-03-04", "1970-01-01", "1970-01-01", - "1000-01-01", + "0001-01-01", "2020-01-01", "2009-01-03", "2020-01-01", @@ -131,7 +129,7 @@ fn test_read_date_text() -> Result<()> { let mut res = vec![]; for _ in 0..expected.len() { - let date = reader.read_date_text(&tz, false)?; + let date = reader.read_date_text(&jiff_tz)?; res.push(format!("{:?}", date)); let _ = reader.ignore_byte(b','); } diff --git a/src/query/expression/Cargo.toml b/src/query/expression/Cargo.toml index 7ecfb7cb3813..5681a549e2d2 100644 --- a/src/query/expression/Cargo.toml +++ b/src/query/expression/Cargo.toml @@ -25,7 +25,6 @@ bumpalo = { workspace = true } chrono = { workspace = true } chrono-tz = { workspace = true } comfy-table = { workspace = true } -dashmap = { workspace = true } databend-common-ast = { workspace = true } databend-common-base = { workspace = true } databend-common-column = { workspace = true } @@ -44,6 +43,7 @@ geozero = { workspace = true } hex = { workspace = true } hyper-util = { workspace = true } itertools = { workspace = true } +jiff = { workspace = true } jsonb = { workspace = true } lexical-core = { workspace = true } log = { workspace = true } diff --git a/src/query/expression/src/filter/selector.rs b/src/query/expression/src/filter/selector.rs index 5ae9959ca4df..644c1a83ce67 100644 --- a/src/query/expression/src/filter/selector.rs +++ b/src/query/expression/src/filter/selector.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// use std::time::Instant; - use std::time::Instant; use databend_common_exception::ErrorCode; diff --git a/src/query/expression/src/function.rs b/src/query/expression/src/function.rs index 0db5ea63bce6..4b9adfb69172 100755 --- a/src/query/expression/src/function.rs +++ b/src/query/expression/src/function.rs @@ -19,8 +19,7 @@ use std::ops::BitOr; use std::ops::Not; use std::sync::Arc; -use chrono::DateTime; -use chrono::Utc; +use chrono_tz::Tz; use databend_common_ast::Span; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; @@ -29,10 +28,11 @@ use databend_common_exception::Result; use databend_common_io::GeometryDataType; use enum_as_inner::EnumAsInner; use itertools::Itertools; +use jiff::tz::TimeZone; +use jiff::Zoned; use serde::Deserialize; use serde::Serialize; -use crate::date_helper::TzLUT; use crate::property::Domain; use crate::property::FunctionProperty; use crate::type_check::try_unify_signature; @@ -97,8 +97,9 @@ pub enum FunctionEval { #[derive(Clone)] pub struct FunctionContext { - pub tz: TzLUT, - pub now: DateTime, + pub tz: Tz, + pub jiff_tz: TimeZone, + pub now: Zoned, pub rounding_mode: bool, pub disable_variant_check: bool, @@ -111,7 +112,6 @@ pub struct FunctionContext { pub geometry_output_format: GeometryDataType, pub parse_datetime_ignore_remainder: bool, - pub enable_dst_hour_fix: bool, pub enable_strict_datetime_parser: bool, pub random_function_seed: bool, } @@ -119,7 +119,8 @@ pub struct FunctionContext { impl Default for FunctionContext { fn default() -> Self { FunctionContext { - tz: Default::default(), + tz: Tz::UTC, + jiff_tz: TimeZone::UTC, now: Default::default(), rounding_mode: false, disable_variant_check: false, @@ -132,7 +133,6 @@ impl Default for FunctionContext { geometry_output_format: Default::default(), parse_datetime_ignore_remainder: false, - enable_dst_hour_fix: false, enable_strict_datetime_parser: true, random_function_seed: false, } diff --git a/src/query/expression/src/types/date.rs b/src/query/expression/src/types/date.rs index 5a7e8ab989a5..0fafeee8b7ec 100644 --- a/src/query/expression/src/types/date.rs +++ b/src/query/expression/src/types/date.rs @@ -17,12 +17,13 @@ use std::fmt::Display; use std::io::Cursor; use std::ops::Range; -use chrono::NaiveDate; -use chrono_tz::Tz; use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::ReadBytesExt; +use jiff::civil::Date; +use jiff::fmt::strtime; +use jiff::tz::TimeZone; use log::error; use num_traits::AsPrimitive; @@ -260,11 +261,10 @@ impl ArgType for DateType { #[inline] pub fn string_to_date( date_str: impl AsRef<[u8]>, - tz: Tz, - enable_dst_hour_fix: bool, -) -> databend_common_exception::Result { + jiff_tz: &TimeZone, +) -> databend_common_exception::Result { let mut reader = Cursor::new(std::str::from_utf8(date_str.as_ref()).unwrap().as_bytes()); - match reader.read_date_text(&tz, enable_dst_hour_fix) { + match reader.read_date_text(jiff_tz) { Ok(d) => match reader.must_eof() { Ok(..) => Ok(d), Err(_) => Err(ErrorCode::BadArguments("unexpected argument")), @@ -277,6 +277,7 @@ pub fn string_to_date( } #[inline] -pub fn date_to_string(date: impl AsPrimitive, tz: Tz) -> impl Display { - date.as_().to_date(tz).format(DATE_FORMAT) +pub fn date_to_string(date: impl AsPrimitive, tz: &TimeZone) -> impl Display { + let res = date.as_().to_date(tz.clone()); + strtime::format(DATE_FORMAT, res).unwrap() } diff --git a/src/query/expression/src/types/timestamp.rs b/src/query/expression/src/types/timestamp.rs index 636f84cdee61..2fea0c1fe3e0 100644 --- a/src/query/expression/src/types/timestamp.rs +++ b/src/query/expression/src/types/timestamp.rs @@ -17,13 +17,14 @@ use std::fmt::Display; use std::io::Cursor; use std::ops::Range; -use chrono::DateTime; -use chrono_tz::Tz; use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::DateTimeResType; use databend_common_io::cursor_ext::ReadBytesExt; +use jiff::fmt::strtime; +use jiff::tz::TimeZone; +use jiff::Zoned; use log::error; use super::number::SimpleDomain; @@ -273,11 +274,10 @@ pub fn microseconds_to_days(micros: i64) -> i32 { #[inline] pub fn string_to_timestamp( ts_str: impl AsRef<[u8]>, - tz: Tz, - enable_dst_hour_fix: bool, -) -> databend_common_exception::Result> { + jiff_tz: &TimeZone, +) -> databend_common_exception::Result { let mut reader = Cursor::new(std::str::from_utf8(ts_str.as_ref()).unwrap().as_bytes()); - match reader.read_timestamp_text(&tz, false, enable_dst_hour_fix) { + match reader.read_timestamp_text(jiff_tz) { Ok(dt) => match dt { DateTimeResType::Datetime(dt) => match reader.must_eof() { Ok(..) => Ok(dt), @@ -293,6 +293,7 @@ pub fn string_to_timestamp( } #[inline] -pub fn timestamp_to_string(ts: i64, tz: Tz) -> impl Display { - ts.to_timestamp(tz).format(TIMESTAMP_FORMAT) +pub fn timestamp_to_string(ts: i64, tz: &TimeZone) -> impl Display { + let zdt = ts.to_timestamp(tz.clone()); + strtime::format(TIMESTAMP_FORMAT, &zdt).unwrap() } diff --git a/src/query/expression/src/types/variant.rs b/src/query/expression/src/types/variant.rs index 1f20d4c14c7d..afe816d0b580 100644 --- a/src/query/expression/src/types/variant.rs +++ b/src/query/expression/src/types/variant.rs @@ -19,6 +19,7 @@ use std::ops::Range; use databend_common_io::deserialize_bitmap; use geozero::wkb::Ewkb; use geozero::ToJson; +use jiff::tz::TimeZone; use jsonb::Value; use super::binary::BinaryColumn; @@ -27,7 +28,6 @@ use super::binary::BinaryColumnIter; use super::date::date_to_string; use super::number::NumberScalar; use super::timestamp::timestamp_to_string; -use crate::date_helper::TzLUT; use crate::property::Domain; use crate::types::map::KvPair; use crate::types::AnyType; @@ -208,8 +208,7 @@ impl VariantType { } } -pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: TzLUT, buf: &mut Vec) { - let inner_tz = tz.tz; +pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: &TimeZone, buf: &mut Vec) { let value = match scalar { ScalarRef::Null => jsonb::Value::Null, ScalarRef::EmptyArray => jsonb::Value::Array(vec![]), @@ -230,8 +229,8 @@ pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: TzLUT, buf: &mut Vec) { ScalarRef::Boolean(b) => jsonb::Value::Bool(b), ScalarRef::Binary(s) => jsonb::Value::String(hex::encode_upper(s).into()), ScalarRef::String(s) => jsonb::Value::String(s.into()), - ScalarRef::Timestamp(ts) => timestamp_to_string(ts, inner_tz).to_string().into(), - ScalarRef::Date(d) => date_to_string(d, inner_tz).to_string().into(), + ScalarRef::Timestamp(ts) => timestamp_to_string(ts, tz).to_string().into(), + ScalarRef::Date(d) => date_to_string(d, tz).to_string().into(), ScalarRef::Array(col) => { let items = cast_scalars_to_variants(col.iter(), tz); jsonb::build_array(items.iter(), buf).expect("failed to build jsonb array"); @@ -246,8 +245,8 @@ pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: TzLUT, buf: &mut Vec) { ScalarRef::Number(v) => v.to_string(), ScalarRef::Decimal(v) => v.to_string(), ScalarRef::Boolean(v) => v.to_string(), - ScalarRef::Timestamp(v) => timestamp_to_string(v, inner_tz).to_string(), - ScalarRef::Date(v) => date_to_string(v, inner_tz).to_string(), + ScalarRef::Timestamp(v) => timestamp_to_string(v, tz).to_string(), + ScalarRef::Date(v) => date_to_string(v, tz).to_string(), _ => unreachable!(), }; let mut val = vec![]; @@ -306,7 +305,7 @@ pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: TzLUT, buf: &mut Vec) { pub fn cast_scalars_to_variants( scalars: impl IntoIterator, - tz: TzLUT, + tz: &TimeZone, ) -> BinaryColumn { let iter = scalars.into_iter(); let mut builder = BinaryColumnBuilder::with_capacity(iter.size_hint().0, 0); diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index dc8a5aa4f6ad..a52a7ea6f136 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -12,250 +12,40 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::LazyLock; - -use chrono::DateTime; use chrono::Datelike; use chrono::Days; -use chrono::Duration; -use chrono::LocalResult; -use chrono::NaiveDate; -use chrono::NaiveDateTime; -use chrono::NaiveTime; -use chrono::Offset; -use chrono::TimeZone; -use chrono::Timelike; +use chrono::TimeZone as ChronoTimeZone; use chrono::Utc; -use chrono::Weekday; -use chrono_tz::Tz; -use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_io::cursor_ext::unwrap_local_time; +use jiff::civil::date; +use jiff::civil::datetime; +use jiff::civil::Date; +use jiff::civil::Weekday; +use jiff::tz::TimeZone; +use jiff::SignedDuration; +use jiff::Timestamp; +use jiff::Unit; +use jiff::Zoned; use num_traits::AsPrimitive; use crate::types::date::clamp_date; use crate::types::timestamp::clamp_timestamp; use crate::types::timestamp::MICROS_PER_SEC; -#[derive(Debug, Clone, Copy)] -pub struct TzLUT { - pub tz: Tz, - // whether the timezone offset is round hour, offset % 3600 == 0 - pub offset_round_hour: bool, - // whether the timezone offset is round minute, offset % 60 == 0 - pub offset_round_minute: bool, -} - -impl Default for TzLUT { - fn default() -> Self { - Self { - tz: Tz::UTC, - offset_round_hour: true, - offset_round_minute: true, - } - } -} - -static TZ_FACTORY: LazyLock = LazyLock::new(|| { - let factory = TzFactory { - luts: dashmap::DashMap::new(), - }; - let _ = factory.get(Tz::UTC); - let _ = factory.get(Tz::Asia__Shanghai); - let _ = factory.get(Tz::Asia__Tokyo); - let _ = factory.get(Tz::America__New_York); - let _ = factory.get(Tz::Europe__London); - factory -}); - -pub struct TzFactory { - pub luts: dashmap::DashMap, -} - -impl TzFactory { - pub fn instance() -> &'static TzFactory { - &TZ_FACTORY - } - - pub fn get_by_name(&self, tz_name: &str) -> Result { - if let Some(lut) = self.luts.get(tz_name) { - return Ok(*lut.value()); - } - - let tz = tz_name.parse::().map_err(|_| { - ErrorCode::InvalidTimezone("Timezone has been checked and should be valid") - })?; - let lut = TzLUT::new(tz); - self.luts.insert(tz_name.to_string(), lut); - Ok(lut) - } - - pub fn get(&self, tz: Tz) -> TzLUT { - let tz_name = tz.name(); - if let Some(lut) = self.luts.get(tz_name) { - return *lut.value(); - } - let lut = TzLUT::new(tz); - self.luts.insert(tz_name.to_string(), lut); - lut - } -} - -impl TzLUT { - // it's very heavy to initial a TzLUT - fn new(tz: Tz) -> Self { - static DATE_LUT_MIN_YEAR: i32 = 1925; - static DATE_LUT_MAX_YEAR: i32 = 2283; - - let mut offset_round_hour = true; - let mut offset_round_minute = true; - - let date = NaiveDate::from_ymd_opt(DATE_LUT_MIN_YEAR, 1, 1).unwrap(); - let mut days = date.num_days_from_ce(); - - loop { - let time = NaiveDateTime::new( - NaiveDate::from_num_days_from_ce_opt(days).unwrap(), - NaiveTime::from_hms_opt(0, 0, 0).unwrap(), - ); - if time.year() > DATE_LUT_MAX_YEAR { - break; - } - - days += 1; - - match tz.offset_from_local_datetime(&time) { - LocalResult::Single(offset) => { - let offset = offset.fix(); - if offset_round_hour && offset.local_minus_utc() % 3600 != 0 { - offset_round_hour = false; - } - if offset_round_minute && offset.local_minus_utc() % 60 != 0 { - offset_round_minute = false; - } - } - _ => { - continue; - } - } - } - Self { - tz, - offset_round_hour, - offset_round_minute, - } - } - - #[allow(dead_code)] - #[inline] - fn start_of_second(&self, us: i64, seconds: i64) -> i64 { - if seconds == 1 { - return us / MICROS_PER_SEC * MICROS_PER_SEC; - } - if seconds % 60 == 0 { - return self.start_of_minutes(us, seconds / 60); - } - self.round_down(us, seconds) - } - - #[inline] - fn start_of_minutes(&self, us: i64, minus: i64) -> i64 { - let us_div = minus * 60 * MICROS_PER_SEC; - if self.offset_round_minute { - return if us > 0 { - us / us_div * us_div - } else { - (us + MICROS_PER_SEC - us_div) / us_div * us_div - }; - } - let datetime = self.to_datetime_from_us(us); - let fix = datetime.offset().fix().local_minus_utc() as i64; - fix + (us - fix * MICROS_PER_SEC) / us_div * us_div - } - - #[inline] - fn round_down(&self, us: i64, seconds: i64) -> i64 { - let us_div = seconds * MICROS_PER_SEC; - if self.offset_round_hour { - return if us > 0 { - us / us_div * us_div - } else { - (us + MICROS_PER_SEC - us_div) / us_div * us_div - }; - } - let datetime = self.to_datetime_from_us(us); - let fix = datetime.offset().fix().local_minus_utc() as i64; - fix + (us - fix * MICROS_PER_SEC) / us_div * us_div - } - - #[inline] - pub fn round_us(&self, us: i64, round: Round) -> i64 { - match round { - Round::Second => self.start_of_second(us, 1), - Round::Minute => self.start_of_minutes(us, 1), - Round::FiveMinutes => self.start_of_minutes(us, 5), - Round::TenMinutes => self.start_of_minutes(us, 10), - Round::FifteenMinutes => self.start_of_minutes(us, 15), - Round::TimeSlot => self.start_of_minutes(us, 30), - Round::Hour => self.round_down(us, 3600), - Round::Day => { - let dt = self.to_datetime_from_us(us); - let dt = self - .tz - .with_ymd_and_hms(dt.year(), dt.month(), dt.day(), 0, 0, 0) - .unwrap(); - dt.timestamp() * MICROS_PER_SEC - } - } - } - - #[inline] - pub fn to_minute(&self, us: i64) -> u8 { - if us >= 0 && self.offset_round_hour { - ((us / MICROS_PER_SEC / 60) % 60) as u8 - } else { - let datetime = self.to_datetime_from_us(us); - datetime.minute() as u8 - } - } - - #[inline] - pub fn to_second(&self, us: i64) -> u8 { - if us >= 0 { - (us / MICROS_PER_SEC % 60) as u8 - } else { - let datetime = self.to_datetime_from_us(us); - datetime.second() as u8 - } - } - - #[inline] - pub fn to_datetime_from_us(&self, us: i64) -> DateTime { - us.to_timestamp(self.tz) - } - - #[inline] - pub fn to_hour(&self, us: i64) -> u8 { - let datetime = self.to_datetime_from_us(us); - datetime.hour() as u8 - } -} - pub trait DateConverter { - fn to_date(&self, tz: Tz) -> NaiveDate; - fn to_timestamp(&self, tz: Tz) -> DateTime; + fn to_date(&self, tz: TimeZone) -> Date; + fn to_timestamp(&self, tz: TimeZone) -> Zoned; } impl DateConverter for T where T: AsPrimitive { - fn to_date(&self, tz: Tz) -> NaiveDate { - let mut dt = tz.with_ymd_and_hms(1970, 1, 1, 0, 0, 0).unwrap(); - dt = dt.checked_add_signed(Duration::days(self.as_())).unwrap(); - dt.date_naive() + fn to_date(&self, _tz: TimeZone) -> Date { + let dur = SignedDuration::from_hours(self.as_() * 24); + date(1970, 1, 1).checked_add(dur).unwrap() } - fn to_timestamp(&self, tz: Tz) -> DateTime { + fn to_timestamp(&self, tz: TimeZone) -> Zoned { // Can't use `tz.timestamp_nanos(self.as_() * 1000)` directly, is may cause multiply with overflow. let micros = self.as_(); let (mut secs, mut nanos) = (micros / MICROS_PER_SEC, (micros % MICROS_PER_SEC) * 1_000); @@ -263,7 +53,16 @@ where T: AsPrimitive secs -= 1; nanos += 1_000_000_000; } - tz.timestamp_opt(secs, nanos as u32).unwrap() + + if secs > 253402207200 { + secs = 253402207200; + nanos = 0; + } else if secs < -377705023201 { + secs = -377705023201; + nanos = 0; + } + let ts = Timestamp::new(secs, nanos as i32).unwrap(); + ts.to_zoned(tz) } } @@ -273,31 +72,28 @@ pub const MICROSECS_PER_DAY: i64 = 86_400_000_000; pub const FACTOR_HOUR: i64 = 3600; pub const FACTOR_MINUTE: i64 = 60; pub const FACTOR_SECOND: i64 = 1; -const LAST_DAY_LUT: [u8; 13] = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; +const LAST_DAY_LUT: [i8; 13] = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; -fn eval_years_base( - year: i32, - month: u32, - day: u32, - delta: i64, -) -> std::result::Result { - let new_year = year + delta as i32; +fn eval_years_base(year: i16, month: i8, day: i8, delta: i64) -> std::result::Result { + let new_year = year as i64 + delta; let mut new_day = day; if std::intrinsics::unlikely(month == 2 && day == 29) { - new_day = last_day_of_year_month(new_year, month); + new_day = last_day_of_year_month(new_year as i16, month); + } + match Date::new(new_year as i16, month, new_day) { + Ok(d) => Ok(d), + Err(e) => Err(format!("Invalid date: {}", e)), } - NaiveDate::from_ymd_opt(new_year, month, new_day) - .ok_or_else(|| format!("Overflow on date YMD {}-{}-{}.", new_year, month, new_day)) } fn eval_months_base( - year: i32, - month: u32, - day: u32, + year: i16, + month: i8, + day: i8, delta: i64, -) -> std::result::Result { - let total_months = month as i64 + delta - 1; - let mut new_year = year + (total_months / 12) as i32; +) -> std::result::Result { + let total_months = (month as i64 + delta - 1) as i16; + let mut new_year = year + (total_months / 12); let mut new_month0 = total_months % 12; if new_month0 < 0 { new_year -= 1; @@ -306,27 +102,23 @@ fn eval_months_base( // Handle month last day overflow, "2020-2-29" + "1 year" should be "2021-2-28", or "1990-1-31" + "3 month" should be "1990-4-30". let new_day = std::cmp::min::( - day, - last_day_of_year_month(new_year, (new_month0 + 1) as u32), + day as u32, + last_day_of_year_month(new_year, (new_month0 + 1) as i8) as u32, ); - NaiveDate::from_ymd_opt(new_year, (new_month0 + 1) as u32, new_day).ok_or_else(|| { - format!( - "Overflow on date YMD {}-{}-{}.", - new_year, - new_month0 + 1, - new_day - ) - }) + match Date::new(new_year, (new_month0 + 1) as i8, new_day as i8) { + Ok(d) => Ok(d), + Err(e) => Err(format!("Invalid date: {}", e)), + } } // Get the last day of the year month, could be 28(non leap Feb), 29(leap year Feb), 30 or 31 -fn last_day_of_year_month(year: i32, month: u32) -> u32 { - let is_leap_year = NaiveDate::from_ymd_opt(year, 2, 29).is_some(); +fn last_day_of_year_month(year: i16, month: i8) -> i8 { + let is_leap_year = (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0); if std::intrinsics::unlikely(month == 2 && is_leap_year) { return 29; } - LAST_DAY_LUT[month as usize] as u32 + LAST_DAY_LUT[month as usize] } macro_rules! impl_interval_year_month { @@ -337,29 +129,34 @@ macro_rules! impl_interval_year_month { impl $name { pub fn eval_date( date: i32, - tz: TzLUT, + tz: TimeZone, delta: impl AsPrimitive, ) -> std::result::Result { - let date = date.to_date(tz.tz); + let date = date.to_date(tz); let new_date = $op(date.year(), date.month(), date.day(), delta.as_())?; + Ok(clamp_date( new_date - .signed_duration_since(NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()) - .num_days(), + .since((Unit::Day, Date::new(1970, 1, 1).unwrap())) + .unwrap() + .get_days() as i64, )) } pub fn eval_timestamp( us: i64, - tz: TzLUT, + tz: TimeZone, delta: impl AsPrimitive, ) -> std::result::Result { - let ts = us.to_timestamp(tz.tz); - let new_ts = $op(ts.year(), ts.month(), ts.day(), delta.as_())?; - let mut ts = NaiveDateTime::new(new_ts, ts.time()) - .and_local_timezone(tz.tz) - .unwrap() - .timestamp_micros(); + let ts = us.to_timestamp(tz.clone()); + let new_date = $op(ts.year(), ts.month(), ts.day(), delta.as_())?; + + let mut ts = new_date + .at(ts.hour(), ts.minute(), ts.second(), ts.subsec_nanosecond()) + .to_zoned(tz) + .map_err(|e| format!("{}", e))? + .timestamp() + .as_microsecond(); clamp_timestamp(&mut ts); Ok(ts) } @@ -371,15 +168,15 @@ impl_interval_year_month!(EvalYearsImpl, eval_years_base); impl_interval_year_month!(EvalMonthsImpl, eval_months_base); impl EvalYearsImpl { - pub fn eval_date_diff(date_start: i32, date_end: i32, tz: TzLUT) -> i32 { - let date_start = date_start.to_date(tz.tz); - let date_end = date_end.to_date(tz.tz); - date_end.year() - date_start.year() + pub fn eval_date_diff(date_start: i32, date_end: i32, tz: TimeZone) -> i32 { + let date_start = date_start.to_date(tz.clone()); + let date_end = date_end.to_date(tz); + (date_end.year() - date_start.year()) as i32 } - pub fn eval_timestamp_diff(date_start: i64, date_end: i64, tz: TzLUT) -> i64 { - let date_start = date_start.to_timestamp(tz.tz); - let date_end = date_end.to_timestamp(tz.tz); + pub fn eval_timestamp_diff(date_start: i64, date_end: i64, tz: TimeZone) -> i64 { + let date_start = date_start.to_timestamp(tz.clone()); + let date_end = date_end.to_timestamp(tz); date_end.year() as i64 - date_start.year() as i64 } } @@ -387,7 +184,7 @@ impl EvalYearsImpl { pub struct EvalQuartersImpl; impl EvalQuartersImpl { - pub fn eval_date_diff(date_start: i32, date_end: i32, tz: TzLUT) -> i32 { + pub fn eval_date_diff(date_start: i32, date_end: i32, tz: TimeZone) -> i32 { EvalQuartersImpl::eval_timestamp_diff( date_start as i64 * MICROSECS_PER_DAY, date_end as i64 * MICROSECS_PER_DAY, @@ -395,23 +192,23 @@ impl EvalQuartersImpl { ) as i32 } - pub fn eval_timestamp_diff(date_start: i64, date_end: i64, tz: TzLUT) -> i64 { - let date_start = date_start.to_timestamp(tz.tz); - let date_end = date_end.to_timestamp(tz.tz); + pub fn eval_timestamp_diff(date_start: i64, date_end: i64, tz: TimeZone) -> i64 { + let date_start = date_start.to_timestamp(tz.clone()); + let date_end = date_end.to_timestamp(tz); (date_end.year() - date_start.year()) as i64 * 4 + ToQuarter::to_number(&date_end) as i64 - ToQuarter::to_number(&date_start) as i64 } } impl EvalMonthsImpl { - pub fn eval_date_diff(date_start: i32, date_end: i32, tz: TzLUT) -> i32 { - let date_start = date_start.to_date(tz.tz); - let date_end = date_end.to_date(tz.tz); - (date_end.year() - date_start.year()) * 12 + date_end.month() as i32 + pub fn eval_date_diff(date_start: i32, date_end: i32, tz: TimeZone) -> i32 { + let date_start = date_start.to_date(tz.clone()); + let date_end = date_end.to_date(tz); + (date_end.year() - date_start.year()) as i32 * 12 + date_end.month() as i32 - date_start.month() as i32 } - pub fn eval_timestamp_diff(date_start: i64, date_end: i64, tz: TzLUT) -> i64 { + pub fn eval_timestamp_diff(date_start: i64, date_end: i64, tz: TimeZone) -> i64 { EvalMonthsImpl::eval_date_diff( (date_start / MICROSECS_PER_DAY) as i32, (date_end / MICROSECS_PER_DAY) as i32, @@ -532,33 +329,32 @@ impl EvalTimesImpl { } #[inline] -pub fn today_date(now: DateTime, tz: TzLUT) -> i32 { - let now = now.with_timezone(&tz.tz); - NaiveDate::from_ymd_opt(now.year(), now.month(), now.day()) +pub fn today_date(now: &Zoned, tz: &TimeZone) -> i32 { + let now = now.with_time_zone(tz.clone()); + now.date() + .since((Unit::Day, Date::new(1970, 1, 1).unwrap())) .unwrap() - .signed_duration_since(NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()) - .num_days() as i32 + .get_days() } pub trait ToNumber { - fn to_number(dt: &DateTime) -> N; + fn to_number(dt: &Zoned) -> N; } pub struct ToNumberImpl; impl ToNumberImpl { - pub fn eval_timestamp, R>(us: i64, tz: TzLUT) -> R { - let dt = us.to_timestamp(tz.tz); + pub fn eval_timestamp, R>(us: i64, tz: TimeZone) -> R { + let dt = us.to_timestamp(tz); T::to_number(&dt) } - pub fn eval_date, R>( - date: i32, - tz: TzLUT, - enable_dst_hour_fix: bool, - ) -> Result { - let naive_dt = date.to_date(tz.tz).and_hms_opt(0, 0, 0).unwrap(); - let dt = unwrap_local_time(&tz.tz, enable_dst_hour_fix, &naive_dt)?; + pub fn eval_date, R>(date: i32, tz: TimeZone) -> Result { + let dt = date + .to_date(tz.clone()) + .at(0, 0, 0, 0) + .to_zoned(tz) + .unwrap(); Ok(T::to_number(&dt)) } } @@ -581,25 +377,25 @@ pub struct ToUnixTimestamp; pub struct ToWeekOfYear; impl ToNumber for ToYYYYMM { - fn to_number(dt: &DateTime) -> u32 { - dt.year() as u32 * 100 + dt.month() + fn to_number(dt: &Zoned) -> u32 { + dt.year() as u32 * 100 + dt.month() as u32 } } impl ToNumber for ToWeekOfYear { - fn to_number(dt: &DateTime) -> u32 { - dt.iso_week().week() + fn to_number(dt: &Zoned) -> u32 { + dt.date().to_iso_week_date().week() as u32 } } impl ToNumber for ToYYYYMMDD { - fn to_number(dt: &DateTime) -> u32 { - dt.year() as u32 * 10_000 + dt.month() * 100 + dt.day() + fn to_number(dt: &Zoned) -> u32 { + dt.year() as u32 * 10_000 + dt.month() as u32 * 100 + dt.day() as u32 } } impl ToNumber for ToYYYYMMDDHH { - fn to_number(dt: &DateTime) -> u64 { + fn to_number(dt: &Zoned) -> u64 { dt.year() as u64 * 1_000_000 + dt.month() as u64 * 10_000 + dt.day() as u64 * 100 @@ -608,7 +404,7 @@ impl ToNumber for ToYYYYMMDDHH { } impl ToNumber for ToYYYYMMDDHHMMSS { - fn to_number(dt: &DateTime) -> u64 { + fn to_number(dt: &Zoned) -> u64 { dt.year() as u64 * 10_000_000_000 + dt.month() as u64 * 100_000_000 + dt.day() as u64 * 1_000_000 @@ -619,44 +415,45 @@ impl ToNumber for ToYYYYMMDDHHMMSS { } impl ToNumber for ToYear { - fn to_number(dt: &DateTime) -> u16 { + fn to_number(dt: &Zoned) -> u16 { dt.year() as u16 } } impl ToNumber for ToQuarter { - fn to_number(dt: &DateTime) -> u8 { - (dt.month0() / 3 + 1) as u8 + fn to_number(dt: &Zoned) -> u8 { + // begin with 0 + ((dt.month() - 1) / 3 + 1) as u8 } } impl ToNumber for ToMonth { - fn to_number(dt: &DateTime) -> u8 { + fn to_number(dt: &Zoned) -> u8 { dt.month() as u8 } } impl ToNumber for ToDayOfYear { - fn to_number(dt: &DateTime) -> u16 { - dt.ordinal() as u16 + fn to_number(dt: &Zoned) -> u16 { + dt.day_of_year() as u16 } } impl ToNumber for ToDayOfMonth { - fn to_number(dt: &DateTime) -> u8 { + fn to_number(dt: &Zoned) -> u8 { dt.day() as u8 } } impl ToNumber for ToDayOfWeek { - fn to_number(dt: &DateTime) -> u8 { - dt.weekday().number_from_monday() as u8 + fn to_number(dt: &Zoned) -> u8 { + dt.weekday().to_monday_one_offset() as u8 } } impl ToNumber for ToUnixTimestamp { - fn to_number(dt: &DateTime) -> i64 { - dt.timestamp() + fn to_number(dt: &Zoned) -> i64 { + dt.with_time_zone(TimeZone::UTC).timestamp().as_second() } } @@ -672,22 +469,108 @@ pub enum Round { Day, } +pub fn round_timestamp(ts: i64, tz: &TimeZone, round: Round) -> i64 { + let dtz = ts.to_timestamp(tz.clone()); + let res = match round { + Round::Second => tz + .to_zoned(datetime( + dtz.year(), + dtz.month(), + dtz.day(), + dtz.hour(), + dtz.minute(), + dtz.second(), + 0, + )) + .unwrap(), + Round::Minute => tz + .to_zoned(datetime( + dtz.year(), + dtz.month(), + dtz.day(), + dtz.hour(), + dtz.minute(), + 0, + 0, + )) + .unwrap(), + Round::FiveMinutes => tz + .to_zoned(datetime( + dtz.year(), + dtz.month(), + dtz.day(), + dtz.hour(), + dtz.minute() / 5 * 5, + 0, + 0, + )) + .unwrap(), + Round::TenMinutes => tz + .to_zoned(datetime( + dtz.year(), + dtz.month(), + dtz.day(), + dtz.hour(), + dtz.minute() / 10 * 10, + 0, + 0, + )) + .unwrap(), + Round::FifteenMinutes => tz + .to_zoned(datetime( + dtz.year(), + dtz.month(), + dtz.day(), + dtz.hour(), + dtz.minute() / 15 * 15, + 0, + 0, + )) + .unwrap(), + Round::TimeSlot => tz + .to_zoned(datetime( + dtz.year(), + dtz.month(), + dtz.day(), + dtz.hour(), + dtz.minute() / 30 * 30, + 0, + 0, + )) + .unwrap(), + Round::Hour => tz + .to_zoned(datetime( + dtz.year(), + dtz.month(), + dtz.day(), + dtz.hour(), + 0, + 0, + 0, + )) + .unwrap(), + Round::Day => tz + .to_zoned(datetime(dtz.year(), dtz.month(), dtz.day(), 0, 0, 0, 0)) + .unwrap(), + }; + res.timestamp().as_microsecond() +} + pub struct DateRounder; impl DateRounder { - pub fn eval_timestamp>(us: i64, tz: TzLUT) -> i32 { - let dt = us.to_timestamp(tz.tz); + pub fn eval_timestamp>(us: i64, tz: TimeZone) -> i32 { + let dt = us.to_timestamp(tz); T::to_number(&dt) } - pub fn eval_date>( - date: i32, - tz: TzLUT, - enable_dst_hour_fix: bool, - ) -> Result { - let naive_dt = date.to_date(tz.tz).and_hms_opt(0, 0, 0).unwrap(); - let dt = unwrap_local_time(&tz.tz, enable_dst_hour_fix, &naive_dt)?; - Ok(T::to_number(&dt)) + pub fn eval_date>(date: i32, tz: TimeZone) -> Result { + let naive_dt = date + .to_date(tz.clone()) + .at(0, 0, 0, 0) + .to_zoned(tz) + .unwrap(); + Ok(T::to_number(&naive_dt)) } } @@ -695,15 +578,11 @@ impl DateRounder { /// /// It's the days since 1970-01-01. #[inline] -fn datetime_to_date_inner_number(date: &DateTime) -> i32 { - date.naive_local() - .signed_duration_since( - NaiveDate::from_ymd_opt(1970, 1, 1) - .unwrap() - // if dt is dst, should respect dt.time - .and_time(date.time()), - ) - .num_days() as i32 +fn datetime_to_date_inner_number(date: &Zoned) -> i32 { + date.date() + .since((Unit::Day, Date::new(1970, 1, 1).unwrap())) + .unwrap() + .get_days() } pub struct ToLastMonday; @@ -732,174 +611,188 @@ pub struct ToNextSaturday; pub struct ToNextSunday; impl ToNumber for ToLastMonday { - fn to_number(dt: &DateTime) -> i32 { + fn to_number(dt: &Zoned) -> i32 { // datetime_to_date_inner_number just calc naive_date, so weekday also need only calc naive_date - datetime_to_date_inner_number(dt) - dt.date_naive().weekday().num_days_from_monday() as i32 + datetime_to_date_inner_number(dt) - dt.date().weekday().to_monday_zero_offset() as i32 } } impl ToNumber for ToLastSunday { - fn to_number(dt: &DateTime) -> i32 { + fn to_number(dt: &Zoned) -> i32 { // datetime_to_date_inner_number just calc naive_date, so weekday also need only calc naive_date - datetime_to_date_inner_number(dt) - dt.date_naive().weekday().num_days_from_sunday() as i32 + datetime_to_date_inner_number(dt) - dt.date().weekday().to_sunday_zero_offset() as i32 } } impl ToNumber for ToStartOfMonth { - fn to_number(dt: &DateTime) -> i32 { - datetime_to_date_inner_number(&dt.with_day(1).unwrap()) + fn to_number(dt: &Zoned) -> i32 { + datetime_to_date_inner_number(&dt.first_of_month().unwrap()) } } impl ToNumber for ToStartOfQuarter { - fn to_number(dt: &DateTime) -> i32 { - let new_month = dt.month0() / 3 * 3 + 1; - datetime_to_date_inner_number(&dt.with_day(1).unwrap().with_month(new_month).unwrap()) + fn to_number(dt: &Zoned) -> i32 { + let new_month = (dt.month() - 1) / 3 * 3 + 1; + let new_day = date(dt.year(), new_month, 1) + .at(0, 0, 0, 0) + .to_zoned(dt.time_zone().clone()) + .unwrap(); + datetime_to_date_inner_number(&new_day) } } impl ToNumber for ToStartOfYear { - fn to_number(dt: &DateTime) -> i32 { - datetime_to_date_inner_number(&dt.with_day(1).unwrap().with_month(1).unwrap()) + fn to_number(dt: &Zoned) -> i32 { + datetime_to_date_inner_number(&dt.first_of_year().unwrap()) } } impl ToNumber for ToStartOfISOYear { - fn to_number(dt: &DateTime) -> i32 { - let iso_year = dt.iso_week().year(); - - let iso_dt = dt - .timezone() - .from_local_datetime( - &NaiveDate::from_isoywd_opt(iso_year, 1, chrono::Weekday::Mon) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap(), - ) - .unwrap(); - datetime_to_date_inner_number(&iso_dt) + fn to_number(dt: &Zoned) -> i32 { + let iso_year = dt.date().to_iso_week_date().year(); + for i in 1..=7 { + let new_dt = date(iso_year, 1, i) + .at(0, 0, 0, 0) + .to_zoned(dt.time_zone().clone()) + .unwrap(); + if new_dt.date().to_iso_week_date().weekday() == Weekday::Monday { + return datetime_to_date_inner_number(&new_dt); + } + } + // Never return 0 + 0 } } impl ToNumber for ToLastOfWeek { - fn to_number(dt: &DateTime) -> i32 { - datetime_to_date_inner_number(dt) - dt.date_naive().weekday().num_days_from_monday() as i32 - + 6 + fn to_number(dt: &Zoned) -> i32 { + datetime_to_date_inner_number(dt) - dt.date().weekday().to_monday_zero_offset() as i32 + 6 } } impl ToNumber for ToLastOfMonth { - fn to_number(dt: &DateTime) -> i32 { + fn to_number(dt: &Zoned) -> i32 { let day = last_day_of_year_month(dt.year(), dt.month()); - datetime_to_date_inner_number(&dt.with_day(day).unwrap()) + let dt = date(dt.year(), dt.month(), day) + .at(dt.hour(), dt.minute(), dt.second(), dt.subsec_nanosecond()) + .to_zoned(dt.time_zone().clone()) + .unwrap(); + datetime_to_date_inner_number(&dt) } } impl ToNumber for ToLastOfQuarter { - fn to_number(dt: &DateTime) -> i32 { - let new_month = dt.month0() / 3 * 3 + 3; + fn to_number(dt: &Zoned) -> i32 { + let new_month = (dt.month() - 1) / 3 * 3 + 3; let day = last_day_of_year_month(dt.year(), new_month); - datetime_to_date_inner_number(&dt.with_month(new_month).unwrap().with_day(day).unwrap()) + let dt = date(dt.year(), new_month, day) + .at(dt.hour(), dt.minute(), dt.second(), dt.subsec_nanosecond()) + .to_zoned(dt.time_zone().clone()) + .unwrap(); + datetime_to_date_inner_number(&dt) } } impl ToNumber for ToLastOfYear { - fn to_number(dt: &DateTime) -> i32 { + fn to_number(dt: &Zoned) -> i32 { let day = last_day_of_year_month(dt.year(), 12); - datetime_to_date_inner_number(&dt.with_month(12).unwrap().with_day(day).unwrap()) + let dt = date(dt.year(), 12, day) + .at(dt.hour(), dt.minute(), dt.second(), dt.subsec_nanosecond()) + .to_zoned(dt.time_zone().clone()) + .unwrap(); + datetime_to_date_inner_number(&dt) } } impl ToNumber for ToPreviousMonday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Mon, true) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Monday, true) } } impl ToNumber for ToPreviousTuesday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Tue, true) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Tuesday, true) } } impl ToNumber for ToPreviousWednesday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Wed, true) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Wednesday, true) } } impl ToNumber for ToPreviousThursday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Thu, true) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Thursday, true) } } impl ToNumber for ToPreviousFriday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Fri, true) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Friday, true) } } impl ToNumber for ToPreviousSaturday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Sat, true) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Saturday, true) } } impl ToNumber for ToPreviousSunday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Sun, true) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Sunday, true) } } impl ToNumber for ToNextMonday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Mon, false) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Monday, false) } } impl ToNumber for ToNextTuesday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Tue, false) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Tuesday, false) } } impl ToNumber for ToNextWednesday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Wed, false) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Wednesday, false) } } impl ToNumber for ToNextThursday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Thu, false) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Thursday, false) } } impl ToNumber for ToNextFriday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Fri, false) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Friday, false) } } impl ToNumber for ToNextSaturday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Sat, false) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Saturday, false) } } impl ToNumber for ToNextSunday { - fn to_number(dt: &DateTime) -> i32 { - previous_or_next_day(dt, Weekday::Sun, false) + fn to_number(dt: &Zoned) -> i32 { + previous_or_next_day(dt, Weekday::Sunday, false) } } -pub fn previous_or_next_day(dt: &DateTime, target: Weekday, is_previous: bool) -> i32 { +pub fn previous_or_next_day(dt: &Zoned, target: Weekday, is_previous: bool) -> i32 { let dir = if is_previous { -1 } else { 1 }; - let mut days_diff = (dir - * (target.num_days_from_monday() as i32 - - dt.date_naive().weekday().num_days_from_monday() as i32) + * (target.to_monday_zero_offset() as i32 + - dt.date().weekday().to_monday_zero_offset() as i32) + 7) % 7; diff --git a/src/query/expression/src/utils/display.rs b/src/query/expression/src/utils/display.rs index b89834ed3362..126a28e95308 100755 --- a/src/query/expression/src/utils/display.rs +++ b/src/query/expression/src/utils/display.rs @@ -16,7 +16,6 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; -use chrono_tz::Tz; use comfy_table::Cell; use comfy_table::Table; use databend_common_ast::ast::quote::display_ident; @@ -28,6 +27,7 @@ use databend_common_io::ewkb_to_geo; use databend_common_io::geo_to_ewkt; use geozero::wkb::Ewkb; use itertools::Itertools; +use jiff::tz::TimeZone; use num_traits::FromPrimitive; use rust_decimal::Decimal; use rust_decimal::RoundingStrategy; @@ -222,8 +222,8 @@ impl<'a> Display for ScalarRef<'a> { Ok(()) } ScalarRef::String(s) => write!(f, "'{s}'"), - ScalarRef::Timestamp(t) => write!(f, "'{}'", timestamp_to_string(*t, Tz::UTC)), - ScalarRef::Date(d) => write!(f, "'{}'", date_to_string(*d as i64, Tz::UTC)), + ScalarRef::Timestamp(t) => write!(f, "'{}'", timestamp_to_string(*t, &TimeZone::UTC)), + ScalarRef::Date(d) => write!(f, "'{}'", date_to_string(*d as i64, &TimeZone::UTC)), ScalarRef::Array(col) => write!(f, "[{}]", col.iter().join(", ")), ScalarRef::Map(col) => { write!(f, "{{")?; diff --git a/src/query/expression/src/utils/serialize.rs b/src/query/expression/src/utils/serialize.rs index 7307ecc58bb8..c33ae1cfa769 100644 --- a/src/query/expression/src/utils/serialize.rs +++ b/src/query/expression/src/utils/serialize.rs @@ -15,8 +15,8 @@ use std::cmp::Ordering; use std::result::Result; -use chrono::Datelike; -use chrono::NaiveDate; +use jiff::civil::Date; +use jiff::Unit; use crate::types::decimal::Decimal; use crate::types::decimal::DecimalSize; @@ -24,8 +24,10 @@ use crate::types::decimal::DecimalSize; pub const EPOCH_DAYS_FROM_CE: i32 = 719_163; #[inline] -pub fn uniform_date(date: NaiveDate) -> i32 { - date.num_days_from_ce() - EPOCH_DAYS_FROM_CE +pub fn uniform_date(date: Date) -> i32 { + date.since((Unit::Day, Date::new(1970, 1, 1).unwrap())) + .unwrap() + .get_days() } // Used in function, so we don't want to return ErrorCode with backtrace @@ -38,7 +40,7 @@ pub fn read_decimal_with_size( // Read one more digit for round let (n, d, e, n_read) = read_decimal::(buf, (size.precision + 1) as u32, size.scale as _, exact)?; - if d as i32 + e > (size.precision - size.scale).into() { + if d as i32 + e > (size.precision - size.scale) as i32 { return Err(decimal_overflow_error()); } let scale_diff = e + size.scale as i32; diff --git a/src/query/expression/tests/it/types.rs b/src/query/expression/tests/it/types.rs index 0ba121d87569..74089ea507dc 100644 --- a/src/query/expression/tests/it/types.rs +++ b/src/query/expression/tests/it/types.rs @@ -13,12 +13,12 @@ // limitations under the License. use arrow_schema::Schema; -use chrono_tz::Tz; use databend_common_expression::arrow::deserialize_column; use databend_common_expression::arrow::serialize_column; use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_expression::DataField; use databend_common_expression::DataSchema; +use jiff::tz::TimeZone; use crate::get_all_test_data_types; use crate::rand_block_for_all_types; @@ -27,10 +27,10 @@ use crate::rand_block_for_all_types; fn test_timestamp_to_string_formats() { // Unix timestamp for "2024-01-01 01:02:03" UTC let ts = 1_704_070_923_000_000; - let tz = Tz::UTC; + let tz = TimeZone::UTC; assert_eq!( - timestamp_to_string(ts, tz).to_string(), + timestamp_to_string(ts, &tz).to_string(), "2024-01-01 01:02:03.000000" ); } diff --git a/src/query/formats/Cargo.toml b/src/query/formats/Cargo.toml index 8f11e6661940..778df19cceac 100644 --- a/src/query/formats/Cargo.toml +++ b/src/query/formats/Cargo.toml @@ -28,6 +28,7 @@ bstr = { workspace = true } chrono-tz = { workspace = true } geozero = { workspace = true } hex = { workspace = true } +jiff = { workspace = true } jsonb = { workspace = true } lexical-core = { workspace = true } match-template = { workspace = true } diff --git a/src/query/formats/src/common_settings.rs b/src/query/formats/src/common_settings.rs index 64ff425d5497..198be30185c5 100644 --- a/src/query/formats/src/common_settings.rs +++ b/src/query/formats/src/common_settings.rs @@ -15,6 +15,7 @@ use chrono_tz::Tz; use databend_common_io::GeometryDataType; use databend_common_meta_app::principal::BinaryFormat; +use jiff::tz::TimeZone; #[derive(Clone)] pub struct InputCommonSettings { @@ -22,6 +23,7 @@ pub struct InputCommonSettings { pub false_bytes: Vec, pub null_if: Vec>, pub timezone: Tz, + pub jiff_timezone: TimeZone, pub disable_variant_check: bool, pub binary_format: BinaryFormat, pub is_rounding_mode: bool, @@ -36,6 +38,7 @@ pub struct OutputCommonSettings { pub nan_bytes: Vec, pub inf_bytes: Vec, pub timezone: Tz, + pub jiff_timezone: TimeZone, pub binary_format: BinaryFormat, pub geometry_format: GeometryDataType, } diff --git a/src/query/formats/src/field_decoder/fast_values.rs b/src/query/formats/src/field_decoder/fast_values.rs index 7e3efd72e7f6..7f5c62583d1c 100644 --- a/src/query/formats/src/field_decoder/fast_values.rs +++ b/src/query/formats/src/field_decoder/fast_values.rs @@ -87,6 +87,7 @@ impl FastFieldDecoderValues { NAN_BYTES_LOWER.as_bytes().to_vec(), ], timezone: format.timezone, + jiff_timezone: format.jiff_timezone, disable_variant_check: false, binary_format: Default::default(), is_rounding_mode, @@ -284,10 +285,7 @@ impl FastFieldDecoderValues { let mut buf = Vec::new(); self.read_string_inner(reader, &mut buf, positions)?; let mut buffer_readr = Cursor::new(&buf); - let date = buffer_readr.read_date_text( - &self.common_settings().timezone, - self.common_settings().enable_dst_hour_fix, - )?; + let date = buffer_readr.read_date_text(&self.common_settings().jiff_timezone)?; let days = uniform_date(date); column.push(clamp_date(days as i64)); Ok(()) @@ -302,11 +300,7 @@ impl FastFieldDecoderValues { let mut buf = Vec::new(); self.read_string_inner(reader, &mut buf, positions)?; let mut buffer_readr = Cursor::new(&buf); - let ts = buffer_readr.read_timestamp_text( - &self.common_settings().timezone, - false, - self.common_settings.enable_dst_hour_fix, - )?; + let ts = buffer_readr.read_timestamp_text(&self.common_settings().jiff_timezone)?; match ts { DateTimeResType::Datetime(ts) => { if !buffer_readr.eof() { @@ -318,7 +312,7 @@ impl FastFieldDecoderValues { ); return Err(ErrorCode::BadBytes(msg)); } - let mut micros = ts.timestamp_micros(); + let mut micros = ts.timestamp().as_microsecond(); clamp_timestamp(&mut micros); column.push(micros.as_()); } diff --git a/src/query/formats/src/field_decoder/json_ast.rs b/src/query/formats/src/field_decoder/json_ast.rs index c258207f70e3..aab0ba8d67e7 100644 --- a/src/query/formats/src/field_decoder/json_ast.rs +++ b/src/query/formats/src/field_decoder/json_ast.rs @@ -15,7 +15,6 @@ use std::any::Any; use std::io::Cursor; -use chrono_tz::Tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::serialize::read_decimal_from_json; @@ -41,6 +40,7 @@ use databend_common_io::cursor_ext::DateTimeResType; use databend_common_io::geography::geography_from_ewkt; use databend_common_io::geometry_from_ewkt; use databend_common_io::parse_bitmap; +use jiff::tz::TimeZone; use lexical_core::FromLexical; use num::cast::AsPrimitive; use num_traits::NumCast; @@ -51,11 +51,10 @@ use crate::FieldDecoder; use crate::FileFormatOptionsExt; pub struct FieldJsonAstDecoder { - timezone: Tz, + jiff_timezone: TimeZone, pub ident_case_sensitive: bool, pub is_select: bool, is_rounding_mode: bool, - enable_dst_hour_fix: bool, } impl FieldDecoder for FieldJsonAstDecoder { @@ -67,11 +66,10 @@ impl FieldDecoder for FieldJsonAstDecoder { impl FieldJsonAstDecoder { pub fn create(options: &FileFormatOptionsExt) -> Self { FieldJsonAstDecoder { - timezone: options.timezone, + jiff_timezone: options.jiff_timezone.clone(), ident_case_sensitive: options.ident_case_sensitive, is_select: options.is_select, is_rounding_mode: options.is_rounding_mode, - enable_dst_hour_fix: options.enable_dst_hour_fix, } } @@ -264,7 +262,7 @@ impl FieldJsonAstDecoder { match value { Value::String(v) => { let mut reader = Cursor::new(v.as_bytes()); - let date = reader.read_date_text(&self.timezone, self.enable_dst_hour_fix)?; + let date = reader.read_date_text(&self.jiff_timezone)?; let days = uniform_date(date); column.push(clamp_date(days as i64)); Ok(()) @@ -285,12 +283,11 @@ impl FieldJsonAstDecoder { Value::String(v) => { let v = v.clone(); let mut reader = Cursor::new(v.as_bytes()); - let ts = - reader.read_timestamp_text(&self.timezone, false, self.enable_dst_hour_fix)?; + let ts = reader.read_timestamp_text(&self.jiff_timezone)?; match ts { DateTimeResType::Datetime(ts) => { - let mut micros = ts.timestamp_micros(); + let mut micros = ts.timestamp().as_microsecond(); clamp_timestamp(&mut micros); column.push(micros.as_()); } diff --git a/src/query/formats/src/field_decoder/nested.rs b/src/query/formats/src/field_decoder/nested.rs index 6d6b515aead3..c4f26296ba7a 100644 --- a/src/query/formats/src/field_decoder/nested.rs +++ b/src/query/formats/src/field_decoder/nested.rs @@ -78,6 +78,7 @@ impl NestedValues { NULL_BYTES_LOWER.as_bytes().to_vec(), ], timezone: options_ext.timezone, + jiff_timezone: options_ext.jiff_timezone.clone(), disable_variant_check: options_ext.disable_variant_check, binary_format: Default::default(), is_rounding_mode: options_ext.is_rounding_mode, @@ -244,10 +245,7 @@ impl NestedValues { let mut buf = Vec::new(); self.read_string_inner(reader, &mut buf)?; let mut buffer_readr = Cursor::new(&buf); - let date = buffer_readr.read_date_text( - &self.common_settings().timezone, - self.common_settings().enable_dst_hour_fix, - )?; + let date = buffer_readr.read_date_text(&self.common_settings().jiff_timezone)?; let days = uniform_date(date); column.push(clamp_date(days as i64)); Ok(()) @@ -264,11 +262,7 @@ impl NestedValues { let mut ts = if !buf.contains(&b'-') { buffer_readr.read_num_text_exact()? } else { - let t = buffer_readr.read_timestamp_text( - &self.common_settings().timezone, - false, - self.common_settings.enable_dst_hour_fix, - )?; + let t = buffer_readr.read_timestamp_text(&self.common_settings().jiff_timezone)?; match t { DateTimeResType::Datetime(t) => { if !buffer_readr.eof() { @@ -280,7 +274,7 @@ impl NestedValues { ); return Err(ErrorCode::BadBytes(msg)); } - t.timestamp_micros() + t.timestamp().as_microsecond() } _ => unreachable!(), } diff --git a/src/query/formats/src/field_decoder/separated_text.rs b/src/query/formats/src/field_decoder/separated_text.rs index 50e7dd9a5cf6..0975ca0f1265 100644 --- a/src/query/formats/src/field_decoder/separated_text.rs +++ b/src/query/formats/src/field_decoder/separated_text.rs @@ -82,6 +82,7 @@ impl SeparatedTextDecoder { false_bytes: FALSE_BYTES_LOWER.as_bytes().to_vec(), null_if: vec![params.null_display.as_bytes().to_vec()], timezone: options_ext.timezone, + jiff_timezone: options_ext.jiff_timezone.clone(), disable_variant_check: options_ext.disable_variant_check, binary_format: params.binary_format, is_rounding_mode: options_ext.is_rounding_mode, @@ -98,6 +99,7 @@ impl SeparatedTextDecoder { true_bytes: TRUE_BYTES_NUM.as_bytes().to_vec(), false_bytes: FALSE_BYTES_NUM.as_bytes().to_vec(), timezone: options_ext.timezone, + jiff_timezone: options_ext.jiff_timezone.clone(), disable_variant_check: options_ext.disable_variant_check, binary_format: Default::default(), is_rounding_mode: options_ext.is_rounding_mode, @@ -251,10 +253,7 @@ impl SeparatedTextDecoder { fn read_date(&self, column: &mut Vec, data: &[u8]) -> Result<()> { let mut buffer_readr = Cursor::new(&data); - let date = buffer_readr.read_date_text( - &self.common_settings().timezone, - self.common_settings().enable_dst_hour_fix, - )?; + let date = buffer_readr.read_date_text(&self.common_settings().jiff_timezone)?; let days = uniform_date(date); column.push(clamp_date(days as i64)); Ok(()) @@ -265,11 +264,7 @@ impl SeparatedTextDecoder { read_num_text_exact(data)? } else { let mut buffer_readr = Cursor::new(&data); - let t = buffer_readr.read_timestamp_text( - &self.common_settings().timezone, - false, - self.common_settings.enable_dst_hour_fix, - )?; + let t = buffer_readr.read_timestamp_text(&self.common_settings().jiff_timezone)?; match t { DateTimeResType::Datetime(t) => { if !buffer_readr.eof() { @@ -281,7 +276,7 @@ impl SeparatedTextDecoder { ); return Err(ErrorCode::BadBytes(msg)); } - t.timestamp_micros() + t.timestamp().as_microsecond() } _ => unreachable!(), } diff --git a/src/query/formats/src/field_encoder/csv.rs b/src/query/formats/src/field_encoder/csv.rs index c4b323c3b831..9e49bef6197c 100644 --- a/src/query/formats/src/field_encoder/csv.rs +++ b/src/query/formats/src/field_encoder/csv.rs @@ -88,6 +88,7 @@ impl FieldEncoderCSV { nan_bytes: params.nan_display.as_bytes().to_vec(), inf_bytes: INF_BYTES_LONG.as_bytes().to_vec(), timezone: options_ext.timezone, + jiff_timezone: options_ext.jiff_timezone.clone(), binary_format: params.binary_format, geometry_format: params.geometry_format, }, @@ -110,6 +111,7 @@ impl FieldEncoderCSV { nan_bytes: params.nan_display.as_bytes().to_vec(), inf_bytes: INF_BYTES_LONG.as_bytes().to_vec(), timezone: options_ext.timezone, + jiff_timezone: options_ext.jiff_timezone.clone(), binary_format: Default::default(), geometry_format: Default::default(), }, diff --git a/src/query/formats/src/field_encoder/json.rs b/src/query/formats/src/field_encoder/json.rs index 6ebe00c5db76..2549d3203e66 100644 --- a/src/query/formats/src/field_encoder/json.rs +++ b/src/query/formats/src/field_encoder/json.rs @@ -44,6 +44,7 @@ impl FieldEncoderJSON { inf_bytes: NULL_BYTES_LOWER.as_bytes().to_vec(), null_bytes: NULL_BYTES_LOWER.as_bytes().to_vec(), timezone: options.timezone, + jiff_timezone: options.jiff_timezone.clone(), binary_format: Default::default(), geometry_format: Default::default(), }, diff --git a/src/query/formats/src/field_encoder/values.rs b/src/query/formats/src/field_encoder/values.rs index 1c3fa889da92..f0fa3a8b2db4 100644 --- a/src/query/formats/src/field_encoder/values.rs +++ b/src/query/formats/src/field_encoder/values.rs @@ -42,6 +42,7 @@ use databend_common_io::geo_to_wkb; use databend_common_io::geo_to_wkt; use databend_common_io::GeometryDataType; use geozero::wkb::Ewkb; +use jiff::tz::TimeZone; use lexical_core::ToLexical; use micromarshal::Marshal; use micromarshal::Unmarshal; @@ -66,6 +67,7 @@ impl FieldEncoderValues { nan_bytes: NAN_BYTES_LOWER.as_bytes().to_vec(), inf_bytes: INF_BYTES_LOWER.as_bytes().to_vec(), timezone: options.timezone, + jiff_timezone: options.jiff_timezone.clone(), binary_format: Default::default(), geometry_format: Default::default(), }, @@ -73,7 +75,11 @@ impl FieldEncoderValues { } } - pub fn create_for_http_handler(timezone: Tz, geometry_format: GeometryDataType) -> Self { + pub fn create_for_http_handler( + jiff_timezone: TimeZone, + timezone: Tz, + geometry_format: GeometryDataType, + ) -> Self { FieldEncoderValues { common_settings: OutputCommonSettings { true_bytes: TRUE_BYTES_NUM.as_bytes().to_vec(), @@ -82,6 +88,7 @@ impl FieldEncoderValues { nan_bytes: NAN_BYTES_SNAKE.as_bytes().to_vec(), inf_bytes: INF_BYTES_LONG.as_bytes().to_vec(), timezone, + jiff_timezone, binary_format: Default::default(), geometry_format, }, @@ -93,7 +100,11 @@ impl FieldEncoderValues { // mysql python client will decode to python float, which is printed as 'nan' and 'inf' // so we still use 'nan' and 'inf' in logic test. // https://github.com/datafuselabs/databend/discussions/8941 - pub fn create_for_mysql_handler(timezone: Tz, geometry_format: GeometryDataType) -> Self { + pub fn create_for_mysql_handler( + jiff_timezone: TimeZone, + timezone: Tz, + geometry_format: GeometryDataType, + ) -> Self { FieldEncoderValues { common_settings: OutputCommonSettings { true_bytes: TRUE_BYTES_NUM.as_bytes().to_vec(), @@ -102,6 +113,7 @@ impl FieldEncoderValues { nan_bytes: NAN_BYTES_SNAKE.as_bytes().to_vec(), inf_bytes: INF_BYTES_LONG.as_bytes().to_vec(), timezone, + jiff_timezone, binary_format: Default::default(), geometry_format, }, @@ -260,7 +272,7 @@ impl FieldEncoderValues { in_nested: bool, ) { let v = unsafe { column.get_unchecked(row_index) }; - let s = date_to_string(*v as i64, self.common_settings().timezone).to_string(); + let s = date_to_string(*v as i64, &self.common_settings().jiff_timezone).to_string(); self.write_string_inner(s.as_bytes(), out_buf, in_nested); } @@ -272,7 +284,7 @@ impl FieldEncoderValues { in_nested: bool, ) { let v = unsafe { column.get_unchecked(row_index) }; - let s = timestamp_to_string(*v, self.common_settings().timezone).to_string(); + let s = timestamp_to_string(*v, &self.common_settings().jiff_timezone).to_string(); self.write_string_inner(s.as_bytes(), out_buf, in_nested); } diff --git a/src/query/formats/src/file_format_type.rs b/src/query/formats/src/file_format_type.rs index b9208ac55406..ba1388e50149 100644 --- a/src/query/formats/src/file_format_type.rs +++ b/src/query/formats/src/file_format_type.rs @@ -20,6 +20,7 @@ use databend_common_io::GeometryDataType; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::StageFileFormatType; use databend_common_settings::Settings; +use jiff::tz::TimeZone; use crate::output_format::CSVOutputFormat; use crate::output_format::CSVWithNamesAndTypesOutputFormat; @@ -45,6 +46,7 @@ pub struct FileFormatOptionsExt { pub json_strings: bool, pub disable_variant_check: bool, pub timezone: Tz, + pub jiff_timezone: TimeZone, pub is_select: bool, pub is_clickhouse: bool, pub is_rounding_mode: bool, @@ -58,6 +60,7 @@ impl FileFormatOptionsExt { is_select: bool, ) -> Result { let timezone = parse_timezone(settings)?; + let jiff_timezone = parse_jiff_timezone(settings)?; let enable_dst_hour_fix = settings.get_enable_dst_hour_fix()?; let geometry_format = settings.get_geometry_output_format()?; let numeric_cast_option = settings @@ -72,6 +75,7 @@ impl FileFormatOptionsExt { json_strings: false, disable_variant_check: false, timezone, + jiff_timezone, is_select, is_clickhouse: false, is_rounding_mode, @@ -86,6 +90,7 @@ impl FileFormatOptionsExt { settings: &Settings, ) -> Result { let timezone = parse_timezone(settings)?; + let jiff_timezone = parse_jiff_timezone(settings)?; let geometry_format = settings.get_geometry_output_format()?; let enable_dst_hour_fix = settings.get_enable_dst_hour_fix()?; let mut options = FileFormatOptionsExt { @@ -95,6 +100,7 @@ impl FileFormatOptionsExt { json_strings: false, disable_variant_check: false, timezone, + jiff_timezone, is_select: false, is_clickhouse: true, is_rounding_mode: true, @@ -217,3 +223,13 @@ pub fn parse_timezone(settings: &Settings) -> Result { tz.parse::() .map_err(|_| ErrorCode::InvalidTimezone("Timezone has been checked and should be valid")) } + +pub fn parse_jiff_timezone(settings: &Settings) -> Result { + let tz = settings.get_timezone()?; + TimeZone::get(&tz).map_err(|e| { + ErrorCode::InvalidTimezone(format!( + "Timezone has been checked and should be valid but got error: {}", + e + )) + }) +} diff --git a/src/query/formats/src/output_format/json.rs b/src/query/formats/src/output_format/json.rs index e23a49313f23..ee1f6231b912 100644 --- a/src/query/formats/src/output_format/json.rs +++ b/src/query/formats/src/output_format/json.rs @@ -21,6 +21,7 @@ use databend_common_io::deserialize_bitmap; use databend_common_io::prelude::FormatSettings; use geozero::wkb::Ewkb; use geozero::ToJson; +use jiff::fmt::strtime; use serde_json::Map as JsonMap; use serde_json::Value as JsonValue; @@ -44,6 +45,7 @@ impl JSONOutputFormat { rows: 0, format_settings: FormatSettings { timezone: options.timezone, + jiff_timezone: options.jiff_timezone.clone(), geometry_format: options.geometry_format, enable_dst_hour_fix: options.enable_dst_hour_fix, format_null_as_str: true, @@ -94,12 +96,12 @@ fn scalar_to_json(s: ScalarRef<'_>, format: &FormatSettings) -> JsonValue { }, ScalarRef::Decimal(x) => serde_json::to_value(x.to_string()).unwrap(), ScalarRef::Date(v) => { - let dt = DateConverter::to_date(&v, format.timezone); - serde_json::to_value(dt.format("%Y-%m-%d").to_string()).unwrap() + let dt = DateConverter::to_date(&v, format.jiff_timezone.clone()); + serde_json::to_value(strtime::format("%Y-%m-%d", dt).unwrap()).unwrap() } ScalarRef::Timestamp(v) => { - let dt = DateConverter::to_timestamp(&v, format.timezone); - serde_json::to_value(dt.format("%Y-%m-%d %H:%M:%S").to_string()).unwrap() + let dt = DateConverter::to_timestamp(&v, format.jiff_timezone.clone()); + serde_json::to_value(strtime::format("%Y-%m-%d %H:%M:%S", &dt).unwrap()).unwrap() } ScalarRef::EmptyArray => JsonValue::Array(vec![]), ScalarRef::EmptyMap => JsonValue::Object(JsonMap::new()), diff --git a/src/query/functions/Cargo.toml b/src/query/functions/Cargo.toml index 652fe5bb59f5..60fcc19f1fb1 100644 --- a/src/query/functions/Cargo.toml +++ b/src/query/functions/Cargo.toml @@ -16,7 +16,6 @@ borsh = { workspace = true, features = ["derive"] } bstr = { workspace = true } bumpalo = { workspace = true } chrono = { workspace = true } -chrono-tz = { workspace = true } crc32fast = { workspace = true } ctor = { workspace = true } @@ -39,6 +38,7 @@ jaq-core = { workspace = true } jaq-interpret = { workspace = true } jaq-parse = { workspace = true } jaq-std = { workspace = true } +jiff = { workspace = true } jsonb = { workspace = true } lexical-core = { workspace = true } libm = { workspace = true } diff --git a/src/query/functions/src/aggregates/aggregate_json_array_agg.rs b/src/query/functions/src/aggregates/aggregate_json_array_agg.rs index b65f7c321b8a..33c525d56c07 100644 --- a/src/query/functions/src/aggregates/aggregate_json_array_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_json_array_agg.rs @@ -20,7 +20,6 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::Result; -use databend_common_expression::date_helper::TzLUT; use databend_common_expression::types::variant::cast_scalar_to_variant; use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; @@ -30,6 +29,7 @@ use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::InputColumns; use databend_common_expression::Scalar; +use jiff::tz::TimeZone; use super::aggregate_function_factory::AggregateFunctionDescription; use super::aggregate_scalar_state::ScalarStateFunc; @@ -100,7 +100,7 @@ where } fn merge_result(&mut self, builder: &mut ColumnBuilder) -> Result<()> { - let tz = TzLUT::default(); + let tz = TimeZone::UTC; let mut items = Vec::with_capacity(self.values.len()); for value in &self.values { let v = T::upcast_scalar(value.clone()); @@ -109,7 +109,7 @@ where continue; } let mut val = vec![]; - cast_scalar_to_variant(v.as_ref(), tz, &mut val); + cast_scalar_to_variant(v.as_ref(), &tz, &mut val); items.push(val); } let mut data = vec![]; diff --git a/src/query/functions/src/aggregates/aggregate_json_object_agg.rs b/src/query/functions/src/aggregates/aggregate_json_object_agg.rs index ac4eaeb6eff6..a3028cd73400 100644 --- a/src/query/functions/src/aggregates/aggregate_json_object_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_json_object_agg.rs @@ -22,7 +22,6 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::date_helper::TzLUT; use databend_common_expression::types::string::StringColumn; use databend_common_expression::types::variant::cast_scalar_to_variant; use databend_common_expression::types::Bitmap; @@ -33,6 +32,7 @@ use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::InputColumns; use databend_common_expression::Scalar; +use jiff::tz::TimeZone; use super::aggregate_function_factory::AggregateFunctionDescription; use super::borsh_deserialize_state; @@ -165,7 +165,7 @@ where } fn merge_result(&mut self, builder: &mut ColumnBuilder) -> Result<()> { - let tz = TzLUT::default(); + let tz = TimeZone::UTC; let mut kvs = Vec::with_capacity(self.kvs.len()); for (key, value) in &self.kvs { let v = V::upcast_scalar(value.clone()); @@ -174,7 +174,7 @@ where continue; } let mut val = vec![]; - cast_scalar_to_variant(v.as_ref(), tz, &mut val); + cast_scalar_to_variant(v.as_ref(), &tz, &mut val); kvs.push((key, val)); } let mut data = vec![]; diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 730f4fa6c9af..87c29543af2f 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -19,9 +19,7 @@ use chrono::format::Parsed; use chrono::format::StrftimeItems; use chrono::prelude::*; use chrono::Datelike; -use chrono::Duration; -use chrono::MappedLocalTime; -use chrono_tz::Tz; +use chrono::TimeZone as ChronoTz; use databend_common_exception::ErrorCode; use databend_common_expression::error_to_null; use databend_common_expression::types::date::clamp_date; @@ -64,8 +62,13 @@ use databend_common_expression::FunctionProperty; use databend_common_expression::FunctionRegistry; use databend_common_expression::Value; use databend_common_expression::ValueRef; -use databend_common_io::cursor_ext::unwrap_local_time; use dtparse::parse; +use jiff::civil::date; +use jiff::civil::datetime; +use jiff::civil::Date; +use jiff::tz::Offset; +use jiff::tz::TimeZone; +use jiff::Unit; use num_traits::AsPrimitive; pub fn register(registry: &mut FunctionRegistry) { @@ -151,9 +154,10 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } } // Convert source timestamp from source timezone to target timezone - let p_src_timestamp = src_timestamp.to_timestamp(ctx.func_ctx.tz.tz); - let src_dst_from_utc = p_src_timestamp.offset().fix().local_minus_utc(); - let t_tz: Tz = match target_tz.parse() { + let p_src_timestamp = src_timestamp.to_timestamp(ctx.func_ctx.jiff_tz.clone()); + let src_dst_from_utc = p_src_timestamp.offset().seconds(); + + let t_tz = match TimeZone::get(target_tz) { Ok(tz) => tz, Err(e) => { ctx.set_error( @@ -165,12 +169,14 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let result_timestamp = p_src_timestamp.with_timezone(&t_tz).timestamp_micros(); + let result_timestamp = p_src_timestamp + .with_time_zone(t_tz.clone()) + .timestamp() + .as_microsecond(); let target_dst_from_utc = p_src_timestamp - .with_timezone(&t_tz) + .with_time_zone(t_tz.clone()) .offset() - .fix() - .local_minus_utc(); + .seconds(); let offset_as_micros_sec = (target_dst_from_utc - src_dst_from_utc) as i64; match offset_as_micros_sec.checked_mul(MICROS_PER_SEC) { Some(offset) => match result_timestamp.checked_add(offset) { @@ -218,11 +224,9 @@ fn register_string_to_timestamp(registry: &mut FunctionRegistry) { ctx: &mut EvalContext, ) -> Value { vectorize_with_builder_1_arg::(|val, output, ctx| { - let tz = ctx.func_ctx.tz.tz; - let enable_dst_hour_fix = ctx.func_ctx.enable_dst_hour_fix; if ctx.func_ctx.enable_strict_datetime_parser { - match string_to_timestamp(val, tz, enable_dst_hour_fix) { - Ok(ts) => output.push(ts.timestamp_micros()), + match string_to_timestamp(val, &ctx.func_ctx.jiff_tz) { + Ok(ts) => output.push(ts.timestamp().as_microsecond()), Err(e) => { ctx.set_error( output.len(), @@ -234,69 +238,30 @@ fn register_string_to_timestamp(registry: &mut FunctionRegistry) { } else { match parse(val) { Ok((naive_dt, parse_tz)) => { - if let Some(parse_tz) = parse_tz { - match naive_dt.and_local_timezone(parse_tz) { - MappedLocalTime::Single(res) => { - output.push(res.with_timezone(&tz).timestamp_micros()) - } - MappedLocalTime::None => { - if enable_dst_hour_fix { - if let Some(res2) = - naive_dt.checked_add_signed(Duration::seconds(3600)) - { - match tz.from_local_datetime(&res2) { - MappedLocalTime::Single(t) => { - output.push(t.timestamp_micros()) - } - MappedLocalTime::Ambiguous(t1, _) => { - output.push(t1.timestamp_micros()) - } - MappedLocalTime::None => { - let err = format!( - "Local Time Error: The local time {:?}, {} can not map to a single unique result with timezone {}", - naive_dt, res2, tz - ); - ctx.set_error( - output.len(), - format!( - "cannot parse to type `TIMESTAMP`. {}", - err - ), - ); - output.push(0); - } - } - } - } else { - let err = format!( - "The time {:?} can not map to a single unique result with timezone {}", - naive_dt, tz - ); - ctx.set_error( - output.len(), - format!("cannot parse to type `TIMESTAMP`. {}", err), - ); - output.push(0); - } - } - MappedLocalTime::Ambiguous(t1, t2) => { - if enable_dst_hour_fix { - output.push(t1.with_timezone(&tz).timestamp_micros()); - } else { - output.push(t2.with_timezone(&tz).timestamp_micros()); - } - } - } + let dt = datetime( + naive_dt.year() as i16, + naive_dt.month() as i8, + naive_dt.day() as i8, + naive_dt.hour() as i8, + naive_dt.minute() as i8, + naive_dt.second() as i8, + naive_dt.nanosecond() as i32, + ); + let tz = if let Some(parse_tz) = parse_tz { + let offset = parse_tz.local_minus_utc(); + let offset = Offset::from_seconds(offset).unwrap(); + offset.to_time_zone() } else { - match unwrap_local_time(&tz, enable_dst_hour_fix, &naive_dt) { - Ok(res) => output.push(res.timestamp_micros()), - Err(e) => { - ctx.set_error( - output.len(), - format!("cannot parse to type `TIMESTAMP`. {}", e), - ); - output.push(0); - } + ctx.func_ctx.jiff_tz.clone() + }; + match dt.to_zoned(tz) { + Ok(res) => output.push(res.timestamp().as_microsecond()), + Err(e) => { + ctx.set_error( + output.len(), + format!("cannot parse to type `TIMESTAMP`. {}", e), + ); + output.push(0); } } } @@ -418,8 +383,6 @@ fn string_to_format_timestamp( let parse_tz = timezone_strftime .iter() .any(|&pattern| format.contains(pattern)); - let enable_dst_hour_fix = ctx.func_ctx.enable_dst_hour_fix; - let tz = ctx.func_ctx.tz.tz; if ctx.func_ctx.parse_datetime_ignore_remainder { let mut parsed = Parsed::new(); if let Err(e) = parse_and_remainder(&mut parsed, timestamp, StrftimeItems::new(format)) { @@ -461,12 +424,24 @@ fn string_to_format_timestamp( parsed .to_naive_datetime_with_offset(0) .map_err(|err| ErrorCode::BadArguments(format!("{err}"))) - .and_then( - |res| match unwrap_local_time(&tz, enable_dst_hour_fix, &res) { - Ok(res) => Ok((res.timestamp_micros(), false)), - Err(e) => Err(e), - }, - ) + .and_then(|res| { + let dt = datetime( + res.year() as i16, + res.month() as i8, + res.day() as i8, + res.hour() as i8, + res.minute() as i8, + res.second() as i8, + res.nanosecond() as i32, + ); + match dt.to_zoned(ctx.func_ctx.jiff_tz.clone()) { + Ok(res) => Ok((res.timestamp().as_microsecond(), false)), + Err(e) => Err(ErrorCode::BadArguments(format!( + "Can not parse timestamp with error: {}", + e + ))), + } + }) } } else if parse_tz { DateTime::parse_from_str(timestamp, format) @@ -475,12 +450,24 @@ fn string_to_format_timestamp( } else { NaiveDateTime::parse_from_str(timestamp, format) .map_err(|err| ErrorCode::BadArguments(format!("{}", err))) - .and_then( - |res| match unwrap_local_time(&tz, enable_dst_hour_fix, &res) { - Ok(res) => Ok((res.timestamp_micros(), false)), - Err(e) => Err(e), - }, - ) + .and_then(|res| { + let dt = datetime( + res.year() as i16, + res.month() as i8, + res.day() as i8, + res.hour() as i8, + res.minute() as i8, + res.second() as i8, + res.nanosecond() as i32, + ); + match dt.to_zoned(ctx.func_ctx.jiff_tz.clone()) { + Ok(res) => Ok((res.timestamp().as_microsecond(), false)), + Err(e) => Err(ErrorCode::BadArguments(format!( + "Can not parse timestamp with error: {}", + e + ))), + } + }) } } @@ -488,10 +475,9 @@ fn register_date_to_timestamp(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "to_timestamp", |ctx, domain| { - let tz = ctx.tz.tz; FunctionDomain::Domain(SimpleDomain { - min: calc_date_to_timestamp(domain.min, tz), - max: calc_date_to_timestamp(domain.max, tz), + min: calc_date_to_timestamp(domain.min, ctx.jiff_tz.clone()), + max: calc_date_to_timestamp(domain.max, ctx.jiff_tz.clone()), }) }, eval_date_to_timestamp, @@ -499,12 +485,11 @@ fn register_date_to_timestamp(registry: &mut FunctionRegistry) { registry.register_combine_nullable_1_arg::( "try_to_timestamp", |ctx, domain| { - let tz = ctx.tz.tz; FunctionDomain::Domain(NullableDomain { has_null: false, value: Some(Box::new(SimpleDomain { - min: calc_date_to_timestamp(domain.min, tz), - max: calc_date_to_timestamp(domain.max, tz), + min: calc_date_to_timestamp(domain.min, ctx.jiff_tz.clone()), + max: calc_date_to_timestamp(domain.max, ctx.jiff_tz.clone()), })), }) }, @@ -516,25 +501,18 @@ fn register_date_to_timestamp(registry: &mut FunctionRegistry) { ctx: &mut EvalContext, ) -> Value { vectorize_with_builder_1_arg::(|val, output, _| { - let tz = ctx.func_ctx.tz.tz; - output.push(calc_date_to_timestamp(val, tz)); + output.push(calc_date_to_timestamp(val, ctx.func_ctx.jiff_tz.clone())); })(val, ctx) } - fn calc_date_to_timestamp(val: i32, tz: Tz) -> i64 { + fn calc_date_to_timestamp(val: i32, tz: TimeZone) -> i64 { let ts = (val as i64) * 24 * 3600 * MICROS_PER_SEC; - let epoch_time_with_ltz = tz - .from_utc_datetime( - &NaiveDate::from_ymd_opt(1970, 1, 1) - .unwrap() - .and_hms_micro_opt(0, 0, 0, 0) - .unwrap(), - ) - .naive_local() - .and_utc() - .timestamp_micros(); - ts - epoch_time_with_ltz + let tz_offset_micros = tz + .to_timestamp(date(1970, 1, 1).at(0, 0, 0, 0)) + .unwrap() + .as_microsecond(); + ts + tz_offset_micros } } @@ -589,8 +567,17 @@ fn register_string_to_date(registry: &mut FunctionRegistry) { fn eval_string_to_date(val: ValueRef, ctx: &mut EvalContext) -> Value { vectorize_with_builder_1_arg::(|val, output, ctx| { if ctx.func_ctx.enable_strict_datetime_parser { - match string_to_date(val, ctx.func_ctx.tz.tz, ctx.func_ctx.enable_dst_hour_fix) { - Ok(d) => output.push(d.num_days_from_ce() - EPOCH_DAYS_FROM_CE), + match string_to_date(val, &ctx.func_ctx.jiff_tz) { + Ok(d) => match d.since((Unit::Day, date(1970, 1, 1))) { + Ok(s) => output.push(s.get_days()), + Err(e) => { + ctx.set_error( + output.len(), + format!("cannot parse to type `DATE`. {}", e), + ); + output.push(0); + } + }, Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `DATE`. {}", e)); output.push(0); @@ -615,10 +602,9 @@ fn register_timestamp_to_date(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "to_date", |ctx, domain| { - let tz = ctx.tz.tz; FunctionDomain::Domain(SimpleDomain { - min: calc_timestamp_to_date(domain.min, tz), - max: calc_timestamp_to_date(domain.max, tz), + min: calc_timestamp_to_date(domain.min, ctx.jiff_tz.clone()), + max: calc_timestamp_to_date(domain.max, ctx.jiff_tz.clone()), }) }, eval_timestamp_to_date, @@ -626,12 +612,11 @@ fn register_timestamp_to_date(registry: &mut FunctionRegistry) { registry.register_combine_nullable_1_arg::( "try_to_date", |ctx, domain| { - let tz = ctx.tz.tz; FunctionDomain::Domain(NullableDomain { has_null: false, value: Some(Box::new(SimpleDomain { - min: calc_timestamp_to_date(domain.min, tz), - max: calc_timestamp_to_date(domain.max, tz), + min: calc_timestamp_to_date(domain.min, ctx.jiff_tz.clone()), + max: calc_timestamp_to_date(domain.max, ctx.jiff_tz.clone()), })), }) }, @@ -643,12 +628,16 @@ fn register_timestamp_to_date(registry: &mut FunctionRegistry) { ctx: &mut EvalContext, ) -> Value { vectorize_with_builder_1_arg::(|val, output, ctx| { - let tz = ctx.func_ctx.tz.tz; + let tz = ctx.func_ctx.jiff_tz.clone(); output.push(calc_timestamp_to_date(val, tz)); })(val, ctx) } - fn calc_timestamp_to_date(val: i64, tz: Tz) -> i32 { - val.to_timestamp(tz).naive_local().num_days_from_ce() - EPOCH_DAYS_FROM_CE + fn calc_timestamp_to_date(val: i64, tz: TimeZone) -> i32 { + val.to_timestamp(tz) + .date() + .since((Unit::Day, Date::new(1970, 1, 1).unwrap())) + .unwrap() + .get_days() } } @@ -690,11 +679,20 @@ fn register_to_string(registry: &mut FunctionRegistry) { "to_string", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::>( - |date, format, output, ctx| { + |micros, format, output, ctx| { if format.is_empty() { output.push_null(); } else { - let ts = date.to_timestamp(ctx.func_ctx.tz.tz); + // Can't use `tz.timestamp_nanos(self.as_() * 1000)` directly, is may cause multiply with overflow. + let (mut secs, mut nanos) = + (micros / MICROS_PER_SEC, (micros % MICROS_PER_SEC) * 1_000); + if nanos < 0 { + secs -= 1; + nanos += 1_000_000_000; + } + let ts = ctx.func_ctx.tz.timestamp_opt(secs, nanos as u32).unwrap(); + // https://github.com/BurntSushi/jiff/issues/155 + // ASCII is currently required in jiff crate let res = ts.format(format).to_string(); output.push(&res); } @@ -709,7 +707,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { write!( output.row_buffer, "{}", - date_to_string(val, ctx.func_ctx.tz.tz) + date_to_string(val, &ctx.func_ctx.jiff_tz) ) .unwrap(); output.commit_row(); @@ -723,7 +721,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { write!( output.row_buffer, "{}", - timestamp_to_string(val, ctx.func_ctx.tz.tz) + timestamp_to_string(val, &ctx.func_ctx.jiff_tz) ) .unwrap(); output.commit_row(); @@ -745,7 +743,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { write!( output.builder.row_buffer, "{}", - date_to_string(val, ctx.func_ctx.tz.tz) + date_to_string(val, &ctx.func_ctx.jiff_tz) ) .unwrap(); output.builder.commit_row(); @@ -769,7 +767,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { write!( output.builder.row_buffer, "{}", - timestamp_to_string(val, ctx.func_ctx.tz.tz) + timestamp_to_string(val, &ctx.func_ctx.jiff_tz) ) .unwrap(); output.builder.commit_row(); @@ -850,7 +848,7 @@ macro_rules! impl_register_arith_functions { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::(|date, delta, builder, ctx| { - match EvalYearsImpl::eval_date(date, ctx.func_ctx.tz, $signed_wrapper!{delta}) { + match EvalYearsImpl::eval_date(date, ctx.func_ctx.jiff_tz.clone(), $signed_wrapper!{delta}) { Ok(t) => builder.push(t), Err(e) => { ctx.set_error(builder.len(), e); @@ -865,7 +863,7 @@ macro_rules! impl_register_arith_functions { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |ts, delta, builder, ctx| { - match EvalYearsImpl::eval_timestamp(ts, ctx.func_ctx.tz, $signed_wrapper!{delta}) { + match EvalYearsImpl::eval_timestamp(ts, ctx.func_ctx.jiff_tz.clone(), $signed_wrapper!{delta}) { Ok(t) => builder.push(t), Err(e) => { ctx.set_error(builder.len(), e); @@ -881,7 +879,7 @@ macro_rules! impl_register_arith_functions { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::(|date, delta, builder, ctx| { - match EvalMonthsImpl::eval_date(date, ctx.func_ctx.tz, $signed_wrapper!{delta} * 3) { + match EvalMonthsImpl::eval_date(date, ctx.func_ctx.jiff_tz.clone(), $signed_wrapper!{delta} * 3) { Ok(t) => builder.push(t), Err(e) => { ctx.set_error(builder.len(), e); @@ -896,7 +894,7 @@ macro_rules! impl_register_arith_functions { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |ts, delta, builder, ctx| { - match EvalMonthsImpl::eval_timestamp(ts, ctx.func_ctx.tz, $signed_wrapper!{delta} * 3) { + match EvalMonthsImpl::eval_timestamp(ts, ctx.func_ctx.jiff_tz.clone(), $signed_wrapper!{delta} * 3) { Ok(t) => builder.push(t), Err(e) => { ctx.set_error(builder.len(), e); @@ -912,7 +910,7 @@ macro_rules! impl_register_arith_functions { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::(|date, delta, builder, ctx| { - match EvalMonthsImpl::eval_date(date, ctx.func_ctx.tz, $signed_wrapper!{delta}) { + match EvalMonthsImpl::eval_date(date, ctx.func_ctx.jiff_tz.clone(), $signed_wrapper!{delta}) { Ok(t) => builder.push(t), Err(e) => { ctx.set_error(builder.len(), e); @@ -927,7 +925,7 @@ macro_rules! impl_register_arith_functions { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |ts, delta, builder, ctx| { - match EvalMonthsImpl::eval_timestamp(ts, ctx.func_ctx.tz, $signed_wrapper!{delta}) { + match EvalMonthsImpl::eval_timestamp(ts, ctx.func_ctx.jiff_tz.clone(), $signed_wrapper!{delta}) { Ok(t) => builder.push(t), Err(e) => { ctx.set_error(builder.len(), e); @@ -1080,8 +1078,11 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |date_end, date_start, builder, ctx| { - let diff_years = - EvalYearsImpl::eval_date_diff(date_start, date_end, ctx.func_ctx.tz); + let diff_years = EvalYearsImpl::eval_date_diff( + date_start, + date_end, + ctx.func_ctx.jiff_tz.clone(), + ); builder.push(diff_years as i64); }, ), @@ -1092,8 +1093,11 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |date_end, date_start, builder, ctx| { - let diff_years = - EvalYearsImpl::eval_timestamp_diff(date_start, date_end, ctx.func_ctx.tz); + let diff_years = EvalYearsImpl::eval_timestamp_diff( + date_start, + date_end, + ctx.func_ctx.jiff_tz.clone(), + ); builder.push(diff_years); }, ), @@ -1104,8 +1108,11 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |date_end, date_start, builder, ctx| { - let diff_years = - EvalQuartersImpl::eval_date_diff(date_start, date_end, ctx.func_ctx.tz); + let diff_years = EvalQuartersImpl::eval_date_diff( + date_start, + date_end, + ctx.func_ctx.jiff_tz.clone(), + ); builder.push(diff_years as i64); }, ), @@ -1116,8 +1123,11 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |date_end, date_start, builder, ctx| { - let diff_years = - EvalQuartersImpl::eval_timestamp_diff(date_start, date_end, ctx.func_ctx.tz); + let diff_years = EvalQuartersImpl::eval_timestamp_diff( + date_start, + date_end, + ctx.func_ctx.jiff_tz.clone(), + ); builder.push(diff_years); }, ), @@ -1128,8 +1138,11 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |date_end, date_start, builder, ctx| { - let diff_months = - EvalMonthsImpl::eval_date_diff(date_start, date_end, ctx.func_ctx.tz); + let diff_months = EvalMonthsImpl::eval_date_diff( + date_start, + date_end, + ctx.func_ctx.jiff_tz.clone(), + ); builder.push(diff_months as i64); }, ), @@ -1140,8 +1153,11 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |date_end, date_start, builder, ctx| { - let diff_months = - EvalMonthsImpl::eval_timestamp_diff(date_start, date_end, ctx.func_ctx.tz); + let diff_months = EvalMonthsImpl::eval_timestamp_diff( + date_start, + date_end, + ctx.func_ctx.jiff_tz.clone(), + ); builder.push(diff_months); }, ), @@ -1326,25 +1342,25 @@ fn register_real_time_functions(registry: &mut FunctionRegistry) { registry.register_0_arg_core::( "now", |_| FunctionDomain::Full, - |ctx| Value::Scalar(ctx.func_ctx.now.timestamp_micros()), + |ctx| Value::Scalar(ctx.func_ctx.now.timestamp().as_microsecond()), ); registry.register_0_arg_core::( "today", |_| FunctionDomain::Full, - |ctx| Value::Scalar(today_date(ctx.func_ctx.now, ctx.func_ctx.tz)), + |ctx| Value::Scalar(today_date(&ctx.func_ctx.now, &ctx.func_ctx.jiff_tz)), ); registry.register_0_arg_core::( "yesterday", |_| FunctionDomain::Full, - |ctx| Value::Scalar(today_date(ctx.func_ctx.now, ctx.func_ctx.tz) - 1), + |ctx| Value::Scalar(today_date(&ctx.func_ctx.now, &ctx.func_ctx.jiff_tz) - 1), ); registry.register_0_arg_core::( "tomorrow", |_| FunctionDomain::Full, - |ctx| Value::Scalar(today_date(ctx.func_ctx.now, ctx.func_ctx.tz) + 1), + |ctx| Value::Scalar(today_date(&ctx.func_ctx.now, &ctx.func_ctx.jiff_tz) + 1), ); } @@ -1354,11 +1370,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_yyyymm", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1371,11 +1383,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_yyyymmdd", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1388,11 +1396,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_yyyymmddhh", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1405,11 +1409,8 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_yyyymmddhhmmss", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) + { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1422,11 +1423,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_year", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1439,11 +1436,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_quarter", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1456,11 +1449,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_month", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1473,11 +1462,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_day_of_year", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1490,11 +1475,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_day_of_month", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1507,11 +1488,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_day_of_week", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1524,11 +1501,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_week_of_year", |_, _| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match ToNumberImpl::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1542,101 +1515,110 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_yyyymm", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_yyyymmdd", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_yyyymmddhh", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_yyyymmddhhmmss", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_year", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_quarter", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_month", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_day_of_year", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_day_of_month", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_day_of_week", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_week_of_year", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_unix_timestamp", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.tz) + ToNumberImpl::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); registry.register_passthrough_nullable_1_arg::( "to_hour", |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| ctx.func_ctx.tz.to_hour(val)), + vectorize_1_arg::(|val, ctx| { + let datetime = val.to_timestamp(ctx.func_ctx.jiff_tz.clone()); + datetime.hour() as u8 + }), ); registry.register_passthrough_nullable_1_arg::( "to_minute", |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| ctx.func_ctx.tz.to_minute(val)), + vectorize_1_arg::(|val, ctx| { + let datetime = val.to_timestamp(ctx.func_ctx.jiff_tz.clone()); + datetime.minute() as u8 + }), ); registry.register_passthrough_nullable_1_arg::( "to_second", |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| ctx.func_ctx.tz.to_second(val)), + vectorize_1_arg::(|val, ctx| { + let datetime = val.to_timestamp(ctx.func_ctx.jiff_tz.clone()); + datetime.second() as u8 + }), ); } @@ -1740,56 +1722,56 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { "to_start_of_second", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.func_ctx.tz.round_us(val, Round::Second) + round_timestamp(val, &ctx.func_ctx.jiff_tz, Round::Second) }), ); registry.register_passthrough_nullable_1_arg::( "to_start_of_minute", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.func_ctx.tz.round_us(val, Round::Minute) + round_timestamp(val, &ctx.func_ctx.jiff_tz, Round::Minute) }), ); registry.register_passthrough_nullable_1_arg::( "to_start_of_five_minutes", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.func_ctx.tz.round_us(val, Round::FiveMinutes) + round_timestamp(val, &ctx.func_ctx.jiff_tz, Round::FiveMinutes) }), ); registry.register_passthrough_nullable_1_arg::( "to_start_of_ten_minutes", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.func_ctx.tz.round_us(val, Round::TenMinutes) + round_timestamp(val, &ctx.func_ctx.jiff_tz, Round::TenMinutes) }), ); registry.register_passthrough_nullable_1_arg::( "to_start_of_fifteen_minutes", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.func_ctx.tz.round_us(val, Round::FifteenMinutes) + round_timestamp(val, &ctx.func_ctx.jiff_tz, Round::FifteenMinutes) }), ); registry.register_passthrough_nullable_1_arg::( "to_start_of_hour", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.func_ctx.tz.round_us(val, Round::Hour) + round_timestamp(val, &ctx.func_ctx.jiff_tz, Round::Hour) }), ); registry.register_passthrough_nullable_1_arg::( "to_start_of_day", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.func_ctx.tz.round_us(val, Round::Day) + round_timestamp(val, &ctx.func_ctx.jiff_tz, Round::Day) }), ); registry.register_passthrough_nullable_1_arg::( "time_slot", |_, _| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.func_ctx.tz.round_us(val, Round::TimeSlot) + round_timestamp(val, &ctx.func_ctx.jiff_tz, Round::TimeSlot) }), ); @@ -1824,11 +1806,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::Full, vectorize_with_builder_2_arg::(|val, mode, output, ctx| { if mode == 0 { - match DateRounder::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match DateRounder::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1836,11 +1814,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { } } } else { - match DateRounder::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + match DateRounder::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1855,9 +1829,9 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::Full, vectorize_2_arg::(|val, mode, ctx| { if mode == 0 { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) + DateRounder::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) } else { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) + DateRounder::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) } }), ); @@ -1868,12 +1842,8 @@ where T: ToNumber { registry.register_passthrough_nullable_1_arg::( name, |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(move |val, output, ctx| { - match DateRounder::eval_date::( - val, - ctx.func_ctx.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { + vectorize_with_builder_1_arg::(|val, output, ctx| { + match DateRounder::eval_date::(val, ctx.func_ctx.jiff_tz.clone()) { Ok(t) => output.push(t), Err(e) => { ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); @@ -1885,8 +1855,8 @@ where T: ToNumber { registry.register_passthrough_nullable_1_arg::( name, |_, _| FunctionDomain::Full, - vectorize_1_arg::(move |val, ctx| { - DateRounder::eval_timestamp::(val, ctx.func_ctx.tz) + vectorize_1_arg::(|val, ctx| { + DateRounder::eval_timestamp::(val, ctx.func_ctx.jiff_tz.clone()) }), ); } diff --git a/src/query/functions/src/scalars/variant.rs b/src/query/functions/src/scalars/variant.rs index cd7c66552a9b..ece9ab1d1f4e 100644 --- a/src/query/functions/src/scalars/variant.rs +++ b/src/query/functions/src/scalars/variant.rs @@ -19,7 +19,6 @@ use std::iter::once; use std::sync::Arc; use bstr::ByteSlice; -use chrono::Datelike; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::date::string_to_date; use databend_common_expression::types::nullable::NullableColumn; @@ -45,7 +44,6 @@ use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::VariantType; use databend_common_expression::types::ALL_NUMERICS_TYPES; -use databend_common_expression::utils::serialize::EPOCH_DAYS_FROM_CE; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; @@ -64,6 +62,8 @@ use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::Value; use databend_common_expression::ValueRef; +use jiff::civil::date; +use jiff::Unit; use jsonb::array_distinct; use jsonb::array_except; use jsonb::array_insert; @@ -347,7 +347,7 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - if idx < 0 || idx > i32::MAX.into() { + if idx < 0 || idx > i32::MAX as i64 { output.push_null(); } else { match get_by_index(val, idx as usize) { @@ -415,7 +415,7 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - if idx < 0 || idx > i32::MAX.into() { + if idx < 0 || idx > i32::MAX as i64 { output.push_null(); } else { match get_by_index(val, idx as usize) { @@ -897,7 +897,7 @@ pub fn register(registry: &mut FunctionRegistry) { ScalarRef::Null => Value::Scalar(Scalar::Null), _ => { let mut buf = Vec::new(); - cast_scalar_to_variant(scalar.clone(), ctx.func_ctx.tz, &mut buf); + cast_scalar_to_variant(scalar.clone(), &ctx.func_ctx.jiff_tz, &mut buf); Value::Scalar(Scalar::Variant(buf)) } }, @@ -909,7 +909,7 @@ pub fn register(registry: &mut FunctionRegistry) { } _ => None, }; - let new_col = cast_scalars_to_variants(col.iter(), ctx.func_ctx.tz); + let new_col = cast_scalars_to_variants(col.iter(), &ctx.func_ctx.jiff_tz); if let Some(validity) = validity { Value::Column(NullableColumn::new_column( Column::Variant(new_col), @@ -941,7 +941,7 @@ pub fn register(registry: &mut FunctionRegistry) { ScalarRef::Null => Value::Scalar(None), _ => { let mut buf = Vec::new(); - cast_scalar_to_variant(scalar, ctx.func_ctx.tz, &mut buf); + cast_scalar_to_variant(scalar, &ctx.func_ctx.jiff_tz, &mut buf); Value::Scalar(Some(buf)) } }, @@ -951,7 +951,7 @@ pub fn register(registry: &mut FunctionRegistry) { Column::Nullable(box ref nullable_column) => nullable_column.validity.clone(), _ => Bitmap::new_constant(true, col.len()), }; - let new_col = cast_scalars_to_variants(col.iter(), ctx.func_ctx.tz); + let new_col = cast_scalars_to_variants(col.iter(), &ctx.func_ctx.jiff_tz); Value::Column(NullableColumn::new(new_col, validity)) } }, @@ -1042,12 +1042,17 @@ pub fn register(registry: &mut FunctionRegistry) { } let val = as_str(val); match val { - Some(val) => match string_to_date( - val.as_bytes(), - ctx.func_ctx.tz.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(d) => output.push(d.num_days_from_ce() - EPOCH_DAYS_FROM_CE), + Some(val) => match string_to_date(val.as_bytes(), &ctx.func_ctx.jiff_tz) { + Ok(d) => match d.since((Unit::Day, date(1970, 1, 1))) { + Ok(s) => output.push(s.get_days()), + Err(e) => { + ctx.set_error( + output.len(), + format!("cannot parse to type `DATE`. {}", e), + ); + output.push(0); + } + }, Err(e) => { ctx.set_error( output.len(), @@ -1076,12 +1081,17 @@ pub fn register(registry: &mut FunctionRegistry) { } let val = as_str(val); match val { - Some(val) => match string_to_date( - val.as_bytes(), - ctx.func_ctx.tz.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(d) => output.push(d.num_days_from_ce() - EPOCH_DAYS_FROM_CE), + Some(val) => match string_to_date(val.as_bytes(), &ctx.func_ctx.jiff_tz) { + Ok(d) => match d.since((Unit::Day, date(1970, 1, 1))) { + Ok(s) => output.push(s.get_days()), + Err(e) => { + ctx.set_error( + output.len(), + format!("cannot parse to type `DATE`. {}", e), + ); + output.push(0); + } + }, Err(_) => output.push_null(), }, None => output.push_null(), @@ -1101,12 +1111,8 @@ pub fn register(registry: &mut FunctionRegistry) { } let val = as_str(val); match val { - Some(val) => match string_to_timestamp( - val.as_bytes(), - ctx.func_ctx.tz.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(ts) => output.push(ts.timestamp_micros()), + Some(val) => match string_to_timestamp(val.as_bytes(), &ctx.func_ctx.jiff_tz) { + Ok(ts) => output.push(ts.timestamp().as_microsecond()), Err(e) => { ctx.set_error( output.len(), @@ -1136,12 +1142,8 @@ pub fn register(registry: &mut FunctionRegistry) { } let val = as_str(val); match val { - Some(val) => match string_to_timestamp( - val.as_bytes(), - ctx.func_ctx.tz.tz, - ctx.func_ctx.enable_dst_hour_fix, - ) { - Ok(ts) => output.push(ts.timestamp_micros()), + Some(val) => match string_to_timestamp(val.as_bytes(), &ctx.func_ctx.jiff_tz) { + Ok(ts) => output.push(ts.timestamp().as_microsecond()), Err(_) => { output.push_null(); } @@ -1758,7 +1760,7 @@ fn json_array_fn(args: &[ValueRef], ctx: &mut EvalContext) -> Value { // if the new value is not a json value, cast it to json. let mut new_val_buf = vec![]; - cast_scalar_to_variant(new_val.clone(), ctx.func_ctx.tz, &mut new_val_buf); + cast_scalar_to_variant(new_val.clone(), &ctx.func_ctx.jiff_tz, &mut new_val_buf); jsonb::object_insert(value, new_key, &new_val_buf, update_flag, &mut builder.data) } }; diff --git a/src/query/functions/tests/it/scalars/testdata/cast.txt b/src/query/functions/tests/it/scalars/testdata/cast.txt index 0355c8e2c42d..4fde3f722315 100644 --- a/src/query/functions/tests/it/scalars/testdata/cast.txt +++ b/src/query/functions/tests/it/scalars/testdata/cast.txt @@ -1311,7 +1311,7 @@ evaluation: | Row 1 | '1969-09-23' | '1969-09-23 00:00:00.000000' | | Row 2 | '1970-01-01' | '1970-01-01 00:00:00.000000' | | Row 3 | '1970-04-11' | '1970-04-11 00:00:00.000000' | -| Row 4 | '9999-12-31' | '9999-12-31 00:00:00.000000' | +| Row 4 | '9999-12-31' | '9999-12-30 22:00:00.000000' | +--------+---------------------+-------------------------------------------+ evaluation (internal): +--------+----------------------------------------------------------------------------+ @@ -1335,7 +1335,7 @@ evaluation: | Row 1 | -100 | '1969-09-23 00:00:00.000000' | | Row 2 | 0 | '1970-01-01 00:00:00.000000' | | Row 3 | 100 | '1970-04-11 00:00:00.000000' | -| Row 4 | 2932896 | '9999-12-31 00:00:00.000000' | +| Row 4 | 2932896 | '9999-12-30 22:00:00.000000' | +--------+---------------------+-------------------------------------------+ evaluation (internal): +--------+----------------------------------------------------------------------------+ @@ -1417,7 +1417,7 @@ error: --> SQL:1:1 | 1 | TO_TIMESTAMP('A NON-TIMESTAMP STR') - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cannot parse to type `TIMESTAMP`. BadBytes. Code: 1046, Text = Date Parsing Error: The value 'A NON-TIME' could not be parsed into a valid Date, cause: input contains invalid characters. while evaluating function `to_timestamp('A NON-TIMESTAMP STR')` in expr `to_timestamp('A NON-TIMESTAMP STR')` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cannot parse to type `TIMESTAMP`. BadBytes. Code: 1046, Text = Date Parsing Error: The value 'A NON-TIME' could not be parsed into a valid Date, cause: failed to parse year in date "A NON-TIME": failed to parse "A NO" as year (a four digit integer): invalid digit, expected 0-9 but got A. while evaluating function `to_timestamp('A NON-TIMESTAMP STR')` in expr `to_timestamp('A NON-TIMESTAMP STR')` @@ -1617,7 +1617,7 @@ error: --> SQL:1:1 | 1 | TO_DATE('A NON-DATE STR') - | ^^^^^^^^^^^^^^^^^^^^^^^^^ cannot parse to type `DATE`. BadBytes. Code: 1046, Text = Date Parsing Error: The value 'A NON-DATE' could not be parsed into a valid Date, cause: input contains invalid characters. while evaluating function `to_date('A NON-DATE STR')` in expr `to_date('A NON-DATE STR')` + | ^^^^^^^^^^^^^^^^^^^^^^^^^ cannot parse to type `DATE`. BadBytes. Code: 1046, Text = Date Parsing Error: The value 'A NON-DATE' could not be parsed into a valid Date, cause: failed to parse year in date "A NON-DATE": failed to parse "A NO" as year (a four digit integer): invalid digit, expected 0-9 but got A. while evaluating function `to_date('A NON-DATE STR')` in expr `to_date('A NON-DATE STR')` @@ -3528,7 +3528,7 @@ evaluation: | Row 1 | '1969-09-23' | '1969-09-23 00:00:00.000000' | | Row 2 | '1970-01-01' | '1970-01-01 00:00:00.000000' | | Row 3 | '1970-04-11' | '1970-04-11 00:00:00.000000' | -| Row 4 | '9999-12-31' | '9999-12-31 00:00:00.000000' | +| Row 4 | '9999-12-31' | '9999-12-30 22:00:00.000000' | +--------+---------------------+-------------------------------------------+ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------+ @@ -3552,7 +3552,7 @@ evaluation: | Row 1 | -100 | '1969-09-23 00:00:00.000000' | | Row 2 | 0 | '1970-01-01 00:00:00.000000' | | Row 3 | 100 | '1970-04-11 00:00:00.000000' | -| Row 4 | 2932896 | '9999-12-31 00:00:00.000000' | +| Row 4 | 2932896 | '9999-12-30 22:00:00.000000' | +--------+---------------------+-------------------------------------------+ evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/query/functions/tests/it/scalars/testdata/datetime.txt b/src/query/functions/tests/it/scalars/testdata/datetime.txt index 90d727817217..3e7b7227f3a5 100644 --- a/src/query/functions/tests/it/scalars/testdata/datetime.txt +++ b/src/query/functions/tests/it/scalars/testdata/datetime.txt @@ -210,13 +210,12 @@ evaluation (internal): +--------+-----------------------------------------+ -ast : add_years(to_date(0), 10000) -raw expr : add_years(to_date(0), 10000) -checked expr : add_years(to_date(to_int64(0_u8)), to_int64(10000_u16)) -optimized expr : -354285 -output type : Date -output domain : {-354285..=-354285} -output : '1000-01-01' +error: + --> SQL:1:1 + | +1 | add_years(to_date(0), 10000) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Invalid date: parameter 'year' with value 11970 is not in the required range of -9999..=9999 while evaluating function `add_years('1970-01-01', 10000)` in expr `add_years(to_date(to_int64(0)), to_int64(10000))` + ast : add_years(to_date(0), 100) @@ -502,13 +501,12 @@ evaluation (internal): +--------+------------------+ -ast : add_years(to_timestamp(0), 10000) -raw expr : add_years(to_timestamp(0), 10000) -checked expr : add_years(to_timestamp(to_int64(0_u8)), to_int64(10000_u16)) -optimized expr : -30610224000000000 -output type : Timestamp -output domain : {-30610224000000000..=-30610224000000000} -output : '1000-01-01 00:00:00.000000' +error: + --> SQL:1:1 + | +1 | add_years(to_timestamp(0), 10000) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Invalid date: parameter 'year' with value 11970 is not in the required range of -9999..=9999 while evaluating function `add_years('1970-01-01 00:00:00.000000', 10000)` in expr `add_years(to_timestamp(to_int64(0)), to_int64(10000))` + ast : add_years(to_timestamp(0), 100) @@ -995,13 +993,12 @@ evaluation (internal): +--------+--------------------------------+ -ast : date_add(year, 10000, to_date(0)) -raw expr : add_years(to_date(0), 10000) -checked expr : add_years(to_date(to_int64(0_u8)), to_int64(10000_u16)) -optimized expr : -354285 -output type : Date -output domain : {-354285..=-354285} -output : '1000-01-01' +error: + --> SQL:1:1 + | +1 | date_add(year, 10000, to_date(0)) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Invalid date: parameter 'year' with value 11970 is not in the required range of -9999..=9999 while evaluating function `add_years('1970-01-01', 10000)` in expr `add_years(to_date(to_int64(0)), to_int64(10000))` + ast : date_add(year, 100, to_date(0)) @@ -1260,13 +1257,12 @@ evaluation (internal): +--------+------------------+ -ast : date_add(year, 10000, to_timestamp(0)) -raw expr : add_years(to_timestamp(0), 10000) -checked expr : add_years(to_timestamp(to_int64(0_u8)), to_int64(10000_u16)) -optimized expr : -30610224000000000 -output type : Timestamp -output domain : {-30610224000000000..=-30610224000000000} -output : '1000-01-01 00:00:00.000000' +error: + --> SQL:1:1 + | +1 | date_add(year, 10000, to_timestamp(0)) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Invalid date: parameter 'year' with value 11970 is not in the required range of -9999..=9999 while evaluating function `add_years('1970-01-01 00:00:00.000000', 10000)` in expr `add_years(to_timestamp(to_int64(0)), to_int64(10000))` + ast : date_add(year, 100, to_timestamp(0)) @@ -1717,13 +1713,12 @@ evaluation (internal): +--------+--------------------------------+ -ast : to_date(0) + interval 10000 year -raw expr : add_years(to_date(0), 10000) -checked expr : add_years(to_date(to_int64(0_u8)), to_int64(10000_u16)) -optimized expr : -354285 -output type : Date -output domain : {-354285..=-354285} -output : '1000-01-01' +error: + --> SQL:1:12 + | +1 | to_date(0) + interval 10000 year + | ^ Invalid date: parameter 'year' with value 11970 is not in the required range of -9999..=9999 while evaluating function `add_years('1970-01-01', 10000)` in expr `add_years(to_date(to_int64(0)), to_int64(10000))` + ast : to_date(0) + interval 100 year @@ -1982,13 +1977,12 @@ evaluation (internal): +--------+------------------+ -ast : to_timestamp(0) + interval 10000 year -raw expr : add_years(to_timestamp(0), 10000) -checked expr : add_years(to_timestamp(to_int64(0_u8)), to_int64(10000_u16)) -optimized expr : -30610224000000000 -output type : Timestamp -output domain : {-30610224000000000..=-30610224000000000} -output : '1000-01-01 00:00:00.000000' +error: + --> SQL:1:17 + | +1 | to_timestamp(0) + interval 10000 year + | ^ Invalid date: parameter 'year' with value 11970 is not in the required range of -9999..=9999 while evaluating function `add_years('1970-01-01 00:00:00.000000', 10000)` in expr `add_years(to_timestamp(to_int64(0)), to_int64(10000))` + ast : to_timestamp(0) + interval 100 year diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index 5106915297f2..0fe25b4dae25 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -128,6 +128,7 @@ http = { workspace = true } humantime = { workspace = true } indicatif = { workspace = true } itertools = { workspace = true } +jiff = { workspace = true } jsonb = { workspace = true } jwt-simple = { workspace = true } log = { workspace = true } diff --git a/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs b/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs index 7082a4429be6..09e6f8a99620 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs @@ -15,7 +15,6 @@ use std::collections::BTreeMap; use std::sync::Arc; -use chrono_tz::Tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::date::date_to_string; @@ -32,6 +31,7 @@ use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::Value; use databend_common_storage::build_operator; +use jiff::tz::TimeZone; use opendal::services::Redis; use opendal::Operator; use sqlx::MySqlPool; @@ -52,8 +52,10 @@ impl DictionaryOperator { fn format_key(&self, key: ScalarRef<'_>) -> String { match key { ScalarRef::String(s) => s.to_string(), - ScalarRef::Date(d) => format!("{}", date_to_string(d as i64, Tz::UTC)), - ScalarRef::Timestamp(t) => format!("{}", timestamp_to_string(t, Tz::UTC)), + ScalarRef::Date(d) => format!("{}", date_to_string(d as i64, &TimeZone::UTC)), + ScalarRef::Timestamp(t) => { + format!("{}", timestamp_to_string(t, &TimeZone::UTC)) + } _ => format!("{}", key), } } diff --git a/src/query/service/src/servers/http/v1/query/string_block.rs b/src/query/service/src/servers/http/v1/query/string_block.rs index be79d13de3a1..09ce7572607d 100644 --- a/src/query/service/src/servers/http/v1/query/string_block.rs +++ b/src/query/service/src/servers/http/v1/query/string_block.rs @@ -47,8 +47,11 @@ pub fn block_to_strings( .collect(); let mut res = Vec::new(); - let encoder = - FieldEncoderValues::create_for_http_handler(format.timezone, format.geometry_format); + let encoder = FieldEncoderValues::create_for_http_handler( + format.jiff_timezone.clone(), + format.timezone, + format.geometry_format, + ); let mut buf = vec![]; for row_index in 0..rows_size { let mut row: Vec> = Vec::with_capacity(block.num_columns()); diff --git a/src/query/service/src/servers/mysql/writers/query_result_writer.rs b/src/query/service/src/servers/mysql/writers/query_result_writer.rs index d0251eefa50a..d7f2ae88d8b3 100644 --- a/src/query/service/src/servers/mysql/writers/query_result_writer.rs +++ b/src/query/service/src/servers/mysql/writers/query_result_writer.rs @@ -236,6 +236,7 @@ impl<'a, W: AsyncWrite + Send + Unpin> DFQueryResultWriter<'a, W> { let num_rows = block.num_rows(); let encoder = FieldEncoderValues::create_for_mysql_handler( + format.jiff_timezone.clone(), format.timezone, format.geometry_format, ); diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index a8dd5e987cf5..9cd8430de768 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -28,7 +28,6 @@ use std::time::Instant; use std::time::SystemTime; use std::time::UNIX_EPOCH; -use chrono::Utc; use chrono_tz::Tz; use dashmap::mapref::multiple::RefMulti; use dashmap::DashMap; @@ -61,7 +60,6 @@ use databend_common_config::GlobalConfig; use databend_common_config::DATABEND_COMMIT_VERSION; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::date_helper::TzFactory; use databend_common_expression::BlockThresholds; use databend_common_expression::DataBlock; use databend_common_expression::Expr; @@ -114,6 +112,8 @@ use databend_common_users::UserApiProvider; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; +use jiff::tz::TimeZone; +use jiff::Zoned; use log::debug; use log::info; use parking_lot::Mutex; @@ -716,11 +716,15 @@ impl TableContext for QueryContext { let timezone = tz.parse::().map_err(|_| { ErrorCode::InvalidTimezone("Timezone has been checked and should be valid") })?; + let jiff_timezone = TimeZone::get(&tz).map_err(|_| { + ErrorCode::InvalidTimezone("Timezone has been checked and should be valid") + })?; let geometry_format = self.get_settings().get_geometry_output_format()?; let format_null_as_str = self.get_settings().get_format_null_as_str()?; let enable_dst_hour_fix = self.get_settings().get_enable_dst_hour_fix()?; let format = FormatSettings { timezone, + jiff_timezone, geometry_format, enable_dst_hour_fix, format_null_as_str, @@ -739,15 +743,22 @@ impl TableContext for QueryContext { fn get_function_context(&self) -> Result { let settings = self.get_settings(); - let tz = settings.get_timezone()?; - let tz = TzFactory::instance().get_by_name(&tz)?; - let now = Utc::now(); + let tz_string = settings.get_timezone()?; + let tz = tz_string.parse::().map_err(|_| { + ErrorCode::InvalidTimezone("Timezone has been checked and should be valid") + })?; + let jiff_tz = TimeZone::get(&tz_string).map_err(|e| { + ErrorCode::InvalidTimezone(format!( + "Timezone has been checked and should be valid but got error: {}", + e + )) + })?; + let now = Zoned::now().with_time_zone(TimeZone::UTC); let numeric_cast_option = settings.get_numeric_cast_option()?; let rounding_mode = numeric_cast_option.as_str() == "rounding"; let disable_variant_check = settings.get_disable_variant_check()?; let geometry_output_format = settings.get_geometry_output_format()?; let parse_datetime_ignore_remainder = settings.get_parse_datetime_ignore_remainder()?; - let enable_dst_hour_fix = settings.get_enable_dst_hour_fix()?; let enable_strict_datetime_parser = settings.get_enable_strict_datetime_parser()?; let query_config = &GlobalConfig::instance().query; let random_function_seed = settings.get_random_function_seed()?; @@ -755,6 +766,7 @@ impl TableContext for QueryContext { Ok(FunctionContext { tz, now, + jiff_tz, rounding_mode, disable_variant_check, @@ -767,7 +779,6 @@ impl TableContext for QueryContext { geometry_output_format, parse_datetime_ignore_remainder, - enable_dst_hour_fix, enable_strict_datetime_parser, random_function_seed, }) diff --git a/src/query/service/src/table_functions/cloud/task_history.rs b/src/query/service/src/table_functions/cloud/task_history.rs index c76fe3034aaa..4949427fce12 100644 --- a/src/query/service/src/table_functions/cloud/task_history.rs +++ b/src/query/service/src/table_functions/cloud/task_history.rs @@ -16,7 +16,6 @@ use std::any::Any; use std::sync::Arc; use chrono::DateTime; -use chrono_tz::Tz::UTC; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::Partitions; @@ -45,6 +44,7 @@ use databend_common_pipeline_sources::AsyncSourcer; use databend_common_sql::plans::task_run_schema; use databend_common_storages_factory::Table; use databend_common_storages_system::parse_task_runs_to_datablock; +use jiff::tz::TimeZone; pub struct TaskHistoryTable { table_info: TableInfo, @@ -249,18 +249,18 @@ fn parse_date_or_timestamp(v: &Scalar) -> Option { if v.as_timestamp().is_some() { Some( v.as_timestamp() - .map(|s| s.to_timestamp(UTC).to_rfc3339()) + .map(|s| s.to_timestamp(TimeZone::UTC).to_string()) .unwrap(), ) } else if v.as_date().is_some() { Some( v.as_date() .map(|s| { - s.to_date(UTC) - .and_hms_opt(0, 0, 0) + s.to_date(TimeZone::UTC) + .at(0, 0, 0, 0) + .intz("UTC") .unwrap() - .and_utc() - .to_rfc3339() + .to_string() }) .unwrap(), ) diff --git a/src/query/storages/common/cache/Cargo.toml b/src/query/storages/common/cache/Cargo.toml index 750cd984cf48..408839f31bd9 100644 --- a/src/query/storages/common/cache/Cargo.toml +++ b/src/query/storages/common/cache/Cargo.toml @@ -12,7 +12,6 @@ test = true [dependencies] arrow = { workspace = true } - databend-common-base = { workspace = true } databend-common-cache = { workspace = true } databend-common-catalog = { workspace = true } diff --git a/src/query/storages/system/Cargo.toml b/src/query/storages/system/Cargo.toml index 79790a6233d0..3b31c6c3966f 100644 --- a/src/query/storages/system/Cargo.toml +++ b/src/query/storages/system/Cargo.toml @@ -17,7 +17,6 @@ jemalloc = ["databend-common-base/jemalloc", "tikv-jemalloc-ctl"] async-backtrace = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } -chrono-tz = { workspace = true } databend-common-ast = { workspace = true } databend-common-base = { workspace = true } databend-common-catalog = { workspace = true } @@ -43,6 +42,7 @@ databend-common-users = { workspace = true } databend-storages-common-cache = { workspace = true } futures = { workspace = true } itertools = { workspace = true } +jiff = { workspace = true } jsonb = { workspace = true } log = { workspace = true } once_cell = { workspace = true } diff --git a/src/query/storages/system/src/task_history_table.rs b/src/query/storages/system/src/task_history_table.rs index de01e385345a..6ee03c8ab9bb 100644 --- a/src/query/storages/system/src/task_history_table.rs +++ b/src/query/storages/system/src/task_history_table.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use chrono_tz::Tz::UTC; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; @@ -41,6 +40,7 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_sql::plans::task_run_schema; +use jiff::tz::TimeZone; use crate::table::AsyncOneBlockSystemTable; use crate::table::AsyncSystemTable; @@ -163,14 +163,16 @@ impl AsyncSystemTable for TaskHistoryTable { find_lt_filter(&expr, &mut |col_name, scalar| { if col_name == "scheduled_time" { if let Scalar::Timestamp(s) = scalar { - scheduled_time_end = Some(s.to_timestamp(UTC).to_rfc3339()); + scheduled_time_end = + Some(s.to_timestamp(TimeZone::UTC).timestamp().to_string()); } } }); find_gt_filter(&expr, &mut |col_name, scalar| { if col_name == "scheduled_time" { if let Scalar::Timestamp(s) = scalar { - scheduled_time_start = Some(s.to_timestamp(UTC).to_rfc3339()); + scheduled_time_start = + Some(s.to_timestamp(TimeZone::UTC).timestamp().to_string()); } } }); diff --git a/src/tests/sqlsmith/Cargo.toml b/src/tests/sqlsmith/Cargo.toml index b87bff0bf4ff..ba3dade807c3 100644 --- a/src/tests/sqlsmith/Cargo.toml +++ b/src/tests/sqlsmith/Cargo.toml @@ -20,6 +20,7 @@ databend-driver = { workspace = true } databend-driver-core = { workspace = true } ethnum = { workspace = true } itertools = { workspace = true } +jiff = { workspace = true } jsonb = { workspace = true } rand = { workspace = true } tokio = { workspace = true } diff --git a/src/tests/sqlsmith/src/sql_gen/dml.rs b/src/tests/sqlsmith/src/sql_gen/dml.rs index 19eaef4421b9..71ff5f27c38a 100644 --- a/src/tests/sqlsmith/src/sql_gen/dml.rs +++ b/src/tests/sqlsmith/src/sql_gen/dml.rs @@ -52,6 +52,7 @@ use databend_common_io::constants::TRUE_BYTES_LOWER; use databend_common_io::deserialize_bitmap; use databend_common_sql::resolve_type_name; use itertools::join; +use jiff::tz::TimeZone; use rand::Rng; use crate::sql_gen::SqlGenerator; @@ -532,6 +533,7 @@ impl<'a, R: Rng + 'a> SqlGenerator<'a, R> { nan_bytes: NAN_BYTES_LOWER.as_bytes().to_vec(), inf_bytes: INF_BYTES_LOWER.as_bytes().to_vec(), timezone: Tz::UTC, + jiff_timezone: TimeZone::UTC, binary_format: Default::default(), geometry_format: Default::default(), }, diff --git a/tests/sqllogictests/suites/base/11_data_type/11_0001_data_type_date_time.test b/tests/sqllogictests/suites/base/11_data_type/11_0001_data_type_date_time.test index 65d1a12bb6fb..42f4641375b6 100644 --- a/tests/sqllogictests/suites/base/11_data_type/11_0001_data_type_date_time.test +++ b/tests/sqllogictests/suites/base/11_data_type/11_0001_data_type_date_time.test @@ -14,7 +14,7 @@ select '0000-00-00 00:00:00.868894'::TIMESTAMP query T select '0099-05-16T03:25:02.868894'::TIMESTAMP ---- -1000-01-01 00:00:00.000000 +0099-05-16 03:25:02.868894 query T select '1022-05-16 03:25:02.868894'::TIMESTAMP diff --git a/tests/sqllogictests/suites/base/11_data_type/11_0005_data_type_date.test b/tests/sqllogictests/suites/base/11_data_type/11_0005_data_type_date.test index 0033ba7916aa..fba91174f031 100644 --- a/tests/sqllogictests/suites/base/11_data_type/11_0005_data_type_date.test +++ b/tests/sqllogictests/suites/base/11_data_type/11_0005_data_type_date.test @@ -14,7 +14,7 @@ select '0000-00-00'::DATE query T select '0099-05-16'::DATE ---- -1000-01-01 +0099-05-16 query T select '1022-05-16'::DATE, date '1022-05-16', timestamp '1022-05-16' @@ -29,7 +29,7 @@ select '0000-00-00 00:00:00.868894'::DATE query T select '0099-05-16T03:25:02.868894'::DATE ---- -1000-01-01 +0099-05-16 query T select '1022-05-16 03:25:02.868894'::DATE diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index ad9f1873ac4a..a5220e9fa180 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -93,6 +93,11 @@ select to_date('9999-12-31') ---- 9999-12-31 +query T +select to_date('9999-12-30') +---- +9999-12-30 + statement error 1006 select to_date('10000-12-31') @@ -100,47 +105,51 @@ select to_date('10000-12-31') query T select to_date('0999-12-31') ---- -1000-01-01 +0999-12-31 query T select to_datetime('1000-01-01 00:00:00') ---- 1000-01-01 00:00:00.000000 -query T +statement error 1046 select to_datetime('9999-12-31 23:59:59') + +# Max datetime in UTC +query T +select to_datetime('9999-12-30 22:00:00') ---- -9999-12-31 23:59:59.000000 +9999-12-30 22:00:00.000000 query T -select to_datetime('9999-12-31 ') +select to_datetime('9999-12-30 ') ---- -9999-12-31 00:00:00.000000 +9999-12-30 00:00:00.000000 query T -select to_datetime('9999-12-31T') +select to_datetime('9999-12-30T') ---- -9999-12-31 00:00:00.000000 +9999-12-30 00:00:00.000000 query T -select to_datetime('9999-12-31 23') +select to_datetime('9999-12-30 22') ---- -9999-12-31 23:00:00.000000 +9999-12-30 22:00:00.000000 query T -select to_datetime('9999-12-31 23:') +select to_datetime('9999-12-30 22:') ---- -9999-12-31 23:00:00.000000 +9999-12-30 22:00:00.000000 query T -select to_datetime('9999-12-31 23:59') +select to_datetime('9999-12-30 21:59') ---- -9999-12-31 23:59:00.000000 +9999-12-30 21:59:00.000000 query T -select to_datetime('9999-12-31 23:59:') +select to_datetime('9999-12-30 21:59:') ---- -9999-12-31 23:59:00.000000 +9999-12-30 21:59:00.000000 query T select to_unix_timestamp('2022-12-31T23:59:59+00:00') @@ -171,7 +180,7 @@ select to_datetime('10000-01-01 00:00:00') query T select to_datetime('0999-12-31 23:59:59') ---- -1000-01-01 00:00:00.000000 +0999-12-31 23:59:59.000000 query T select to_datetime('2022-12-31T23:59:59+00:00') @@ -662,15 +671,16 @@ select add_years(to_datetime(1582970400000000), cast(-50, INT8)) 1970-02-28 10:00:00.000000 +# years gap: -9999..=9999 query T -select add_years(to_date('9999-12-31'), 1) +select add_years(to_date('9998-12-30'), 1) ---- -1000-01-01 +9999-12-30 query T -select add_years(to_datetime('9999-12-31 23:59:59'), 1) +select add_years(to_datetime('9998-12-30 21:59:59'), 1) ---- -1000-01-01 00:00:00.000000 +9999-12-30 21:59:59.000000 # 2020-2-29 - 13 months query T @@ -708,14 +718,14 @@ select add_days(to_datetime(1582970400000000), cast(-1, INT16)) 2020-02-28 10:00:00.000000 query T -select add_days(to_date('9999-12-31'), 1) +select add_days(to_date('9999-12-30'), 1) ---- -1000-01-01 +9999-12-31 query T -select add_days(to_datetime('9999-12-31 23:59:59'), 1) +select add_days(to_datetime('9999-12-30 21:59:59'), 1) ---- -1000-01-01 00:00:00.000000 +9999-12-30 22:00:00.000000 # 2020-2-29T10:00:00 + 25 hours query T @@ -731,14 +741,14 @@ select add_hours(to_date(18321), 1) query T -select add_hours(to_date('9999-12-31'), 24) +select add_hours(to_date('9999-12-30'), 24) ---- -1000-01-01 00:00:00.000000 +9999-12-30 22:00:00.000000 query T -select add_hours(to_datetime('9999-12-31 23:59:59'), 1) +select add_hours(to_datetime('9999-12-29 23:59:59'), 1) ---- -1000-01-01 00:00:00.000000 +9999-12-30 00:59:59.000000 # 2020-2-29T10:00:00 - 1 minutes query T @@ -1328,6 +1338,7 @@ insert into t values('2022-04-02T15:10:28-08:13', '2022-04-02T15:10:28.223+08:00 statement error 1006 insert into t values('2022-04-02 15:10:28.221', '2022-04-02 15:10:28.221', '10000-10-10') +# ignore minute offset query TTT select * from t order by b ---- diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test index 19c70da87074..ef51800ed713 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test @@ -593,13 +593,15 @@ select count_if(y = true) from (select to_timestamp(to_date(number)) as ts, to_ statement ok set timezone='Europe/London'; -statement error 1006 ----- +query T select to_date(to_timestamp('2021-03-28 01:00')); - -statement error 1006 ---- +2021-03-28 + +query T select '2021-03-28 01:59:59'::timestamp; +---- +2021-03-28 02:59:59.000000 statement ok set timezone='Asia/Shanghai'; @@ -609,8 +611,10 @@ select to_date('1941-03-15'); ---- 1941-03-15 -statement error 1006 +query T select to_date('1941-03-15 00:00:00'); +---- +1941-03-15 query T select to_date('1941-03-15 02:00:00'); @@ -728,29 +732,35 @@ set timezone='Asia/Shanghai'; statement ok unset enable_dst_hour_fix; -statement error 1006 +query T select to_timestamp('1947-04-15 00:00:00') +---- +1947-04-15 01:00:00.000000 query T select try_to_timestamp('1947-04-15 00:00:00') ---- -NULL +1947-04-15 01:00:00.000000 -statement error 1006 +query T select to_timestamp('1947-04-15 00:00:00', '%Y-%m-%d %H:%M:%S') +---- +1947-04-15 01:00:00.000000 query T select try_to_timestamp('1947-04-15 00:00:00', '%Y-%m-%d %H:%M:%S') ---- -NULL +1947-04-15 01:00:00.000000 query T select to_date('1947-04-15') ---- 1947-04-15 -statement error 1006 +query T select to_date('1947-04-15 00:00:00') +---- +1947-04-15 query T select to_timestamp('1990-09-16 01:00:00'); @@ -895,11 +905,15 @@ select to_year(to_date('1919-04-13','%Y-%m-%d')); statement ok unset enable_dst_hour_fix; -statement error 1006 +query T SELECT DATE_ADD(month, 1, '1941-03-15 00:00:00'::timestamp); +---- +1941-04-15 01:00:00.000000 -statement error 1006 +query T SELECT DATE_ADD(year, 1, '1941-03-15 00:00:00'::timestamp); +---- +1942-03-15 01:00:00.000000 statement ok unset timezone; @@ -968,8 +982,10 @@ statement ok set timezone='Asia/Shanghai'; -statement error 1006 +query T select convert_timezone('Asia/Shanghai', '1947-04-15 00:00:00'); +---- +1947-04-15 01:00:00.000000 statement ok set enable_dst_hour_fix=1; @@ -989,3 +1005,47 @@ drop table if exists t; statement ok unset timezone; + +# UTC +5:45 +statement ok +set timezone='Asia/Kathmandu'; + +query T +select to_start_of_second('2024-11-14 11:54:33.667269'); +---- +2024-11-14 11:54:33.000000 + +query T +select to_start_of_minute('2024-11-14 11:54:33.667269'); +---- +2024-11-14 11:54:00.000000 + +query T +select to_start_of_five_minutes('2024-11-14 11:54:33.667269'); +---- +2024-11-14 11:50:00.000000 + +query T +select to_start_of_ten_minutes('2024-11-14 11:54:33.667269'); +---- +2024-11-14 11:50:00.000000 + +query T +select to_start_of_fifteen_minutes('2024-11-14 11:54:33.667269'); +---- +2024-11-14 11:45:00.000000 + +query T +select time_slot('2024-11-14 11:54:33.667269'); +---- +2024-11-14 11:30:00.000000 + +query T +select to_start_of_hour('2024-11-14 11:54:33.667269'); +---- +2024-11-14 11:00:00.000000 + +query T +select to_start_of_day('2024-11-14 11:54:33.667269'); +---- +2024-11-14 00:00:00.000000 diff --git a/tests/sqllogictests/suites/query/functions/02_0075_function_datetimes_tz.test b/tests/sqllogictests/suites/query/functions/02_0075_function_datetimes_tz.test index 069f4869cdb8..f323c770c5d7 100644 --- a/tests/sqllogictests/suites/query/functions/02_0075_function_datetimes_tz.test +++ b/tests/sqllogictests/suites/query/functions/02_0075_function_datetimes_tz.test @@ -431,13 +431,15 @@ select count_if(y = true) from (select to_timestamp(to_date(number)) as ts, to_ statement ok set timezone='Europe/London'; -statement error 1006 ----- +query T select to_date(to_timestamp('2021-03-28 01:00')); - -statement error 1006 ---- +2021-03-28 + +query T select '2021-03-28 01:59:59'::timestamp; +---- +2021-03-28 02:59:59.000000 statement ok set timezone='Asia/Shanghai'; @@ -574,21 +576,25 @@ set timezone='Asia/Shanghai'; statement ok unset enable_dst_hour_fix; -statement error 1006 +query T select to_timestamp('1947-04-15 00:00:00') +---- +1947-04-15 01:00:00.000000 query T select try_to_timestamp('1947-04-15 00:00:00') ---- -NULL +1947-04-15 01:00:00.000000 -statement error 1006 +query T select to_timestamp('1947-04-15 00:00:00', '%Y-%m-%d %H:%M:%S') +---- +1947-04-15 01:00:00.000000 query T select try_to_timestamp('1947-04-15 00:00:00', '%Y-%m-%d %H:%M:%S') ---- -NULL +1947-04-15 01:00:00.000000 query T select to_date('1947-04-15') @@ -763,11 +769,15 @@ SELECT substr(DATE_ADD(month, 1, now())::String, 1,4)=substr(now()::String, 1,4) statement ok unset enable_dst_hour_fix; -statement error 1006 +query T SELECT DATE_ADD(month, 1, '1941-03-15 00:00:00'::timestamp); +---- +1941-04-15 01:00:00.000000 -statement error 1006 +query T SELECT DATE_ADD(year, 1, '1941-03-15 00:00:00'::timestamp); +---- +1942-03-15 01:00:00.000000 statement ok unset timezone; From 5ea43fff244d96d82f8af95e0d8171d3644ff5fb Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 20 Nov 2024 17:48:16 +0800 Subject: [PATCH 63/92] chore(ci): adjust runner size (#16879) * chore(ci): adjust runner size * z * z * z * z * z * z * z * z * z * z --- .github/workflows/bindings.python.yml | 2 +- .github/workflows/cloud.yml | 4 +- .github/workflows/meta.yml | 4 +- .github/workflows/release.yml | 22 ++-- .github/workflows/reuse.benchmark.yml | 2 +- .github/workflows/reuse.linux.hive.yml | 4 +- .github/workflows/reuse.linux.yml | 44 ++++---- .github/workflows/reuse.sqllogic.yml | 100 ++++++++---------- Cargo.toml | 8 +- tests/sqllogictests/suites/query/set.test | 12 +-- .../suites/tpch_iceberg/prune.test | 6 +- 11 files changed, 96 insertions(+), 112 deletions(-) diff --git a/.github/workflows/bindings.python.yml b/.github/workflows/bindings.python.yml index d8a0a2fbc829..2a0fc2ba34c0 100644 --- a/.github/workflows/bindings.python.yml +++ b/.github/workflows/bindings.python.yml @@ -22,7 +22,7 @@ concurrency: jobs: linux: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c16g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c16g, aws] strategy: matrix: include: diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml index 6bb7c37488be..d899dce3c1be 100644 --- a/.github/workflows/cloud.yml +++ b/.github/workflows/cloud.yml @@ -45,7 +45,7 @@ jobs: build: needs: info - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws] strategy: matrix: include: @@ -71,7 +71,7 @@ jobs: docker: needs: [info, build] timeout-minutes: 10 - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] outputs: tag: ${{ steps.prepare.outputs.tag }} steps: diff --git a/.github/workflows/meta.yml b/.github/workflows/meta.yml index 34c694744c25..bba330a4efe2 100644 --- a/.github/workflows/meta.yml +++ b/.github/workflows/meta.yml @@ -36,7 +36,7 @@ jobs: build: needs: info - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws] strategy: matrix: include: @@ -56,7 +56,7 @@ jobs: chaos: needs: [info, build] - runs-on: [self-hosted, X64, Linux, 8c16g, aws] + runs-on: [self-hosted, X64, Linux, 4c16g, aws] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b956abaec2a8..366a7c7a3d13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -99,7 +99,7 @@ jobs: delete-branch: true build_default: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws] needs: create_release strategy: fail-fast: false @@ -132,7 +132,7 @@ jobs: bash ./scripts/ci/ci-run-sqllogic-tests.sh base build_musl: - runs-on: [self-hosted, X64, Linux, 16c32g, aws] + runs-on: [self-hosted, X64, Linux, 8c32g, aws] needs: create_release strategy: fail-fast: false @@ -157,7 +157,7 @@ jobs: artifacts: query,meta,metactl build_udf: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws] needs: create_release strategy: fail-fast: false @@ -184,7 +184,7 @@ jobs: category: udf publish: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_default, build_musl] strategy: fail-fast: false @@ -251,7 +251,7 @@ jobs: category: ${{ matrix.category }} publish_testsuite: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_default] strategy: fail-fast: false @@ -295,7 +295,7 @@ jobs: category: testsuite docker_all_in_one: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_default] steps: - name: Checkout @@ -370,7 +370,7 @@ jobs: readme-filepath: ./docker/README.md docker_service: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_udf] strategy: fail-fast: false @@ -441,7 +441,7 @@ jobs: file: ./docker/service/${{ matrix.service }}.Dockerfile distribution: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_default] strategy: matrix: @@ -592,7 +592,7 @@ jobs: sqlsmith: needs: [create_release, notify] - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] steps: - uses: actions/checkout@v4 with: @@ -617,7 +617,7 @@ jobs: metachaos: needs: [create_release, notify] - runs-on: [self-hosted, X64, Linux, 8c16g, aws] + runs-on: [self-hosted, X64, Linux, 4c16g, aws] steps: - uses: actions/checkout@v4 with: @@ -643,7 +643,7 @@ jobs: await script({context, core}) # sharing: - # runs-on: [self-hosted, X64, Linux, 4c8g, aws] + # runs-on: [self-hosted, X64, Linux, 2c8g, aws] # needs: [create_release, notify] # steps: # - uses: actions/checkout@v4 diff --git a/.github/workflows/reuse.benchmark.yml b/.github/workflows/reuse.benchmark.yml index 32042aacd5cf..d8097d9297d0 100644 --- a/.github/workflows/reuse.benchmark.yml +++ b/.github/workflows/reuse.benchmark.yml @@ -44,7 +44,7 @@ env: jobs: local: timeout-minutes: 60 - runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"] strategy: matrix: dataset: diff --git a/.github/workflows/reuse.linux.hive.yml b/.github/workflows/reuse.linux.hive.yml index eeae250f4921..e189608c6a1b 100644 --- a/.github/workflows/reuse.linux.hive.yml +++ b/.github/workflows/reuse.linux.hive.yml @@ -20,7 +20,7 @@ env: jobs: build: - runs-on: [self-hosted, X64, Linux, 8c16g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"] strategy: matrix: include: @@ -43,7 +43,7 @@ jobs: test_stateful_hive_standalone: needs: build - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateful_hive_standalone diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml index 116c6c719995..91652bed3739 100644 --- a/.github/workflows/reuse.linux.yml +++ b/.github/workflows/reuse.linux.yml @@ -25,7 +25,7 @@ env: jobs: check: - runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 with: @@ -41,7 +41,7 @@ jobs: - self-hosted - "${{ matrix.runner }}" - Linux - - 16c32g + - 8c32g - "${{ inputs.runner_provider }}" strategy: fail-fast: false @@ -67,7 +67,7 @@ jobs: - self-hosted - "${{ matrix.runner }}" - Linux - - 16c32g + - 8c32g - "${{ inputs.runner_provider }}" strategy: fail-fast: false @@ -96,7 +96,7 @@ jobs: # - self-hosted # - "${{ matrix.runner }}" # - Linux - # - 16c32g + # - 8c32g # - "${{ inputs.runner_provider }}" # strategy: # fail-fast: false @@ -115,7 +115,7 @@ jobs: # artifacts: query test_unit: - runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 with: @@ -125,7 +125,7 @@ jobs: timeout-minutes: 60 test_metactl: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -133,7 +133,7 @@ jobs: timeout-minutes: 10 test_compat_meta_query: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -141,7 +141,7 @@ jobs: timeout-minutes: 10 test_compat_fuse: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -149,7 +149,7 @@ jobs: timeout-minutes: 20 test_compat_meta_meta: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -157,7 +157,7 @@ jobs: timeout-minutes: 20 test_logs: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -165,7 +165,7 @@ jobs: timeout-minutes: 20 test_meta_cluster: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -173,7 +173,7 @@ jobs: timeout-minutes: 10 test_stateless_standalone: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -181,7 +181,7 @@ jobs: timeout-minutes: 15 test_stateless_cluster: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -193,7 +193,7 @@ jobs: timeout-minutes: 15 test_stateful_standalone: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -206,7 +206,7 @@ jobs: name: test-stateful-standalone-linux test_stateful_cluster: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -224,7 +224,7 @@ jobs: test_stateful_large_data: if: contains(github.event.pull_request.labels.*.name, 'ci-largedata') - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -232,7 +232,7 @@ jobs: timeout-minutes: 60 test_stateful_iceberg_rest: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] needs: [build, check] steps: - uses: actions/checkout@v4 @@ -245,7 +245,7 @@ jobs: name: test-stateful-iceberg-rest-standalone # test_fuzz_standalone: - # runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + # runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] # needs: [build, check] # steps: # - uses: actions/checkout@v4 @@ -255,7 +255,7 @@ jobs: test_ee_standalone: needs: [build, check] - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_license @@ -272,7 +272,7 @@ jobs: test_ee_standalone_background: needs: [build, check] - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_bendsql @@ -295,7 +295,7 @@ jobs: # # test_ee_standalone_fake_time: # needs: [build, check] - # runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + # runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] # steps: # - uses: actions/checkout@v4 # - uses: ./.github/actions/setup_license @@ -312,7 +312,7 @@ jobs: test_ee_management_mode: needs: [build, check] - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_bendsql diff --git a/.github/workflows/reuse.sqllogic.yml b/.github/workflows/reuse.sqllogic.yml index ea56077fad83..a959a38c470c 100644 --- a/.github/workflows/reuse.sqllogic.yml +++ b/.github/workflows/reuse.sqllogic.yml @@ -25,7 +25,7 @@ env: jobs: management_mode: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_sqllogic_management_mode_linux @@ -35,18 +35,24 @@ jobs: handlers: mysql,http standalone: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: + - self-hosted + - X64 + - Linux + - "${{ matrix.tests.runner }}" + - "${{ inputs.runner_provider }}" strategy: fail-fast: false matrix: - dirs: - - "standalone" - - "crdb" - - "duckdb" - - "base" - - "ydb" - - "tpcds" - - "tpch" + tests: + - { dirs: "query", runner: "4c16g" } + - { dirs: "duckdb", runner: "4c16g" } + - { dirs: "crdb", runner: "2c8g" } + - { dirs: "base", runner: "2c8g" } + - { dirs: "ydb", runner: "2c8g" } + - { dirs: "tpcds", runner: "2c8g" } + - { dirs: "tpch", runner: "2c8g" } + - { dirs: "standalone", runner: "2c8g" } handler: - "mysql" - "http" @@ -55,44 +61,17 @@ jobs: - uses: ./.github/actions/test_sqllogic_standalone_linux timeout-minutes: 15 with: - dirs: ${{ matrix.dirs }} + dirs: ${{ matrix.tests.dirs }} handlers: ${{ matrix.handler }} storage-format: all - name: Upload failure if: failure() uses: ./.github/actions/artifact_failure with: - name: test-sqllogic-standalone-${{ matrix.dirs }}-${{ matrix.handler }} - - standalone_query: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] - strategy: - fail-fast: false - matrix: - dirs: - - "query" - handler: - - "mysql" - - "http" - format: - - "native" - - "parquet" - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/test_sqllogic_standalone_linux - timeout-minutes: 15 - with: - dirs: ${{ matrix.dirs }} - handlers: ${{ matrix.handler }} - storage-format: ${{ matrix.format }} - - name: Upload failure - if: failure() - uses: ./.github/actions/artifact_failure - with: - name: test-sqllogic-standalone-query-${{ matrix.handler }}-${{ matrix.format }} + name: test-sqllogic-standalone-${{ matrix.tests.dirs }}-${{ matrix.handler }} standalone_udf_server: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - name: Start UDF Server @@ -113,7 +92,7 @@ jobs: name: test-sqllogic-standalone-udf-server standalone_cloud: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - name: Start Cloud Control Server @@ -134,7 +113,7 @@ jobs: name: test-sqllogic-standalone-cloud standalone_minio: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -161,7 +140,7 @@ jobs: name: test-sqllogic-standalone-minio-${{ matrix.dirs }}-${{ matrix.handler }}-${{ matrix.format }} standalone_iceberg_tpch: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -180,19 +159,24 @@ jobs: name: test-sqllogic-standalone-iceberg-tpch cluster: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: + - self-hosted + - X64 + - Linux + - "${{ matrix.tests.runner }}" + - "${{ inputs.runner_provider }}" strategy: fail-fast: false matrix: - dirs: - - "query" - - "crdb" - - "duckdb" - - "base" - - "ydb" - - "tpcds" - - "tpch" - - "cluster" + tests: + - { dirs: "query", runner: "4c16g" } + - { dirs: "duckdb", runner: "4c16g" } + - { dirs: "crdb", runner: "2c8g" } + - { dirs: "base", runner: "2c8g" } + - { dirs: "ydb", runner: "2c8g" } + - { dirs: "tpcds", runner: "2c8g" } + - { dirs: "tpch", runner: "2c8g" } + - { dirs: "cluster", runner: "2c8g" } handler: - "mysql" - "http" @@ -205,16 +189,16 @@ jobs: - uses: ./.github/actions/test_sqllogic_cluster_linux timeout-minutes: 15 with: - dirs: ${{ matrix.dirs }} + dirs: ${{ matrix.tests.dirs }} handlers: ${{ matrix.handler }} - name: Upload failure if: failure() uses: ./.github/actions/artifact_failure with: - name: test-sqllogic-cluster-${{ matrix.dirs }}-${{ matrix.handler }} + name: test-sqllogic-cluster-${{ matrix.tests.dirs }}-${{ matrix.handler }} stage: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -236,7 +220,7 @@ jobs: name: test-sqllogic-stage-${{ matrix.storage }} standalone_no_table_meta_cache: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -260,7 +244,7 @@ jobs: name: test-sqllogic-standalone-no-table-meta-cache-${{ matrix.dirs }}-${{ matrix.handler }} ee: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: diff --git a/Cargo.toml b/Cargo.toml index af99aca4b21c..bca41bfd82e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -581,10 +581,6 @@ debug-assertions = true # databend-query = { codegen-units = 4 } # databend-binaries = { codegen-units = 4 } -[profile.bench] -debug = true -overflow-checks = false - [profile.dev] split-debuginfo = "unpacked" overflow-checks = false @@ -600,6 +596,10 @@ object = { opt-level = 3 } rustc-demangle = { opt-level = 3 } databend-common-exception = { opt-level = 3 } +[profile.bench] +debug = true +overflow-checks = false + [profile.test] opt-level = 0 debug = true diff --git a/tests/sqllogictests/suites/query/set.test b/tests/sqllogictests/suites/query/set.test index 162072166377..540135f8795f 100644 --- a/tests/sqllogictests/suites/query/set.test +++ b/tests/sqllogictests/suites/query/set.test @@ -1,14 +1,14 @@ statement ok -SET GLOBAL (max_threads, storage_io_min_bytes_for_seek) = (4, 56); +SET GLOBAL (max_threads, storage_io_min_bytes_for_seek) = (13, 56); query TT select value, default = value from system.settings where name in ('max_threads', 'storage_io_min_bytes_for_seek') order by value; ---- -4 0 +13 0 56 0 statement ok -set variable (a, b) = (select 3, 55) +set variable (a, b) = (select 12, 55) statement ok @@ -17,7 +17,7 @@ SET GLOBAL (max_threads, storage_io_min_bytes_for_seek) = select $a + 1, $b + 1; query TT select value, default = value from system.settings where name in ('max_threads', 'storage_io_min_bytes_for_seek') order by value; ---- -4 0 +13 0 56 0 statement ok @@ -36,7 +36,7 @@ statement ok set variable (b, c) = ('yy', 'zz'); query ITT -select $a + getvariable('a') + $a, getvariable('b'), getvariable('c'), getvariable('d') +select $a + getvariable('a') + $a, getvariable('b'), getvariable('c'), getvariable('d') ---- 3 yy zz NULL @@ -44,7 +44,7 @@ statement ok unset variable (a, b) query ITT -select getvariable('a'), getvariable('b'), 'xx' || 'yy' || getvariable('c') , getvariable('d') +select getvariable('a'), getvariable('b'), 'xx' || 'yy' || getvariable('c') , getvariable('d') ---- NULL NULL xxyyzz NULL diff --git a/tests/sqllogictests/suites/tpch_iceberg/prune.test b/tests/sqllogictests/suites/tpch_iceberg/prune.test index f79f367d7145..c32b0ae6101f 100644 --- a/tests/sqllogictests/suites/tpch_iceberg/prune.test +++ b/tests/sqllogictests/suites/tpch_iceberg/prune.test @@ -134,8 +134,8 @@ EvalScalar ├── table: ctl.tpch.lineitem ├── output columns: [l_orderkey (#0), l_commitdate (#11)] ├── read rows: 600572 - ├── read size: 14.41 MiB - ├── partitions total: 6 - ├── partitions scanned: 6 + ├── read size: 14.27 MiB + ├── partitions total: 4 + ├── partitions scanned: 4 ├── push downs: [filters: [(NOT is_not_null(lineitem.l_orderkey (#0)) OR is_not_null(lineitem.l_commitdate (#11)))], limit: NONE] └── estimated rows: 0.00 From 008485ae1e981566713bb52f5af8290cdce78bee Mon Sep 17 00:00:00 2001 From: Jk Xu <54522439+Dousir9@users.noreply.github.com> Date: Wed, 20 Nov 2024 20:52:28 +0800 Subject: [PATCH 64/92] chore(settings): enable spill (#16888) --- src/query/settings/src/settings_default.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index 90e50c14782f..e44da8e74c72 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -288,7 +288,7 @@ impl DefaultSettings { range: Some(SettingRange::Numeric(0..=1)), }), ("join_spilling_memory_ratio", DefaultSettingValue { - value: UserSettingValue::UInt64(0), + value: UserSettingValue::UInt64(60), desc: "Sets the maximum memory ratio in bytes that hash join can use before spilling data to storage during query execution, 0 is unlimited", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=100)), @@ -463,7 +463,7 @@ impl DefaultSettings { range: Some(SettingRange::Numeric(0..=u64::MAX)), }), ("aggregate_spilling_memory_ratio", DefaultSettingValue { - value: UserSettingValue::UInt64(0), + value: UserSettingValue::UInt64(60), desc: "Sets the maximum memory ratio in bytes that an aggregator can use before spilling data to storage during query execution.", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=100)), @@ -475,7 +475,7 @@ impl DefaultSettings { range: Some(SettingRange::Numeric(0..=u64::MAX)), }), ("window_partition_spilling_memory_ratio", DefaultSettingValue { - value: UserSettingValue::UInt64(0), + value: UserSettingValue::UInt64(60), desc: "Sets the maximum memory ratio in bytes that a window partitioner can use before spilling data to storage during query execution.", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=100)), @@ -511,7 +511,7 @@ impl DefaultSettings { range: Some(SettingRange::Numeric(0..=u64::MAX)), }), ("sort_spilling_memory_ratio", DefaultSettingValue { - value: UserSettingValue::UInt64(0), + value: UserSettingValue::UInt64(60), desc: "Sets the maximum memory ratio in bytes that a sorter can use before spilling data to storage during query execution.", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=100)), From 45a58cdadcf2d2f4c47515bfa7ee28934e49481f Mon Sep 17 00:00:00 2001 From: baishen Date: Wed, 20 Nov 2024 23:00:41 +0800 Subject: [PATCH 65/92] fix(query): fix distinct set-returning function (#16883) * fix(query): fix distinct set-returning function * add tests * fix * fix tests --------- Co-authored-by: sundyli <543950155@qq.com> --- .../sql/src/planner/binder/bind_query/bind_select.rs | 2 +- src/query/sql/src/planner/binder/distinct.rs | 11 ++++++++++- src/query/sql/src/planner/binder/select.rs | 10 ++++++---- src/query/sql/src/planner/dataframe.rs | 2 +- .../base/03_common/03_0003_select_group_by.test | 7 +++++++ .../query/functions/02_0062_function_unnest.test | 6 ++++++ 6 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind_select.rs b/src/query/sql/src/planner/binder/bind_query/bind_select.rs index 82907fff517b..bd8fb366b9c5 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind_select.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind_select.rs @@ -225,7 +225,7 @@ impl Binder { if stmt.distinct { s_expr = self.bind_distinct( stmt.span, - &from_context, + &mut from_context, &projections, &mut scalar_items, s_expr, diff --git a/src/query/sql/src/planner/binder/distinct.rs b/src/query/sql/src/planner/binder/distinct.rs index 621bc1ea0ca6..df1d26e6cedd 100644 --- a/src/query/sql/src/planner/binder/distinct.rs +++ b/src/query/sql/src/planner/binder/distinct.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use databend_common_ast::Span; use databend_common_exception::Result; +use crate::binder::project_set::SetReturningRewriter; use crate::binder::Binder; use crate::binder::ColumnBinding; use crate::optimizer::SExpr; @@ -37,11 +38,19 @@ impl Binder { pub fn bind_distinct( &self, span: Span, - bind_context: &BindContext, + bind_context: &mut BindContext, projections: &[ColumnBinding], scalar_items: &mut HashMap, child: SExpr, ) -> Result { + if !bind_context.srf_info.srfs.is_empty() { + // Rewrite the Set-returning functions as columns. + let mut srf_rewriter = SetReturningRewriter::new(bind_context, false); + for (_, item) in scalar_items.iter_mut() { + srf_rewriter.visit(&mut item.scalar)?; + } + } + let scalar_items: Vec = scalar_items .drain() .map(|(_, item)| { diff --git a/src/query/sql/src/planner/binder/select.rs b/src/query/sql/src/planner/binder/select.rs index 2b8df21fc3fb..3c3fd840bf3a 100644 --- a/src/query/sql/src/planner/binder/select.rs +++ b/src/query/sql/src/planner/binder/select.rs @@ -296,10 +296,11 @@ impl Binder { ); if distinct { + let columns = new_bind_context.all_column_bindings().to_vec(); new_expr = self.bind_distinct( left_span, - &new_bind_context, - new_bind_context.all_column_bindings(), + &mut new_bind_context, + &columns, &mut HashMap::new(), new_expr, )?; @@ -359,10 +360,11 @@ impl Binder { right_expr: SExpr, join_type: JoinType, ) -> Result<(SExpr, BindContext)> { + let columns = left_context.all_column_bindings().to_vec(); let left_expr = self.bind_distinct( left_span, - &left_context, - left_context.all_column_bindings(), + &mut left_context, + &columns, &mut HashMap::new(), left_expr, )?; diff --git a/src/query/sql/src/planner/dataframe.rs b/src/query/sql/src/planner/dataframe.rs index 97681ee9cb14..a590b9bbad8d 100644 --- a/src/query/sql/src/planner/dataframe.rs +++ b/src/query/sql/src/planner/dataframe.rs @@ -313,7 +313,7 @@ impl Dataframe { )?; self.s_expr = self.binder.bind_distinct( None, - &self.bind_context, + &mut self.bind_context, &projections, &mut scalar_items, self.s_expr.clone(), diff --git a/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test b/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test index a5a5e685429d..5d68e422f37d 100644 --- a/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test +++ b/tests/sqllogictests/suites/base/03_common/03_0003_select_group_by.test @@ -268,6 +268,13 @@ a1 a2 a3 +query T +SELECT distinct unnest(split(col2, ',')) AS col3 FROM t_str ORDER BY col3; +---- +a1 +a2 +a3 + statement ok DROP TABLE t_str diff --git a/tests/sqllogictests/suites/query/functions/02_0062_function_unnest.test b/tests/sqllogictests/suites/query/functions/02_0062_function_unnest.test index 52b1b5fecec2..a3792a8718bc 100644 --- a/tests/sqllogictests/suites/query/functions/02_0062_function_unnest.test +++ b/tests/sqllogictests/suites/query/functions/02_0062_function_unnest.test @@ -503,6 +503,12 @@ select max(unnest([11,12])) ---- 12 +query T +SELECT distinct unnest(split(coalesce(NULL, 'a,b'), ',')) AS c1 +---- +a +b + statement error 1065 select unnest(first_value('aa') OVER (PARTITION BY 'bb')) From 8498ad97ecb65f704dd59b9785073b08e2156380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Thu, 21 Nov 2024 12:12:25 +0800 Subject: [PATCH 66/92] feat: databend-meta adds more metrics about raft-log (#16884) - "raft_log_cache_items" : number of items in raft log cache; - "raft_log_cache_used_size" : size of used space in raft log cache; - "raft_log_wal_open_chunk_size" : size of open chunk in raft log wal; - "raft_log_wal_offset" : global offset of raft log WAL; - "raft_log_wal_closed_chunk_count" : number of closed chunks in raft log WAL; - "raft_log_wal_closed_chunk_total_size" : total size of closed chunks in raft log WAL; Example output of these new metrics: ```text metasrv_server_raft_log_cache_items 35496 metasrv_server_raft_log_cache_used_size 659024620 metasrv_server_raft_log_wal_open_chunk_size 803 metasrv_server_raft_log_wal_offset 3536913654 metasrv_server_raft_log_wal_closed_chunk_count 3 metasrv_server_raft_log_wal_closed_chunk_total_size 584045971 metasrv_server_raft_log_size 584046774 ``` --- Cargo.lock | 4 +- Cargo.toml | 2 +- src/meta/raft-store/src/raft_log_v004/mod.rs | 1 + .../service/src/meta_service/meta_node.rs | 10 +++ src/meta/service/src/metrics/meta_metrics.rs | 68 +++++++++++++++++++ 5 files changed, 82 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 835bbf246e22..48fb0ad2367f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11998,9 +11998,9 @@ dependencies = [ [[package]] name = "raft-log" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d5edc50b7d30e6b04683575129a996cf210951fc36c6e5856eb8dc746fef77a" +checksum = "b2e7582480ea22f8268d681f4c5f8cf9b626e7f6c917724da68125c24820b055" dependencies = [ "byteorder", "clap", diff --git a/Cargo.toml b/Cargo.toml index bca41bfd82e7..d8bafb180e45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -405,7 +405,7 @@ prost = { version = "0.13" } prost-build = { version = "0.13" } prqlc = "0.11.3" quanta = "0.11.1" -raft-log = { version = "0.2.3" } +raft-log = { version = "0.2.5" } rand = { version = "0.8.5", features = ["small_rng"] } rayon = "1.9.0" recursive = "0.1.1" diff --git a/src/meta/raft-store/src/raft_log_v004/mod.rs b/src/meta/raft-store/src/raft_log_v004/mod.rs index f0f50c8a6676..4cfb94141122 100644 --- a/src/meta/raft-store/src/raft_log_v004/mod.rs +++ b/src/meta/raft-store/src/raft_log_v004/mod.rs @@ -27,6 +27,7 @@ pub const TREE_RAFT_LOG: &str = "raft_log"; pub type RaftLogV004 = raft_log::RaftLog; pub type RaftLogConfig = raft_log::Config; +pub type RaftLogStat = raft_log::Stat; pub use callback::Callback; pub use callback_data::CallbackData; diff --git a/src/meta/service/src/meta_service/meta_node.rs b/src/meta/service/src/meta_service/meta_node.rs index 02ea28e34208..b38aff9cf0e3 100644 --- a/src/meta/service/src/meta_service/meta_node.rs +++ b/src/meta/service/src/meta_service/meta_node.rs @@ -32,6 +32,7 @@ use databend_common_meta_client::reply_to_api_result; use databend_common_meta_client::RequestFor; use databend_common_meta_raft_store::config::RaftConfig; use databend_common_meta_raft_store::ondisk::DATA_VERSION; +use databend_common_meta_raft_store::raft_log_v004::RaftLogStat; use databend_common_meta_sled_store::openraft; use databend_common_meta_sled_store::openraft::ChangeMembers; use databend_common_meta_stoerr::MetaStorageError; @@ -446,6 +447,11 @@ impl MetaNode { server_metrics::set_proposals_applied(mm.last_applied.unwrap_or_default().index); server_metrics::set_last_seq(meta_node.get_last_seq().await); + { + let st = meta_node.get_raft_log_stat().await; + server_metrics::set_raft_log_stat(st); + } + // metrics about server storage server_metrics::set_raft_log_size(meta_node.get_raft_log_size().await); server_metrics::set_snapshot_key_count(meta_node.get_snapshot_key_count().await); @@ -822,6 +828,10 @@ impl MetaNode { self.sto.log.read().await.on_disk_size() } + async fn get_raft_log_stat(&self) -> RaftLogStat { + self.sto.log.read().await.stat() + } + async fn get_snapshot_key_count(&self) -> u64 { self.sto .try_get_snapshot_key_count() diff --git a/src/meta/service/src/metrics/meta_metrics.rs b/src/meta/service/src/metrics/meta_metrics.rs index 022cb1f5c832..73b65aabdc1d 100644 --- a/src/meta/service/src/metrics/meta_metrics.rs +++ b/src/meta/service/src/metrics/meta_metrics.rs @@ -33,6 +33,7 @@ use prometheus_client::encoding::text::encode as prometheus_encode; pub mod server_metrics { use std::sync::LazyLock; + use databend_common_meta_raft_store::raft_log_v004::RaftLogStat; use databend_common_meta_types::raft_types::NodeId; use prometheus_client::metrics::counter::Counter; use prometheus_client::metrics::family::Family; @@ -53,6 +54,14 @@ pub mod server_metrics { leader_changes: Counter, applying_snapshot: Gauge, snapshot_key_count: Gauge, + + raft_log_cache_items: Gauge, + raft_log_cache_used_size: Gauge, + raft_log_wal_open_chunk_size: Gauge, + raft_log_wal_offset: Gauge, + raft_log_wal_closed_chunk_count: Gauge, + raft_log_wal_closed_chunk_total_size: Gauge, + raft_log_size: Gauge, last_log_index: Gauge, last_seq: Gauge, @@ -74,6 +83,12 @@ pub mod server_metrics { leader_changes: Counter::default(), applying_snapshot: Gauge::default(), snapshot_key_count: Gauge::default(), + raft_log_cache_items: Gauge::default(), + raft_log_cache_used_size: Gauge::default(), + raft_log_wal_open_chunk_size: Gauge::default(), + raft_log_wal_offset: Gauge::default(), + raft_log_wal_closed_chunk_count: Gauge::default(), + raft_log_wal_closed_chunk_total_size: Gauge::default(), raft_log_size: Gauge::default(), last_log_index: Gauge::default(), last_seq: Gauge::default(), @@ -113,6 +128,38 @@ pub mod server_metrics { "number of keys in the last snapshot", metrics.snapshot_key_count.clone(), ); + + registry.register( + key!("raft_log_cache_items"), + "number of items in raft log cache", + metrics.raft_log_cache_items.clone(), + ); + registry.register( + key!("raft_log_cache_used_size"), + "size of used space in raft log cache", + metrics.raft_log_cache_used_size.clone(), + ); + registry.register( + key!("raft_log_wal_open_chunk_size"), + "size of open chunk in raft log wal", + metrics.raft_log_wal_open_chunk_size.clone(), + ); + registry.register( + key!("raft_log_wal_offset"), + "global offset of raft log WAL", + metrics.raft_log_wal_offset.clone(), + ); + registry.register( + key!("raft_log_wal_closed_chunk_count"), + "number of closed chunks in raft log WAL", + metrics.raft_log_wal_closed_chunk_count.clone(), + ); + registry.register( + key!("raft_log_wal_closed_chunk_total_size"), + "total size of closed chunks in raft log WAL", + metrics.raft_log_wal_closed_chunk_total_size.clone(), + ); + registry.register( key!("raft_log_size"), "the size in bytes of the on disk data of raft log", @@ -182,6 +229,27 @@ pub mod server_metrics { SERVER_METRICS.snapshot_key_count.set(n as i64); } + pub fn set_raft_log_stat(st: RaftLogStat) { + SERVER_METRICS + .raft_log_cache_items + .set(st.payload_cache_item_count as i64); + SERVER_METRICS + .raft_log_cache_used_size + .set(st.payload_cache_size as i64); + SERVER_METRICS + .raft_log_wal_open_chunk_size + .set(st.open_chunk.size as i64); + SERVER_METRICS + .raft_log_wal_offset + .set(st.open_chunk.global_end as i64); + SERVER_METRICS + .raft_log_wal_closed_chunk_count + .set(st.closed_chunks.len() as i64); + SERVER_METRICS + .raft_log_wal_closed_chunk_total_size + .set(st.closed_chunks.iter().map(|v| v.size).sum::() as i64); + } + pub fn set_raft_log_size(raft_log_size: u64) { SERVER_METRICS.raft_log_size.set(raft_log_size as i64); } From 22a8a82bc3fd532988ed661084eeeb1ef3bf3066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Thu, 21 Nov 2024 15:09:15 +0800 Subject: [PATCH 67/92] refactor: add new raft-log metrics to "metactl status" response (#16899) These new metrics are added to gRPC API `GetClusterStatus` and corresponding databend-metactl CLI command. Newly added metrics are: ``` $ databend-metactl status # ... RaftLog: - CacheItems: 287 - CacheUsedSize: 4959439 - WALTotalSize: 4378076 - WALOpenChunkSize: 4378058 - WALOffset: 4378076 - WALClosedChunkCount: 1 - WALClosedChunkTotalSize: 18 - WALClosedChunkSizes: - ChunkId(00_000_000_000_000_000_000): 18 ``` --- Cargo.lock | 1 - src/meta/binaries/metactl/main.rs | 17 ++++++++ src/meta/service/src/api/grpc/grpc_service.rs | 15 ++++++- .../service/src/meta_service/meta_node.rs | 8 ++-- .../src/meta_service/meta_node_status.rs | 41 ++++++++++++++++++- src/meta/types/build.rs | 1 + src/meta/types/proto/meta.proto | 19 +++++++-- 7 files changed, 91 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48fb0ad2367f..9a70b9c64379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3325,7 +3325,6 @@ dependencies = [ "chrono-tz 0.8.6", "comfy-table", "criterion", - "dashmap 6.1.0", "databend-common-ast", "databend-common-base", "databend-common-column", diff --git a/src/meta/binaries/metactl/main.rs b/src/meta/binaries/metactl/main.rs index bffe0aae683c..2e218806e542 100644 --- a/src/meta/binaries/metactl/main.rs +++ b/src/meta/binaries/metactl/main.rs @@ -237,6 +237,23 @@ impl App { println!("BinaryVersion: {}", res.binary_version); println!("DataVersion: {}", res.data_version); println!("RaftLogSize: {}", res.raft_log_size); + if let Some(s) = res.raft_log_status { + println!("RaftLog:"); + println!(" - CacheItems: {}", s.cache_items); + println!(" - CacheUsedSize: {}", s.cache_used_size); + println!(" - WALTotalSize: {}", s.wal_total_size); + println!(" - WALOpenChunkSize: {}", s.wal_open_chunk_size); + println!(" - WALOffset: {}", s.wal_offset); + println!(" - WALClosedChunkCount: {}", s.wal_closed_chunk_count); + println!( + " - WALClosedChunkTotalSize: {}", + s.wal_closed_chunk_total_size + ); + println!(" - WALClosedChunkSizes:"); + for (k, v) in s.wal_closed_chunk_sizes { + println!(" - {}: {}", k, v); + } + } println!("SnapshotKeyCount: {}", res.snapshot_key_count); println!("Node: id={} raft={}", res.id, res.endpoint); println!("State: {}", res.state); diff --git a/src/meta/service/src/api/grpc/grpc_service.rs b/src/meta/service/src/api/grpc/grpc_service.rs index 1ca6b2e458ef..4e29a019f50c 100644 --- a/src/meta/service/src/api/grpc/grpc_service.rs +++ b/src/meta/service/src/api/grpc/grpc_service.rs @@ -462,7 +462,20 @@ impl MetaService for MetaServiceImpl { binary_version: status.binary_version, data_version: status.data_version.to_string(), endpoint: status.endpoint, - raft_log_size: status.raft_log_size, + + raft_log_size: status.raft_log.wal_total_size, + + raft_log_status: Some(pb::RaftLogStatus { + cache_items: status.raft_log.cache_items, + cache_used_size: status.raft_log.cache_used_size, + wal_total_size: status.raft_log.wal_total_size, + wal_open_chunk_size: status.raft_log.wal_open_chunk_size, + wal_offset: status.raft_log.wal_offset, + wal_closed_chunk_count: status.raft_log.wal_closed_chunk_count, + wal_closed_chunk_total_size: status.raft_log.wal_closed_chunk_total_size, + wal_closed_chunk_sizes: status.raft_log.wal_closed_chunk_sizes, + }), + snapshot_key_count: status.snapshot_key_count as u64, state: status.state, is_leader: status.is_leader, diff --git a/src/meta/service/src/meta_service/meta_node.rs b/src/meta/service/src/meta_service/meta_node.rs index b38aff9cf0e3..cb0937651522 100644 --- a/src/meta/service/src/meta_service/meta_node.rs +++ b/src/meta/service/src/meta_service/meta_node.rs @@ -852,8 +852,8 @@ impl MetaNode { let endpoint = self.sto.get_node_raft_endpoint(&self.sto.id).await?; - let raft_log_size = self.get_raft_log_size().await; - let key_count = self.get_snapshot_key_count().await; + let raft_log_status = self.get_raft_log_stat().await.into(); + let snapshot_key_count = self.get_snapshot_key_count().await; let metrics = self.raft.metrics().borrow().clone(); @@ -870,8 +870,8 @@ impl MetaNode { binary_version: METASRV_COMMIT_VERSION.as_str().to_string(), data_version: DATA_VERSION, endpoint: endpoint.to_string(), - raft_log_size, - snapshot_key_count: key_count, + raft_log: raft_log_status, + snapshot_key_count, state: format!("{:?}", metrics.state), is_leader: metrics.state == openraft::ServerState::Leader, current_term: metrics.current_term, diff --git a/src/meta/service/src/meta_service/meta_node_status.rs b/src/meta/service/src/meta_service/meta_node_status.rs index cd995fb3784f..08a7f617120e 100644 --- a/src/meta/service/src/meta_service/meta_node_status.rs +++ b/src/meta/service/src/meta_service/meta_node_status.rs @@ -15,6 +15,7 @@ use std::collections::BTreeMap; use databend_common_meta_raft_store::ondisk::DataVersion; +use databend_common_meta_raft_store::raft_log_v004::RaftLogStat; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Node; @@ -32,8 +33,8 @@ pub struct MetaNodeStatus { /// The raft service endpoint for internal communication pub endpoint: String, - /// The size in bytes of the raft-log on disk data. - pub raft_log_size: u64, + /// The status about local raft-log + pub raft_log: RaftLogStatus, /// Total number of keys in current snapshot pub snapshot_key_count: u64, @@ -78,3 +79,39 @@ pub struct MetaNodeStatus { /// `seq` is a monotonically incremental integer for every value that is inserted or updated. pub last_seq: u64, } + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct RaftLogStatus { + pub cache_items: u64, + pub cache_used_size: u64, + pub wal_total_size: u64, + pub wal_open_chunk_size: u64, + pub wal_offset: u64, + pub wal_closed_chunk_count: u64, + pub wal_closed_chunk_total_size: u64, + pub wal_closed_chunk_sizes: BTreeMap, +} + +impl From for RaftLogStatus { + fn from(s: RaftLogStat) -> Self { + let closed_sizes = s + .closed_chunks + .iter() + .map(|c| (c.chunk_id.to_string(), c.size)) + .collect(); + + let closed_total_size = s.closed_chunks.iter().map(|c| c.size).sum::(); + let wal_total_size = closed_total_size + s.open_chunk.size; + + Self { + cache_items: s.payload_cache_item_count, + cache_used_size: s.payload_cache_size, + wal_total_size, + wal_open_chunk_size: s.open_chunk.size, + wal_offset: s.open_chunk.global_end, + wal_closed_chunk_count: s.closed_chunks.len() as u64, + wal_closed_chunk_total_size: closed_total_size, + wal_closed_chunk_sizes: closed_sizes, + } + } +} diff --git a/src/meta/types/build.rs b/src/meta/types/build.rs index dcf50cda01a2..3f5a4e4d1185 100644 --- a/src/meta/types/build.rs +++ b/src/meta/types/build.rs @@ -45,6 +45,7 @@ fn build_proto() { let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); tonic_build::configure() + .btree_map(["RaftLogStatus.wal_closed_chunk_sizes"]) .file_descriptor_set_path(out_dir.join("meta_descriptor.bin")) .type_attribute( "SeqV", diff --git a/src/meta/types/proto/meta.proto b/src/meta/types/proto/meta.proto index ab3a304e3aae..90d830d4e8cc 100644 --- a/src/meta/types/proto/meta.proto +++ b/src/meta/types/proto/meta.proto @@ -36,9 +36,9 @@ message TransferLeaderRequest { LogId last_log_id = 3; } -message MemberListRequest { string data = 1; } +message MemberListRequest {string data = 1;} -message MemberListReply { repeated string data = 1; } +message MemberListReply {repeated string data = 1;} message HandshakeRequest { uint64 protocol_version = 1; @@ -94,7 +94,7 @@ message Event { optional SeqV prev = 3; } -message WatchResponse { Event event = 1; } +message WatchResponse {Event event = 1;} // messages for txn message TxnCondition { @@ -181,6 +181,19 @@ message ClusterStatus { repeated string non_voters = 16; uint64 last_seq = 17; uint64 snapshot_key_count = 18; + RaftLogStatus raft_log_status = 19; +} + +// Status about local raft-log storage +message RaftLogStatus { + uint64 cache_items = 1; + uint64 cache_used_size = 2; + uint64 wal_total_size = 3; + uint64 wal_open_chunk_size = 4; + uint64 wal_offset = 5; + uint64 wal_closed_chunk_count = 6; + uint64 wal_closed_chunk_total_size = 7; + map wal_closed_chunk_sizes = 8; } message ClientInfo { From bb5e2e6c4cf74710c607a11e993a9f7f06059e2d Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Thu, 21 Nov 2024 15:41:21 +0800 Subject: [PATCH 68/92] fix(query): fix incorrect total_bytes_len in string view (#16877) * update * update * update * update * update --- src/common/column/src/binview/builder.rs | 4 +- src/common/column/src/binview/mod.rs | 56 +++++---------- .../src/kernels/group_by_hash/utils.rs | 2 + .../11_0002_data_type_string.test | 69 +++++++++++++++++++ 4 files changed, 91 insertions(+), 40 deletions(-) diff --git a/src/common/column/src/binview/builder.rs b/src/common/column/src/binview/builder.rs index 7ea031694a1c..1ff5df5fe84d 100644 --- a/src/common/column/src/binview/builder.rs +++ b/src/common/column/src/binview/builder.rs @@ -196,9 +196,7 @@ impl BinaryViewColumnBuilder { self.push_value(value); let value = self.views.pop().unwrap(); - - self.total_bytes_len += - (self.total_bytes_len - old_bytes_len) * additional.saturating_sub(1); + self.total_bytes_len = old_bytes_len + value.length as usize * additional; self.views.extend(std::iter::repeat(value).take(additional)); } diff --git a/src/common/column/src/binview/mod.rs b/src/common/column/src/binview/mod.rs index 58e52b234f33..f963a0ca99d7 100644 --- a/src/common/column/src/binview/mod.rs +++ b/src/common/column/src/binview/mod.rs @@ -20,8 +20,6 @@ mod view; use std::fmt::Debug; use std::marker::PhantomData; -use std::sync::atomic::AtomicU64; -use std::sync::atomic::Ordering; use std::sync::Arc; use arrow_data::ArrayData; @@ -52,8 +50,6 @@ mod private { impl Sealed for [u8] {} } -const UNKNOWN_LEN: u64 = u64::MAX; - pub trait ViewType: Sealed + 'static + PartialEq + AsRef { const IS_UTF8: bool; type Owned: Debug + Clone + Sync + Send + AsRef; @@ -119,7 +115,7 @@ pub struct BinaryViewColumnGeneric { buffers: Arc<[Buffer]>, phantom: PhantomData, /// Total bytes length if we would concat them all - total_bytes_len: AtomicU64, + total_bytes_len: usize, /// Total bytes in the buffer (exclude remaining capacity) total_buffer_len: usize, } @@ -131,7 +127,7 @@ impl Clone for BinaryViewColumnGeneric { buffers: self.buffers.clone(), phantom: Default::default(), - total_bytes_len: AtomicU64::new(self.total_bytes_len.load(Ordering::Relaxed)), + total_bytes_len: self.total_bytes_len, total_buffer_len: self.total_buffer_len, } } @@ -151,26 +147,19 @@ impl BinaryViewColumnGeneric { ) -> Self { #[cfg(debug_assertions)] { - if total_bytes_len != UNKNOWN_LEN as usize { - let total = views.iter().map(|v| v.length as usize).sum::(); - assert_eq!(total, total_bytes_len); - } + let total = views.iter().map(|v| v.length as usize).sum::(); + assert_eq!(total, total_bytes_len); - if total_buffer_len != UNKNOWN_LEN as usize { - let total = buffers.iter().map(|v| v.len()).sum::(); - assert_eq!(total, total_buffer_len); - } + let total = buffers.iter().map(|v| v.len()).sum::(); + assert_eq!(total, total_buffer_len); } - // # Safety - // The caller must ensure - // - the data is valid utf8 (if required) - // - the offsets match the buffers. + Self { views, buffers, phantom: Default::default(), - total_bytes_len: AtomicU64::new(total_bytes_len as u64), + total_bytes_len, total_buffer_len, } } @@ -181,12 +170,11 @@ impl BinaryViewColumnGeneric { pub unsafe fn new_unchecked_unknown_md( views: Buffer, buffers: Arc<[Buffer]>, - total_buffer_len: Option, ) -> Self { - let total_bytes_len = UNKNOWN_LEN as usize; let total_buffer_len = - total_buffer_len.unwrap_or_else(|| buffers.iter().map(|b| b.len()).sum()); + total_buffer_len.unwrap_or(buffers.iter().map(|v| v.len()).sum::()); + let total_bytes_len = views.iter().map(|v| v.length as usize).sum::(); Self::new_unchecked(views, buffers, total_bytes_len, total_buffer_len) } @@ -305,14 +293,7 @@ impl BinaryViewColumnGeneric { /// Get the total length of bytes that it would take to concatenate all binary/str values in this array. pub fn total_bytes_len(&self) -> usize { - let total = self.total_bytes_len.load(Ordering::Relaxed); - if total == UNKNOWN_LEN { - let total = self.len_iter().map(|v| v as usize).sum::(); - self.total_bytes_len.store(total as u64, Ordering::Relaxed); - total - } else { - total as usize - } + self.total_bytes_len } pub fn memory_size(&self) -> usize { @@ -378,7 +359,7 @@ impl BinaryViewColumnGeneric { unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { debug_assert!(offset + length <= self.len()); self.views.slice_unchecked(offset, length); - self.total_bytes_len.store(UNKNOWN_LEN, Ordering::Relaxed) + self.total_bytes_len = self.views.iter().map(|v| v.length as usize).sum::(); } impl_sliced!(); @@ -418,13 +399,14 @@ impl BinaryViewColumnGeneric { pub fn make_mut(self) -> BinaryViewColumnBuilder { let views = self.views.make_mut(); let completed_buffers = self.buffers.to_vec(); + BinaryViewColumnBuilder { views, completed_buffers, in_progress_buffer: vec![], phantom: Default::default(), - total_bytes_len: self.total_bytes_len.load(Ordering::Relaxed) as usize, + total_bytes_len: self.total_bytes_len, total_buffer_len: self.total_buffer_len, } } @@ -440,19 +422,19 @@ impl BinaryViewColumnGeneric { completed_buffers: self.buffers.to_vec(), in_progress_buffer: vec![], phantom: Default::default(), - total_bytes_len: self.total_bytes_len.load(Ordering::Relaxed) as usize, + total_bytes_len: self.total_bytes_len, total_buffer_len: self.total_buffer_len, }), (Right(views), false) => Left(Self::new_unchecked( views.into(), self.buffers, - self.total_bytes_len.load(Ordering::Relaxed) as usize, + self.total_bytes_len, self.total_buffer_len, )), (Left(views), _) => Left(Self::new_unchecked( views, self.buffers, - self.total_bytes_len.load(Ordering::Relaxed) as usize, + self.total_bytes_len, self.total_buffer_len, )), } @@ -518,7 +500,7 @@ impl BinaryViewColumn { Utf8ViewColumn::new_unchecked( self.views.clone(), self.buffers.clone(), - self.total_bytes_len.load(Ordering::Relaxed) as usize, + self.total_bytes_len, self.total_buffer_len, ) } @@ -529,7 +511,7 @@ impl Utf8ViewColumn { BinaryViewColumn::new_unchecked( self.views.clone(), self.buffers.clone(), - self.total_bytes_len.load(Ordering::Relaxed) as usize, + self.total_bytes_len, self.total_buffer_len, ) } diff --git a/src/query/expression/src/kernels/group_by_hash/utils.rs b/src/query/expression/src/kernels/group_by_hash/utils.rs index 8b844452a798..280edbe6e850 100644 --- a/src/query/expression/src/kernels/group_by_hash/utils.rs +++ b/src/query/expression/src/kernels/group_by_hash/utils.rs @@ -40,6 +40,8 @@ pub fn serialize_group_columns( } builder.commit_row(); } + // For nulllable column it will only serialize valid row data + debug_assert!(builder.data.len() <= serialize_size); builder.build() } diff --git a/tests/sqllogictests/suites/base/11_data_type/11_0002_data_type_string.test b/tests/sqllogictests/suites/base/11_data_type/11_0002_data_type_string.test index e69de29bb2d1..0ef9b2985d30 100644 --- a/tests/sqllogictests/suites/base/11_data_type/11_0002_data_type_string.test +++ b/tests/sqllogictests/suites/base/11_data_type/11_0002_data_type_string.test @@ -0,0 +1,69 @@ +query T +SELECT '1'::string from numbers(3) +---- +1 +1 +1 + +query T +SELECT {'w1vY1t5':-15239676694.972677,'6UfU721':-4905646705.765232} from numbers(3); +---- +{'w1vY1t5':-15239676694.972677,'6UfU721':-4905646705.765232} +{'w1vY1t5':-15239676694.972677,'6UfU721':-4905646705.765232} +{'w1vY1t5':-15239676694.972677,'6UfU721':-4905646705.765232} + + +# String concatenation +query T +SELECT 'hello' || ' ' || 'world' FROM numbers(1); +---- +hello world + +query T +SELECT '!@#$%^&*()'::string, '你好'::string, '🌟'::string; +---- +!@#$%^&*() 你好 🌟 + +# String with escape sequences +query T +SELECT 'line1-line2'::string, 'tab\there'::string; +---- +line1-line2 tab here + +query T +SELECT UPPER('hello'), LOWER('WORLD'), LENGTH('databend') FROM numbers(1); +---- +HELLO world 8 + +# String with JSON objects +query T +SELECT {'key': 'value'::string, 'numbers': 123::string} FROM numbers(2); +---- +{'key':'value','numbers':'123'} +{'key':'value','numbers':'123'} + + +# String with scientific notation +query T +SELECT {'scientific': 1.23e-4, 'regular': 123.456} FROM numbers(1); +---- +{'scientific':0.000123,'regular':123.456000} + + +# String with very long content +query T +SELECT repeat('a', 100)::string FROM numbers(1); +---- +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + +# String with whitespace handling +query T +SELECT TRIM(' spaced ')::string, LTRIM(' left')::string, RTRIM('right ')::string; +---- +spaced left right + +# String with NULL values +query T +SELECT NULL::string, COALESCE(NULL::string, 'default') FROM numbers(1); +---- +NULL default From 1d50bfe2dcc130e6d7f686f7c07621b93f4195ca Mon Sep 17 00:00:00 2001 From: Sky Fan <3374614481@qq.com> Date: Thu, 21 Nov 2024 17:48:14 +0800 Subject: [PATCH 69/92] =?UTF-8?q?fix:=20StreamVersionMismatched=20when=20u?= =?UTF-8?q?sing=20CTAS=20in=20multi=20statement=20trans=E2=80=A6=20(#16889?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix: StreamVersionMismatched when using CTAS in multi statement transaction --- Cargo.lock | 2 + src/query/ast/src/ast/statements/statement.rs | 170 ++++++++++++++++++ .../service/src/interpreters/interpreter.rs | 14 +- .../interpreters/interpreter_txn_commit.rs | 75 +------- .../tests/it/sql/exec/get_table_bind_test.rs | 2 +- src/query/sql/Cargo.toml | 2 + src/query/sql/src/planner/binder/binder.rs | 119 ++++++++++++ src/query/sql/src/planner/binder/mod.rs | 1 + src/query/sql/src/planner/mod.rs | 1 + .../06_ee_stream/06_0007_stream_ddl_txn.test | 36 ++++ 10 files changed, 335 insertions(+), 87 deletions(-) create mode 100644 tests/sqllogictests/suites/ee/06_ee_stream/06_0007_stream_ddl_txn.test diff --git a/Cargo.lock b/Cargo.lock index 9a70b9c64379..cc7a1b0ff3b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4076,6 +4076,8 @@ dependencies = [ "databend-common-users", "databend-enterprise-data-mask-feature", "databend-storages-common-cache", + "databend-storages-common-io", + "databend-storages-common-session", "databend-storages-common-table-meta", "derive-visitor", "educe 0.4.23", diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index 0c520fbd0b44..496313aab318 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -376,6 +376,176 @@ impl Statement { _ => format!("{}", self), } } + + pub fn allowed_in_multi_statement(&self) -> bool { + match self { + Statement::Query(..) + | Statement::Explain { .. } + | Statement::ExplainAnalyze { .. } + | Statement::CopyIntoTable(..) + | Statement::CopyIntoLocation(..) + | Statement::Call(..) + | Statement::ShowSettings { .. } + | Statement::ShowProcessList { .. } + | Statement::ShowMetrics { .. } + | Statement::ShowEngines { .. } + | Statement::ShowFunctions { .. } + | Statement::ShowUserFunctions { .. } + | Statement::ShowTableFunctions { .. } + | Statement::ShowIndexes { .. } + | Statement::ShowLocks(..) + | Statement::SetPriority { .. } + | Statement::System(..) + | Statement::KillStmt { .. } + | Statement::SetStmt { .. } + | Statement::UnSetStmt { .. } + | Statement::ShowVariables { .. } + | Statement::SetRole { .. } + | Statement::SetSecondaryRoles { .. } + | Statement::Insert(..) + | Statement::InsertMultiTable(..) + | Statement::Replace(..) + | Statement::MergeInto(..) + | Statement::Delete(..) + | Statement::Update(..) + | Statement::ShowCatalogs(..) + | Statement::ShowCreateCatalog(..) + | Statement::ShowDatabases(..) + | Statement::ShowDropDatabases(..) + | Statement::ShowCreateDatabase(..) + | Statement::UseDatabase { .. } + | Statement::ShowTables(..) + | Statement::ShowCreateTable(..) + | Statement::DescribeTable(..) + | Statement::ShowTablesStatus(..) + | Statement::ShowDropTables(..) + | Statement::OptimizeTable(..) + | Statement::VacuumTable(..) + | Statement::VacuumDropTable(..) + | Statement::VacuumTemporaryFiles(..) + | Statement::AnalyzeTable(..) + | Statement::ExistsTable(..) + | Statement::ShowCreateDictionary(..) + | Statement::ShowDictionaries(..) + | Statement::ShowColumns(..) + | Statement::ShowViews(..) + | Statement::DescribeView(..) + | Statement::ShowStreams(..) + | Statement::DescribeStream(..) + | Statement::RefreshIndex(..) + | Statement::RefreshInvertedIndex(..) + | Statement::RefreshVirtualColumn(..) + | Statement::ShowVirtualColumns(..) + | Statement::ShowUsers + | Statement::DescribeUser { .. } + | Statement::ShowRoles + | Statement::ShowGrants { .. } + | Statement::ShowObjectPrivileges(..) + | Statement::ShowStages + | Statement::DescribeStage { .. } + | Statement::RemoveStage { .. } + | Statement::ListStage { .. } + | Statement::DescribeConnection(..) + | Statement::ShowConnections(..) + | Statement::ShowFileFormats + | Statement::Presign(..) + | Statement::DescDatamaskPolicy(..) + | Statement::DescNetworkPolicy(..) + | Statement::ShowNetworkPolicies + | Statement::DescPasswordPolicy(..) + | Statement::ShowPasswordPolicies { .. } + | Statement::ExecuteTask(..) + | Statement::DescribeTask(..) + | Statement::ShowTasks(..) + | Statement::DescribePipe(..) + | Statement::Begin + | Statement::Commit + | Statement::Abort + | Statement::DescribeNotification(..) + | Statement::ExecuteImmediate(..) + | Statement::ShowProcedures { .. } + | Statement::DescProcedure(..) + | Statement::CallProcedure(..) => true, + + Statement::CreateDatabase(..) + | Statement::CreateTable(..) + | Statement::CreateView(..) + | Statement::CreateIndex(..) + | Statement::CreateStage(..) + | Statement::CreateSequence(..) + | Statement::CreateDictionary(..) + | Statement::CreateConnection(..) + | Statement::CreatePipe(..) + | Statement::AlterTable(..) + | Statement::AlterView(..) + | Statement::AlterUser(..) + | Statement::AlterDatabase(..) + | Statement::DropDatabase(..) + | Statement::DropTable(..) + | Statement::DropView(..) + | Statement::DropIndex(..) + | Statement::DropSequence(..) + | Statement::DropDictionary(..) + | Statement::TruncateTable(..) + | Statement::AttachTable(..) + | Statement::RenameTable(..) + | Statement::CreateCatalog(..) + | Statement::DropCatalog(..) + | Statement::UndropDatabase(..) + | Statement::UndropTable(..) + | Statement::RenameDictionary(..) + | Statement::CreateStream(..) + | Statement::DropStream(..) + | Statement::CreateInvertedIndex(..) + | Statement::DropInvertedIndex(..) + | Statement::CreateVirtualColumn(..) + | Statement::AlterVirtualColumn(..) + | Statement::DropVirtualColumn(..) + | Statement::CreateUser(..) + | Statement::DropUser { .. } + | Statement::CreateRole { .. } + | Statement::DropRole { .. } + | Statement::Grant(..) + | Statement::Revoke(..) + | Statement::CreateUDF(..) + | Statement::DropUDF { .. } + | Statement::AlterUDF(..) + | Statement::DropStage { .. } + | Statement::DropConnection(..) + | Statement::CreateFileFormat { .. } + | Statement::DropFileFormat { .. } + | Statement::CreateDatamaskPolicy(..) + | Statement::DropDatamaskPolicy(..) + | Statement::CreateNetworkPolicy(..) + | Statement::AlterNetworkPolicy(..) + | Statement::DropNetworkPolicy(..) + | Statement::CreatePasswordPolicy(..) + | Statement::AlterPasswordPolicy(..) + | Statement::DropPasswordPolicy(..) + | Statement::CreateTask(..) + | Statement::AlterTask(..) + | Statement::DropTask(..) + | Statement::CreateDynamicTable(..) + | Statement::DropPipe(..) + | Statement::AlterPipe(..) + | Statement::CreateNotification(..) + | Statement::AlterNotification(..) + | Statement::DropNotification(..) + | Statement::CreateProcedure(..) + | Statement::DropProcedure(..) => false, + + Statement::StatementWithSettings { stmt, settings: _ } => { + stmt.allowed_in_multi_statement() + } + } + } + + pub fn is_transaction_command(&self) -> bool { + matches!( + self, + Statement::Commit | Statement::Abort | Statement::Begin + ) + } } impl Display for Statement { diff --git a/src/query/service/src/interpreters/interpreter.rs b/src/query/service/src/interpreters/interpreter.rs index f7b8d2771e5a..861d9d754f3d 100644 --- a/src/query/service/src/interpreters/interpreter.rs +++ b/src/query/service/src/interpreters/interpreter.rs @@ -51,7 +51,6 @@ use md5::Md5; use super::hook::vacuum_hook::hook_disk_temp_dir; use super::hook::vacuum_hook::hook_vacuum_temp_files; -use super::interpreter_txn_commit::CommitInterpreter; use super::InterpreterMetrics; use super::InterpreterQueryLog; use crate::pipelines::executor::ExecutorSettings; @@ -100,18 +99,7 @@ pub trait Interpreter: Sync + Send { ctx.set_status_info("building pipeline"); ctx.check_aborting().with_context(make_error)?; - if self.is_ddl() { - CommitInterpreter::try_create(ctx.clone())? - .execute2() - .await?; - ctx.clear_tables_cache(); - } - if !self.is_txn_command() && ctx.txn_mgr().lock().is_fail() { - let err = ErrorCode::CurrentTransactionIsAborted( - "current transaction is aborted, commands ignored until end of transaction block", - ); - return Err(err); - } + let mut build_res = match self.execute2().await { Ok(build_res) => build_res, Err(err) => { diff --git a/src/query/service/src/interpreters/interpreter_txn_commit.rs b/src/query/service/src/interpreters/interpreter_txn_commit.rs index 175c72f656e7..f685f7ca8f0c 100644 --- a/src/query/service/src/interpreters/interpreter_txn_commit.rs +++ b/src/query/service/src/interpreters/interpreter_txn_commit.rs @@ -14,16 +14,11 @@ use std::sync::Arc; -use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_storages_fuse::TableContext; -use databend_storages_common_session::TxnManagerRef; -use log::error; -use log::info; +use databend_common_sql::execute_commit_statement; use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; -use crate::pipelines::PipelineBuilder; use crate::sessions::QueryContext; pub struct CommitInterpreter { ctx: Arc, @@ -51,73 +46,7 @@ impl Interpreter for CommitInterpreter { #[async_backtrace::framed] async fn execute2(&self) -> Result { - // After commit statement, current session should be in auto commit mode, no matter update meta success or not. - // Use this guard to clear txn manager before return. - let _guard = ClearTxnManagerGuard(self.ctx.txn_mgr().clone()); - let is_active = self.ctx.txn_mgr().lock().is_active(); - if is_active { - let catalog = self.ctx.get_default_catalog()?; - - let req = self.ctx.txn_mgr().lock().req(); - - let update_summary = { - let table_descriptions = req - .update_table_metas - .iter() - .map(|(req, _)| (req.table_id, req.seq, req.new_table_meta.engine.clone())) - .collect::>(); - let stream_descriptions = req - .update_stream_metas - .iter() - .map(|s| (s.stream_id, s.seq, "stream")) - .collect::>(); - (table_descriptions, stream_descriptions) - }; - - let mismatched_tids = { - self.ctx.txn_mgr().lock().set_auto_commit(); - let ret = catalog.retryable_update_multi_table_meta(req).await; - if let Err(ref e) = ret { - // other errors may occur, especially the version mismatch of streams, - // let's log it here for the convenience of diagnostics - error!( - "Non-recoverable fault occurred during updating tables. {}", - e - ); - } - ret? - }; - - match &mismatched_tids { - Ok(_) => { - info!( - "COMMIT: Commit explicit transaction success, targets updated {:?}", - update_summary - ); - } - Err(e) => { - let err_msg = format!( - "COMMIT: Table versions mismatched in multi statement transaction, conflict tables: {:?}", - e.iter() - .map(|(tid, seq, meta)| (tid, seq, &meta.engine)) - .collect::>() - ); - return Err(ErrorCode::TableVersionMismatched(err_msg)); - } - } - let need_purge_files = self.ctx.txn_mgr().lock().need_purge_files(); - for (stage_info, files) in need_purge_files { - PipelineBuilder::try_purge_files(self.ctx.clone(), &stage_info, &files).await; - } - } + execute_commit_statement(self.ctx.clone()).await?; Ok(PipelineBuildResult::create()) } } - -struct ClearTxnManagerGuard(TxnManagerRef); - -impl Drop for ClearTxnManagerGuard { - fn drop(&mut self) { - self.0.lock().clear(); - } -} diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index ea8d1476bc25..c2b2fa719e94 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -499,7 +499,7 @@ impl TableContext for CtxDelegation { } fn txn_mgr(&self) -> TxnManagerRef { - todo!() + self.ctx.txn_mgr() } fn incr_total_scan_value(&self, _value: ProgressValues) { diff --git a/src/query/sql/Cargo.toml b/src/query/sql/Cargo.toml index b0453ffd87ea..1f017a82dac1 100644 --- a/src/query/sql/Cargo.toml +++ b/src/query/sql/Cargo.toml @@ -36,6 +36,8 @@ databend-common-storages-view = { workspace = true } databend-common-users = { workspace = true } databend-enterprise-data-mask-feature = { workspace = true } databend-storages-common-cache = { workspace = true } +databend-storages-common-io = { workspace = true } +databend-storages-common-session = { workspace = true } databend-storages-common-table-meta = { workspace = true } ahash = { workspace = true, features = ["no-rng"] } diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index 6b0661f4e3bc..e21f85497de4 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -42,6 +42,14 @@ use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_app::principal::FileFormatOptionsReader; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::StageFileFormatType; +use databend_common_meta_app::principal::StageInfo; +use databend_common_metrics::storage::metrics_inc_copy_purge_files_cost_milliseconds; +use databend_common_metrics::storage::metrics_inc_copy_purge_files_counter; +use databend_common_storage::init_stage_operator; +use databend_storages_common_io::Files; +use databend_storages_common_session::TxnManagerRef; +use log::error; +use log::info; use log::warn; use super::Finder; @@ -139,6 +147,15 @@ impl<'a> Binder { #[async_backtrace::framed] #[fastrace::trace] pub async fn bind(mut self, stmt: &Statement) -> Result { + if !stmt.allowed_in_multi_statement() { + execute_commit_statement(self.ctx.clone()).await?; + } + if !stmt.is_transaction_command() && self.ctx.txn_mgr().lock().is_fail() { + let err = ErrorCode::CurrentTransactionIsAborted( + "current transaction is aborted, commands ignored until end of transaction block", + ); + return Err(err); + } let start = Instant::now(); self.ctx.set_status_info("binding"); let mut bind_context = BindContext::new(); @@ -953,3 +970,105 @@ impl<'a> Binder { Ok(s_expr) } } + +struct ClearTxnManagerGuard(TxnManagerRef); + +impl Drop for ClearTxnManagerGuard { + fn drop(&mut self) { + self.0.lock().clear(); + } +} + +pub async fn execute_commit_statement(ctx: Arc) -> Result<()> { + // After commit statement, current session should be in auto commit mode, no matter update meta success or not. + // Use this guard to clear txn manager before return. + let _guard = ClearTxnManagerGuard(ctx.txn_mgr().clone()); + let is_active = ctx.txn_mgr().lock().is_active(); + if is_active { + let catalog = ctx.get_default_catalog()?; + + let req = ctx.txn_mgr().lock().req(); + + let update_summary = { + let table_descriptions = req + .update_table_metas + .iter() + .map(|(req, _)| (req.table_id, req.seq, req.new_table_meta.engine.clone())) + .collect::>(); + let stream_descriptions = req + .update_stream_metas + .iter() + .map(|s| (s.stream_id, s.seq, "stream")) + .collect::>(); + (table_descriptions, stream_descriptions) + }; + + let mismatched_tids = { + ctx.txn_mgr().lock().set_auto_commit(); + let ret = catalog.retryable_update_multi_table_meta(req).await; + if let Err(ref e) = ret { + // other errors may occur, especially the version mismatch of streams, + // let's log it here for the convenience of diagnostics + error!( + "Non-recoverable fault occurred during updating tables. {}", + e + ); + } + ret? + }; + + match &mismatched_tids { + Ok(_) => { + info!( + "COMMIT: Commit explicit transaction success, targets updated {:?}", + update_summary + ); + } + Err(e) => { + let err_msg = format!( + "COMMIT: Table versions mismatched in multi statement transaction, conflict tables: {:?}", + e.iter() + .map(|(tid, seq, meta)| (tid, seq, &meta.engine)) + .collect::>() + ); + return Err(ErrorCode::TableVersionMismatched(err_msg)); + } + } + let need_purge_files = ctx.txn_mgr().lock().need_purge_files(); + for (stage_info, files) in need_purge_files { + try_purge_files(ctx.clone(), &stage_info, &files).await; + } + } + Ok(()) +} + +#[async_backtrace::framed] +async fn try_purge_files(ctx: Arc, stage_info: &StageInfo, files: &[String]) { + let start = Instant::now(); + let op = init_stage_operator(stage_info); + + match op { + Ok(op) => { + let file_op = Files::create(ctx, op); + if let Err(e) = file_op.remove_file_in_batch(files).await { + error!("Failed to delete file: {:?}, error: {}", files, e); + } + } + Err(e) => { + error!("Failed to get stage table op, error: {}", e); + } + } + + let elapsed = start.elapsed(); + info!( + "purged files: number {}, time used {:?} ", + files.len(), + elapsed + ); + + // Perf. + { + metrics_inc_copy_purge_files_counter(files.len() as u32); + metrics_inc_copy_purge_files_cost_milliseconds(elapsed.as_millis() as u32); + } +} diff --git a/src/query/sql/src/planner/binder/mod.rs b/src/query/sql/src/planner/binder/mod.rs index 07944deb8db9..0271f172f231 100644 --- a/src/query/sql/src/planner/binder/mod.rs +++ b/src/query/sql/src/planner/binder/mod.rs @@ -63,6 +63,7 @@ pub use bind_mutation::MutationStrategy; pub use bind_mutation::MutationType; pub use bind_query::bind_values; pub use bind_table_reference::parse_result_scan_args; +pub use binder::execute_commit_statement; pub use binder::Binder; pub use builders::*; pub use column_binding::ColumnBinding; diff --git a/src/query/sql/src/planner/mod.rs b/src/query/sql/src/planner/mod.rs index 1bf403ca2f88..73e35305a158 100644 --- a/src/query/sql/src/planner/mod.rs +++ b/src/query/sql/src/planner/mod.rs @@ -29,6 +29,7 @@ pub mod plans; mod stream_column; mod udf_validator; +pub use binder::execute_commit_statement; pub use binder::parse_result_scan_args; pub use binder::BindContext; pub use binder::Binder; diff --git a/tests/sqllogictests/suites/ee/06_ee_stream/06_0007_stream_ddl_txn.test b/tests/sqllogictests/suites/ee/06_ee_stream/06_0007_stream_ddl_txn.test new file mode 100644 index 000000000000..a2d001a08052 --- /dev/null +++ b/tests/sqllogictests/suites/ee/06_ee_stream/06_0007_stream_ddl_txn.test @@ -0,0 +1,36 @@ +statement ok +create or replace table t(a int); + +statement ok +create or replace table t1(a int); + +statement ok +create or replace stream s on table t; + +statement ok +insert into t values(1); + +statement ok +begin; + +statement ok +insert into t1 select a from s; + +statement ok +create or replace table t2 as select a from s; + +statement ok +commit; + +query I +select * from t1; +---- +1 + + +query I +select count(*) from t2; +---- +0 + + From 1cb0ebe648958f5e72f08a1324c640c3f143dbd9 Mon Sep 17 00:00:00 2001 From: zhya Date: Thu, 21 Nov 2024 18:37:29 +0800 Subject: [PATCH 70/92] fix(ci): flaky test (#16898) flaky test --- src/query/service/tests/it/sessions/queue_mgr.rs | 2 ++ src/query/settings/src/settings_getter_setter.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/query/service/tests/it/sessions/queue_mgr.rs b/src/query/service/tests/it/sessions/queue_mgr.rs index ceb92b064af5..fd90f874c09c 100644 --- a/src/query/service/tests/it/sessions/queue_mgr.rs +++ b/src/query/service/tests/it/sessions/queue_mgr.rs @@ -18,6 +18,7 @@ use std::time::Instant; use std::time::SystemTime; use std::time::UNIX_EPOCH; +use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::Planner; @@ -307,6 +308,7 @@ async fn test_heavy_actions() -> Result<()> { let fixture = TestFixture::setup().await?; let ctx = fixture.new_query_ctx().await?; + ctx.get_settings().set_enable_table_lock(0)?; // Create table and stage. { diff --git a/src/query/settings/src/settings_getter_setter.rs b/src/query/settings/src/settings_getter_setter.rs index fcaead2fe2c6..d26d4511063c 100644 --- a/src/query/settings/src/settings_getter_setter.rs +++ b/src/query/settings/src/settings_getter_setter.rs @@ -504,6 +504,10 @@ impl Settings { Ok(self.try_get_u64("enable_table_lock")? != 0) } + pub fn set_enable_table_lock(&self, value: u64) -> Result<()> { + self.try_set_u64("enable_table_lock", value) + } + pub fn get_enable_experimental_rbac_check(&self) -> Result { Ok(self.try_get_u64("enable_experimental_rbac_check")? != 0) } From d38c91c7ef829d0dec88563e07f24be13c775ec5 Mon Sep 17 00:00:00 2001 From: zhya Date: Thu, 21 Nov 2024 19:34:04 +0800 Subject: [PATCH 71/92] fix(storge): write progress in compact hook (#16901) * fix write progress in compact hook * fix test --- .../src/interpreters/hook/compact_hook.rs | 61 ++++++++++--------- .../09_0041_auto_compaction.test | 19 +++--- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/src/query/service/src/interpreters/hook/compact_hook.rs b/src/query/service/src/interpreters/hook/compact_hook.rs index ffddf8863c9c..48b416c929fa 100644 --- a/src/query/service/src/interpreters/hook/compact_hook.rs +++ b/src/query/service/src/interpreters/hook/compact_hook.rs @@ -78,43 +78,50 @@ async fn do_hook_compact( } pipeline.set_on_finished(move |info: &ExecutionInfo| { - let compaction_limits = match compact_target.mutation_kind { - MutationKind::Insert => { - let compaction_num_block_hint = ctx.get_compaction_num_block_hint(&compact_target.table); - info!("table {} hint number of blocks need to be compacted {}", compact_target.table, compaction_num_block_hint); - if compaction_num_block_hint == 0 { - return Ok(()); - } - CompactionLimits { - segment_limit: None, - block_limit: Some(compaction_num_block_hint as usize), + if info.res.is_ok() { + let op_name = &trace_ctx.operation_name; + metrics_inc_compact_hook_main_operation_time_ms(op_name, trace_ctx.start.elapsed().as_millis() as u64); + info!("execute {op_name} finished successfully. running table optimization job."); + + let compact_start_at = Instant::now(); + let compaction_limits = match compact_target.mutation_kind { + MutationKind::Insert => { + let compaction_num_block_hint = ctx.get_compaction_num_block_hint(&compact_target.table); + info!("table {} hint number of blocks need to be compacted {}", compact_target.table, compaction_num_block_hint); + if compaction_num_block_hint == 0 { + return Ok(()); + } + CompactionLimits { + segment_limit: None, + block_limit: Some(compaction_num_block_hint as usize), + } } - } - _ => { - let auto_compaction_segments_limit = ctx.get_settings().get_auto_compaction_segments_limit()?; - CompactionLimits { - segment_limit: Some(auto_compaction_segments_limit as usize), - block_limit: None, + _ => { + let auto_compaction_segments_limit = ctx.get_settings().get_auto_compaction_segments_limit()?; + CompactionLimits { + segment_limit: Some(auto_compaction_segments_limit as usize), + block_limit: None, + } } - } - }; + }; - let op_name = &trace_ctx.operation_name; - metrics_inc_compact_hook_main_operation_time_ms(op_name, trace_ctx.start.elapsed().as_millis() as u64); + // keep the original progress value + let progress = ctx.get_write_progress(); + let progress_value = progress.as_ref().get_values(); - let compact_start_at = Instant::now(); - if info.res.is_ok() { - info!("execute {op_name} finished successfully. running table optimization job."); match GlobalIORuntime::instance().block_on({ compact_table(ctx, compact_target, compaction_limits, lock_opt) }) { Ok(_) => { info!("execute {op_name} finished successfully. table optimization job finished."); } - Err(e) => { info!("execute {op_name} finished successfully. table optimization job failed. {:?}", e) } + Err(e) => { info!("execute {op_name} finished successfully. table optimization job failed. {:?}", e); } } + + // reset the progress value + progress.set(&progress_value); + metrics_inc_compact_hook_compact_time_ms(&trace_ctx.operation_name, compact_start_at.elapsed().as_millis() as u64); } - metrics_inc_compact_hook_compact_time_ms(&trace_ctx.operation_name, compact_start_at.elapsed().as_millis() as u64); Ok(()) }); @@ -139,8 +146,6 @@ async fn compact_table( ) .await?; let settings = ctx.get_settings(); - // keep the original progress value - let progress_value = ctx.get_write_progress_value(); let do_recluster = !table.cluster_keys(ctx.clone()).is_empty(); let do_compact = compaction_limits.block_limit.is_some() || !do_recluster; @@ -203,7 +208,5 @@ async fn compact_table( assert!(build_res.main_pipeline.is_empty()); } - // reset the progress value - ctx.get_write_progress().set(&progress_value); Ok(()) } diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test index db96ae7968df..e11c03b61be6 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test @@ -90,14 +90,19 @@ insert into t1 values(3),(5),(8); statement ok insert into t1 values(4),(6); -query III -select segment_count, block_count, row_count from fuse_snapshot('i15760', 't1') limit 10; +query II +select segment_count, row_count from fuse_snapshot('i15760', 't1') limit 10; ---- -1 1 8 -1 2 8 -3 3 8 -2 2 6 -1 1 3 +1 8 +1 8 +3 8 +2 6 +1 3 + +query F +select average_depth from clustering_information('i15760', 't1') +---- +1.0 statement ok drop table t1 all; From 110437a5c675ceb725e7dcc22ec927cd4db27139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Thu, 21 Nov 2024 21:44:23 +0800 Subject: [PATCH 72/92] chore: remove unused config_id from RaftConfig (#16902) * chore: optimize meta-service initialization Try initialize instead of check `is_open` flag. This flag may not be accurate: the raft storage is opend but no initialization is done yet. And remove trait `Opened` and refactor related struct names * chore: remove unused config_id from RaftConfig `config_id` is used for separate sled-db key spaces for parallel tests. Since sled db is removed, this `config_id` is useless. --- src/meta/binaries/meta/entry.rs | 2 +- src/meta/binaries/metactl/export_from_disk.rs | 4 +- src/meta/service/src/api/grpc/grpc_service.rs | 4 +- src/meta/service/src/configs/outer_v0.rs | 11 +- src/meta/service/src/lib.rs | 6 - .../service/src/meta_service/forwarder.rs | 2 +- .../service/src/meta_service/meta_leader.rs | 2 +- .../service/src/meta_service/meta_node.rs | 128 +++++++++++------- .../src/meta_service/raft_service_impl.rs | 6 +- src/meta/service/src/store/mod.rs | 2 +- src/meta/service/src/store/store.rs | 20 ++- src/meta/service/src/store/store_inner.rs | 34 ++--- src/meta/service/tests/it/configs.rs | 1 - .../it/meta_node/meta_node_kv_api_expire.rs | 2 +- .../tests/it/meta_node/meta_node_lifecycle.rs | 40 ++++-- .../it/meta_node/meta_node_replication.rs | 2 +- src/meta/service/tests/it/store.rs | 5 +- src/meta/service/tests/it/tests/meta_node.rs | 9 +- 18 files changed, 140 insertions(+), 140 deletions(-) diff --git a/src/meta/binaries/meta/entry.rs b/src/meta/binaries/meta/entry.rs index 1bbb062cdd49..e4dc56a77c09 100644 --- a/src/meta/binaries/meta/entry.rs +++ b/src/meta/binaries/meta/entry.rs @@ -197,7 +197,7 @@ pub async fn entry(conf: Config) -> anyhow::Result<()> { } async fn do_register(meta_node: &Arc, conf: &Config) -> Result<(), MetaAPIError> { - let node_id = meta_node.sto.id; + let node_id = meta_node.raft_store.id; let raft_endpoint = conf.raft_config.raft_api_advertise_host_endpoint(); let node = Node::new(node_id, raft_endpoint) .with_grpc_advertise_address(conf.grpc_api_advertise_address()); diff --git a/src/meta/binaries/metactl/export_from_disk.rs b/src/meta/binaries/metactl/export_from_disk.rs index 9db46e2222a2..9f0e8269747c 100644 --- a/src/meta/binaries/metactl/export_from_disk.rs +++ b/src/meta/binaries/metactl/export_from_disk.rs @@ -17,7 +17,7 @@ use std::io::Write; use std::sync::Arc; use databend_common_meta_raft_store::config::RaftConfig; -use databend_meta::store::StoreInner; +use databend_meta::store::RaftStoreInner; use futures::TryStreamExt; use crate::upgrade; @@ -36,7 +36,7 @@ pub async fn export_from_dir(args: &ExportArgs) -> anyhow::Result<()> { eprintln!(); eprintln!("Export:"); - let sto_inn = StoreInner::open(&raft_config).await?; + let sto_inn = RaftStoreInner::open(&raft_config).await?; let mut lines = Arc::new(sto_inn).export(); eprintln!(" From: {}", raft_config.raft_dir); diff --git a/src/meta/service/src/api/grpc/grpc_service.rs b/src/meta/service/src/api/grpc/grpc_service.rs index 4e29a019f50c..d2a31b29df55 100644 --- a/src/meta/service/src/api/grpc/grpc_service.rs +++ b/src/meta/service/src/api/grpc/grpc_service.rs @@ -362,7 +362,7 @@ impl MetaService for MetaServiceImpl { let _guard = RequestInFlight::guard(); let meta_node = &self.meta_node; - let strm = meta_node.sto.inner().export(); + let strm = meta_node.raft_store.inner().export(); let chunk_size = 32; // - Chunk up upto 32 Ok items inside a Vec; @@ -390,7 +390,7 @@ impl MetaService for MetaServiceImpl { let _guard = RequestInFlight::guard(); let meta_node = &self.meta_node; - let strm = meta_node.sto.inner().export(); + let strm = meta_node.raft_store.inner().export(); let chunk_size = request.get_ref().chunk_size.unwrap_or(32) as usize; // - Chunk up upto `chunk_size` Ok items inside a Vec; diff --git a/src/meta/service/src/configs/outer_v0.rs b/src/meta/service/src/configs/outer_v0.rs index c4560284a0f7..49ad99ca8af2 100644 --- a/src/meta/service/src/configs/outer_v0.rs +++ b/src/meta/service/src/configs/outer_v0.rs @@ -279,7 +279,6 @@ pub struct ConfigViaEnv { pub grpc_tls_server_cert: String, pub grpc_tls_server_key: String, - pub config_id: String, pub kvsrv_listen_host: String, pub kvsrv_advertise_host: String, pub kvsrv_api_port: u16, @@ -338,7 +337,6 @@ impl From for ConfigViaEnv { metasrv_grpc_api_advertise_host: cfg.grpc_api_advertise_host, grpc_tls_server_cert: cfg.grpc_tls_server_cert, grpc_tls_server_key: cfg.grpc_tls_server_key, - config_id: cfg.raft_config.config_id, kvsrv_listen_host: cfg.raft_config.raft_listen_host, kvsrv_advertise_host: cfg.raft_config.raft_advertise_host, kvsrv_api_port: cfg.raft_config.raft_api_port, @@ -377,7 +375,6 @@ impl From for ConfigViaEnv { impl Into for ConfigViaEnv { fn into(self) -> Config { let raft_config = RaftConfig { - config_id: self.config_id, raft_listen_host: self.kvsrv_listen_host, raft_advertise_host: self.kvsrv_advertise_host, raft_api_port: self.kvsrv_api_port, @@ -457,11 +454,6 @@ impl Into for ConfigViaEnv { #[clap(about, version, author)] #[serde(default)] pub struct RaftConfig { - /// Identify a config. - /// This is only meant to make debugging easier with more than one Config involved. - #[clap(long, default_value = "")] - pub config_id: String, - /// The local listening host for metadata communication. /// This config does not need to be stored in raft-store, /// only used when metasrv startup and listen to. @@ -610,7 +602,7 @@ impl Default for RaftConfig { impl From for InnerRaftConfig { fn from(x: RaftConfig) -> InnerRaftConfig { InnerRaftConfig { - config_id: x.config_id, + config_id: "".to_string(), raft_listen_host: x.raft_listen_host, raft_advertise_host: x.raft_advertise_host, raft_api_port: x.raft_api_port, @@ -649,7 +641,6 @@ impl From for InnerRaftConfig { impl From for RaftConfig { fn from(inner: InnerRaftConfig) -> Self { Self { - config_id: inner.config_id, raft_listen_host: inner.raft_listen_host, raft_advertise_host: inner.raft_advertise_host, raft_api_port: inner.raft_api_port, diff --git a/src/meta/service/src/lib.rs b/src/meta/service/src/lib.rs index d15ad7fc309c..9b0a48aa3da5 100644 --- a/src/meta/service/src/lib.rs +++ b/src/meta/service/src/lib.rs @@ -27,9 +27,3 @@ pub(crate) mod request_handling; pub mod store; pub mod version; pub mod watcher; - -pub trait Opened { - /// Return true if it is opened from a previous persistent state. - /// Otherwise it is just created. - fn is_opened(&self) -> bool; -} diff --git a/src/meta/service/src/meta_service/forwarder.rs b/src/meta/service/src/meta_service/forwarder.rs index 705f58ebf510..844680aae1f0 100644 --- a/src/meta/service/src/meta_service/forwarder.rs +++ b/src/meta/service/src/meta_service/forwarder.rs @@ -47,7 +47,7 @@ pub struct MetaForwarder<'a> { impl<'a> MetaForwarder<'a> { pub fn new(meta_node: &'a MetaNode) -> Self { Self { - sto: &meta_node.sto, + sto: &meta_node.raft_store, raft: &meta_node.raft, } } diff --git a/src/meta/service/src/meta_service/meta_leader.rs b/src/meta/service/src/meta_service/meta_leader.rs index 719401559a43..be3c8c837227 100644 --- a/src/meta/service/src/meta_service/meta_leader.rs +++ b/src/meta/service/src/meta_service/meta_leader.rs @@ -167,7 +167,7 @@ impl<'a> Handler for MetaLeader<'a> { impl<'a> MetaLeader<'a> { pub fn new(meta_node: &'a MetaNode) -> MetaLeader { MetaLeader { - sto: &meta_node.sto, + sto: &meta_node.raft_store, raft: &meta_node.raft, } } diff --git a/src/meta/service/src/meta_service/meta_node.rs b/src/meta/service/src/meta_service/meta_node.rs index cb0937651522..fc6fbdae2e54 100644 --- a/src/meta/service/src/meta_service/meta_node.rs +++ b/src/meta/service/src/meta_service/meta_node.rs @@ -34,6 +34,7 @@ use databend_common_meta_raft_store::config::RaftConfig; use databend_common_meta_raft_store::ondisk::DATA_VERSION; use databend_common_meta_raft_store::raft_log_v004::RaftLogStat; use databend_common_meta_sled_store::openraft; +use databend_common_meta_sled_store::openraft::error::RaftError; use databend_common_meta_sled_store::openraft::ChangeMembers; use databend_common_meta_stoerr::MetaStorageError; use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; @@ -41,6 +42,7 @@ use databend_common_meta_types::protobuf::raft_service_server::RaftServiceServer use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::raft_types::CommittedLeaderId; use databend_common_meta_types::raft_types::ForwardToLeader; +use databend_common_meta_types::raft_types::InitializeError; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::MembershipNode; use databend_common_meta_types::raft_types::NodeId; @@ -95,17 +97,16 @@ use crate::watcher::EventDispatcher; use crate::watcher::EventDispatcherHandle; use crate::watcher::Watcher; use crate::watcher::WatcherSender; -use crate::Opened; pub type LogStore = RaftStore; pub type SMStore = RaftStore; -/// MetaRaft is a implementation of the generic Raft handling meta data R/W. +/// MetaRaft is an implementation of the generic Raft handling metadata R/W. pub type MetaRaft = Raft; -/// MetaNode is the container of meta data related components and threads, such as storage, the raft node and a raft-state monitor. +/// MetaNode is the container of metadata related components and threads, such as storage, the raft node and a raft-state monitor. pub struct MetaNode { - pub sto: RaftStore, + pub raft_store: RaftStore, pub dispatcher_handle: EventDispatcherHandle, pub raft: MetaRaft, pub running_tx: watch::Sender<()>, @@ -118,18 +119,12 @@ impl Drop for MetaNode { fn drop(&mut self) { info!( "MetaNode(id={}, raft={}) is dropping", - self.sto.id, - self.sto.config.raft_api_advertise_host_string() + self.raft_store.id, + self.raft_store.config.raft_api_advertise_host_string() ); } } -impl Opened for MetaNode { - fn is_opened(&self) -> bool { - self.sto.is_opened() - } -} - pub struct MetaNodeBuilder { node_id: Option, raft_config: Option, @@ -171,7 +166,7 @@ impl MetaNodeBuilder { .set_subscriber(Box::new(DispatcherSender(dispatcher_tx.clone()))); let meta_node = Arc::new(MetaNode { - sto: sto.clone(), + raft_store: sto.clone(), dispatcher_handle: EventDispatcherHandle::new(dispatcher_tx), raft: raft.clone(), running_tx: tx, @@ -290,7 +285,7 @@ impl MetaNode { info!("about to start raft grpc on: {}", ip_port); let socket_addr = ip_port.parse::()?; - let node_id = meta_node.sto.id; + let node_id = meta_node.raft_store.id; let srv = tonic::transport::Server::builder().add_service(raft_server); @@ -331,13 +326,17 @@ impl MetaNode { config.no_sync = true; } - let sto = RaftStore::open(&config).await?; + let log_store = RaftStore::open(&config).await?; // config.id only used for the first time - let self_node_id = if sto.is_opened() { sto.id } else { config.id }; + let self_node_id = if log_store.is_opened { + log_store.id + } else { + config.id + }; let builder = MetaNode::builder(&config) - .sto(sto.clone()) + .sto(log_store.clone()) .node_id(self_node_id) .raft_service_endpoint(config.raft_api_listen_host_endpoint()); let mn = builder.build().await?; @@ -404,7 +403,7 @@ impl MetaNode { .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } - info!("shutdown: id={}", self.sto.id); + info!("shutdown: id={}", self.raft_store.id); let joined = self.joined_tasks.load(std::sync::atomic::Ordering::Relaxed); Ok(joined) } @@ -439,7 +438,7 @@ impl MetaNode { server_metrics::incr_leader_change(); } server_metrics::set_current_leader(mm.current_leader.unwrap_or_default()); - server_metrics::set_is_leader(mm.current_leader == Some(meta_node.sto.id)); + server_metrics::set_is_leader(mm.current_leader == Some(meta_node.raft_store.id)); // metrics about raft log and state machine. server_metrics::set_current_term(mm.current_term); @@ -630,7 +629,7 @@ impl MetaNode { Err(MetaManagementError::Join(AnyError::error(format!( "fail to join node-{} to cluster via {:?}, errors: {}", - self.sto.id, + self.raft_store.id, addrs, errors.into_iter().map(|e| e.to_string()).join(", ") )))) @@ -724,20 +723,23 @@ impl MetaNode { /// Only when the membership is committed, this node can be sure it is in a cluster. async fn is_in_cluster(&self) -> Result, MetaStorageError> { let membership = { - let sm = self.sto.get_state_machine().await; + let sm = self.raft_store.get_state_machine().await; sm.sys_data_ref().last_membership_ref().membership().clone() }; info!("is_in_cluster: membership: {:?}", membership); let voter_ids = membership.voter_ids().collect::>(); - if voter_ids.contains(&self.sto.id) { - return Ok(Ok(format!("node {} already in cluster", self.sto.id))); + if voter_ids.contains(&self.raft_store.id) { + return Ok(Ok(format!( + "node {} already in cluster", + self.raft_store.id + ))); } Ok(Err(format!( "node {} has membership but not in it", - self.sto.id + self.raft_store.id ))) } @@ -768,14 +770,9 @@ impl MetaNode { /// - Adding current node into the meta data. #[fastrace::trace] pub async fn init_cluster(&self, node: Node) -> Result<(), MetaStartupError> { - info!("init_cluster: node: {:?}", node); + info!("Initialize node as single node cluster: {:?}", node); - if self.is_opened() { - info!("It is opened, skip initializing cluster"); - return Ok(()); - } - - let node_id = self.sto.id; + let node_id = self.raft_store.id; let mut cluster_node_ids = BTreeSet::new(); cluster_node_ids.insert(node_id); @@ -783,19 +780,45 @@ impl MetaNode { // initialize() and add_node() are not done atomically. // There is an issue that just after initializing the cluster, // the node will be used but no node info is found. - // Thus meta-server can only be initialized with a single node. + // Thus, meta-server can only be initialized with a single node. // // We do not store node info in membership config, // because every start a meta-server node updates its latest configured address. - self.raft.initialize(cluster_node_ids).await?; - - info!("initialized cluster"); + let res = self.raft.initialize(cluster_node_ids.clone()).await; + match res { + Ok(_) => { + info!("Initialized with: {:?}", cluster_node_ids); + } + Err(e) => match e { + RaftError::APIError(e) => match e { + InitializeError::NotAllowed(e) => { + info!("Already initialized: {}", e); + } + InitializeError::NotInMembers(e) => { + return Err(MetaStartupError::InvalidConfig(e.to_string())); + } + }, + RaftError::Fatal(fatal) => { + return Err(MetaStartupError::MetaServiceError(fatal.to_string())); + } + }, + } - self.add_node(node_id, node) - .await - .map_err(|e| MetaStartupError::AddNodeError { - source: AnyError::new(&e), + if self.get_node(&node_id).await.is_none() { + info!( + "This node not found in state-machine; add node: {}:{:?}", + node_id, node + ); + self.add_node(node_id, node.clone()).await.map_err(|e| { + MetaStartupError::AddNodeError { + source: AnyError::new(&e), + } })?; + } else { + info!("This node already in state-machine; No need to add"); + } + + info!("Done initializing node as single node cluster: {:?}", node); Ok(()) } @@ -804,7 +827,7 @@ impl MetaNode { pub async fn get_node(&self, node_id: &NodeId) -> Option { // inconsistent get: from local state machine - let sm = self.sto.state_machine.read().await; + let sm = self.raft_store.state_machine.read().await; let n = sm.sys_data_ref().nodes_ref().get(node_id).cloned(); n } @@ -813,7 +836,7 @@ impl MetaNode { pub async fn get_nodes(&self) -> Vec { // inconsistent get: from local state machine - let sm = self.sto.state_machine.read().await; + let sm = self.raft_store.state_machine.read().await; let nodes = sm .sys_data_ref() .nodes_ref() @@ -825,15 +848,15 @@ impl MetaNode { /// Get the size in bytes of the on disk files of the raft log storage. async fn get_raft_log_size(&self) -> u64 { - self.sto.log.read().await.on_disk_size() + self.raft_store.log.read().await.on_disk_size() } async fn get_raft_log_stat(&self) -> RaftLogStat { - self.sto.log.read().await.stat() + self.raft_store.log.read().await.stat() } async fn get_snapshot_key_count(&self) -> u64 { - self.sto + self.raft_store .try_get_snapshot_key_count() .await .unwrap_or_default() @@ -841,16 +864,19 @@ impl MetaNode { pub async fn get_status(&self) -> Result { let voters = self - .sto + .raft_store .get_nodes(|ms| ms.voter_ids().collect::>()) .await; let learners = self - .sto + .raft_store .get_nodes(|ms| ms.learner_ids().collect::>()) .await; - let endpoint = self.sto.get_node_raft_endpoint(&self.sto.id).await?; + let endpoint = self + .raft_store + .get_node_raft_endpoint(&self.raft_store.id) + .await?; let raft_log_status = self.get_raft_log_stat().await.into(); let snapshot_key_count = self.get_snapshot_key_count().await; @@ -866,7 +892,7 @@ impl MetaNode { let last_seq = self.get_last_seq().await; Ok(MetaNodeStatus { - id: self.sto.id, + id: self.raft_store.id, binary_version: METASRV_COMMIT_VERSION.as_str().to_string(), data_version: DATA_VERSION, endpoint: endpoint.to_string(), @@ -890,7 +916,7 @@ impl MetaNode { } pub(crate) async fn get_last_seq(&self) -> u64 { - let sm = self.sto.state_machine.read().await; + let sm = self.raft_store.state_machine.read().await; sm.sys_data_ref().curr_seq() } @@ -899,7 +925,7 @@ impl MetaNode { // Maybe stale get: from local state machine let nodes = { - let sm = self.sto.state_machine.read().await; + let sm = self.raft_store.state_machine.read().await; sm.sys_data_ref() .nodes_ref() .values() @@ -1036,7 +1062,7 @@ impl MetaNode { debug!("curr_leader_id: {:?}", leader_id); - if leader_id == Some(self.sto.id) { + if leader_id == Some(self.raft_store.id) { return Ok(MetaLeader::new(self)); } diff --git a/src/meta/service/src/meta_service/raft_service_impl.rs b/src/meta/service/src/meta_service/raft_service_impl.rs index 0fe2e3f9bd88..3e130499e1ef 100644 --- a/src/meta/service/src/meta_service/raft_service_impl.rs +++ b/src/meta/service/src/meta_service/raft_service_impl.rs @@ -148,7 +148,7 @@ impl RaftServiceImpl { let resp = InstallSnapshotResponse { vote: my_vote }; - let ss_store = self.meta_node.sto.snapshot_store(); + let ss_store = self.meta_node.raft_store.snapshot_store(); let finished_snapshot = { let mut receiver_v1 = self.receiver_v1.lock().await; @@ -194,7 +194,7 @@ impl RaftServiceImpl { .. } = received; - let raft_config = &self.meta_node.sto.config; + let raft_config = &self.meta_node.raft_store.config; let db = DB::open_snapshot(&temp_path, snapshot_meta.snapshot_id.clone(), raft_config) .map_err(|e| { @@ -235,7 +235,7 @@ impl RaftServiceImpl { let _guard = snapshot_recv_inflight(&addr).counter_guard(); - let ss_store = self.meta_node.sto.snapshot_store(); + let ss_store = self.meta_node.raft_store.snapshot_store(); let mut receiver_v003 = ss_store.new_receiver(&addr).map_err(io_err_to_status)?; receiver_v003.set_on_recv_callback(new_incr_recvfrom_bytes(addr.clone())); diff --git a/src/meta/service/src/store/mod.rs b/src/meta/service/src/store/mod.rs index 257c6d07f0b6..9a3c2439ce64 100644 --- a/src/meta/service/src/store/mod.rs +++ b/src/meta/service/src/store/mod.rs @@ -19,4 +19,4 @@ mod store; mod store_inner; pub use store::RaftStore; -pub use store_inner::StoreInner; +pub use store_inner::RaftStoreInner; diff --git a/src/meta/service/src/store/store.rs b/src/meta/service/src/store/store.rs index c908db4edb4b..b814380664de 100644 --- a/src/meta/service/src/store/store.rs +++ b/src/meta/service/src/store/store.rs @@ -18,36 +18,32 @@ use std::sync::Arc; use databend_common_meta_raft_store::config::RaftConfig; use databend_common_meta_types::MetaStartupError; -use crate::store::StoreInner; +use crate::store::RaftStoreInner; /// A store that implements `RaftStorage` trait and provides full functions. /// /// It is designed to be cloneable in order to be shared by MetaNode and Raft. #[derive(Clone)] pub struct RaftStore { - pub(crate) inner: Arc, + pub(crate) inner: Arc, } impl RaftStore { - pub fn new(sto: StoreInner) -> Self { - Self { - inner: Arc::new(sto), - } - } - #[fastrace::trace] pub async fn open(config: &RaftConfig) -> Result { - let sto = StoreInner::open(config).await?; - Ok(Self::new(sto)) + let inner = RaftStoreInner::open(config).await?; + Ok(Self { + inner: Arc::new(inner), + }) } - pub fn inner(&self) -> Arc { + pub fn inner(&self) -> Arc { self.inner.clone() } } impl Deref for RaftStore { - type Target = StoreInner; + type Target = RaftStoreInner; fn deref(&self) -> &Self::Target { &self.inner diff --git a/src/meta/service/src/store/store_inner.rs b/src/meta/service/src/store/store_inner.rs index 80701815ba22..ce83950e7e26 100644 --- a/src/meta/service/src/store/store_inner.rs +++ b/src/meta/service/src/store/store_inner.rs @@ -54,14 +54,8 @@ use log::info; use raft_log::api::raft_log_writer::RaftLogWriter; use tokio::time::sleep; -use crate::Opened; - -/// This is the inner store that provides support utilities for implementing the raft storage API. -/// -/// This store include two parts: -/// log(including id, vote, committed, and purged) -/// state_machine -pub struct StoreInner { +/// This is the inner store that implements the raft log storage API. +pub struct RaftStoreInner { /// The ID of the Raft node for which this storage instances is configured. /// ID is also stored in raft-log. /// @@ -71,7 +65,7 @@ pub struct StoreInner { pub(crate) config: RaftConfig, /// If the instance is opened from an existent state(e.g. load from fs) or created. - is_opened: bool, + pub is_opened: bool, /// A series of raft logs. pub log: Arc>, @@ -80,27 +74,17 @@ pub struct StoreInner { pub state_machine: Arc>, } -impl AsRef for StoreInner { - fn as_ref(&self) -> &StoreInner { +impl AsRef for RaftStoreInner { + fn as_ref(&self) -> &RaftStoreInner { self } } -impl Opened for StoreInner { - /// If the instance is opened(true) from an existent state(e.g. load from fs) or created(false). - fn is_opened(&self) -> bool { - self.is_opened - } -} - -impl StoreInner { +impl RaftStoreInner { /// Open an existent raft-store or create a new one. #[fastrace::trace] - pub async fn open(config: &RaftConfig) -> Result { - info!( - "open_or_create StoreInner: id={}, config_id={}", - config.id, config.config_id - ); + pub async fn open(config: &RaftConfig) -> Result { + info!("open_or_create StoreInner: id={}", config.id); fn to_startup_err(e: impl std::error::Error + 'static) -> MetaStartupError { let ae = AnyError::new(&e); @@ -315,7 +299,7 @@ impl StoreInner { /// /// Returns a `BoxStream<'a, Result>` that yields a series of JSON strings. #[futures_async_stream::try_stream(boxed, ok = String, error = io::Error)] - pub async fn export(self: Arc) { + pub async fn export(self: Arc) { info!("StoreInner::export start"); // Convert an error occurred during export to `io::Error(InvalidData)`. diff --git a/src/meta/service/tests/it/configs.rs b/src/meta/service/tests/it/configs.rs index 19c18203ef27..4f4f2daba9a5 100644 --- a/src/meta/service/tests/it/configs.rs +++ b/src/meta/service/tests/it/configs.rs @@ -78,7 +78,6 @@ cluster_name = "foo_cluster" assert_eq!(cfg.grpc_api_address, "127.0.0.1:10000"); assert_eq!(cfg.grpc_tls_server_cert, "grpc server cert"); assert_eq!(cfg.grpc_tls_server_key, "grpc server key"); - assert_eq!(cfg.raft_config.config_id, "raft config id"); assert_eq!(cfg.raft_config.raft_listen_host, "127.0.0.1"); assert_eq!(cfg.raft_config.raft_api_port, 11000); assert_eq!(cfg.raft_config.raft_dir, "raft dir"); diff --git a/src/meta/service/tests/it/meta_node/meta_node_kv_api_expire.rs b/src/meta/service/tests/it/meta_node/meta_node_kv_api_expire.rs index 38874a91e6da..ef8b451a57e2 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_kv_api_expire.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_kv_api_expire.rs @@ -115,7 +115,7 @@ async fn test_meta_node_replicate_kv_with_expire() -> anyhow::Result<()> { // This way on every node applying a log always get the same result. info!("--- get updated kv with new expire, assert the updated value"); { - let sm = learner.sto.state_machine.read().await; + let sm = learner.raft_store.state_machine.read().await; let resp = sm.kv_api().get_kv(key).await.unwrap(); let seq_v = resp.unwrap(); assert_eq!(Some(KVMeta::new_expire(now_sec + 1000)), seq_v.meta); diff --git a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs index 4eba750aa52a..a2114f8d940b 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs @@ -130,7 +130,10 @@ async fn test_meta_node_join() -> anyhow::Result<()> { for mn in all.iter() { mn.raft .wait(timeout()) - .voter_ids(btreeset! {0,2}, format!("node-2 is joined: {}", mn.sto.id)) + .voter_ids( + btreeset! {0,2}, + format!("node-2 is joined: {}", mn.raft_store.id), + ) .await?; } } @@ -165,7 +168,7 @@ async fn test_meta_node_join() -> anyhow::Result<()> { .wait(timeout()) .voter_ids( btreeset! {0,2,3}, - format!("node-3 is joined: {}", mn.sto.id), + format!("node-3 is joined: {}", mn.raft_store.id), ) .await?; } @@ -198,7 +201,10 @@ async fn test_meta_node_join() -> anyhow::Result<()> { for mn in all.iter() { mn.raft .wait(timeout()) - .voter_ids(btreeset! {0,2,3}, format!("node-{} membership", mn.sto.id)) + .voter_ids( + btreeset! {0,2,3}, + format!("node-{} membership", mn.raft_store.id), + ) .await?; } @@ -242,7 +248,10 @@ async fn test_meta_node_join_rejoin() -> anyhow::Result<()> { for mn in all.iter() { mn.raft .wait(timeout()) - .voter_ids(btreeset! {0,1}, format!("node-1 is joined: {}", mn.sto.id)) + .voter_ids( + btreeset! {0,1}, + format!("node-1 is joined: {}", mn.raft_store.id), + ) .await?; } } @@ -284,7 +293,7 @@ async fn test_meta_node_join_rejoin() -> anyhow::Result<()> { .wait(timeout()) .voter_ids( btreeset! {0,1,2}, - format!("node-2 is joined: {}", mn.sto.id), + format!("node-2 is joined: {}", mn.raft_store.id), ) .await?; } @@ -497,7 +506,10 @@ async fn test_meta_node_leave() -> anyhow::Result<()> { for mn in all.iter() { mn.raft .wait(timeout()) - .voter_ids(btreeset! {0,2}, format!("node-{} membership", mn.sto.id)) + .voter_ids( + btreeset! {0,2}, + format!("node-{} membership", mn.raft_store.id), + ) .await?; } @@ -605,8 +617,8 @@ async fn test_meta_node_restart() -> anyhow::Result<()> { // add node, update membership log_index += 2; - let sto0 = mn0.sto.clone(); - let sto1 = mn1.sto.clone(); + let sto0 = mn0.raft_store.clone(); + let sto1 = mn1.raft_store.clone(); let meta_nodes = vec![mn0.clone(), mn1.clone()]; @@ -714,7 +726,7 @@ async fn test_meta_node_restart_single_node() -> anyhow::Result<()> { .await?; log_index += 1; - want_hs = leader.sto.clone().read_vote().await?; + want_hs = leader.raft_store.clone().read_vote().await?; leader.stop().await?; } @@ -741,27 +753,27 @@ async fn test_meta_node_restart_single_node() -> anyhow::Result<()> { Some(log_index), format!( "reopened: applied index at {} for node-{}", - log_index, leader.sto.id + log_index, leader.raft_store.id ), ) .await?; info!("--- check hard state"); { - let hs = leader.sto.clone().read_vote().await?; + let hs = leader.raft_store.clone().read_vote().await?; assert_eq!(want_hs, hs); } info!("--- check logs"); { - let logs = leader.sto.clone().try_get_log_entries(..).await?; + let logs = leader.raft_store.clone().try_get_log_entries(..).await?; info!("logs: {:?}", logs); assert_eq!(log_index as usize + 1, logs.len()); } info!("--- check state machine: nodes"); { - let node = leader.sto.get_node(&0).await.unwrap(); + let node = leader.raft_store.get_node(&0).await.unwrap(); assert_eq!( tc.config.raft_config.raft_api_advertise_host_endpoint(), node.endpoint @@ -816,7 +828,7 @@ async fn assert_upsert_kv_synced(meta_nodes: Vec>, key: &str) -> a format!( "check upsert-kv has applied index at {} for node-{}", last_applied.next_index(), - mn.sto.id + mn.raft_store.id ), ) .await?; diff --git a/src/meta/service/tests/it/meta_node/meta_node_replication.rs b/src/meta/service/tests/it/meta_node/meta_node_replication.rs index 236fdfb6a9db..e70261f6f3b3 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_replication.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_replication.rs @@ -143,7 +143,7 @@ async fn test_meta_node_snapshot_replication() -> anyhow::Result<()> { for i in 0..n_req { let key = format!("test_meta_node_snapshot_replication-key-{}", i); - let sm = mn1.sto.get_state_machine().await; + let sm = mn1.raft_store.get_state_machine().await; let got = sm.get_maybe_expired_kv(&key).await?; match got { None => { diff --git a/src/meta/service/tests/it/store.rs b/src/meta/service/tests/it/store.rs index d64400495c3c..faebd665aca1 100644 --- a/src/meta/service/tests/it/store.rs +++ b/src/meta/service/tests/it/store.rs @@ -36,7 +36,6 @@ use databend_common_meta_types::snapshot_db::DB; use databend_meta::meta_service::meta_node::LogStore; use databend_meta::meta_service::meta_node::SMStore; use databend_meta::store::RaftStore; -use databend_meta::Opened; use futures::TryStreamExt; use log::debug; use log::info; @@ -83,7 +82,7 @@ async fn test_meta_store_restart() -> anyhow::Result<()> { { let mut sto = RaftStore::open(&tc.config.raft_config).await?; assert_eq!(id, sto.id); - assert!(!sto.is_opened()); + assert!(!sto.is_opened); assert_eq!(None, sto.read_vote().await?); info!("--- update metasrv"); @@ -102,7 +101,7 @@ async fn test_meta_store_restart() -> anyhow::Result<()> { { let mut sto = RaftStore::open(&tc.config.raft_config).await?; assert_eq!(id, sto.id); - assert!(sto.is_opened()); + assert!(sto.is_opened); assert_eq!(Some(Vote::new(10, 5)), sto.read_vote().await?); assert_eq!(log_id(1, 2, 1), sto.get_log_id(1).await?); diff --git a/src/meta/service/tests/it/tests/meta_node.rs b/src/meta/service/tests/it/tests/meta_node.rs index fec6ea5e5b03..778f757f0a78 100644 --- a/src/meta/service/tests/it/tests/meta_node.rs +++ b/src/meta/service/tests/it/tests/meta_node.rs @@ -23,7 +23,6 @@ use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::AppliedState; use databend_common_meta_types::Node; use databend_meta::meta_service::MetaNode; -use databend_meta::Opened; use log::info; use maplit::btreeset; @@ -106,7 +105,7 @@ pub(crate) async fn start_meta_node_cluster( .wait(timeout()) .state( ServerState::Follower, - format!("check follower-{} state", item.meta_node().sto.id), + format!("check follower-{} state", item.meta_node().raft_store.id), ) .await?; } @@ -120,7 +119,7 @@ pub(crate) async fn start_meta_node_cluster( .wait(timeout()) .state( ServerState::Learner, - format!("check learner-{} state", item.meta_node().sto.id), + format!("check learner-{} state", item.meta_node().raft_store.id), ) .await?; } @@ -137,7 +136,7 @@ pub(crate) async fn start_meta_node_cluster( format!( "check applied index: {} for node-{}", log_index, - tc.meta_node().sto.id + tc.meta_node().raft_store.id ), ) .await?; @@ -199,7 +198,7 @@ pub(crate) async fn start_meta_node_non_voter( // // Log index becomes non-deterministic. // mn.raft.enable_heartbeat(false); - assert!(!mn.is_opened()); + assert!(!mn.raft_store.is_opened); tc.meta_node = Some(mn.clone()); From 6bc928e3df6e28bc8c0459f14ec772e672ad3c7e Mon Sep 17 00:00:00 2001 From: zhya Date: Fri, 22 Nov 2024 12:07:24 +0800 Subject: [PATCH 73/92] fix: ensure atomicity in create_lock_revision (#16907) * fix: double lock ownership due to concurrent race * fix review commend * fmt --- src/meta/api/src/schema_api_impl.rs | 55 +++++++++++++++++++---------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/meta/api/src/schema_api_impl.rs b/src/meta/api/src/schema_api_impl.rs index 6070e4cac9aa..b7320ae499f2 100644 --- a/src/meta/api/src/schema_api_impl.rs +++ b/src/meta/api/src/schema_api_impl.rs @@ -2738,30 +2738,49 @@ impl + ?Sized> SchemaApi for KV { &self, req: CreateLockRevReq, ) -> Result { - debug!(req :? =(&req); "SchemaApi: {}", func_name!()); + let ctx = func_name!(); + debug!(req :? =(&req); "SchemaApi: {}", ctx); let lock_key = &req.lock_key; + let id_generator = IdGenerator::table_lock_id(); - let revision = fetch_id(self, IdGenerator::table_lock_id()).await?; - let key = lock_key.gen_key(revision); + let mut trials = txn_backoff(None, ctx); + loop { + trials.next().unwrap()?.await; - let lock_meta = LockMeta { - user: req.user.clone(), - node: req.node.clone(), - query_id: req.query_id.clone(), - created_on: Utc::now(), - acquired_on: None, - lock_type: lock_key.lock_type(), - extra_info: lock_key.get_extra_info(), - }; + let current_rev = self.get_seq(&id_generator).await?; + let revision = current_rev + 1; + let key = lock_key.gen_key(revision); + let lock_meta = LockMeta { + user: req.user.clone(), + node: req.node.clone(), + query_id: req.query_id.clone(), + created_on: Utc::now(), + acquired_on: None, + lock_type: lock_key.lock_type(), + extra_info: lock_key.get_extra_info(), + }; - // Revision is unique. if it presents, consider it as success. - // Thus, we could just ignore create result - let _ = self - .crud_try_insert(&key, lock_meta, Some(req.ttl), || Ok::<(), Infallible>(())) - .await?; + let condition = vec![ + txn_cond_seq(&id_generator, Eq, current_rev), + // assumes lock are absent. + txn_cond_seq(&key, Eq, 0), + ]; + let if_then = vec![ + txn_op_put(&id_generator, b"".to_vec()), + txn_op_put_pb(&key, &lock_meta, Some(req.ttl))?, + ]; + let txn_req = TxnRequest { + condition, + if_then, + else_then: vec![], + }; + let (succ, _responses) = send_txn(self, txn_req).await?; - Ok(CreateLockRevReply { revision }) + if succ { + return Ok(CreateLockRevReply { revision }); + } + } } #[logcall::logcall] From e4bf8b2aaaf9a7f439e8f9430c1786b75cb75d72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Fri, 22 Nov 2024 14:39:54 +0800 Subject: [PATCH 74/92] refactor: remove obsolete gRPC read API (#16909) * refactor: list_background_tasks() use structured return values * chore: simplify list_udf * refactor: remove obsolete gRPC read API * chore: skip damaged version 1.2.287. Use 1.2.288 instead * chore: try 300 * chore: adapt to new version of sqllogictest * chore: fixup --- src/meta/README.md | 3 +- src/meta/api/src/background_api.rs | 3 +- src/meta/api/src/background_api_impl.rs | 20 ++---------- src/meta/api/src/background_api_test_suite.rs | 4 +-- src/meta/client/src/grpc_action.rs | 14 --------- src/meta/client/src/lib.rs | 5 +++ src/meta/service/src/api/grpc/grpc_service.rs | 21 ------------- src/meta/service/src/version.rs | 4 ++- src/query/management/src/udf/udf_mgr.rs | 29 ++++++----------- .../src/servers/admin/v1/background_tasks.rs | 21 +++++++------ .../system/src/background_tasks_table.rs | 31 ++++++++++++------- tests/compat/test-compat.sh | 26 +++++++++++++--- 12 files changed, 78 insertions(+), 103 deletions(-) diff --git a/src/meta/README.md b/src/meta/README.md index 2140f7bc8990..0e127278c0d4 100644 --- a/src/meta/README.md +++ b/src/meta/README.md @@ -86,7 +86,8 @@ The following is an illustration of the latest query-meta compatibility: | [1.1.32, 1.2.63) | ✅ | ✅ | ❌ |❌ | | [1.2.63, 1.2.226) | ✅ | ✅ | ❌ |❌ | | [1.2.226, 1.2.258) | ✅ | ✅ | ✅ |❌ | -| [1.2.258, +∞) | ✅ | ✅ | ✅ |✅ | +| [1.2.258, 1.2.663) | ✅ | ✅ | ✅ |✅ | +| [1.2.663, +∞) | ❌ | ❌ | ✅ |✅ | History versions that are not included in the above chart: diff --git a/src/meta/api/src/background_api.rs b/src/meta/api/src/background_api.rs index e0c48fda1d37..738a8f849ece 100644 --- a/src/meta/api/src/background_api.rs +++ b/src/meta/api/src/background_api.rs @@ -13,6 +13,7 @@ // limitations under the License. use databend_common_meta_app::background::BackgroundJobInfo; +use databend_common_meta_app::background::BackgroundTaskIdent; use databend_common_meta_app::background::BackgroundTaskInfo; use databend_common_meta_app::background::CreateBackgroundJobReply; use databend_common_meta_app::background::CreateBackgroundJobReq; @@ -70,7 +71,7 @@ pub trait BackgroundApi: Send + Sync { async fn list_background_tasks( &self, req: ListBackgroundTasksReq, - ) -> Result, KVAppError>; + ) -> Result)>, KVAppError>; async fn update_background_task( &self, diff --git a/src/meta/api/src/background_api_impl.rs b/src/meta/api/src/background_api_impl.rs index cb939559ed25..43355dc160d6 100644 --- a/src/meta/api/src/background_api_impl.rs +++ b/src/meta/api/src/background_api_impl.rs @@ -42,7 +42,6 @@ use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqValue; -use databend_common_meta_types::InvalidReply; use databend_common_meta_types::MatchSeq::Any; use databend_common_meta_types::MetaError; use databend_common_meta_types::MetaSpec; @@ -54,7 +53,6 @@ use futures::TryStreamExt; use log::debug; use crate::background_api::BackgroundApi; -use crate::deserialize_struct; use crate::fetch_id; use crate::kv_app_error::KVAppError; use crate::kv_pb_api::KVPbApi; @@ -243,22 +241,10 @@ impl> BackgroundApi for KV { async fn list_background_tasks( &self, req: ListBackgroundTasksReq, - ) -> Result, KVAppError> { + ) -> Result)>, KVAppError> { let ident = BackgroundTaskIdent::new(&req.tenant, "dummy"); - let prefix = ident.tenant_prefix(); - - let reply = self.prefix_list_kv(&prefix).await?; - let mut res = vec![]; - for (k, v) in reply { - let ident = BackgroundTaskIdent::from_str_key(k.as_str()).map_err(|e| { - KVAppError::MetaError(MetaError::from(InvalidReply::new( - "list_background_tasks", - &e, - ))) - })?; - let val: BackgroundTaskInfo = deserialize_struct(&v.data)?; - res.push((v.seq, ident.name().to_string(), val)); - } + let dir = DirName::new(ident); + let res = self.list_pb_vec(&dir).await?; Ok(res) } diff --git a/src/meta/api/src/background_api_test_suite.rs b/src/meta/api/src/background_api_test_suite.rs index b986798d7956..26ffe1f32ec4 100644 --- a/src/meta/api/src/background_api_test_suite.rs +++ b/src/meta/api/src/background_api_test_suite.rs @@ -180,10 +180,10 @@ impl BackgroundApiTestSuite { info!("update log res: {:?}", res); let res = res.unwrap(); assert_eq!(1, res.len(), "there is one task"); - assert_eq!(task_id, res[0].1, "task name"); + assert_eq!(task_id, res[0].0.name(), "task name"); assert_eq!( BackgroundTaskState::DONE, - res[0].2.task_state, + res[0].1.task_state, "first state is done" ); } diff --git a/src/meta/client/src/grpc_action.rs b/src/meta/client/src/grpc_action.rs index 5c6256d35653..d9c4eeb2866a 100644 --- a/src/meta/client/src/grpc_action.rs +++ b/src/meta/client/src/grpc_action.rs @@ -53,10 +53,6 @@ pub trait RequestFor: Clone + fmt::Debug { #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, derive_more::From)] pub enum MetaGrpcReq { UpsertKV(UpsertKVReq), - - GetKV(GetKVReq), - MGetKV(MGetKVReq), - ListKV(ListKVReq), } impl TryInto for Request { @@ -125,16 +121,6 @@ impl RequestFor for MetaGrpcReadReq { type Reply = BoxStream; } -impl From for MetaGrpcReq { - fn from(v: MetaGrpcReadReq) -> Self { - match v { - MetaGrpcReadReq::GetKV(v) => MetaGrpcReq::GetKV(v), - MetaGrpcReadReq::MGetKV(v) => MetaGrpcReq::MGetKV(v), - MetaGrpcReadReq::ListKV(v) => MetaGrpcReq::ListKV(v), - } - } -} - impl From for RaftRequest { fn from(v: MetaGrpcReadReq) -> Self { let raft_request = RaftRequest { diff --git a/src/meta/client/src/lib.rs b/src/meta/client/src/lib.rs index d7c805673cdd..4fe33fab7f89 100644 --- a/src/meta/client/src/lib.rs +++ b/src/meta/client/src/lib.rs @@ -105,6 +105,11 @@ pub static METACLI_COMMIT_SEMVER: LazyLock = LazyLock::new(|| { /// - 2024-03-04: since: 1.2.361 /// 👥 client: `MetaSpec` use `ttl`, remove `expire_at`, require 1.2.258 /// +/// - 2024-11-22: since 1.2.663 +/// 🖥 server: remove `MetaGrpcReq::GetKV/MGetKV/ListKV`, +/// require the client to call kv_read_v1 for get/mget/list, +/// which is added `2024-01-07: since 1.2.287` +/// /// Server feature set: /// ```yaml /// server_features: diff --git a/src/meta/service/src/api/grpc/grpc_service.rs b/src/meta/service/src/api/grpc/grpc_service.rs index d2a31b29df55..0692f25c0af5 100644 --- a/src/meta/service/src/api/grpc/grpc_service.rs +++ b/src/meta/service/src/api/grpc/grpc_service.rs @@ -120,27 +120,6 @@ impl MetaServiceImpl { .await; RaftReply::from(res) } - MetaGrpcReq::GetKV(a) => { - let res = m - .get_kv(&a.key) - .log_elapsed_info(format!("GetKV: {:?}", a)) - .await; - RaftReply::from(res) - } - MetaGrpcReq::MGetKV(a) => { - let res = m - .mget_kv(&a.keys) - .log_elapsed_info(format!("MGetKV: {:?}", a)) - .await; - RaftReply::from(res) - } - MetaGrpcReq::ListKV(a) => { - let res = m - .prefix_list_kv(&a.prefix) - .log_elapsed_info(format!("ListKV: {:?}", a)) - .await; - RaftReply::from(res) - } }; network_metrics::incr_request_result(reply.error.is_empty()); diff --git a/src/meta/service/src/version.rs b/src/meta/service/src/version.rs index df1c81c0cf4b..d3fa9ff5cc61 100644 --- a/src/meta/service/src/version.rs +++ b/src/meta/service/src/version.rs @@ -54,7 +54,9 @@ pub static METASRV_SEMVER: LazyLock = LazyLock::new(|| { }); /// Oldest compatible nightly meta-client version -pub static MIN_METACLI_SEMVER: Version = Version::new(0, 9, 41); +/// +/// It should be 1.2.287 but 1.2.287 does not contain complete binaries +pub static MIN_METACLI_SEMVER: Version = Version::new(1, 2, 288); /// The min meta-server version that can be deployed together in a cluster, /// i.e., the network APIs are compatible. diff --git a/src/query/management/src/udf/udf_mgr.rs b/src/query/management/src/udf/udf_mgr.rs index 683f8aef157d..afd2eedf2aa4 100644 --- a/src/query/management/src/udf/udf_mgr.rs +++ b/src/query/management/src/udf/udf_mgr.rs @@ -24,7 +24,6 @@ use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; -use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaError; @@ -135,25 +134,15 @@ impl UdfMgr { #[async_backtrace::framed] #[fastrace::trace] pub async fn list_udf_fallback(&self) -> Result, ErrorCode> { - let key = UdfIdent::new(&self.tenant, ""); - let values = self.kv_api.prefix_list_kv(&key.to_string_key()).await?; - - let mut udfs = Vec::with_capacity(values.len()); - // At begin udf is serialize to json. https://github.com/datafuselabs/databend/pull/12729/files#diff-9c992028e59caebc313d761b8488b17f142618fb89db64c51c1655689d68c41b - // But we can not deserialize the UserDefinedFunction from json now, - // because add a new field created_on and the field `definition` refactor to a ENUM type. - for (name, value) in values { - let udf = crate::deserialize_struct(&value.data, ErrorCode::IllegalUDFFormat, || { - format!( - "Encountered invalid json data for LambdaUDF '{}', \ - please drop this invalid udf and re-create it. \n\ - Example: `DROP FUNCTION ;` then `CREATE FUNCTION AS ;`\n\ - ", - name - ) - })?; - udfs.push(udf); - } + let key = UdfIdent::new(&self.tenant, "dummy"); + let dir = DirName::new(key); + let udfs = self + .kv_api + .list_pb_values(&dir) + .await? + .try_collect::>() + .await?; + Ok(udfs) } diff --git a/src/query/service/src/servers/admin/v1/background_tasks.rs b/src/query/service/src/servers/admin/v1/background_tasks.rs index ddb2276cad90..6e2455e32939 100644 --- a/src/query/service/src/servers/admin/v1/background_tasks.rs +++ b/src/query/service/src/servers/admin/v1/background_tasks.rs @@ -61,30 +61,33 @@ async fn load_background_tasks( tasks.len() ); let mut task_infos = Vec::with_capacity(tasks.len()); - for (_, name, task) in tasks { - if params.task_state.is_some() && task.task_state != *params.task_state.as_ref().unwrap() { + for (name, seq_task) in tasks { + if params.task_state.is_some() + && seq_task.task_state != *params.task_state.as_ref().unwrap() + { continue; } - if params.task_type.is_some() && task.task_type != *params.task_type.as_ref().unwrap() { + if params.task_type.is_some() && seq_task.task_type != *params.task_type.as_ref().unwrap() { continue; } - if task.task_type == BackgroundTaskType::COMPACTION { - if task.compaction_task_stats.is_none() { + if seq_task.task_type == BackgroundTaskType::COMPACTION { + if seq_task.compaction_task_stats.is_none() { continue; } if params.table_id.is_some() - && task.compaction_task_stats.as_ref().unwrap().table_id != params.table_id.unwrap() + && seq_task.compaction_task_stats.as_ref().unwrap().table_id + != params.table_id.unwrap() { continue; } } if params.timestamp.as_ref().is_some() - && task.last_updated.is_some() - && task.last_updated.unwrap() < params.timestamp.unwrap() + && seq_task.last_updated.is_some() + && seq_task.last_updated.unwrap() < params.timestamp.unwrap() { continue; } - task_infos.push((name, task)); + task_infos.push((name.name().to_string(), seq_task.data)); } Ok(ListBackgroundTasksResponse { task_infos, diff --git a/src/query/storages/system/src/background_tasks_table.rs b/src/query/storages/system/src/background_tasks_table.rs index 6c807891a5bd..b53c980878e4 100644 --- a/src/query/storages/system/src/background_tasks_table.rs +++ b/src/query/storages/system/src/background_tasks_table.rs @@ -74,22 +74,24 @@ impl AsyncSystemTable for BackgroundTaskTable { let mut trigger = Vec::with_capacity(tasks.len()); let mut create_timestamps = Vec::with_capacity(tasks.len()); let mut update_timestamps = Vec::with_capacity(tasks.len()); - for (_, name, task) in tasks { - names.push(name); - types.push(task.task_type.to_string()); - stats.push(task.task_state.to_string()); - messages.push(task.message); + for (name, seq_task) in tasks { + names.push(name.name().to_string()); + types.push(seq_task.task_type.to_string()); + stats.push(seq_task.task_state.to_string()); + messages.push(seq_task.message.to_string()); compaction_stats.push( - task.compaction_task_stats + seq_task + .compaction_task_stats .as_ref() .map(|s| serde_json::to_vec(s).unwrap_or_default()), ); vacuum_stats.push( - task.vacuum_stats + seq_task + .vacuum_stats .as_ref() .map(|s| serde_json::to_vec(s).unwrap_or_default()), ); - if let Some(compact_stats) = task.compaction_task_stats.as_ref() { + if let Some(compact_stats) = seq_task.compaction_task_stats.as_ref() { database_ids.push(compact_stats.db_id); table_ids.push(compact_stats.table_id); task_run_secs.push(compact_stats.total_compaction_time.map(|s| s.as_secs())); @@ -98,10 +100,15 @@ impl AsyncSystemTable for BackgroundTaskTable { table_ids.push(0); task_run_secs.push(None); } - creators.push(task.creator.map(|s| s.to_string())); - trigger.push(task.manual_trigger.map(|s| s.trigger.display().to_string())); - create_timestamps.push(task.created_at.timestamp_micros()); - update_timestamps.push(task.last_updated.unwrap_or_default().timestamp_micros()); + creators.push(seq_task.creator.as_ref().map(|s| s.to_string())); + trigger.push( + seq_task + .manual_trigger + .as_ref() + .map(|s| s.trigger.display().to_string()), + ); + create_timestamps.push(seq_task.created_at.timestamp_micros()); + update_timestamps.push(seq_task.last_updated.unwrap_or_default().timestamp_micros()); } Ok(DataBlock::new_from_columns(vec![ StringType::from_data(names), diff --git a/tests/compat/test-compat.sh b/tests/compat/test-compat.sh index 14eb9c888510..8e513fa7238c 100755 --- a/tests/compat/test-compat.sh +++ b/tests/compat/test-compat.sh @@ -29,6 +29,11 @@ binary_url() { echo "https://github.com/datafuselabs/databend/releases/download/v${ver}-nightly/databend-v${ver}-nightly-x86_64-unknown-linux-gnu.tar.gz" } +test_suite_url() { + local ver="$1" + echo "https://github.com/databendlabs/databend/releases/download/v${ver}-nightly/databend-testsuite-v${ver}-nightly-x86_64-unknown-linux-gnu.tar.gz" +} + # output: 0.7.58 # Without prefix `v` and `-nightly` find_min_query_ver() { @@ -104,9 +109,20 @@ git_partial_clone() { download_test_suite() { local ver="$1" - echo " === Download test suites from $ver:$query_test_path" + echo " === Download test suites $ver" + + local url="$(test_suite_url $ver)" + local fn="databend-testsuite-v${ver}.tar.gz" - git_partial_clone "$bend_repo_url" "v$ver-nightly" "$query_test_path" old_suite + curl --connect-timeout 5 --retry-all-errors --retry 5 --retry-delay 1 -L "$url" -o "$fn" + + mkdir -p ./testsuite/$ver + tar -xf "$fn" -C ./testsuite/$ver + + echo " === Extracted test suites to ./testsuite/$ver:" + ls ./testsuite/$ver + + chmod +x ./testsuite/$ver/bin/* } # Download config.toml for a specific version of query. @@ -221,15 +237,15 @@ run_test() { $sqllogictests --handlers mysql --run_dir 05_ddl else ( - # download suites into ./old_suite + # download suites into ./testsuite/$query_ver/{bin/,suites/} download_test_suite $query_ver ) # Replace suites rm -rf "tests/sqllogictests/suites" - mv "old_suite/tests/sqllogictests/suites" "tests/sqllogictests/suites" + mv "./testsuite/$query_ver/suites" "tests/sqllogictests/suites" - $sqllogictests --handlers mysql --run_dir 05_ddl + ./testsuite/$query_ver/bin/databend-sqllogictests --handlers mysql --run_dir 05_ddl cd - fi } From 38c44115d8ca5bd4759e1905a132893d900d95c9 Mon Sep 17 00:00:00 2001 From: zhya Date: Fri, 22 Nov 2024 19:32:09 +0800 Subject: [PATCH 75/92] chore(ci): flaky test (#16917) flaky test --- .../09_fuse_engine/09_0041_auto_compaction.test | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test index e11c03b61be6..873d343096d5 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0041_auto_compaction.test @@ -88,16 +88,15 @@ statement ok insert into t1 values(3),(5),(8); statement ok -insert into t1 values(4),(6); +insert into t1 values(4),(6),(10),(11); -query II -select segment_count, row_count from fuse_snapshot('i15760', 't1') limit 10; +statement ok +insert into t1 values(0),(9),(12); + +query I +select count() from fuse_snapshot('i15760', 't1'); ---- -1 8 -1 8 -3 8 -2 6 -1 3 +6 query F select average_depth from clustering_information('i15760', 't1') From 6f0ae45cd0550e470a5b57bdc92691f602b67bf6 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Fri, 22 Nov 2024 20:22:07 +0800 Subject: [PATCH 76/92] chore(ci): enable overflow-checks in ci profile (#16895) * chore(ci): enable overflow-checks in ci profile * update * update * update * update * update * update * update * update --- Cargo.toml | 4 +- src/common/column/src/buffer/iterator.rs | 50 +++++------ src/query/expression/benches/bench.rs | 82 +++++++++++++++++++ src/query/expression/src/filter/like.rs | 10 +-- src/query/expression/src/utils/block_debug.rs | 6 +- src/query/service/src/sessions/session_ctx.rs | 2 +- .../src/sessions/session_mgr_status.rs | 2 +- .../1_stateful/02_query/02_0000_kill_query.py | 2 +- 8 files changed, 115 insertions(+), 43 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d8bafb180e45..3c74b37602d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -573,7 +573,7 @@ incremental = true [profile.ci] inherits = "release" -overflow-checks = false +overflow-checks = true incremental = false debug-assertions = true @@ -583,7 +583,7 @@ debug-assertions = true [profile.dev] split-debuginfo = "unpacked" -overflow-checks = false +overflow-checks = true # Report to https://github.com/rust-lang/rust/issues/100142 if incremental works well incremental = true diff --git a/src/common/column/src/buffer/iterator.rs b/src/common/column/src/buffer/iterator.rs index 91b4d351b5d5..5771de1ec7d3 100644 --- a/src/common/column/src/buffer/iterator.rs +++ b/src/common/column/src/buffer/iterator.rs @@ -20,20 +20,20 @@ use super::Buffer; /// This crates' equivalent of [`std::vec::IntoIter`] for [`Buffer`]. #[derive(Debug, Clone)] pub struct IntoIter { - values: Buffer, - index: usize, - end: usize, + buffer: Buffer, + current: *const T, + end: *const T, } impl IntoIter { - /// Creates a new [`Buffer`] #[inline] - pub fn new(values: Buffer) -> Self { - let end = values.len(); + pub fn new(buffer: Buffer) -> Self { + let ptr = buffer.as_ptr(); + let len = buffer.len(); Self { - values, - index: 0, - end, + current: ptr, + end: unsafe { ptr.add(len) }, + buffer, } } } @@ -43,40 +43,30 @@ impl Iterator for IntoIter { #[inline] fn next(&mut self) -> Option { - if self.index == self.end { - return None; + if self.current == self.end { + None + } else { + let value = unsafe { *self.current }; + self.current = unsafe { self.current.add(1) }; + Some(value) } - let old = self.index; - self.index += 1; - Some(*unsafe { self.values.get_unchecked(old) }) } #[inline] fn size_hint(&self) -> (usize, Option) { - (self.end - self.index, Some(self.end - self.index)) - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - let new_index = self.index + n; - if new_index > self.end { - self.index = self.end; - None - } else { - self.index = new_index; - self.next() - } + let len = unsafe { self.end.offset_from(self.current) } as usize; + (len, Some(len)) } } impl DoubleEndedIterator for IntoIter { #[inline] fn next_back(&mut self) -> Option { - if self.index == self.end { + if self.current == self.end { None } else { - self.end -= 1; - Some(*unsafe { self.values.get_unchecked(self.end) }) + self.end = unsafe { self.end.sub(1) }; + Some(unsafe { *self.end }) } } } diff --git a/src/query/expression/benches/bench.rs b/src/query/expression/benches/bench.rs index b0e18083fc80..8ff91219fa5c 100644 --- a/src/query/expression/benches/bench.rs +++ b/src/query/expression/benches/bench.rs @@ -16,6 +16,7 @@ extern crate criterion; use criterion::Criterion; +use databend_common_column::buffer::Buffer; use databend_common_expression::arrow::deserialize_column; use databend_common_expression::arrow::serialize_column; use databend_common_expression::types::BinaryType; @@ -130,6 +131,81 @@ fn bench(c: &mut Criterion) { }); } } + + for length in [10240, 102400] { + let (left, right) = generate_random_int_data(&mut rng, length); + + group.bench_function(format!("function_iterator_iterator_v1/{length}"), |b| { + b.iter(|| { + let left = left.clone(); + let right = right.clone(); + + let _c = left + .into_iter() + .zip(right.into_iter()) + .map(|(a, b)| a + b) + .collect::>(); + }) + }); + + group.bench_function(format!("function_iterator_iterator_v2/{length}"), |b| { + b.iter(|| { + let _c = left + .iter() + .cloned() + .zip(right.iter().cloned()) + .map(|(a, b)| a + b) + .collect::>(); + }) + }); + + group.bench_function( + format!("function_buffer_index_unchecked_iterator/{length}"), + |b| { + b.iter(|| { + let _c = (0..length) + .map(|i| unsafe { left.get_unchecked(i) + right.get_unchecked(i) }) + .collect::>(); + }) + }, + ); + + group.bench_function( + format!("function_slice_index_unchecked_iterator/{length}"), + |b| { + b.iter(|| { + let left = left.as_slice(); + let right = right.as_slice(); + + let _c = (0..length) + .map(|i| unsafe { left.get_unchecked(i) + right.get_unchecked(i) }) + .collect::>(); + }) + }, + ); + + let left_vec: Vec<_> = left.iter().cloned().collect(); + let right_vec: Vec<_> = right.iter().cloned().collect(); + + group.bench_function( + format!("function_vec_index_unchecked_iterator/{length}"), + |b| { + b.iter(|| { + let _c = (0..length) + .map(|i| unsafe { left_vec.get_unchecked(i) + right_vec.get_unchecked(i) }) + .collect::>(); + }) + }, + ); + + group.bench_function(format!("function_buffer_index_iterator/{length}"), |b| { + b.iter(|| { + let _c = (0..length) + .map(|i| left[i] + right[i]) + .collect::>(); + }) + }); + } } criterion_group!(benches, bench); @@ -155,3 +231,9 @@ fn generate_random_string_data(rng: &mut StdRng, length: usize) -> (Vec, (iter_str, iter_binary) } + +fn generate_random_int_data(rng: &mut StdRng, length: usize) -> (Buffer, Buffer) { + let s: Buffer = (0..length).map(|_| rng.gen_range(-1000..1000)).collect(); + let b: Buffer = (0..length).map(|_| rng.gen_range(-1000..1000)).collect(); + (s, b) +} diff --git a/src/query/expression/src/filter/like.rs b/src/query/expression/src/filter/like.rs index de5a0ba9d17e..0de572a947cc 100644 --- a/src/query/expression/src/filter/like.rs +++ b/src/query/expression/src/filter/like.rs @@ -304,13 +304,13 @@ fn find(mut haystack: &[u8], needle: &[u8]) -> Option { return None; } // Inspired by fast_strstr (https://github.com/RaphaelJ/fast_strstr). - let mut checksum = 0; + let mut checksum: i64 = 0; for i in 0..needle_len { // # Safety // `needle_len` <= haystack_len unsafe { - checksum += haystack.get_unchecked(i); - checksum -= needle.get_unchecked(i); + checksum += *haystack.get_unchecked(i) as i64; + checksum -= *needle.get_unchecked(i) as i64; } } let mut idx = 0; @@ -331,8 +331,8 @@ fn find(mut haystack: &[u8], needle: &[u8]) -> Option { // # Safety // `idx` < `haystack_len` and `idx` + `needle_len` < `haystack_len`. unsafe { - checksum -= haystack.get_unchecked(idx); - checksum += haystack.get_unchecked(idx + needle_len); + checksum -= *haystack.get_unchecked(idx) as i64; + checksum += *haystack.get_unchecked(idx + needle_len) as i64; } idx += 1; } diff --git a/src/query/expression/src/utils/block_debug.rs b/src/query/expression/src/utils/block_debug.rs index 05f877215ea5..d6ffa2191cdc 100644 --- a/src/query/expression/src/utils/block_debug.rs +++ b/src/query/expression/src/utils/block_debug.rs @@ -186,7 +186,7 @@ fn create_box_table( let row_count: usize = results.iter().map(|block| block.num_rows()).sum(); let mut rows_to_render = row_count.min(max_rows); - if row_count <= max_rows + 3 { + if row_count <= max_rows.saturating_add(3) { // hiding rows adds 3 extra rows // so hiding rows makes no sense if we are only slightly over the limit // if we are 1 row over the limit hiding rows will actually increase the number of lines we display! @@ -228,7 +228,7 @@ fn create_box_table( // "..." take up three lengths if max_width > 0 { (widths, column_map) = - compute_render_widths(schema, max_width, max_col_width + 3, &res_vec); + compute_render_widths(schema, max_width, max_col_width.saturating_add(3), &res_vec); } let mut header = Vec::with_capacity(schema.fields().len()); @@ -355,7 +355,7 @@ fn compute_render_widths( for field in schema.fields() { // head_name = field_name + "\n" + field_data_type let col_length = field.name().len().max(field.data_type().to_string().len()); - widths.push(col_length + 3); + widths.push(col_length.saturating_add(3)); } for values in results { diff --git a/src/query/service/src/sessions/session_ctx.rs b/src/query/service/src/sessions/session_ctx.rs index 0a0e03b6d4fc..f79f5c5c076e 100644 --- a/src/query/service/src/sessions/session_ctx.rs +++ b/src/query/service/src/sessions/session_ctx.rs @@ -297,7 +297,7 @@ impl SessionContext { index }; - if idx < 0 || idx > (query_ids_len - 1) as i32 { + if query_ids_len < 1 || idx < 0 || idx > (query_ids_len - 1) as i32 { return "".to_string(); } diff --git a/src/query/service/src/sessions/session_mgr_status.rs b/src/query/service/src/sessions/session_mgr_status.rs index 69e43bea1afa..2db67041972a 100644 --- a/src/query/service/src/sessions/session_mgr_status.rs +++ b/src/query/service/src/sessions/session_mgr_status.rs @@ -31,7 +31,7 @@ impl SessionManagerStatus { } pub(crate) fn query_finish(&mut self, now: SystemTime) { - self.running_queries_count -= 1; + self.running_queries_count = self.running_queries_count.saturating_sub(1); if self.running_queries_count == 0 { self.last_query_finished_at = Some(now) } diff --git a/tests/suites/1_stateful/02_query/02_0000_kill_query.py b/tests/suites/1_stateful/02_query/02_0000_kill_query.py index 590a8821f1cc..d538411d4662 100755 --- a/tests/suites/1_stateful/02_query/02_0000_kill_query.py +++ b/tests/suites/1_stateful/02_query/02_0000_kill_query.py @@ -37,7 +37,7 @@ res = mycursor.fetchone() kill_query = "kill query " + str(res[0]) + ";" mycursor.execute(kill_query) - time.sleep(1) + time.sleep(10) mycursor.execute( "SELECT * FROM system.processes WHERE extra_info LIKE '%SELECT max(number)%' AND extra_info NOT LIKE '%system.processes%';" ) From 7d09e1e05b7aebab238468660e43c6c96234fbb3 Mon Sep 17 00:00:00 2001 From: Liu Zhenlong <49094455+Dragonliu2018@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:04:45 +0800 Subject: [PATCH 77/92] feat(query): speed up fetching redis data from dictionaries via mget. (#16766) * support redis dic batch * update code according to reviews * update code according to reviews * format Cargo.toml * add mget to redis mock * update code --- Cargo.lock | 43 +- Cargo.toml | 2 +- src/common/exception/Cargo.toml | 1 + src/common/exception/src/exception_into.rs | 6 + src/meta/app/src/schema/dictionary.rs | 12 - src/query/service/Cargo.toml | 1 + .../transforms/transform_dictionary.rs | 372 +++++++++++++----- .../sql/src/planner/plans/scalar_expr.rs | 4 +- .../sql/src/planner/semantic/type_check.rs | 17 +- .../src/mock_source/redis_source.rs | 31 +- .../functions/02_0077_function_dict_get.test | 163 ++++++-- 11 files changed, 461 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc7a1b0ff3b0..8221d4ae2777 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1343,18 +1343,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bb8" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" -dependencies = [ - "async-trait", - "futures-util", - "parking_lot 0.12.3", - "tokio", -] - [[package]] name = "beef" version = "0.5.2" @@ -2636,12 +2624,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" -[[package]] -name = "crc16" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" - [[package]] name = "crc32c" version = "0.6.8" @@ -3294,6 +3276,7 @@ dependencies = [ "parquet", "paste", "prost", + "redis", "reqwest", "rustc-demangle", "serde", @@ -5164,6 +5147,7 @@ dependencies = [ "prost", "rand", "recursive", + "redis", "regex", "reqwest", "rmp-serde", @@ -10488,7 +10472,6 @@ dependencies = [ "async-trait", "backon", "base64 0.22.1", - "bb8", "bytes", "chrono", "crc32c", @@ -10506,7 +10489,6 @@ dependencies = [ "prometheus-client", "prost", "quick-xml 0.36.1", - "redis", "reqsign", "reqwest", "serde", @@ -12165,25 +12147,17 @@ dependencies = [ "async-trait", "bytes", "combine", - "crc16", "futures", "futures-util", "itoa", - "log", "num-bigint", "percent-encoding", "pin-project-lite", - "rand", - "rustls 0.23.12", - "rustls-native-certs 0.7.1", - "rustls-pemfile 2.1.3", - "rustls-pki-types", "ryu", "sha1_smol", "socket2", "tokio", "tokio-retry2", - "tokio-rustls 0.26.0", "tokio-util", "url", ] @@ -12787,19 +12761,6 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-native-certs" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" -dependencies = [ - "openssl-probe", - "rustls-pemfile 2.1.3", - "rustls-pki-types", - "schannel", - "security-framework", -] - [[package]] name = "rustls-native-certs" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 3c74b37602d5..1a6632f910e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -369,7 +369,6 @@ opendal = { version = "0.50.1", features = [ "services-moka", "services-webhdfs", "services-huggingface", - "services-redis", ] } openraft = { version = "0.10.0", features = [ "serde", @@ -409,6 +408,7 @@ raft-log = { version = "0.2.5" } rand = { version = "0.8.5", features = ["small_rng"] } rayon = "1.9.0" recursive = "0.1.1" +redis = { version = "0.27.5", features = ["tokio-comp", "connection-manager"] } regex = "1.8.1" replace_with = "0.1.7" reqwest = { version = "0.12", default-features = false, features = [ diff --git a/src/common/exception/Cargo.toml b/src/common/exception/Cargo.toml index b1950b19735c..bec53de42686 100644 --- a/src/common/exception/Cargo.toml +++ b/src/common/exception/Cargo.toml @@ -28,6 +28,7 @@ opendal = { workspace = true } parquet = { workspace = true } paste = { workspace = true } prost = { workspace = true } +redis = { workspace = true } reqwest = { workspace = true } rustc-demangle = { workspace = true } serde = { workspace = true } diff --git a/src/common/exception/src/exception_into.rs b/src/common/exception/src/exception_into.rs index 6d0819f1d609..f2d42093c023 100644 --- a/src/common/exception/src/exception_into.rs +++ b/src/common/exception/src/exception_into.rs @@ -410,3 +410,9 @@ impl From for ErrorCode { ErrorCode::DictionarySourceError(format!("Dictionary Sqlx Error, cause: {}", error)) } } + +impl From for ErrorCode { + fn from(error: redis::RedisError) -> Self { + ErrorCode::DictionarySourceError(format!("Dictionary Redis Error, cause: {}", error)) + } +} diff --git a/src/meta/app/src/schema/dictionary.rs b/src/meta/app/src/schema/dictionary.rs index 15be7c77019d..b52bbd279c3b 100644 --- a/src/meta/app/src/schema/dictionary.rs +++ b/src/meta/app/src/schema/dictionary.rs @@ -108,18 +108,6 @@ impl DictionaryMeta { username, password, host, port, db )) } - - pub fn build_redis_connection_url(&self) -> Result { - let host = self - .options - .get("host") - .ok_or_else(|| ErrorCode::BadArguments("Miss option `host`"))?; - let port = self - .options - .get("port") - .ok_or_else(|| ErrorCode::BadArguments("Miss option `port`"))?; - Ok(format!("tcp://{}:{}", host, port)) - } } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index 0fe25b4dae25..1d3b3a4acdfb 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -150,6 +150,7 @@ poem = { workspace = true } prost = { workspace = true } rand = { workspace = true } recursive = { workspace = true } +redis = { workspace = true } regex = { workspace = true } reqwest = { workspace = true } rustls = { workspace = true } diff --git a/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs b/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs index 09e6f8a99620..806091ffaf1b 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs @@ -13,16 +13,23 @@ // limitations under the License. use std::collections::BTreeMap; +use std::collections::HashMap; +use std::string::String; use std::sync::Arc; +use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::date::date_to_string; use databend_common_expression::types::timestamp::timestamp_to_string; +use databend_common_expression::types::AnyType; use databend_common_expression::types::DataType; use databend_common_expression::types::Number; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::StringColumn; +use databend_common_expression::types::StringType; +use databend_common_expression::types::ValueType; use databend_common_expression::with_integer_mapped_type; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; @@ -30,10 +37,13 @@ use databend_common_expression::DataBlock; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::Value; -use databend_common_storage::build_operator; use jiff::tz::TimeZone; -use opendal::services::Redis; -use opendal::Operator; +use redis::aio::ConnectionManager; +use redis::AsyncCommands; +use redis::Client; +use redis::ConnectionInfo; +use redis::ProtocolVersion; +use redis::RedisConnectionInfo; use sqlx::MySqlPool; use crate::pipelines::processors::transforms::TransformAsyncFunction; @@ -44,94 +54,255 @@ use crate::sql::plans::DictionarySource; use crate::sql::IndexType; pub(crate) enum DictionaryOperator { - Operator(Operator), + Redis(ConnectionManager), Mysql((MySqlPool, String)), } impl DictionaryOperator { - fn format_key(&self, key: ScalarRef<'_>) -> String { - match key { - ScalarRef::String(s) => s.to_string(), - ScalarRef::Date(d) => format!("{}", date_to_string(d as i64, &TimeZone::UTC)), - ScalarRef::Timestamp(t) => { - format!("{}", timestamp_to_string(t, &TimeZone::UTC)) - } - _ => format!("{}", key), + async fn dict_get( + &self, + value: &Value, + data_type: &DataType, + default_value: &Scalar, + ) -> Result> { + match self { + DictionaryOperator::Redis(connection) => match value { + Value::Scalar(scalar) => { + self.get_scalar_value_from_redis(scalar, connection, default_value) + .await + } + Value::Column(column) => { + let (_, validity) = column.validity(); + let column = match StringType::try_downcast_column(&column.remove_nullable()) { + Some(col) => col, + None => { + return Err(ErrorCode::DictionarySourceError(format!( + "Redis dictionary operator currently does not support column type {}", + column.data_type(), + ))); + } + }; + self.get_column_values_from_redis( + &column, + validity, + data_type, + connection, + default_value, + ) + .await + } + }, + DictionaryOperator::Mysql((pool, sql)) => match value { + Value::Scalar(scalar) => { + let value = self + .get_data_from_mysql(scalar.as_ref(), data_type, pool, sql) + .await? + .unwrap_or(default_value.clone()); + Ok(Value::Scalar(value)) + } + Value::Column(column) => { + let mut builder = ColumnBuilder::with_capacity(data_type, column.len()); + for scalar_ref in column.iter() { + let value = self + .get_data_from_mysql(scalar_ref, data_type, pool, sql) + .await? + .unwrap_or(default_value.clone()); + builder.push(value.as_ref()); + } + Ok(Value::Column(builder.build())) + } + }, } } - async fn dict_get(&self, key: ScalarRef<'_>, data_type: &DataType) -> Result> { - if key == ScalarRef::Null { - return Ok(None); - } - match self { - DictionaryOperator::Operator(op) => { - if let ScalarRef::String(key) = key { - let buffer = op.read(key).await; - match buffer { - Ok(res) => { - let value = - unsafe { String::from_utf8_unchecked(res.current().to_vec()) }; - Ok(Some(Scalar::String(value))) - } - Err(e) => { - if e.kind() == opendal::ErrorKind::NotFound { - Ok(None) - } else { - Err(ErrorCode::DictionarySourceError(format!( - "dictionary source error: {e}" - ))) - } - } + async fn get_scalar_value_from_redis( + &self, + key: &Scalar, + connection: &ConnectionManager, + default_value: &Scalar, + ) -> Result> { + match key { + Scalar::String(str) => { + let mut conn = connection.clone(); + let redis_val: redis::Value = match conn.get(str).await { + Ok(response) => response, + Err(err) => { + return Err(ErrorCode::DictionarySourceError(format!("{}", err))); } + }; + let res = Self::from_redis_value_to_scalar(&redis_val, default_value)?; + if res.is_empty() { + Err(ErrorCode::DictionarySourceError(format!( + "from_redis_value_to_scalar gets an empty vector", + ))) } else { - Ok(None) + Ok(Value::Scalar(res[0].clone())) } } - DictionaryOperator::Mysql((pool, sql)) => match data_type.remove_nullable() { - DataType::Boolean => { - let value: Option = sqlx::query_scalar(sql) - .bind(self.format_key(key)) - .fetch_optional(pool) - .await?; - Ok(value.map(Scalar::Boolean)) + Scalar::Null => Ok(Value::Scalar(default_value.clone())), + _ => Err(ErrorCode::DictionarySourceError(format!( + "Redis dictionary operator currently does not support value type {}", + key.as_ref().infer_data_type(), + ))), + } + } + + async fn get_column_values_from_redis( + &self, + str_col: &StringColumn, + validity: Option<&Bitmap>, + data_type: &DataType, + connection: &ConnectionManager, + default_value: &Scalar, + ) -> Result> { + // step-1: deduplicate the keys in the column. + let key_cnt = str_col.len(); + let mut keys = Vec::with_capacity(key_cnt); + let mut key_map = HashMap::with_capacity(key_cnt); + for key in str_col.option_iter(validity).flatten() { + if !key_map.contains_key(key) { + keys.push(key); + let index = key_map.len(); + key_map.insert(key, index); + } + } + + // step-2: get the values from redis via mget. + let mut builder = ColumnBuilder::with_capacity(data_type, key_cnt); + if keys.is_empty() { + // keys in the column only have null. + for _ in 0..key_cnt { + builder.push(default_value.as_ref()); + } + } else { + let mut conn = connection.clone(); + let redis_val: redis::Value = match conn.get(keys).await { + Ok(response) => response, + Err(err) => { + return Err(ErrorCode::DictionarySourceError(format!("{}", err))); } - DataType::String => { - let value: Option = sqlx::query_scalar(sql) - .bind(self.format_key(key)) - .fetch_optional(pool) - .await?; - Ok(value.map(Scalar::String)) + }; + let res = Self::from_redis_value_to_scalar(&redis_val, default_value)?; + if res.is_empty() { + return Err(ErrorCode::DictionarySourceError(format!( + "from_redis_value_to_scalar gets an empty vector", + ))); + } else { + for key in str_col.option_iter(validity) { + if let Some(key) = key { + let index = key_map[key]; + builder.push(res[index].as_ref()); + } else { + builder.push(default_value.as_ref()); + } } - DataType::Number(num_ty) => { - with_integer_mapped_type!(|NUM_TYPE| match num_ty { - NumberDataType::NUM_TYPE => { - let value: Option = sqlx::query_scalar(&sql) - .bind(self.format_key(key)) - .fetch_optional(pool) - .await?; - Ok(value.map(|v| Scalar::Number(NUM_TYPE::upcast_scalar(v)))) - } - NumberDataType::Float32 => { - let value: Option = sqlx::query_scalar(sql) - .bind(self.format_key(key)) - .fetch_optional(pool) - .await?; - Ok(value.map(|v| Scalar::Number(NumberScalar::Float32(v.into())))) + } + } + Ok(Value::Column(builder.build())) + } + + #[inline] + fn from_redis_value_to_scalar( + rv: &redis::Value, + default_value: &Scalar, + ) -> Result> { + match rv { + redis::Value::BulkString(bs) => { + let str = unsafe { String::from_utf8_unchecked(bs.to_vec()) }; + Ok(vec![Scalar::String(str)]) + } + redis::Value::Array(arr) => { + let mut scalar_vec = Vec::with_capacity(arr.len()); + for item in arr { + let scalar = match item { + redis::Value::BulkString(bs) => { + let str = unsafe { String::from_utf8_unchecked(bs.to_vec()) }; + Scalar::String(str) } - NumberDataType::Float64 => { - let value: Option = sqlx::query_scalar(sql) - .bind(self.format_key(key)) - .fetch_optional(pool) - .await?; - Ok(value.map(|v| Scalar::Number(NumberScalar::Float64(v.into())))) + redis::Value::Nil => default_value.clone(), + _ => { + return Err(ErrorCode::DictionarySourceError(format!( + "from_redis_value_to_scalar currently does not support redis::Value type {:?}", + item, + ))); } - }) + }; + scalar_vec.push(scalar); } - _ => Err(ErrorCode::DictionarySourceError(format!( - "unsupported value type {data_type}" - ))), - }, + Ok(scalar_vec) + } + redis::Value::Nil => Ok(vec![default_value.clone()]), + _ => Err(ErrorCode::DictionarySourceError(format!( + "from_redis_value_to_scalar currently does not support redis::Value type {:?}", + rv, + ))), + } + } + + async fn get_data_from_mysql( + &self, + key: ScalarRef<'_>, + data_type: &DataType, + pool: &MySqlPool, + sql: &String, + ) -> Result> { + if key == ScalarRef::Null { + return Ok(None); + } + match data_type.remove_nullable() { + DataType::Boolean => { + let value: Option = sqlx::query_scalar(sql) + .bind(self.format_key(key)) + .fetch_optional(pool) + .await?; + Ok(value.map(Scalar::Boolean)) + } + DataType::String => { + let value: Option = sqlx::query_scalar(sql) + .bind(self.format_key(key)) + .fetch_optional(pool) + .await?; + Ok(value.map(Scalar::String)) + } + DataType::Number(num_ty) => { + with_integer_mapped_type!(|NUM_TYPE| match num_ty { + NumberDataType::NUM_TYPE => { + let value: Option = sqlx::query_scalar(&sql) + .bind(self.format_key(key)) + .fetch_optional(pool) + .await?; + Ok(value.map(|v| Scalar::Number(NUM_TYPE::upcast_scalar(v)))) + } + NumberDataType::Float32 => { + let value: Option = sqlx::query_scalar(sql) + .bind(self.format_key(key)) + .fetch_optional(pool) + .await?; + Ok(value.map(|v| Scalar::Number(NumberScalar::Float32(v.into())))) + } + NumberDataType::Float64 => { + let value: Option = sqlx::query_scalar(sql) + .bind(self.format_key(key)) + .fetch_optional(pool) + .await?; + Ok(value.map(|v| Scalar::Number(NumberScalar::Float64(v.into())))) + } + }) + } + _ => Err(ErrorCode::DictionarySourceError(format!( + "MySQL dictionary operator currently does not support value type {data_type}" + ))), + } + } + + fn format_key(&self, key: ScalarRef<'_>) -> String { + match key { + ScalarRef::String(s) => s.to_string(), + ScalarRef::Date(d) => format!("{}", date_to_string(d as i64, &TimeZone::UTC)), + ScalarRef::Timestamp(t) => { + format!("{}", timestamp_to_string(t, &TimeZone::UTC)) + } + _ => format!("{}", key), } } } @@ -145,18 +316,23 @@ impl TransformAsyncFunction { if let AsyncFunctionArgument::DictGetFunction(dict_arg) = &async_func_desc.func_arg { match &dict_arg.dict_source { DictionarySource::Redis(redis_source) => { - let mut builder = Redis::default().endpoint(&redis_source.connection_url); - if let Some(ref username) = redis_source.username { - builder = builder.username(username); - } - if let Some(ref password) = redis_source.password { - builder = builder.password(password); - } - if let Some(db_index) = redis_source.db_index { - builder = builder.db(db_index); - } - let op = build_operator(builder)?; - operators.insert(i, Arc::new(DictionaryOperator::Operator(op))); + let connection_info = ConnectionInfo { + addr: redis::ConnectionAddr::Tcp( + redis_source.host.clone(), + redis_source.port, + ), + redis: RedisConnectionInfo { + db: redis_source.db_index.unwrap_or(0), + username: redis_source.username.clone(), + password: redis_source.password.clone(), + protocol: ProtocolVersion::RESP2, + }, + }; + let client = Client::open(connection_info)?; + let conn = databend_common_base::runtime::block_on( + ConnectionManager::new(client), + )?; + operators.insert(i, Arc::new(DictionaryOperator::Redis(conn))); } DictionarySource::Mysql(sql_source) => { let mysql_pool = databend_common_base::runtime::block_on( @@ -187,26 +363,8 @@ impl TransformAsyncFunction { // only support one key field. let arg_index = arg_indices[0]; let entry = data_block.get_by_offset(arg_index); - let value = match &entry.value { - Value::Scalar(scalar) => { - let value = op - .dict_get(scalar.as_ref(), data_type) - .await? - .unwrap_or(dict_arg.default_value.clone()); - Value::Scalar(value) - } - Value::Column(column) => { - let mut builder = ColumnBuilder::with_capacity(data_type, column.len()); - for scalar_ref in column.iter() { - let value = op - .dict_get(scalar_ref, data_type) - .await? - .unwrap_or(dict_arg.default_value.clone()); - builder.push(value.as_ref()); - } - Value::Column(builder.build()) - } - }; + let default_value = dict_arg.default_value.clone(); + let value = op.dict_get(&entry.value, data_type, &default_value).await?; let entry = BlockEntry { data_type: data_type.clone(), value, diff --git a/src/query/sql/src/planner/plans/scalar_expr.rs b/src/query/sql/src/planner/plans/scalar_expr.rs index ea4f0201cd61..d7d165cfb8c5 100644 --- a/src/query/sql/src/planner/plans/scalar_expr.rs +++ b/src/query/sql/src/planner/plans/scalar_expr.rs @@ -801,8 +801,8 @@ pub enum AsyncFunctionArgument { #[derive(Clone, Debug, Educe, serde::Serialize, serde::Deserialize)] #[educe(PartialEq, Eq, Hash)] pub struct RedisSource { - // Redis source connection URL, like `tcp://127.0.0.1:6379` - pub connection_url: String, + pub host: String, + pub port: u16, pub username: Option, pub password: Option, pub db_index: Option, diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index ea049fe045b4..38350db16026 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -4095,7 +4095,7 @@ impl<'a> TypeChecker<'a> { let mut args = Vec::with_capacity(1); let box (key_scalar, key_type) = self.resolve(key_arg)?; - if primary_type != key_type { + if primary_type != key_type.remove_nullable() { args.push(wrap_cast(&key_scalar, &primary_type)); } else { args.push(key_scalar); @@ -4115,7 +4115,17 @@ impl<'a> TypeChecker<'a> { }) } "redis" => { - let connection_url = dictionary.build_redis_connection_url()?; + let host = dictionary + .options + .get("host") + .ok_or_else(|| ErrorCode::BadArguments("Miss option `host`"))?; + let port_str = dictionary + .options + .get("port") + .ok_or_else(|| ErrorCode::BadArguments("Miss option `port`"))?; + let port = port_str + .parse() + .expect("Failed to parse String port to u16"); let username = dictionary.options.get("username").cloned(); let password = dictionary.options.get("password").cloned(); let db_index = dictionary @@ -4123,7 +4133,8 @@ impl<'a> TypeChecker<'a> { .get("db_index") .map(|i| i.parse::().unwrap()); DictionarySource::Redis(RedisSource { - connection_url, + host: host.to_string(), + port, username, password, db_index, diff --git a/tests/sqllogictests/src/mock_source/redis_source.rs b/tests/sqllogictests/src/mock_source/redis_source.rs index a36b13e21f06..72c23d0e706e 100644 --- a/tests/sqllogictests/src/mock_source/redis_source.rs +++ b/tests/sqllogictests/src/mock_source/redis_source.rs @@ -59,6 +59,23 @@ async fn process(stream: TcpStream) { }; ret_values.push_back(ret_value); } + Command::MGet(keys) => { + // Process a batch of keys, and each key is handled in the same way as Get above. + let mut responses = Vec::new(); + for key in keys { + let response = + if key.starts_with(|c: char| c.is_ascii_alphanumeric()) { + let v = format!("{}_value", key); + format!("${}\r\n{}\r\n", v.len(), v) + } else { + "$-1\r\n".to_string() + }; + responses.push(response); + } + let ret_value = + format!("*{}\r\n{}", responses.len(), responses.concat()); + ret_values.push_back(ret_value); + } Command::Ping => { let ret_value = "+PONG\r\n".to_string(); ret_values.push_back(ret_value); @@ -96,9 +113,10 @@ async fn process(stream: TcpStream) { } } -// Redis command, only support get, other commands are ignored. +// Redis command, only support get and mget, other commands are ignored. enum Command { Get(String), + MGet(Vec), Ping, Invalid, Other, @@ -120,10 +138,19 @@ fn parse_resp(request: String) -> Vec { cmds.push(Command::Invalid); return cmds; } - // only parse GET command and ingore other commands + // only parse GET & MGET and ingore other commands if lines[2] == "GET" { let cmd = Command::Get(lines[4].to_string()); cmds.push(cmd); + } else if lines[2] == "MGET" { + let args: Vec = lines + .iter() + .enumerate() + .filter(|(index, _)| *index >= 4 && (index - 4) % 2 == 0) + .map(|(_, &line)| line.to_string()) + .collect(); + let cmd = Command::MGet(args); + cmds.push(cmd); } else if lines[2] == "PING" { cmds.push(Command::Ping) } else { diff --git a/tests/sqllogictests/suites/query/functions/02_0077_function_dict_get.test b/tests/sqllogictests/suites/query/functions/02_0077_function_dict_get.test index 9c5585786fad..bdbe8df06a0b 100644 --- a/tests/sqllogictests/suites/query/functions/02_0077_function_dict_get.test +++ b/tests/sqllogictests/suites/query/functions/02_0077_function_dict_get.test @@ -1,56 +1,173 @@ statement ok -create or replace table t(a string) +CREATE OR REPLACE DICTIONARY redis_d1(key string not null, value string not null) PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) statement ok -insert into t values('a'),('b'),('%c') +CREATE OR REPLACE DICTIONARY redis_d2(key string not null, value string null) PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) statement ok -CREATE OR REPLACE DICTIONARY d(key string not null, value string not null) PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) +CREATE OR REPLACE DICTIONARY redis_d3(key string null, value string not null) PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) -query T -select a, dict_get(d, 'value', a) from t ----- -a a_value -b b_value -%c (empty) +statement ok +CREATE OR REPLACE DICTIONARY redis_d4(key string null, value string null) PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) + +statement ok +CREATE OR REPLACE DICTIONARY redis_d5(key string not null, value string not null default 'default_value') PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) + +statement ok +CREATE OR REPLACE DICTIONARY redis_d6(key string not null, value string null default 'default_value') PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) + +statement ok +CREATE OR REPLACE DICTIONARY redis_d7(key string null, value string not null default 'default_value') PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) + +statement ok +CREATE OR REPLACE DICTIONARY redis_d8(key string null, value string null default 'default_value') PRIMARY KEY key SOURCE(redis(host='127.0.0.1' port='6179')) query T -SELECT dict_get(d, 'value', 'b') +SELECT dict_get(redis_d1, 'value', 'b') ---- b_value statement error 1006 -select dict_get(d, 'value11', 'a') +select dict_get(redis_d1, 'value11', 'a') statement error 3114 select dict_get(test, 'value', 'b') query T -SELECT dict_get(d, 'value', 1) +SELECT dict_get(redis_d1, 'value', 1) ---- 1_value statement ok -create or replace table t2(id int, name string) +create or replace table redis_t1(key string null) + +statement ok +insert into redis_t1 values('a'),('b'),('%c'),(null) + +query T +select key, dict_get(redis_d1, 'value', key) from redis_t1 +---- +a a_value +b b_value +%c (empty) +NULL (empty) + +query T +select key, dict_get(redis_d2, 'value', key) from redis_t1 +---- +a a_value +b b_value +%c NULL +NULL NULL + +query T +select key, dict_get(redis_d3, 'value', key) from redis_t1 +---- +a a_value +b b_value +%c (empty) +NULL (empty) + +query T +select key, dict_get(redis_d4, 'value', key) from redis_t1 +---- +a a_value +b b_value +%c NULL +NULL NULL + +query T +select key, dict_get(redis_d5, 'value', key) from redis_t1 +---- +a a_value +b b_value +%c default_value +NULL default_value + +query T +select key, dict_get(redis_d6, 'value', key) from redis_t1 +---- +a a_value +b b_value +%c default_value +NULL default_value + +query T +select key, dict_get(redis_d7, 'value', key) from redis_t1 +---- +a a_value +b b_value +%c default_value +NULL default_value + +query T +select key, dict_get(redis_d8, 'value', key) from redis_t1 +---- +a a_value +b b_value +%c default_value +NULL default_value + +statement ok +create or replace table redis_t2(key string not null) + +statement ok +insert into redis_t2 values('a'),('a'),('b'),('b'),('%c') + +query T +select key, dict_get(redis_d1, 'value', key) from redis_t2 +---- +a a_value +a a_value +b b_value +b b_value +%c (empty) + +statement ok +create or replace table redis_t3(key string null) + +statement ok +insert into redis_t3 values(null),(null) + +query T +select key, dict_get(redis_d2, 'value', key) from redis_t3 +---- +NULL NULL +NULL NULL + +query T +select key, dict_get(redis_d3, 'value', key) from redis_t3 +---- +NULL (empty) +NULL (empty) + +query T +select key, dict_get(redis_d7, 'value', key) from redis_t3 +---- +NULL default_value +NULL default_value + +statement ok +create or replace table mysql_t1(id int, name string) statement ok -insert into t2 values(1, 'Alice'),(2, 'Bob'),(3, 'Lily'),(4, 'Tom'),(5, 'Tim') +insert into mysql_t1 values(1, 'Alice'),(2, 'Bob'),(3, 'Lily'),(4, 'Tom'),(5, 'Tim') statement ok -CREATE OR REPLACE DICTIONARY d2(id int, name string, age uint16, salary float, active bool) PRIMARY KEY id SOURCE(mysql(host='localhost' port='3106' username='root' password='123456' db='test' table='user')); +CREATE OR REPLACE DICTIONARY mysql_d1(id int, name string, age uint16, salary float, active bool) PRIMARY KEY id SOURCE(mysql(host='localhost' port='3106' username='root' password='123456' db='test' table='user')); query TIFT -select dict_get(d2, 'name', 1), dict_get(d2, 'age', 1), dict_get(d2, 'salary', 1), dict_get(d2, 'active', 1) +select dict_get(mysql_d1, 'name', 1), dict_get(mysql_d1, 'age', 1), dict_get(mysql_d1, 'salary', 1), dict_get(mysql_d1, 'active', 1) ---- Alice 24 100.0 1 query TIFT -select dict_get(d2, 'name', 5), dict_get(d2, 'age', 5), dict_get(d2, 'salary', 5), dict_get(d2, 'active', 5) +select dict_get(mysql_d1, 'name', 5), dict_get(mysql_d1, 'age', 5), dict_get(mysql_d1, 'salary', 5), dict_get(mysql_d1, 'active', 5) ---- NULL NULL NULL NULL query ITIFT -select id, dict_get(d2, 'name', id), dict_get(d2, 'age', id), dict_get(d2, 'salary', id), dict_get(d2, 'active', id) from t2 +select id, dict_get(mysql_d1, 'name', id), dict_get(mysql_d1, 'age', id), dict_get(mysql_d1, 'salary', id), dict_get(mysql_d1, 'active', id) from mysql_t1 ---- 1 Alice 24 100.0 1 2 Bob 35 200.1 0 @@ -59,26 +176,26 @@ select id, dict_get(d2, 'name', id), dict_get(d2, 'age', id), dict_get(d2, 'sala 5 NULL NULL NULL NULL query ITI -select id, name, dict_get(d2, 'age', id) as age from t2 where age > 35 +select id, name, dict_get(mysql_d1, 'age', id) as age from mysql_t1 where age > 35 ---- 3 Lily 41 4 Tom 55 statement ok -CREATE OR REPLACE DICTIONARY d3(id int, name string, age uint16, salary float, active bool) PRIMARY KEY name SOURCE(mysql(host='localhost' port='3106' username='root' password='123456' db='test' table='user')); +CREATE OR REPLACE DICTIONARY mysql_d2(id int, name string, age uint16, salary float, active bool) PRIMARY KEY name SOURCE(mysql(host='localhost' port='3106' username='root' password='123456' db='test' table='user')); query TIFT -select dict_get(d3, 'id', 'Alice'), dict_get(d3, 'age', 'Alice'), dict_get(d3, 'salary', 'Alice'), dict_get(d3, 'active', 'Alice') +select dict_get(mysql_d2, 'id', 'Alice'), dict_get(mysql_d2, 'age', 'Alice'), dict_get(mysql_d2, 'salary', 'Alice'), dict_get(mysql_d2, 'active', 'Alice') ---- 1 24 100.0 1 query TIFT -select dict_get(d3, 'id', 'Nancy'), dict_get(d3, 'age', 'Nancy'), dict_get(d3, 'salary', 'Nancy'), dict_get(d3, 'active', 'Nancy') +select dict_get(mysql_d2, 'id', 'Nancy'), dict_get(mysql_d2, 'age', 'Nancy'), dict_get(mysql_d2, 'salary', 'Nancy'), dict_get(mysql_d2, 'active', 'Nancy') ---- NULL NULL NULL NULL query ITIFT -select name, dict_get(d3, 'id', name), dict_get(d3, 'age', name), dict_get(d3, 'salary', name), dict_get(d3, 'active', name) from t2 +select name, dict_get(mysql_d2, 'id', name), dict_get(mysql_d2, 'age', name), dict_get(mysql_d2, 'salary', name), dict_get(mysql_d2, 'active', name) from mysql_t1 ---- Alice 1 24 100.0 1 Bob 2 35 200.1 0 From 5a934a6716d03e3e2b0ce8f38267c529528f4737 Mon Sep 17 00:00:00 2001 From: Sky Fan <3374614481@qq.com> Date: Sat, 23 Nov 2024 00:14:04 +0800 Subject: [PATCH 78/92] fix: dropped temporary table should not be shown in system.temporary_tables (#16911) * fix: dropped temporary table should not be shown in system.temporary_tables * add test * fix ctas and rename --- .../storages/common/session/src/temp_table.rs | 15 ++++++-- .../suites/temp_table/rename_temp_tables.sql | 6 ++++ .../temp_table/temporary_tables_table.sql | 36 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/query/storages/common/session/src/temp_table.rs b/src/query/storages/common/session/src/temp_table.rs index b090c70f2288..fd7d9f6d4468 100644 --- a/src/query/storages/common/session/src/temp_table.rs +++ b/src/query/storages/common/session/src/temp_table.rs @@ -123,7 +123,10 @@ impl TempTblMgr { .as_ref() .map(|o| format!("{}.{}", name_ident.db_name, o)) .unwrap_or(desc); - self.name_to_id.insert(desc, table_id); + let old_id = self.name_to_id.insert(desc, table_id); + if let Some(old_id) = old_id { + self.id_to_table.remove(&old_id); + } self.id_to_table.insert(table_id, TempTable { db_name: name_ident.db_name, table_name: orphan_table_name.clone().unwrap_or(name_ident.table_name), @@ -157,7 +160,9 @@ impl TempTblMgr { let desc = format!("{}.{}", req.name_ident.db_name, req.name_ident.table_name); match self.name_to_id.remove(&orphan_desc) { Some(id) => { - self.name_to_id.insert(desc, id); + if let Some(old_id) = self.name_to_id.insert(desc, id) { + self.id_to_table.remove(&old_id); + } let table = self.id_to_table.get_mut(&id).unwrap(); table.db_name = req.name_ident.db_name.clone(); table.table_name = req.name_ident.table_name.clone(); @@ -182,6 +187,12 @@ impl TempTblMgr { match self.name_to_id.remove(&desc) { Some(id) => { let new_desc = format!("{}.{}", new_db_name, new_table_name); + if self.name_to_id.contains_key(&new_desc) { + return Err(ErrorCode::TableAlreadyExists(format!( + "Temporary table {} already exists", + new_desc + ))); + } self.name_to_id.insert(new_desc, id); let table = self.id_to_table.get_mut(&id).unwrap(); table.db_name = new_db_name.clone(); diff --git a/tests/sqllogictests/suites/temp_table/rename_temp_tables.sql b/tests/sqllogictests/suites/temp_table/rename_temp_tables.sql index 051018db0d69..be33f148e60c 100644 --- a/tests/sqllogictests/suites/temp_table/rename_temp_tables.sql +++ b/tests/sqllogictests/suites/temp_table/rename_temp_tables.sql @@ -84,6 +84,12 @@ SELECT * FROM t1 ---- 1 +statement ok +CREATE TEMP TABLE t2(c int) + +statement error 2302 +RENAME TABLE t1 to t2 + statement error 1006 RENAME TABLE t1 to system.t1 diff --git a/tests/sqllogictests/suites/temp_table/temporary_tables_table.sql b/tests/sqllogictests/suites/temp_table/temporary_tables_table.sql index 8529f2e300a9..1dc212b71112 100644 --- a/tests/sqllogictests/suites/temp_table/temporary_tables_table.sql +++ b/tests/sqllogictests/suites/temp_table/temporary_tables_table.sql @@ -20,3 +20,39 @@ default t0 4611686018427407904 FUSE d1 t1 4611686018427407905 FUSE d2 t2 4611686018427407906 MEMORY +statement ok +CREATE OR REPLACE TEMP TABLE d1.t1(a int not null, b int not null); + +query T +select * from system.temporary_tables order by table_id; +---- +default t0 4611686018427407904 FUSE +d2 t2 4611686018427407906 MEMORY +d1 t1 4611686018427407907 FUSE + +statement ok +drop table d2.t2; + +query T +select * from system.temporary_tables order by table_id; +---- +default t0 4611686018427407904 FUSE +d1 t1 4611686018427407907 FUSE + +statement ok +CREATE OR REPLACE TEMP TABLE d1.t1(a int not null, b int not null) as select * from d1.t1; + +query T +select * from system.temporary_tables order by table_id; +---- +default t0 4611686018427407904 FUSE +d1 t1 4611686018427407908 FUSE + +statement ok +CREATE OR REPLACE TEMP TABLE d1.t1(a int not null, b int not null) as select * from d1.t1; + +query T +select * from system.temporary_tables order by table_id; +---- +default t0 4611686018427407904 FUSE +d1 t1 4611686018427407909 FUSE From a2ce45c721e851c35860c9f82f21f21558d9292c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Sat, 23 Nov 2024 10:15:46 +0800 Subject: [PATCH 79/92] refactor: deprecate: Operation::AsIs and will be removed (#16913) * refactor: remove put_with_expire_at; Use ttl instead of expire_at for BackgroundTask * refactor: simplify ClusterApi::heartbeat. heartbeat is barely an `upsert` operation no matter the node key present or not. * refactor: deprecate: Operation::AsIs and will be removed * chore: fix deprecated * M src/query/management/src/cluster/cluster_api.rs --- src/meta/api/src/background_api_impl.rs | 10 +- src/meta/api/src/background_api_test_suite.rs | 8 +- src/meta/api/src/kv_pb_api/codec.rs | 4 +- src/meta/api/src/kv_pb_api/mod.rs | 4 +- src/meta/api/src/kv_pb_api/upsert_pb.rs | 4 - src/meta/api/src/name_id_value_api.rs | 4 +- src/meta/api/src/schema_api_test_suite.rs | 5 +- src/meta/api/src/util.rs | 4 +- .../app/src/background/background_task.rs | 5 +- src/meta/binaries/meta/kvapi.rs | 8 +- src/meta/binaries/metabench/main.rs | 4 +- src/meta/binaries/metaverifier/main.rs | 6 +- src/meta/client/src/grpc_action.rs | 6 +- src/meta/client/src/kv_api_impl.rs | 4 +- src/meta/client/src/lib.rs | 3 + src/meta/client/src/message.rs | 4 +- src/meta/kvapi/src/kvapi/api.rs | 3 +- src/meta/kvapi/src/kvapi/message.rs | 3 - src/meta/kvapi/src/kvapi/mod.rs | 1 - src/meta/kvapi/src/kvapi/test_suite.rs | 91 +++++++++---------- src/meta/process/src/kv_processor.rs | 1 + .../raft-store/src/state_machine_api_ext.rs | 1 + .../tests/it/grpc/metasrv_connection_error.rs | 4 +- .../service/tests/it/grpc/metasrv_grpc_api.rs | 8 +- .../tests/it/grpc/metasrv_grpc_export.rs | 4 +- .../metasrv_grpc_kv_api_restart_cluster.rs | 8 +- .../tests/it/grpc/metasrv_grpc_kv_read_v1.rs | 10 +- .../tests/it/grpc/metasrv_grpc_watch.rs | 83 ++++++++++------- src/meta/store/src/lib.rs | 4 +- src/meta/types/src/operation.rs | 4 + src/meta/types/src/proto_ext/txn_ext.rs | 11 +-- .../src/background_service/compaction_job.rs | 6 +- src/query/management/src/client_session.rs | 6 +- .../management/src/cluster/cluster_api.rs | 46 ++++++++-- .../management/src/cluster/cluster_mgr.rs | 42 ++------- src/query/management/src/role/role_mgr.rs | 10 +- src/query/management/src/serde/pb_serde.rs | 4 +- .../management/src/setting/setting_mgr.rs | 8 +- src/query/management/src/user/user_mgr.rs | 8 +- src/query/management/tests/it/cluster.rs | 2 +- src/query/management/tests/it/role.rs | 6 +- src/query/management/tests/it/user.rs | 18 ++-- src/query/service/src/clusters/cluster.rs | 2 +- 43 files changed, 244 insertions(+), 233 deletions(-) diff --git a/src/meta/api/src/background_api_impl.rs b/src/meta/api/src/background_api_impl.rs index 43355dc160d6..7dacefcf1beb 100644 --- a/src/meta/api/src/background_api_impl.rs +++ b/src/meta/api/src/background_api_impl.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::time::Instant; + use chrono::Utc; use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::background::background_job_id_ident::BackgroundJobId; @@ -40,7 +42,6 @@ use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::Key; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqValue; use databend_common_meta_types::MatchSeq::Any; use databend_common_meta_types::MetaError; @@ -48,6 +49,7 @@ use databend_common_meta_types::MetaSpec; use databend_common_meta_types::Operation; use databend_common_meta_types::SeqV; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use fastrace::func_name; use futures::TryStreamExt; use log::debug; @@ -222,18 +224,18 @@ impl> BackgroundApi for KV { let meta = req.task_info.clone(); let resp = self - .upsert_kv(UpsertKVReq::new( + .upsert_kv(UpsertKV::new( name_key.to_string_key().as_str(), Any, Operation::Update(serialize_struct(&meta)?), - Some(MetaSpec::new_expire(req.expire_at)), + Some(MetaSpec::new_ttl(req.ttl)), )) .await?; // confirm a successful update assert!(resp.is_changed()); Ok(UpdateBackgroundTaskReply { last_updated: Utc::now(), - expire_at: req.expire_at, + expire_at: Instant::now() + req.ttl, }) } diff --git a/src/meta/api/src/background_api_test_suite.rs b/src/meta/api/src/background_api_test_suite.rs index 26ffe1f32ec4..9a19cb86791e 100644 --- a/src/meta/api/src/background_api_test_suite.rs +++ b/src/meta/api/src/background_api_test_suite.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::time::Duration; + use chrono::DateTime; use chrono::Utc; use databend_common_meta_app::background::BackgroundJobIdent; @@ -127,13 +129,11 @@ impl BackgroundApiTestSuite { info!("--- create a background task"); let create_on = Utc::now(); - // expire after 5 secs - let expire_at = create_on + chrono::Duration::seconds(5); { let req = UpdateBackgroundTaskReq { task_name: task_ident.clone(), task_info: new_background_task(BackgroundTaskState::STARTED, create_on), - expire_at: expire_at.timestamp() as u64, + ttl: Duration::from_secs(5), }; let res = mt.update_background_task(req).await; @@ -155,7 +155,7 @@ impl BackgroundApiTestSuite { let req = UpdateBackgroundTaskReq { task_name: task_ident.clone(), task_info: new_background_task(BackgroundTaskState::DONE, create_on), - expire_at: expire_at.timestamp() as u64, + ttl: Duration::from_secs(5), }; let res = mt.update_background_task(req).await; diff --git a/src/meta/api/src/kv_pb_api/codec.rs b/src/meta/api/src/kv_pb_api/codec.rs index 0eb00bacb35e..d7f182cb4019 100644 --- a/src/meta/api/src/kv_pb_api/codec.rs +++ b/src/meta/api/src/kv_pb_api/codec.rs @@ -36,7 +36,9 @@ where T: FromToProto { Ok(Operation::Update(buf)) } Operation::Delete => Ok(Operation::Delete), - Operation::AsIs => Ok(Operation::AsIs), + _ => { + unreachable!("Operation::AsIs is not supported") + } } } diff --git a/src/meta/api/src/kv_pb_api/mod.rs b/src/meta/api/src/kv_pb_api/mod.rs index 67b01bea97fb..1e1ced7223b6 100644 --- a/src/meta/api/src/kv_pb_api/mod.rs +++ b/src/meta/api/src/kv_pb_api/mod.rs @@ -516,13 +516,13 @@ mod tests { use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::UpsertKVReply; - use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::seq_value::SeqValue; use databend_common_meta_types::MetaError; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; + use databend_common_meta_types::UpsertKV; use databend_common_proto_conv::FromToProto; use futures::StreamExt; use futures::TryStreamExt; @@ -541,7 +541,7 @@ mod tests { impl KVApi for FooKV { type Error = MetaError; - async fn upsert_kv(&self, _req: UpsertKVReq) -> Result { + async fn upsert_kv(&self, _req: UpsertKV) -> Result { unimplemented!() } diff --git a/src/meta/api/src/kv_pb_api/upsert_pb.rs b/src/meta/api/src/kv_pb_api/upsert_pb.rs index cb7614f02450..6f1cbabe05d6 100644 --- a/src/meta/api/src/kv_pb_api/upsert_pb.rs +++ b/src/meta/api/src/kv_pb_api/upsert_pb.rs @@ -90,10 +90,6 @@ impl UpsertPB { } } - pub fn with_expire_sec(self, expire_at_sec: u64) -> Self { - self.with(MetaSpec::new_expire(expire_at_sec)) - } - /// Set the time to last for the value. /// When the ttl is passed, the value is deleted. pub fn with_ttl(self, ttl: Duration) -> Self { diff --git a/src/meta/api/src/name_id_value_api.rs b/src/meta/api/src/name_id_value_api.rs index 6309fe40330f..46117cb9cf29 100644 --- a/src/meta/api/src/name_id_value_api.rs +++ b/src/meta/api/src/name_id_value_api.rs @@ -388,12 +388,12 @@ mod tests { use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::UpsertKVReply; - use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MetaError; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; + use databend_common_meta_types::UpsertKV; use databend_common_proto_conv::FromToProto; use futures::StreamExt; use prost::Message; @@ -408,7 +408,7 @@ mod tests { impl KVApi for Foo { type Error = MetaError; - async fn upsert_kv(&self, _req: UpsertKVReq) -> Result { + async fn upsert_kv(&self, _req: UpsertKV) -> Result { unimplemented!() } diff --git a/src/meta/api/src/schema_api_test_suite.rs b/src/meta/api/src/schema_api_test_suite.rs index daa218786f87..eb4cb386e32b 100644 --- a/src/meta/api/src/schema_api_test_suite.rs +++ b/src/meta/api/src/schema_api_test_suite.rs @@ -130,7 +130,6 @@ use databend_common_meta_app::tenant::ToTenant; use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaError; use databend_common_meta_types::Operation; @@ -238,7 +237,7 @@ async fn upsert_test_data( value: Vec, ) -> Result { let res = kv_api - .upsert_kv(UpsertKVReq { + .upsert_kv(UpsertKV { key: key.to_string_key(), seq: MatchSeq::GE(0), value: Operation::Update(value), @@ -255,7 +254,7 @@ async fn delete_test_data( key: &impl kvapi::Key, ) -> Result<(), KVAppError> { let _res = kv_api - .upsert_kv(UpsertKVReq { + .upsert_kv(UpsertKV { key: key.to_string_key(), seq: MatchSeq::GE(0), value: Operation::Delete, diff --git a/src/meta/api/src/util.rs b/src/meta/api/src/util.rs index 40ea4f39887c..569ce2a47dac 100644 --- a/src/meta/api/src/util.rs +++ b/src/meta/api/src/util.rs @@ -25,7 +25,6 @@ use databend_common_meta_app::primitive::Id; use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::TableNameIdent; use databend_common_meta_kvapi::kvapi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::txn_condition::Target; use databend_common_meta_types::ConditionResult; @@ -40,6 +39,7 @@ use databend_common_meta_types::TxnGetResponse; use databend_common_meta_types::TxnOp; use databend_common_meta_types::TxnOpResponse; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use databend_common_proto_conv::FromToProto; use log::debug; @@ -200,7 +200,7 @@ pub async fn fetch_id( generator: T, ) -> Result { let res = kv_api - .upsert_kv(UpsertKVReq { + .upsert_kv(UpsertKV { key: generator.to_string_key(), seq: MatchSeq::GE(0), value: Operation::Update(b"".to_vec()), diff --git a/src/meta/app/src/background/background_task.rs b/src/meta/app/src/background/background_task.rs index 03814caddc28..4fcb306c5850 100644 --- a/src/meta/app/src/background/background_task.rs +++ b/src/meta/app/src/background/background_task.rs @@ -16,6 +16,7 @@ use std::fmt; use std::fmt::Display; use std::fmt::Formatter; use std::time::Duration; +use std::time::Instant; use chrono::DateTime; use chrono::Utc; @@ -156,7 +157,7 @@ impl BackgroundTaskInfo { pub struct UpdateBackgroundTaskReq { pub task_name: BackgroundTaskIdent, pub task_info: BackgroundTaskInfo, - pub expire_at: u64, + pub ttl: Duration, } impl Display for UpdateBackgroundTaskReq { @@ -176,7 +177,7 @@ impl Display for UpdateBackgroundTaskReq { #[derive(Clone, Debug, PartialEq, Eq)] pub struct UpdateBackgroundTaskReply { pub last_updated: DateTime, - pub expire_at: u64, + pub expire_at: Instant, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/src/meta/binaries/meta/kvapi.rs b/src/meta/binaries/meta/kvapi.rs index ae20fae1cc57..64ef2b5b5b00 100644 --- a/src/meta/binaries/meta/kvapi.rs +++ b/src/meta/binaries/meta/kvapi.rs @@ -15,15 +15,15 @@ use std::sync::Arc; use databend_common_meta_kvapi::kvapi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MetaError; use databend_common_meta_types::MetaSpec; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::With; use databend_meta::configs::Config; pub enum KvApiCommand { Get(String), - Upsert(UpsertKVReq), + Upsert(UpsertKV), MGet(Vec), List(String), } @@ -36,7 +36,7 @@ impl KvApiCommand { return Err("The number of keys must be 1".to_string()); } - let req = UpsertKVReq::update(config.key[0].as_str(), config.value.as_bytes()); + let req = UpsertKV::update(config.key[0].as_str(), config.value.as_bytes()); let req = if let Some(expire_after) = config.expire_after { req.with(MetaSpec::new_ttl(std::time::Duration::from_secs( @@ -52,7 +52,7 @@ impl KvApiCommand { if config.key.len() != 1 { return Err("The number of keys must be 1".to_string()); } - Self::Upsert(UpsertKVReq::delete(&config.key[0])) + Self::Upsert(UpsertKV::delete(&config.key[0])) } "get" => { if config.key.len() != 1 { diff --git a/src/meta/binaries/metabench/main.rs b/src/meta/binaries/metabench/main.rs index ef3c9808daf1..8cfc2fa97420 100644 --- a/src/meta/binaries/metabench/main.rs +++ b/src/meta/binaries/metabench/main.rs @@ -41,10 +41,10 @@ use databend_common_meta_app::tenant::Tenant; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::Operation; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use databend_common_tracing::init_logging; use databend_common_tracing::FileConfig; use databend_common_tracing::StderrConfig; @@ -171,7 +171,7 @@ async fn benchmark_upsert(client: &Arc, prefix: u64, client_num: u let value = Operation::Update(node_key().as_bytes().to_vec()); let res = client - .upsert_kv(UpsertKVReq::new(node_key(), seq, value, None)) + .upsert_kv(UpsertKV::new(node_key(), seq, value, None)) .await; print_res(i, "upsert_kv", &res); diff --git a/src/meta/binaries/metaverifier/main.rs b/src/meta/binaries/metaverifier/main.rs index 04a1ee43043a..794adbb427bf 100644 --- a/src/meta/binaries/metaverifier/main.rs +++ b/src/meta/binaries/metaverifier/main.rs @@ -31,9 +31,9 @@ use databend_common_exception::Result; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::Operation; +use databend_common_meta_types::UpsertKV; use databend_common_tracing::init_logging; use databend_common_tracing::FileConfig; use databend_common_tracing::StderrConfig; @@ -229,7 +229,7 @@ async fn verifier( let value = Operation::Update(node_key.as_bytes().to_vec()); let _res = client - .upsert_kv(UpsertKVReq::new(&node_key, seq, value, None)) + .upsert_kv(UpsertKV::new(&node_key, seq, value, None)) .await?; let n: u64 = rng.gen_range(0..=100); @@ -238,7 +238,7 @@ async fn verifier( let value = Operation::Delete; let _res = client - .upsert_kv(UpsertKVReq::new(&node_key, seq, value, None)) + .upsert_kv(UpsertKV::new(&node_key, seq, value, None)) .await?; } else { kv.insert(node_key); diff --git a/src/meta/client/src/grpc_action.rs b/src/meta/client/src/grpc_action.rs index d9c4eeb2866a..c2e99fa7ddb6 100644 --- a/src/meta/client/src/grpc_action.rs +++ b/src/meta/client/src/grpc_action.rs @@ -23,7 +23,6 @@ use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReply; use databend_common_meta_kvapi::kvapi::MGetKVReq; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::RaftRequest; @@ -33,6 +32,7 @@ use databend_common_meta_types::protobuf::WatchResponse; use databend_common_meta_types::InvalidArgument; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use log::debug; use tonic::codegen::BoxStream; use tonic::Request; @@ -52,7 +52,7 @@ pub trait RequestFor: Clone + fmt::Debug { #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, derive_more::From)] pub enum MetaGrpcReq { - UpsertKV(UpsertKVReq), + UpsertKV(UpsertKV), } impl TryInto for Request { @@ -177,7 +177,7 @@ impl RequestFor for Streamed { type Reply = BoxStream; } -impl RequestFor for UpsertKVReq { +impl RequestFor for UpsertKV { type Reply = UpsertKVReply; } diff --git a/src/meta/client/src/kv_api_impl.rs b/src/meta/client/src/kv_api_impl.rs index fa03eefc9319..90dda6a9ee52 100644 --- a/src/meta/client/src/kv_api_impl.rs +++ b/src/meta/client/src/kv_api_impl.rs @@ -17,10 +17,10 @@ use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReq; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MetaError; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use futures::StreamExt; use futures::TryStreamExt; @@ -32,7 +32,7 @@ impl kvapi::KVApi for ClientHandle { type Error = MetaError; #[fastrace::trace] - async fn upsert_kv(&self, act: UpsertKVReq) -> Result { + async fn upsert_kv(&self, act: UpsertKV) -> Result { let reply = self.request(act).await?; Ok(reply) } diff --git a/src/meta/client/src/lib.rs b/src/meta/client/src/lib.rs index 4fe33fab7f89..7a6ed44a5c56 100644 --- a/src/meta/client/src/lib.rs +++ b/src/meta/client/src/lib.rs @@ -110,6 +110,9 @@ pub static METACLI_COMMIT_SEMVER: LazyLock = LazyLock::new(|| { /// require the client to call kv_read_v1 for get/mget/list, /// which is added `2024-01-07: since 1.2.287` /// +/// - 2024-11-2*: since 1.2.6** +/// 👥 client: remove use of `Operation::AsIs` +/// /// Server feature set: /// ```yaml /// server_features: diff --git a/src/meta/client/src/message.rs b/src/meta/client/src/message.rs index 5bb710210946..cccbd3c73d43 100644 --- a/src/meta/client/src/message.rs +++ b/src/meta/client/src/message.rs @@ -20,7 +20,6 @@ use databend_common_base::runtime::TrackingPayload; use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReq; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::ExportedChunk; @@ -31,6 +30,7 @@ use databend_common_meta_types::MetaClientError; use databend_common_meta_types::MetaError; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use fastrace::Span; use tonic::codegen::BoxStream; @@ -81,7 +81,7 @@ pub enum Request { StreamList(Streamed), /// Update or insert KV - Upsert(UpsertKVReq), + Upsert(UpsertKV), /// Run a transaction on remote Txn(TxnRequest), diff --git a/src/meta/kvapi/src/kvapi/api.rs b/src/meta/kvapi/src/kvapi/api.rs index 62c62c9f6a05..1e516ac807d6 100644 --- a/src/meta/kvapi/src/kvapi/api.rs +++ b/src/meta/kvapi/src/kvapi/api.rs @@ -32,7 +32,6 @@ use crate::kvapi::GetKVReply; use crate::kvapi::ListKVReply; use crate::kvapi::MGetKVReply; use crate::kvapi::UpsertKVReply; -use crate::kvapi::UpsertKVReq; /// Build an API impl instance or a cluster of API impl #[async_trait] @@ -139,7 +138,7 @@ pub trait KVApi: Send + Sync { impl + Send + Sync> kvapi::KVApi for T { type Error = U::Error; - async fn upsert_kv(&self, act: UpsertKVReq) -> Result { + async fn upsert_kv(&self, act: UpsertKV) -> Result { self.deref().upsert_kv(act).await } diff --git a/src/meta/kvapi/src/kvapi/message.rs b/src/meta/kvapi/src/kvapi/message.rs index 8f37e5cb3cf9..de43587ce804 100644 --- a/src/meta/kvapi/src/kvapi/message.rs +++ b/src/meta/kvapi/src/kvapi/message.rs @@ -17,11 +17,8 @@ use std::fmt::Formatter; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::Change; -use databend_common_meta_types::UpsertKV; use databend_common_meta_types::VecDisplay; -pub type UpsertKVReq = UpsertKV; - #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct GetKVReq { pub key: String, diff --git a/src/meta/kvapi/src/kvapi/mod.rs b/src/meta/kvapi/src/kvapi/mod.rs index d083d1598fb3..c73b534b3d10 100644 --- a/src/meta/kvapi/src/kvapi/mod.rs +++ b/src/meta/kvapi/src/kvapi/mod.rs @@ -48,7 +48,6 @@ pub use message::ListKVReq; pub use message::MGetKVReply; pub use message::MGetKVReq; pub use message::UpsertKVReply; -pub use message::UpsertKVReq; pub use pair::BasicPair; pub use pair::Pair; pub use pair::SeqPair; diff --git a/src/meta/kvapi/src/kvapi/test_suite.rs b/src/meta/kvapi/src/kvapi/test_suite.rs index bfc8c9918dd0..6e62bbcd66a7 100644 --- a/src/meta/kvapi/src/kvapi/test_suite.rs +++ b/src/meta/kvapi/src/kvapi/test_suite.rs @@ -36,6 +36,7 @@ use databend_common_meta_types::TxnOpResponse; use databend_common_meta_types::TxnPutResponse; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::With; use fastrace::func_name; use fastrace::func_path; @@ -43,7 +44,6 @@ use log::debug; use log::info; use crate::kvapi; -use crate::kvapi::UpsertKVReq; pub struct TestSuite {} @@ -108,7 +108,7 @@ impl kvapi::TestSuite { info!("--- kvapi::KVApiTestSuite::kv_write_read() start"); { // write - let res = kv.upsert_kv(UpsertKVReq::update("foo", b"bar")).await?; + let res = kv.upsert_kv(UpsertKV::update("foo", b"bar")).await?; assert_eq!(None, res.prev); assert_eq!(Some(SeqV::with_meta(1, None, b"bar".to_vec())), res.result); } @@ -116,7 +116,7 @@ impl kvapi::TestSuite { { // write fails with unmatched seq let res = kv - .upsert_kv(UpsertKVReq::update("foo", b"bar").with(MatchSeq::Exact(2))) + .upsert_kv(UpsertKV::update("foo", b"bar").with(MatchSeq::Exact(2))) .await?; assert_eq!( ( @@ -131,7 +131,7 @@ impl kvapi::TestSuite { { // write done with matching seq let res = kv - .upsert_kv(UpsertKVReq::update("foo", b"wow").with(MatchSeq::Exact(1))) + .upsert_kv(UpsertKV::update("foo", b"wow").with(MatchSeq::Exact(1))) .await?; assert_eq!( Some(SeqV::with_meta(1, None, b"bar".to_vec())), @@ -152,19 +152,14 @@ impl kvapi::TestSuite { pub async fn kv_delete(&self, kv: &KV) -> anyhow::Result<()> { info!("--- kvapi::KVApiTestSuite::kv_delete() start"); let test_key = "test_key"; - kv.upsert_kv(UpsertKVReq::update(test_key, b"v1")).await?; + kv.upsert_kv(UpsertKV::update(test_key, b"v1")).await?; let current = kv.get_kv(test_key).await?; if let Some(SeqV { seq, .. }) = current { // seq mismatch let wrong_seq = MatchSeq::Exact(seq + 1); let res = kv - .upsert_kv(UpsertKVReq::new( - test_key, - wrong_seq, - Operation::Delete, - None, - )) + .upsert_kv(UpsertKV::new(test_key, wrong_seq, Operation::Delete, None)) .await?; assert_eq!(res.prev, res.result); @@ -173,7 +168,7 @@ impl kvapi::TestSuite { // seq match let res = kv - .upsert_kv(UpsertKVReq::delete(test_key).with(MatchSeq::Exact(seq))) + .upsert_kv(UpsertKV::delete(test_key).with(MatchSeq::Exact(seq))) .await?; assert!(res.result.is_none()); @@ -185,17 +180,17 @@ impl kvapi::TestSuite { } // key not exist - let res = kv.upsert_kv(UpsertKVReq::delete("not exists")).await?; + let res = kv.upsert_kv(UpsertKV::delete("not exists")).await?; // dbg!("delete non-exist key", &res); assert_eq!(None, res.prev); assert_eq!(None, res.result); // do not care seq - let _res = kv.upsert_kv(UpsertKVReq::update(test_key, b"v2")).await?; + let _res = kv.upsert_kv(UpsertKV::update(test_key, b"v2")).await?; // dbg!("update with v2", &res); - let res = kv.upsert_kv(UpsertKVReq::delete(test_key)).await?; + let res = kv.upsert_kv(UpsertKV::delete(test_key)).await?; // dbg!("delete", &res); assert_eq!( @@ -212,31 +207,31 @@ impl kvapi::TestSuite { let test_key = "test_key_for_update"; let r = kv - .upsert_kv(UpsertKVReq::update(test_key, b"v1").with(MatchSeq::GE(1))) + .upsert_kv(UpsertKV::update(test_key, b"v1").with(MatchSeq::GE(1))) .await?; assert_eq!((None, None), (r.prev, r.result), "not changed"); - let r = kv.upsert_kv(UpsertKVReq::update(test_key, b"v1")).await?; + let r = kv.upsert_kv(UpsertKV::update(test_key, b"v1")).await?; assert_eq!(Some(SeqV::with_meta(1, None, b"v1".to_vec())), r.result); let seq = r.result.unwrap().seq; // unmatched seq let r = kv - .upsert_kv(UpsertKVReq::update(test_key, b"v2").with(MatchSeq::Exact(seq + 1))) + .upsert_kv(UpsertKV::update(test_key, b"v2").with(MatchSeq::Exact(seq + 1))) .await?; assert_eq!(Some(SeqV::with_meta(1, None, b"v1".to_vec())), r.prev); assert_eq!(Some(SeqV::with_meta(1, None, b"v1".to_vec())), r.result); // matched seq let r = kv - .upsert_kv(UpsertKVReq::update(test_key, b"v2").with(MatchSeq::Exact(seq))) + .upsert_kv(UpsertKV::update(test_key, b"v2").with(MatchSeq::Exact(seq))) .await?; assert_eq!(Some(SeqV::with_meta(1, None, b"v1".to_vec())), r.prev); assert_eq!(Some(SeqV::with_meta(2, None, b"v2".to_vec())), r.result); // blind update let r = kv - .upsert_kv(UpsertKVReq::update(test_key, b"v3").with(MatchSeq::GE(1))) + .upsert_kv(UpsertKV::update(test_key, b"v3").with(MatchSeq::GE(1))) .await?; assert_eq!(Some(SeqV::with_meta(2, None, b"v2".to_vec())), r.prev); assert_eq!(Some(SeqV::with_meta(3, None, b"v3".to_vec())), r.result); @@ -263,7 +258,7 @@ impl kvapi::TestSuite { let _res = kv .upsert_kv( - UpsertKVReq::update("k1", b"v1").with(MetaSpec::new_ttl(Duration::from_secs(2))), + UpsertKV::update("k1", b"v1").with(MetaSpec::new_ttl(Duration::from_secs(2))), ) .await?; // dbg!("upsert non expired k1", _res); @@ -292,7 +287,7 @@ impl kvapi::TestSuite { // dbg!("start upsert expired k1"); let _res = kv .upsert_kv( - UpsertKVReq::update("k1", b"v1") + UpsertKV::update("k1", b"v1") .with(MatchSeq::Exact(0)) .with(MetaSpec::new_expire(now_sec - 1)), ) @@ -301,7 +296,7 @@ impl kvapi::TestSuite { let _res = kv .upsert_kv( - UpsertKVReq::update("k2", b"v2") + UpsertKV::update("k2", b"v2") .with(MatchSeq::Exact(0)) .with(MetaSpec::new_ttl(Duration::from_secs(10))), ) @@ -334,7 +329,7 @@ impl kvapi::TestSuite { info!("--- update expire"); { kv.upsert_kv( - UpsertKVReq::update("k2", b"v2") + UpsertKV::update("k2", b"v2") .with(MatchSeq::Exact(3)) .with(MetaSpec::new_expire(now_sec - 1)), ) @@ -355,8 +350,7 @@ impl kvapi::TestSuite { let _res = kv .upsert_kv( - UpsertKVReq::update("k1", b"v1") - .with(MetaSpec::new_ttl(Duration::from_millis(2_000))), + UpsertKV::update("k1", b"v1").with(MetaSpec::new_ttl(Duration::from_millis(2_000))), ) .await?; @@ -384,17 +378,17 @@ impl kvapi::TestSuite { let now_sec = SeqV::<()>::now_sec(); - let r = kv.upsert_kv(UpsertKVReq::update(test_key, b"v1")).await?; + let r = kv.upsert_kv(UpsertKV::update(test_key, b"v1")).await?; assert_eq!(Some(SeqV::with_meta(1, None, b"v1".to_vec())), r.result); let seq = r.result.unwrap().seq; info!("--- mismatching seq does nothing"); let r = kv - .upsert_kv(UpsertKVReq::new( + .upsert_kv(UpsertKV::new( test_key, MatchSeq::Exact(seq + 1), - Operation::AsIs, + Operation::Update(b"v1".to_vec()), Some(MetaSpec::new_ttl(Duration::from_secs(20))), )) .await?; @@ -404,10 +398,10 @@ impl kvapi::TestSuite { info!("--- matching seq only update meta"); let r = kv - .upsert_kv(UpsertKVReq::new( + .upsert_kv(UpsertKV::new( test_key, MatchSeq::Exact(seq), - Operation::AsIs, + Operation::Update(b"v1".to_vec()), Some(MetaSpec::new_ttl(Duration::from_secs(20))), )) .await?; @@ -450,16 +444,15 @@ impl kvapi::TestSuite { let mut values = vec![]; { - kv.upsert_kv(UpsertKVReq::update("t", b"")).await?; + kv.upsert_kv(UpsertKV::update("t", b"")).await?; for i in 0..9 { let key = format!("__users/{}", i); let val = format!("val_{}", i); values.push(val.clone()); - kv.upsert_kv(UpsertKVReq::update(&key, val.as_bytes())) - .await?; + kv.upsert_kv(UpsertKV::update(&key, val.as_bytes())).await?; } - kv.upsert_kv(UpsertKVReq::update("v", b"")).await?; + kv.upsert_kv(UpsertKV::update("v", b"")).await?; } let res = kv.prefix_list_kv("__users/").await?; @@ -479,8 +472,8 @@ impl kvapi::TestSuite { pub async fn kv_mget(&self, kv: &KV) -> anyhow::Result<()> { info!("--- kvapi::KVApiTestSuite::kv_mget() start"); - kv.upsert_kv(UpsertKVReq::update("k1", b"v1")).await?; - kv.upsert_kv(UpsertKVReq::update("k2", b"v2")).await?; + kv.upsert_kv(UpsertKV::update("k1", b"v1")).await?; + kv.upsert_kv(UpsertKV::update("k2", b"v2")).await?; let res = kv.mget_kv(&["k1".to_string(), "k2".to_string()]).await?; assert_eq!(res, vec![ @@ -567,7 +560,7 @@ impl kvapi::TestSuite { let unmatch_keys = vec!["teskey4".to_string()]; for key in [match_keys.clone(), unmatch_keys.clone()].concat().iter() { - kv.upsert_kv(UpsertKVReq::update(key, b"v1")).await?; + kv.upsert_kv(UpsertKV::update(key, b"v1")).await?; let current = kv.get_kv(key).await?; assert!(current.is_some()); @@ -671,7 +664,7 @@ impl kvapi::TestSuite { let val1 = b"v1".to_vec(); // first insert k1 value - kv.upsert_kv(UpsertKVReq::update(k1, &val1)).await?; + kv.upsert_kv(UpsertKV::update(k1, &val1)).await?; // transaction by k1 condition let txn_key = k1.to_string(); @@ -707,7 +700,7 @@ impl kvapi::TestSuite { let k1 = "txn_2_K1"; let k2 = "txn_2_K2"; - kv.upsert_kv(UpsertKVReq::update(k1, b"v1")).await?; + kv.upsert_kv(UpsertKV::update(k1, b"v1")).await?; // transaction by k1 and k2 condition let txn_key1 = k1.to_string(); @@ -752,8 +745,8 @@ impl kvapi::TestSuite { let val2 = b"v1".to_vec(); // first insert k1 and k2 value - kv.upsert_kv(UpsertKVReq::update(k1, &val1)).await?; - kv.upsert_kv(UpsertKVReq::update(k2, &val2)).await?; + kv.upsert_kv(UpsertKV::update(k1, &val1)).await?; + kv.upsert_kv(UpsertKV::update(k2, &val2)).await?; // transaction by k1 and k2 condition let txn_key1 = k1.to_string(); @@ -865,7 +858,7 @@ impl kvapi::TestSuite { let val1_new = b"v1_new".to_vec(); // first insert k1 value - kv.upsert_kv(UpsertKVReq::update(k1, &val1)).await?; + kv.upsert_kv(UpsertKV::update(k1, &val1)).await?; // transaction by k1 condition let txn_key1 = k1.to_string(); @@ -966,7 +959,7 @@ impl kvapi::TestSuite { let key = || "txn_1_K1".to_string(); let val = || b"v1".to_vec(); - kv.upsert_kv(UpsertKVReq::update(key(), &val())).await?; + kv.upsert_kv(UpsertKV::update(key(), &val())).await?; let txn = TxnRequest { condition: vec![], @@ -1000,7 +993,7 @@ impl kvapi::TestSuite { let key = || "txn_1_K1".to_string(); let val = || b"v1".to_vec(); - kv.upsert_kv(UpsertKVReq::update(key(), &val())).await?; + kv.upsert_kv(UpsertKV::update(key(), &val())).await?; let txn = TxnRequest { condition: vec![], @@ -1038,7 +1031,7 @@ impl kvapi::TestSuite { let key = || "txn_1_K1".to_string(); let val = || b"v1".to_vec(); - kv.upsert_kv(UpsertKVReq::update(key(), &val())).await?; + kv.upsert_kv(UpsertKV::update(key(), &val())).await?; let txn = TxnRequest { condition: vec![], @@ -1073,19 +1066,19 @@ impl kvapi::TestSuite { ) -> anyhow::Result<()> { let mut values = vec![]; { - kv1.upsert_kv(UpsertKVReq::update("t", b"t")).await?; + kv1.upsert_kv(UpsertKV::update("t", b"t")).await?; for i in 0..9 { let key = format!("__users/{}", i); let val = format!("val_{}", i); values.push(val.clone()); info!("--- Start upsert-kv: {}", key); - kv1.upsert_kv(UpsertKVReq::update(&key, val.as_bytes())) + kv1.upsert_kv(UpsertKV::update(&key, val.as_bytes())) .await?; info!("--- Done upsert-kv: {}", key); } - kv1.upsert_kv(UpsertKVReq::update("v", b"v")).await?; + kv1.upsert_kv(UpsertKV::update("v", b"v")).await?; } info!("--- test get on other node"); diff --git a/src/meta/process/src/kv_processor.rs b/src/meta/process/src/kv_processor.rs index 0a9dab5f0647..cad80935c285 100644 --- a/src/meta/process/src/kv_processor.rs +++ b/src/meta/process/src/kv_processor.rs @@ -187,6 +187,7 @@ where F: Fn(&str, Vec) -> Result, anyhow::Error> })) } Operation::Delete => Ok(None), + #[allow(deprecated)] Operation::AsIs => Ok(None), } } diff --git a/src/meta/raft-store/src/state_machine_api_ext.rs b/src/meta/raft-store/src/state_machine_api_ext.rs index 5da204110602..5679db88c35b 100644 --- a/src/meta/raft-store/src/state_machine_api_ext.rs +++ b/src/meta/raft-store/src/state_machine_api_ext.rs @@ -60,6 +60,7 @@ pub trait StateMachineApiExt: StateMachineApi { .await? } Operation::Delete => self.map_mut().set(upsert_kv.key.clone(), None).await?, + #[allow(deprecated)] Operation::AsIs => { MapApiExt::update_meta(self.map_mut(), upsert_kv.key.clone(), kv_meta.clone()) .await? diff --git a/src/meta/service/tests/it/grpc/metasrv_connection_error.rs b/src/meta/service/tests/it/grpc/metasrv_connection_error.rs index 579adc09e009..95e9b8dc6556 100644 --- a/src/meta/service/tests/it/grpc/metasrv_connection_error.rs +++ b/src/meta/service/tests/it/grpc/metasrv_connection_error.rs @@ -23,8 +23,8 @@ use databend_common_base::base::Stoppable; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MetaClientError; +use databend_common_meta_types::UpsertKV; use log::info; use test_harness::test; @@ -162,7 +162,7 @@ async fn test_write_read(client: &Arc, key: impl Display) -> anyho info!("--- test write/read: {}", key); let k = key.to_string(); - let res = client.upsert_kv(UpsertKVReq::update(&k, &b(&k))).await?; + let res = client.upsert_kv(UpsertKV::update(&k, &b(&k))).await?; info!("--- upsert {} res: {:?}", k, res); diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs index c125903aa299..e685fa02d1dc 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs @@ -19,8 +19,8 @@ use databend_common_base::base::tokio; use databend_common_base::base::Stoppable; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqV; +use databend_common_meta_types::UpsertKV; use log::debug; use log::info; use pretty_assertions::assert_eq; @@ -46,7 +46,7 @@ async fn test_restart() -> anyhow::Result<()> { info!("--- upsert kv"); { - let res = client.upsert_kv(UpsertKVReq::update("foo", b"bar")).await; + let res = client.upsert_kv(UpsertKV::update("foo", b"bar")).await; debug!("set kv res: {:?}", res); let res = res?; @@ -171,7 +171,7 @@ async fn test_join() -> anyhow::Result<()> { let k = format!("join-{}", i); let res = cli - .upsert_kv(UpsertKVReq::update(k.as_str(), k.as_bytes())) + .upsert_kv(UpsertKV::update(k.as_str(), k.as_bytes())) .await; debug!("set kv res: {:?}", res); @@ -240,7 +240,7 @@ async fn test_auto_sync_addr() -> anyhow::Result<()> { { let k = "join-k".to_string(); - let res = client.upsert_kv(UpsertKVReq::update(&k, &b(&k))).await; + let res = client.upsert_kv(UpsertKV::update(&k, &b(&k))).await; debug!("set kv res: {:?}", res); let res = res?; diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index a81646be9af7..07d62c64ef61 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -16,8 +16,8 @@ use std::time::Duration; use databend_common_base::base::tokio::time::sleep; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf as pb; +use databend_common_meta_types::UpsertKV; use log::info; use pretty_assertions::assert_eq; use regex::Regex; @@ -42,7 +42,7 @@ async fn test_export() -> anyhow::Result<()> { info!("--- upsert kv"); { for k in ["foo", "bar", "wow"] { - client.upsert_kv(UpsertKVReq::update(k, &b(k))).await?; + client.upsert_kv(UpsertKV::update(k, &b(k))).await?; } } diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs index 0368eb5d4f5d..f0729213704c 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs @@ -21,7 +21,7 @@ use databend_common_base::base::Stoppable; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; +use databend_common_meta_types::UpsertKV; use log::info; use test_harness::test; @@ -52,7 +52,7 @@ async fn test_kv_api_restart_cluster_write_read() -> anyhow::Result<()> { let client = tc.grpc_client().await?; let k = make_key(tc, key_suffix); - let res = client.upsert_kv(UpsertKVReq::update(&k, &b(&k))).await?; + let res = client.upsert_kv(UpsertKV::update(&k, &b(&k))).await?; info!("--- upsert res: {:?}", res); @@ -138,11 +138,11 @@ async fn test_kv_api_restart_cluster_token_expired() -> anyhow::Result<()> { for (i, tc) in tcs.iter().enumerate() { let k = make_key(tc, key_suffix); if i == 0 { - let res = client.upsert_kv(UpsertKVReq::update(&k, &b(&k))).await?; + let res = client.upsert_kv(UpsertKV::update(&k, &b(&k))).await?; info!("--- upsert res: {:?}", res); } else { let client = tc.grpc_client().await.unwrap(); - let res = client.upsert_kv(UpsertKVReq::update(&k, &b(&k))).await?; + let res = client.upsert_kv(UpsertKV::update(&k, &b(&k))).await?; info!("--- upsert res: {:?}", res); } diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs index f510c5563cd4..48b4d987f001 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs @@ -22,11 +22,11 @@ use databend_common_meta_client::Streamed; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReq; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf as pb; use databend_common_meta_types::protobuf::KvMeta; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MetaSpec; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::With; use futures::stream::StreamExt; use futures::TryStreamExt; @@ -130,10 +130,10 @@ async fn initialize_kvs(client: &Arc) -> anyhow::Result<()> { info!("--- prepare keys: a(meta),c,c1,c2"); let updates = vec![ - UpsertKVReq::insert("a", &b("a")).with(MetaSpec::new_ttl(Duration::from_secs(10))), - UpsertKVReq::insert("c", &b("c")), - UpsertKVReq::insert("c1", &b("c1")), - UpsertKVReq::insert("c2", &b("c2")), + UpsertKV::insert("a", &b("a")).with(MetaSpec::new_ttl(Duration::from_secs(10))), + UpsertKV::insert("c", &b("c")), + UpsertKV::insert("c1", &b("c1")), + UpsertKV::insert("c2", &b("c2")), ]; for update in updates { diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs index 51030c2b4e6d..d7c02d9e68ad 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs @@ -23,7 +23,6 @@ use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf::watch_request::FilterType; use databend_common_meta_types::protobuf::Event; use databend_common_meta_types::protobuf::KvMeta; @@ -38,6 +37,7 @@ use databend_common_meta_types::Operation; use databend_common_meta_types::TxnCondition; use databend_common_meta_types::TxnDeleteByPrefixRequest; use databend_common_meta_types::TxnOp; +use databend_common_meta_types::UpsertKV; use databend_meta::meta_service::MetaNode; use log::info; use test_harness::test; @@ -48,7 +48,7 @@ async fn test_watch_main( addr: String, watch: WatchRequest, mut watch_events: Vec, - updates: Vec, + updates: Vec, ) -> anyhow::Result<()> { let client = make_client(&addr)?; let mut watch_stream = client.request(watch).await?; @@ -171,12 +171,12 @@ async fn test_watch() -> anyhow::Result<()> { seq = 4; // update kv let updates = vec![ - UpsertKVReq::new("a", MatchSeq::GE(0), Operation::Update(val_a), None), + UpsertKV::new("a", MatchSeq::GE(0), Operation::Update(val_a), None), // upsert key z, because z in key_end and the range is [key_start, key_end), so key z MUST not be notified in watch events. - UpsertKVReq::new("z", MatchSeq::GE(0), Operation::Update(val_z), None), - UpsertKVReq::new("b", MatchSeq::GE(0), Operation::Update(val_b), None), - UpsertKVReq::new("b", MatchSeq::GE(0), Operation::Update(val_new), None), - UpsertKVReq::new("b", MatchSeq::GE(0), Operation::Delete, None), + UpsertKV::new("z", MatchSeq::GE(0), Operation::Update(val_z), None), + UpsertKV::new("b", MatchSeq::GE(0), Operation::Update(val_b), None), + UpsertKV::new("b", MatchSeq::GE(0), Operation::Update(val_new), None), + UpsertKV::new("b", MatchSeq::GE(0), Operation::Delete, None), ]; test_watch_main(addr.clone(), watch, watch_events, updates).await?; } @@ -213,10 +213,10 @@ async fn test_watch() -> anyhow::Result<()> { // update and delete twice let updates = vec![ - UpsertKVReq::new(key_str, MatchSeq::GE(0), Operation::Update(val), None), - UpsertKVReq::new(key_str, MatchSeq::GE(0), Operation::Delete, None), - UpsertKVReq::new(key_str, MatchSeq::GE(0), Operation::Update(val_new), None), - UpsertKVReq::new(key_str, MatchSeq::GE(0), Operation::Delete, None), + UpsertKV::new(key_str, MatchSeq::GE(0), Operation::Update(val), None), + UpsertKV::new(key_str, MatchSeq::GE(0), Operation::Delete, None), + UpsertKV::new(key_str, MatchSeq::GE(0), Operation::Update(val_new), None), + UpsertKV::new(key_str, MatchSeq::GE(0), Operation::Delete, None), ]; test_watch_main(addr.clone(), watch, watch_events, updates).await?; } @@ -230,8 +230,8 @@ async fn test_watch() -> anyhow::Result<()> { let client = make_client(&addr)?; let updates = vec![ - UpsertKVReq::update(delete_key, &b(delete_key)), - UpsertKVReq::update(watch_delete_by_prefix_key, &b(watch_delete_by_prefix_key)), + UpsertKV::update(delete_key, &b(delete_key)), + UpsertKV::update(watch_delete_by_prefix_key, &b(watch_delete_by_prefix_key)), ]; for update in updates { @@ -332,24 +332,30 @@ async fn test_watch_expired_events() -> anyhow::Result<()> { for i in 0..(32 + 1) { let k = format!("w_auto_gc_{}", i); txn.if_then - .push(TxnOp::put_with_expire(&k, b(&k), Some(expire - 10))); + .push(TxnOp::put_with_ttl(&k, b(&k), Some(Duration::from_secs(1)))); } - // Expired key wont be cleaned when they are read, although read returns None. + // Expired key won't be cleaned when they are read, although read returns None. - txn.if_then - .push(TxnOp::put_with_expire("w_b1", b("w_b1"), Some(expire - 5))); - txn.if_then - .push(TxnOp::put_with_expire("w_b2", b("w_b2"), Some(expire - 5))); - txn.if_then.push(TxnOp::put_with_expire( + txn.if_then.push(TxnOp::put_with_ttl( + "w_b1", + b("w_b1"), + Some(Duration::from_secs(6)), + )); + txn.if_then.push(TxnOp::put_with_ttl( + "w_b2", + b("w_b2"), + Some(Duration::from_secs(6)), + )); + txn.if_then.push(TxnOp::put_with_ttl( "w_b3a", b("w_b3a"), - Some(expire - 5), + Some(Duration::from_secs(6)), )); - txn.if_then.push(TxnOp::put_with_expire( + txn.if_then.push(TxnOp::put_with_ttl( "w_b3b", b("w_b3b"), - Some(expire + 5), + Some(Duration::from_secs(11)), )); client.transaction(txn).await?; @@ -368,7 +374,7 @@ async fn test_watch_expired_events() -> anyhow::Result<()> { }; info!("--- sleep {} for expiration", expire - now_sec); - tokio::time::sleep(Duration::from_secs(expire - now_sec)).await; + tokio::time::sleep(Duration::from_secs(10)).await; info!("--- apply another txn in another thread to override keys"); { @@ -397,7 +403,7 @@ async fn test_watch_expired_events() -> anyhow::Result<()> { // 32 expired keys are auto cleaned. for i in 0..(32 + 1) { let k = format!("w_auto_gc_{}", i); - let want = del_event(&k, 1 + i, &k, Some(KvMeta::new_expire(expire - 10))); + let want = del_event(&k, 1 + i, &k, Some(KvMeta::new_expire(now_sec + 1))); let msg = client_stream.message().await?.unwrap(); assert_eq!(Some(want), msg.event); } @@ -406,31 +412,46 @@ async fn test_watch_expired_events() -> anyhow::Result<()> { let seq = 34; let watch_events = vec![ - del_event("w_b1", seq, "w_b1", Some(KvMeta::new_expire(expire - 5))), // expired + del_event("w_b1", seq, "w_b1", Some(KvMeta::new_expire(now_sec + 6))), // expired del_event( "w_b2", seq + 1, "w_b2", - Some(KvMeta::new_expire(expire - 5)), + Some(KvMeta::new_expire(now_sec + 6)), ), // expired del_event( "w_b3a", seq + 2, "w_b3a", - Some(KvMeta::new_expire(expire - 5)), + Some(KvMeta::new_expire(now_sec + 6)), ), // expired - add_event("w_b1", seq + 4, "w_b1_override", Some(KvMeta::default())), // override + add_event("w_b1", seq + 4, "w_b1_override", Some(KvMeta::default())), // override del_event( "w_b3b", seq + 3, "w_b3b", - Some(KvMeta::new_expire(expire + 5)), + Some(KvMeta::new_expire(now_sec + 16)), ), // expired ]; + // remove the millisecond part of expire_at + fn tidy(mut ev: Event) -> Event { + if let Some(ref mut prev) = ev.prev { + if let Some(ref mut meta) = prev.meta { + meta.expire_at = meta.expire_at.map(|x| x / 1000 * 1000); + } + } + if let Some(ref mut current) = ev.current { + if let Some(ref mut meta) = current.meta { + meta.expire_at = meta.expire_at.map(|x| x / 1000 * 1000); + } + } + ev + } + for ev in watch_events { let msg = client_stream.message().await?.unwrap(); - assert_eq!(Some(ev), msg.event); + assert_eq!(Some(tidy(ev)), msg.event.map(tidy)); } } diff --git a/src/meta/store/src/lib.rs b/src/meta/store/src/lib.rs index 2daa684e9cb0..e60840e42d8e 100644 --- a/src/meta/store/src/lib.rs +++ b/src/meta/store/src/lib.rs @@ -24,12 +24,12 @@ use databend_common_meta_embedded::MemMeta; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; use databend_common_meta_types::MetaError; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use log::info; use tokio_stream::Stream; @@ -85,7 +85,7 @@ impl MetaStore { impl kvapi::KVApi for MetaStore { type Error = MetaError; - async fn upsert_kv(&self, act: UpsertKVReq) -> Result { + async fn upsert_kv(&self, act: UpsertKV) -> Result { match self { MetaStore::L(x) => x.upsert_kv(act).await, MetaStore::R(x) => x.upsert_kv(act).await, diff --git a/src/meta/types/src/operation.rs b/src/meta/types/src/operation.rs index 62d136c1b826..ce5772ffaee7 100644 --- a/src/meta/types/src/operation.rs +++ b/src/meta/types/src/operation.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(deprecated)] + //! This crate defines data types used in meta data storage service. use std::fmt::Debug; @@ -24,6 +26,7 @@ pub type MetaId = u64; pub enum Operation { Update(T), Delete, + #[deprecated(note = "not supported and will be removed from server side")] AsIs, } @@ -32,6 +35,7 @@ impl Debug for Operation { match self { Operation::Update(_) => f.debug_tuple("Update").field(&"[binary]").finish(), Operation::Delete => f.debug_tuple("Delete").finish(), + #[allow(deprecated)] Operation::AsIs => f.debug_tuple("AsIs").finish(), } } diff --git a/src/meta/types/src/proto_ext/txn_ext.rs b/src/meta/types/src/proto_ext/txn_ext.rs index 1c9e579b13d2..ea65c18b54ac 100644 --- a/src/meta/types/src/proto_ext/txn_ext.rs +++ b/src/meta/types/src/proto_ext/txn_ext.rs @@ -49,21 +49,12 @@ impl pb::TxnCondition { impl pb::TxnOp { /// Create a txn operation that puts a record. pub fn put(key: impl ToString, value: Vec) -> pb::TxnOp { - Self::put_with_expire(key, value, None) - } - - /// Create a txn operation that puts a record with expiration time. - pub fn put_with_expire( - key: impl ToString, - value: Vec, - expire_at: Option, - ) -> pb::TxnOp { pb::TxnOp { request: Some(pb::txn_op::Request::Put(pb::TxnPutRequest { key: key.to_string(), value, prev_value: true, - expire_at, + expire_at: None, ttl_ms: None, })), } diff --git a/src/query/ee/src/background_service/compaction_job.rs b/src/query/ee/src/background_service/compaction_job.rs index 3fc35b1c5e99..8e728687ab2f 100644 --- a/src/query/ee/src/background_service/compaction_job.rs +++ b/src/query/ee/src/background_service/compaction_job.rs @@ -332,7 +332,7 @@ impl CompactionJob { .update_background_task(UpdateBackgroundTaskReq { task_name: task_ident.clone(), task_info: info.clone(), - expire_at: Utc::now().timestamp() as u64 + EXPIRE_SEC, + ttl: Duration::from_secs(EXPIRE_SEC), }) .await?; @@ -359,7 +359,7 @@ impl CompactionJob { .update_background_task(UpdateBackgroundTaskReq { task_name: task_ident, task_info: info.clone(), - expire_at: Utc::now().timestamp() as u64 + EXPIRE_SEC, + ttl: Duration::from_secs(EXPIRE_SEC), }) .await?; } @@ -370,7 +370,7 @@ impl CompactionJob { .update_background_task(UpdateBackgroundTaskReq { task_name: task_ident, task_info: info.clone(), - expire_at: Utc::now().timestamp() as u64 + EXPIRE_SEC, + ttl: Duration::from_secs(EXPIRE_SEC), }) .await?; } diff --git a/src/query/management/src/client_session.rs b/src/query/management/src/client_session.rs index 684edde84cf2..467fb0702b19 100644 --- a/src/query/management/src/client_session.rs +++ b/src/query/management/src/client_session.rs @@ -26,10 +26,10 @@ use databend_common_meta_app::principal::user_token_ident::TokenIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaError; use databend_common_meta_types::Operation; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::With; pub struct ClientSessionMgr { @@ -94,7 +94,7 @@ impl ClientSessionMgr { // simply ignore the result self.kv_api - .upsert_kv(UpsertKVReq::new( + .upsert_kv(UpsertKV::new( &key, MatchSeq::GE(0), Operation::Delete, @@ -150,7 +150,7 @@ impl ClientSessionMgr { // simply ignore the result self.kv_api - .upsert_kv(UpsertKVReq::new( + .upsert_kv(UpsertKV::new( &key, MatchSeq::GE(0), Operation::Delete, diff --git a/src/query/management/src/cluster/cluster_api.rs b/src/query/management/src/cluster/cluster_api.rs index cb3609539c9c..e8d1046fe424 100644 --- a/src/query/management/src/cluster/cluster_api.rs +++ b/src/query/management/src/cluster/cluster_api.rs @@ -12,23 +12,55 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_meta_types::Change; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::NodeInfo; +/// Databend-query cluster management API #[async_trait::async_trait] pub trait ClusterApi: Sync + Send { - // Add a new node info to /tenant/cluster_id/node-name. - async fn add_node(&self, node: NodeInfo) -> Result; + /// Add or update a node info to /tenant/cluster_id/node-name. + /// + /// - To update, use `SeqMatch::GE(1)` to match any present record. + /// - To add, use `SeqMatch::Exact(0)` to match no present record. + async fn upsert_node(&self, node: NodeInfo, seq: MatchSeq) -> Result>>; - // Get the tenant's cluster all nodes. + /// Get the tenant's cluster all nodes. async fn get_nodes(&self) -> Result>; - // Drop the tenant's cluster one node by node.id. + /// Drop the tenant's cluster one node by node.id. async fn drop_node(&self, node_id: String, seq: MatchSeq) -> Result<()>; - // Keep the tenant's cluster node alive. - async fn heartbeat(&self, node: &NodeInfo, seq: MatchSeq) -> Result; - async fn get_local_addr(&self) -> Result>; + + /// Add a new node. + async fn add_node(&self, node: NodeInfo) -> Result { + let res = self.upsert_node(node.clone(), MatchSeq::Exact(0)).await?; + + let res_seq = res.added_seq_or_else(|_v| { + ErrorCode::ClusterNodeAlreadyExists(format!( + "Node with ID '{}' already exists in the cluster.", + node.id + )) + })?; + + Ok(res_seq) + } + + /// Keep the tenant's cluster node alive. + async fn heartbeat(&self, node: &NodeInfo) -> Result { + // Update or insert the node with GE(0). + let transition = self.upsert_node(node.clone(), MatchSeq::GE(0)).await?; + + let Some(res) = transition.result else { + return Err(ErrorCode::MetaServiceError(format!( + "Unexpected None result returned when upsert heartbeat node {}", + node.id + ))); + }; + + Ok(res.seq) + } } diff --git a/src/query/management/src/cluster/cluster_mgr.rs b/src/query/management/src/cluster/cluster_mgr.rs index 0b35309d18c9..c6d07a7689c4 100644 --- a/src/query/management/src/cluster/cluster_mgr.rs +++ b/src/query/management/src/cluster/cluster_mgr.rs @@ -20,13 +20,13 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_store::MetaStore; -use databend_common_meta_types::seq_value::SeqV; +use databend_common_meta_types::Change; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaSpec; use databend_common_meta_types::NodeInfo; use databend_common_meta_types::Operation; +use databend_common_meta_types::UpsertKV; use crate::cluster::ClusterApi; @@ -72,24 +72,16 @@ impl ClusterMgr { impl ClusterApi for ClusterMgr { #[async_backtrace::framed] #[fastrace::trace] - async fn add_node(&self, node: NodeInfo) -> Result { - // Only when there are no record, i.e. seq=0 - let seq = MatchSeq::Exact(0); + async fn upsert_node(&self, node: NodeInfo, seq: MatchSeq) -> Result>> { let meta = Some(self.new_lift_time()); let value = Operation::Update(serde_json::to_vec(&node)?); let node_key = format!("{}/{}", self.cluster_prefix, escape_for_key(&node.id)?); let upsert_node = self .metastore - .upsert_kv(UpsertKVReq::new(&node_key, seq, value, meta)); + .upsert_kv(UpsertKV::new(&node_key, seq, value, meta)); - let res_seq = upsert_node.await?.added_seq_or_else(|_v| { - ErrorCode::ClusterNodeAlreadyExists(format!( - "Node with ID '{}' already exists in the cluster.", - node.id - )) - })?; - - Ok(res_seq) + let transition = upsert_node.await?; + Ok(transition) } #[async_backtrace::framed] @@ -114,7 +106,7 @@ impl ClusterApi for ClusterMgr { let node_key = format!("{}/{}", self.cluster_prefix, escape_for_key(&node_id)?); let upsert_node = self.metastore - .upsert_kv(UpsertKVReq::new(&node_key, seq, Operation::Delete, None)); + .upsert_kv(UpsertKV::new(&node_key, seq, Operation::Delete, None)); match upsert_node.await? { UpsertKVReply { @@ -129,26 +121,6 @@ impl ClusterApi for ClusterMgr { } } - #[async_backtrace::framed] - #[fastrace::trace] - async fn heartbeat(&self, node: &NodeInfo, seq: MatchSeq) -> Result { - let meta = Some(self.new_lift_time()); - let node_key = format!("{}/{}", self.cluster_prefix, escape_for_key(&node.id)?); - - let upsert_meta = - self.metastore - .upsert_kv(UpsertKVReq::new(&node_key, seq, Operation::AsIs, meta)); - - match upsert_meta.await? { - UpsertKVReply { - ident: None, - prev: Some(_), - result: Some(SeqV { seq: s, .. }), - } => Ok(s), - UpsertKVReply { .. } => self.add_node(node.clone()).await, - } - } - #[async_backtrace::framed] #[fastrace::trace] async fn get_local_addr(&self) -> Result> { diff --git a/src/query/management/src/role/role_mgr.rs b/src/query/management/src/role/role_mgr.rs index 81db047555c7..c0a79e9395bc 100644 --- a/src/query/management/src/role/role_mgr.rs +++ b/src/query/management/src/role/role_mgr.rs @@ -35,7 +35,6 @@ use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_kvapi::kvapi::ListKVReply; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::ConditionResult::Eq; use databend_common_meta_types::MatchSeq; @@ -43,6 +42,7 @@ use databend_common_meta_types::MatchSeqExt; use databend_common_meta_types::MetaError; use databend_common_meta_types::Operation; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use enumflags2::make_bitflags; use fastrace::func_name; use log::debug; @@ -87,7 +87,7 @@ impl RoleMgr { let res = self .kv_api - .upsert_kv(UpsertKVReq::new(&key, seq, Operation::Update(value), None)) + .upsert_kv(UpsertKV::new(&key, seq, Operation::Update(value), None)) .await?; match res.result { Some(SeqV { seq: s, .. }) => Ok(s), @@ -106,7 +106,7 @@ impl RoleMgr { seq: MatchSeq, ) -> Result { self.kv_api - .upsert_kv(UpsertKVReq::new(&key, seq, Operation::Update(value), None)) + .upsert_kv(UpsertKV::new(&key, seq, Operation::Update(value), None)) .await } @@ -145,7 +145,7 @@ impl RoleApi for RoleMgr { let key = self.role_ident(role_info.identity()).to_string_key(); let value = serialize_struct(&role_info, ErrorCode::IllegalUserInfoFormat, || "")?; - let upsert_kv = self.kv_api.upsert_kv(UpsertKVReq::new( + let upsert_kv = self.kv_api.upsert_kv(UpsertKV::new( &key, match_seq, Operation::Update(value), @@ -493,7 +493,7 @@ impl RoleApi for RoleMgr { let res = self .kv_api - .upsert_kv(UpsertKVReq::new(&key, seq, Operation::Delete, None)) + .upsert_kv(UpsertKV::new(&key, seq, Operation::Delete, None)) .await?; res.removed_or_else(|_p| { diff --git a/src/query/management/src/serde/pb_serde.rs b/src/query/management/src/serde/pb_serde.rs index 08031a571b3f..a4127c650075 100644 --- a/src/query/management/src/serde/pb_serde.rs +++ b/src/query/management/src/serde/pb_serde.rs @@ -18,7 +18,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; use databend_common_meta_kvapi::kvapi; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::seq_value::SeqValue; use databend_common_meta_types::InvalidReply; @@ -26,6 +25,7 @@ use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaError; use databend_common_meta_types::MetaNetworkError; use databend_common_meta_types::Operation; +use databend_common_meta_types::UpsertKV; use databend_common_proto_conv::FromToProto; use crate::serde::Quota; @@ -102,7 +102,7 @@ where let value = databend_common_meta_api::serialize_struct(&data)?; let res = kv_api - .upsert_kv(UpsertKVReq::new( + .upsert_kv(UpsertKV::new( key, MatchSeq::Exact(seq_value.seq), Operation::Update(value), diff --git a/src/query/management/src/setting/setting_mgr.rs b/src/query/management/src/setting/setting_mgr.rs index 6d310c2bf636..43c064140048 100644 --- a/src/query/management/src/setting/setting_mgr.rs +++ b/src/query/management/src/setting/setting_mgr.rs @@ -21,13 +21,13 @@ use databend_common_meta_app::principal::UserSetting; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::seq_value::SeqValue; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MatchSeqExt; use databend_common_meta_types::MetaError; use databend_common_meta_types::Operation; +use databend_common_meta_types::UpsertKV; use crate::setting::SettingApi; @@ -67,9 +67,7 @@ impl SettingApi for SettingMgr { let seq = MatchSeq::GE(0); let val = Operation::Update(serde_json::to_vec(&setting)?); let key = self.setting_key(&setting.name); - let upsert = self - .kv_api - .upsert_kv(UpsertKVReq::new(&key, seq, val, None)); + let upsert = self.kv_api.upsert_kv(UpsertKV::new(&key, seq, val, None)); let (_prev, curr) = upsert.await?.unpack(); let res_seq = curr.seq(); @@ -115,7 +113,7 @@ impl SettingApi for SettingMgr { let key = self.setting_key(name); let _res = self .kv_api - .upsert_kv(UpsertKVReq::new(&key, seq, Operation::Delete, None)) + .upsert_kv(UpsertKV::new(&key, seq, Operation::Delete, None)) .await?; Ok(()) diff --git a/src/query/management/src/user/user_mgr.rs b/src/query/management/src/user/user_mgr.rs index f58e073a9165..a6e1aefd5faf 100644 --- a/src/query/management/src/user/user_mgr.rs +++ b/src/query/management/src/user/user_mgr.rs @@ -25,12 +25,12 @@ use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_kvapi::kvapi::ListKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MatchSeqExt; use databend_common_meta_types::MetaError; use databend_common_meta_types::Operation; +use databend_common_meta_types::UpsertKV; use crate::serde::deserialize_struct; use crate::serde::serialize_struct; @@ -73,7 +73,7 @@ impl UserMgr { let kv_api = self.kv_api.clone(); let res = kv_api - .upsert_kv(UpsertKVReq::new(&key, seq, Operation::Update(value), None)) + .upsert_kv(UpsertKV::new(&key, seq, Operation::Update(value), None)) .await?; match res.result { @@ -101,7 +101,7 @@ impl UserApi for UserMgr { let kv_api = &self.kv_api; let seq = MatchSeq::from(*create_option); let res = kv_api - .upsert_kv(UpsertKVReq::new(&key, seq, Operation::Update(value), None)) + .upsert_kv(UpsertKV::new(&key, seq, Operation::Update(value), None)) .await?; if let CreateOption::Create = create_option { @@ -187,7 +187,7 @@ impl UserApi for UserMgr { let key = self.user_key(&user.username, &user.hostname); let res = self .kv_api - .upsert_kv(UpsertKVReq::new(&key, seq, Operation::Delete, None)) + .upsert_kv(UpsertKV::new(&key, seq, Operation::Delete, None)) .await?; if res.prev.is_some() && res.result.is_none() { Ok(()) diff --git a/src/query/management/tests/it/cluster.rs b/src/query/management/tests/it/cluster.rs index 567627849ba9..cace6c5b9e76 100644 --- a/src/query/management/tests/it/cluster.rs +++ b/src/query/management/tests/it/cluster.rs @@ -130,7 +130,7 @@ async fn test_successfully_heartbeat_node() -> Result<()> { assert!(expire_ms - now_ms >= 59_000); let now_ms = SeqV::<()>::now_ms(); - cluster_api.heartbeat(&node_info, MatchSeq::GE(1)).await?; + cluster_api.heartbeat(&node_info).await?; let value = kv_api .get_kv("__fd_clusters_v4/test%2dtenant%2did/test%2dcluster%2did/databend_query/test_node") diff --git a/src/query/management/tests/it/role.rs b/src/query/management/tests/it/role.rs index e662269c9f7c..e4495241b81c 100644 --- a/src/query/management/tests/it/role.rs +++ b/src/query/management/tests/it/role.rs @@ -18,8 +18,8 @@ use databend_common_base::base::tokio; use databend_common_management::*; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_embedded::MemMeta; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::MatchSeq; +use databend_common_meta_types::UpsertKV; use mockall::predicate::*; fn make_role_key(role: &str) -> String { @@ -45,7 +45,7 @@ mod add { let v = serde_json::to_vec(&role_info)?; let kv_api = kv_api.clone(); let _upsert_kv = kv_api - .upsert_kv(UpsertKVReq::new( + .upsert_kv(UpsertKV::new( &role_key, MatchSeq::Exact(0), Operation::Update(v), @@ -69,7 +69,7 @@ mod add { let v = serde_json::to_vec(&role_info)?; let kv_api = kv_api.clone(); let _upsert_kv = kv_api - .upsert_kv(UpsertKVReq::new( + .upsert_kv(UpsertKV::new( &role_key, MatchSeq::Exact(0), Operation::Update(v), diff --git a/src/query/management/tests/it/user.rs b/src/query/management/tests/it/user.rs index 0346b92e293e..6a375691213e 100644 --- a/src/query/management/tests/it/user.rs +++ b/src/query/management/tests/it/user.rs @@ -28,13 +28,13 @@ use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::ListKVReply; use databend_common_meta_kvapi::kvapi::MGetKVReply; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_kvapi::kvapi::UpsertKVReq; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaError; use databend_common_meta_types::Operation; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use mockall::predicate::*; use mockall::*; @@ -47,7 +47,7 @@ mock! { async fn upsert_kv( &self, - act: UpsertKVReq, + act: UpsertKV, ) -> Result; async fn get_kv(&self, key: &str) -> Result; @@ -112,7 +112,7 @@ mod add { let test_key = test_key.clone(); let mut api = MockKV::new(); api.expect_upsert_kv() - .with(predicate::eq(UpsertKVReq::new( + .with(predicate::eq(UpsertKV::new( &test_key, test_seq, value.clone(), @@ -132,7 +132,7 @@ mod add { let test_key = test_key.clone(); let mut api = MockKV::new(); api.expect_upsert_kv() - .with(predicate::eq(UpsertKVReq::new( + .with(predicate::eq(UpsertKV::new( &test_key, test_seq, value.clone(), @@ -408,7 +408,7 @@ mod drop { escape_for_key(&format_user_key(test_user, test_hostname))? ); kv.expect_upsert_kv() - .with(predicate::eq(UpsertKVReq::new( + .with(predicate::eq(UpsertKV::new( &test_key, MatchSeq::GE(1), Operation::Delete, @@ -434,7 +434,7 @@ mod drop { escape_for_key(&format_user_key(test_user, test_hostname))? ); kv.expect_upsert_kv() - .with(predicate::eq(UpsertKVReq::new( + .with(predicate::eq(UpsertKV::new( &test_key, MatchSeq::GE(1), Operation::Delete, @@ -509,7 +509,7 @@ mod update { serialize_struct(&new_user_info, ErrorCode::IllegalUserInfoFormat, || "")?; kv.expect_upsert_kv() - .with(predicate::eq(UpsertKVReq::new( + .with(predicate::eq(UpsertKV::new( &test_key, MatchSeq::Exact(1), Operation::Update(new_value_with_old_salt.clone()), @@ -583,7 +583,7 @@ mod update { // upsert should be called kv.expect_upsert_kv() - .with(predicate::function(move |act: &UpsertKVReq| { + .with(predicate::function(move |act: &UpsertKV| { act.key == test_key.as_str() && act.seq == MatchSeq::Exact(2) })) .times(1) @@ -638,7 +638,7 @@ mod set_user_privileges { let new_value = serialize_struct(&user_info, ErrorCode::IllegalUserInfoFormat, || "")?; kv.expect_upsert_kv() - .with(predicate::eq(UpsertKVReq::new( + .with(predicate::eq(UpsertKV::new( &test_key, MatchSeq::Exact(1), Operation::Update(new_value), diff --git a/src/query/service/src/clusters/cluster.rs b/src/query/service/src/clusters/cluster.rs index 039ce494402b..9c66a8b5770c 100644 --- a/src/query/service/src/clusters/cluster.rs +++ b/src/query/service/src/clusters/cluster.rs @@ -476,7 +476,7 @@ impl ClusterHeartbeat { } Either::Right((_, new_shutdown_notified)) => { shutdown_notified = new_shutdown_notified; - let heartbeat = cluster_api.heartbeat(&node, MatchSeq::GE(1)); + let heartbeat = cluster_api.heartbeat(&node); if let Err(failure) = heartbeat.await { metric_incr_cluster_heartbeat_count( &node.id, From 221018b70c661bd12c4803179398f4fca8ca061a Mon Sep 17 00:00:00 2001 From: baishen Date: Sat, 23 Nov 2024 17:24:51 +0800 Subject: [PATCH 80/92] refactor(sqlsmith): refactor sqlsmith using http client (#16890) * refactor(sqlsmith): refactor sqlsmith using http client * fix * rewrite HttpClient --- Cargo.lock | 148 +---------- Cargo.toml | 3 - src/query/expression/src/row/row_converter.rs | 18 +- src/tests/sqlsmith/Cargo.toml | 10 +- src/tests/sqlsmith/src/bin/main.rs | 32 ++- src/tests/sqlsmith/src/http_client.rs | 231 ++++++++++++++++++ src/tests/sqlsmith/src/lib.rs | 1 + src/tests/sqlsmith/src/reducer.rs | 127 +++++----- src/tests/sqlsmith/src/runner.rs | 224 +++++++++-------- src/tests/sqlsmith/src/sql_gen/expr.rs | 115 ++++++++- 10 files changed, 580 insertions(+), 329 deletions(-) create mode 100644 src/tests/sqlsmith/src/http_client.rs diff --git a/Cargo.lock b/Cargo.lock index 8221d4ae2777..bc38613c7e9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -708,8 +708,6 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "zstd 0.13.2", - "zstd-safe 7.2.1", ] [[package]] @@ -2981,28 +2979,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "databend-client" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0819048a792e2eac58b455bbbcf6077c81e2780b3cc4f565a6e72e92dde56bd1" -dependencies = [ - "async-trait", - "log", - "once_cell", - "parking_lot 0.12.3", - "percent-encoding", - "reqwest", - "serde", - "serde_json", - "tokio", - "tokio-retry", - "tokio-stream", - "tokio-util", - "url", - "uuid", -] - [[package]] name = "databend-codegen" version = "0.1.0" @@ -3266,7 +3242,7 @@ dependencies = [ "backtrace", "bincode 2.0.0-rc.3", "databend-common-ast", - "geozero 0.14.0", + "geozero", "gimli 0.31.1", "http 1.1.0", "libc", @@ -3322,7 +3298,7 @@ dependencies = [ "ethnum", "futures", "geo", - "geozero 0.14.0", + "geozero", "goldenfile", "hex", "hyper-util", @@ -3368,7 +3344,7 @@ dependencies = [ "databend-common-settings", "databend-storages-common-blocks", "databend-storages-common-table-meta", - "geozero 0.14.0", + "geozero", "hex", "jiff", "jsonb", @@ -3409,7 +3385,7 @@ dependencies = [ "ethnum", "geo", "geohash", - "geozero 0.14.0", + "geozero", "goldenfile", "h3o", "hex", @@ -3507,7 +3483,7 @@ dependencies = [ "enumflags2", "ethnum", "geo", - "geozero 0.14.0", + "geozero", "hex", "jiff", "lexical-core", @@ -4660,64 +4636,6 @@ dependencies = [ "ndarray", ] -[[package]] -name = "databend-driver" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbea10312fa203003b572b701b6121cecd8eaf260e7ec7687796b552044541c6" -dependencies = [ - "arrow", - "async-compression 0.4.12", - "async-trait", - "chrono", - "csv", - "databend-client", - "databend-driver-core", - "databend-driver-macros", - "dyn-clone", - "glob", - "log", - "once_cell", - "percent-encoding", - "serde_json", - "tokio", - "tokio-stream", - "url", -] - -[[package]] -name = "databend-driver-core" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b8f3bf7bb87d0f2fce8a1992eea4a0b442c01165a0d68aab2727a386c945ab" -dependencies = [ - "arrow", - "chrono", - "databend-client", - "geozero 0.13.0", - "glob", - "hex", - "itertools 0.12.1", - "jsonb", - "lexical-core", - "memchr", - "roaring", - "serde", - "serde_json", - "tokio-stream", - "url", -] - -[[package]] -name = "databend-driver-macros" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40bed5c66f36e79baea7c95d2d0dc8433671606ca7152012562c7145e3b909a1" -dependencies = [ - "quote", - "syn 2.0.58", -] - [[package]] name = "databend-enterprise-aggregating-index" version = "0.1.0" @@ -5212,24 +5130,26 @@ version = "0.1.0" dependencies = [ "chrono-tz 0.8.6", "clap", - "databend-client", + "cookie", "databend-common-ast", + "databend-common-exception", "databend-common-expression", "databend-common-formats", "databend-common-functions", "databend-common-io", "databend-common-sql", - "databend-driver", - "databend-driver-core", "ethnum", "itertools 0.13.0", "jiff", "jsonb", "rand", + "reqwest", + "serde", + "serde_json", "tokio", - "tokio-stream", "tracing", "tracing-subscriber", + "url", ] [[package]] @@ -6883,21 +6803,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "geozero" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cd8fb67347739a057fd607b6d8b43ba4ed93619ed84b8f429fa3296f8ae504c" -dependencies = [ - "geo-types", - "geojson", - "log", - "scroll 0.11.0", - "serde_json", - "thiserror", - "wkt 0.10.3", -] - [[package]] name = "geozero" version = "0.14.0" @@ -9715,16 +9620,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "minijinja" version = "0.31.0" @@ -12325,7 +12220,6 @@ dependencies = [ "js-sys", "log", "mime", - "mime_guess", "native-tls", "once_cell", "percent-encoding", @@ -14584,17 +14478,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-retry" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" -dependencies = [ - "pin-project", - "rand", - "tokio", -] - [[package]] name = "tokio-retry2" version = "0.5.6" @@ -15093,15 +14976,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.15" diff --git a/Cargo.toml b/Cargo.toml index 1a6632f910e1..0b82af877c9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -536,9 +536,6 @@ tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json", "valuable"] } # Databend Integration Test -databend-client = { version = "0.22" } -databend-driver = { version = "0.22" } -databend-driver-core = { version = "0.22" } msql-srv = "0.11.0" mysql_common = "0.32.4" quickcheck = "1.0" diff --git a/src/query/expression/src/row/row_converter.rs b/src/query/expression/src/row/row_converter.rs index f259df073192..549221e8384b 100644 --- a/src/query/expression/src/row/row_converter.rs +++ b/src/query/expression/src/row/row_converter.rs @@ -57,15 +57,17 @@ impl RowConverter { fn support_data_type(d: &DataType) -> bool { match d { - DataType::Array(_) - | DataType::EmptyArray - | DataType::EmptyMap - | DataType::Map(_) - | DataType::Bitmap - | DataType::Tuple(_) - | DataType::Generic(_) => false, + DataType::Null + | DataType::Boolean + | DataType::Number(_) + | DataType::Decimal(_) + | DataType::Timestamp + | DataType::Date + | DataType::Binary + | DataType::String + | DataType::Variant => true, DataType::Nullable(inner) => Self::support_data_type(inner.as_ref()), - _ => true, + _ => false, } } diff --git a/src/tests/sqlsmith/Cargo.toml b/src/tests/sqlsmith/Cargo.toml index ba3dade807c3..bfaff3e693d8 100644 --- a/src/tests/sqlsmith/Cargo.toml +++ b/src/tests/sqlsmith/Cargo.toml @@ -9,24 +9,26 @@ edition = { workspace = true } [dependencies] chrono-tz = { workspace = true } clap = { workspace = true } -databend-client = { workspace = true } +cookie = { workspace = true } databend-common-ast = { workspace = true } +databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } databend-common-formats = { workspace = true } databend-common-functions = { workspace = true } databend-common-io = { workspace = true } databend-common-sql = { workspace = true } -databend-driver = { workspace = true } -databend-driver-core = { workspace = true } ethnum = { workspace = true } itertools = { workspace = true } jiff = { workspace = true } jsonb = { workspace = true } rand = { workspace = true } +reqwest = { workspace = true, features = ["cookies"] } +serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true } -tokio-stream = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +url = { workspace = true } [[bin]] name = "databend-sqlsmith" diff --git a/src/tests/sqlsmith/src/bin/main.rs b/src/tests/sqlsmith/src/bin/main.rs index 62ca1b5e1f05..840098bc4745 100644 --- a/src/tests/sqlsmith/src/bin/main.rs +++ b/src/tests/sqlsmith/src/bin/main.rs @@ -13,7 +13,10 @@ // limitations under the License. use clap::Parser; +use databend_common_exception::Result; use databend_sqlsmith::Runner; +use tracing::metadata::LevelFilter; +use tracing_subscriber::EnvFilter; #[derive(Clone, Debug, PartialEq, Eq, Parser)] #[clap(about, author)] @@ -27,7 +30,7 @@ pub struct Args { port: u16, /// The test database. - #[clap(long, default_value = "default")] + #[clap(long, default_value = "sqlsmith_test")] db: String, /// The username. @@ -48,15 +51,26 @@ pub struct Args { } #[tokio::main(flavor = "multi_thread", worker_threads = 5)] -async fn main() { - tracing_subscriber::fmt::init(); +async fn main() -> Result<()> { + let filter = EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .parse("") + .unwrap(); + tracing_subscriber::fmt().with_env_filter(filter).init(); let args = Args::parse(); + let host = format!("http://{}:{}", args.host, args.port); + let mut runner = Runner::try_new( + host, + args.user.clone(), + args.pass.clone(), + args.db.clone(), + args.count, + None, + args.timeout, + ) + .await?; + runner.run().await?; - let dsn = format!( - "databend://{}:{}@{}:{}/{}?sslmode=disable", - args.user, args.pass, args.host, args.port, args.db - ); - let runner = Runner::new(dsn, args.count, None, args.timeout); - runner.run().await; + Ok(()) } diff --git a/src/tests/sqlsmith/src/http_client.rs b/src/tests/sqlsmith/src/http_client.rs new file mode 100644 index 000000000000..4811f8379ae1 --- /dev/null +++ b/src/tests/sqlsmith/src/http_client.rs @@ -0,0 +1,231 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::RwLock; +use std::time::Duration; + +use cookie::Cookie; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use reqwest::cookie::CookieStore; +use reqwest::header::HeaderMap; +use reqwest::header::HeaderValue; +use reqwest::Client; +use reqwest::ClientBuilder; +use serde::Deserialize; +use serde::Serialize; +use url::Url; + +struct GlobalCookieStore { + cookies: RwLock>>, +} + +impl GlobalCookieStore { + pub fn new() -> Self { + GlobalCookieStore { + cookies: RwLock::new(HashMap::new()), + } + } +} + +impl CookieStore for GlobalCookieStore { + fn set_cookies(&self, cookie_headers: &mut dyn Iterator, _url: &Url) { + let iter = cookie_headers + .filter_map(|val| std::str::from_utf8(val.as_bytes()).ok()) + .filter_map(|kv| Cookie::parse(kv).map(|c| c.into_owned()).ok()); + + let mut guard = self.cookies.write().unwrap(); + for cookie in iter { + guard.insert(cookie.name().to_string(), cookie); + } + } + + fn cookies(&self, _url: &Url) -> Option { + let guard = self.cookies.read().unwrap(); + let s: String = guard + .values() + .map(|cookie| cookie.name_value()) + .map(|(name, value)| format!("{name}={value}")) + .collect::>() + .join("; "); + + if s.is_empty() { + return None; + } + + HeaderValue::from_str(&s).ok() + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +pub(crate) struct ServerInfo { + pub(crate) id: String, + pub(crate) start_time: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +pub(crate) struct HttpSessionConf { + pub(crate) database: Option, + pub(crate) role: Option, + pub(crate) secondary_roles: Option>, + pub(crate) settings: Option>, + pub(crate) txn_state: Option, + pub(crate) last_server_info: Option, + #[serde(default)] + pub(crate) last_query_ids: Vec, + pub(crate) internal: Option, +} + +#[derive(serde::Deserialize, Debug)] +pub(crate) struct QueryResponse { + pub(crate) session: Option, + pub(crate) data: Option, + next_uri: Option, + + pub(crate) error: Option, +} + +#[derive(Deserialize)] +struct TokenInfo { + session_token: String, +} + +#[derive(Deserialize)] +struct LoginResponse { + tokens: Option, +} + +pub(crate) struct HttpClient { + pub(crate) host: String, + pub(crate) client: Client, + pub(crate) session_token: String, + pub(crate) session: Option, +} + +impl HttpClient { + pub(crate) async fn create(host: String, username: String, password: String) -> Result { + let mut header = HeaderMap::new(); + header.insert( + "Content-Type", + HeaderValue::from_str("application/json").unwrap(), + ); + header.insert("Accept", HeaderValue::from_str("application/json").unwrap()); + let cookie_provider = GlobalCookieStore::new(); + let cookie = HeaderValue::from_str("cookie_enabled=true").unwrap(); + let mut initial_cookies = [&cookie].into_iter(); + cookie_provider.set_cookies(&mut initial_cookies, &Url::parse("https://a.com").unwrap()); + let client = ClientBuilder::new() + .cookie_provider(Arc::new(cookie_provider)) + .default_headers(header) + // https://github.com/hyperium/hyper/issues/2136#issuecomment-589488526 + .http2_keep_alive_timeout(Duration::from_secs(15)) + .pool_max_idle_per_host(0) + .build()?; + + let url = format!("{}/v1/session/login", host); + + let login_resp = client + .post(&url) + .body("{}") + .basic_auth(username, Some(password)) + .send() + .await + .inspect_err(|e| { + println!("fail to send to {}: {:?}", url, e); + })? + .json::() + .await + .inspect_err(|e| { + println!("fail to decode json when call {}: {:?}", url, e); + })?; + let session_token = match login_resp.tokens { + Some(tokens) => tokens.session_token, + None => { + return Err(ErrorCode::AuthenticateFailure( + "failed to get session token", + )); + } + }; + + Ok(Self { + host, + client, + session_token, + session: None, + }) + } + + pub(crate) async fn query(&mut self, sql: &str) -> Result> { + let url = format!("{}/v1/query", self.host); + let mut responses = vec![]; + let response = self.post_query(sql, &url).await?; + let mut next_uri_opt = response.next_uri.clone(); + responses.push(response); + while let Some(next_uri) = &next_uri_opt { + let url = format!("{}{}", self.host, next_uri); + let new_response = self.poll_query_result(&url).await?; + if new_response.session.is_some() { + self.session = new_response.session.clone(); + } + next_uri_opt = new_response.next_uri.clone(); + responses.push(new_response); + } + Ok(responses) + } + + // Send request and get response by json format + async fn post_query(&self, sql: &str, url: &str) -> Result { + let mut query = HashMap::new(); + query.insert("sql", serde_json::to_value(sql)?); + if let Some(session) = &self.session { + query.insert("session", serde_json::to_value(session)?); + } + + Ok(self + .client + .post(url) + .json(&query) + .bearer_auth(&self.session_token) + .send() + .await + .inspect_err(|e| { + println!("fail to send to {}: {:?}", url, e); + })? + .json::() + .await + .inspect_err(|e| { + println!("fail to decode json when call {}: {:?}", url, e); + })?) + } + + async fn poll_query_result(&self, url: &str) -> Result { + Ok(self + .client + .get(url) + .bearer_auth(&self.session_token) + .send() + .await + .inspect_err(|e| { + println!("fail to send to {}: {:?}", url, e); + })? + .json::() + .await + .inspect_err(|e| { + println!("fail to decode json when call {}: {:?}", url, e); + })?) + } +} diff --git a/src/tests/sqlsmith/src/lib.rs b/src/tests/sqlsmith/src/lib.rs index 33ca3141e6cb..9dacb7236384 100644 --- a/src/tests/sqlsmith/src/lib.rs +++ b/src/tests/sqlsmith/src/lib.rs @@ -14,6 +14,7 @@ #![feature(box_patterns)] +mod http_client; mod reducer; mod runner; mod sql_gen; diff --git a/src/tests/sqlsmith/src/reducer.rs b/src/tests/sqlsmith/src/reducer.rs index 0dbd65e342be..3b2eac238caa 100644 --- a/src/tests/sqlsmith/src/reducer.rs +++ b/src/tests/sqlsmith/src/reducer.rs @@ -12,82 +12,83 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_client::error::Error as ClientError; use databend_common_ast::ast::Query; use databend_common_ast::ast::SetExpr; -use databend_driver::Connection; -use databend_driver::Error; -pub(crate) async fn try_reduce_query( - conn: Box, - code: u16, - mut query: Query, -) -> Query { - // reduce limit and offset - if !query.limit.is_empty() || query.offset.is_some() { - let mut reduced_query = query.clone(); - reduced_query.limit = vec![]; - reduced_query.offset = None; - if execute_reduce_query(conn.clone(), code, &reduced_query).await { - query = reduced_query; - } - } +use crate::Runner; - // reduce order by - if !query.order_by.is_empty() { - let mut reduced_query = query.clone(); - reduced_query.order_by = vec![]; - if execute_reduce_query(conn.clone(), code, &reduced_query).await { - query = reduced_query; +impl Runner { + pub(crate) async fn try_reduce_query(&mut self, code: u64, mut query: Query) -> Query { + // reduce limit and offset + if !query.limit.is_empty() || query.offset.is_some() { + let mut reduced_query = query.clone(); + reduced_query.limit = vec![]; + reduced_query.offset = None; + if self.execute_reduce_query(code, &reduced_query).await { + query = reduced_query; + } } - } - if let SetExpr::Select(select_stmt) = query.body.clone() { - let mut select_stmt = select_stmt.clone(); - let mut reduced_query = query.clone(); - // TODO: reduce expr - if select_stmt.selection.is_some() { - let mut reduced_select_stmt = select_stmt.clone(); - reduced_select_stmt.selection = None; - reduced_query.body = SetExpr::Select(reduced_select_stmt.clone()); - if execute_reduce_query(conn.clone(), code, &reduced_query).await { - select_stmt = reduced_select_stmt; + // reduce order by + if !query.order_by.is_empty() { + let mut reduced_query = query.clone(); + reduced_query.order_by = vec![]; + if self.execute_reduce_query(code, &reduced_query).await { + query = reduced_query; } } - if select_stmt.having.is_some() { - let mut reduced_select_stmt = select_stmt.clone(); - reduced_select_stmt.having = None; - reduced_query.body = SetExpr::Select(reduced_select_stmt.clone()); - if execute_reduce_query(conn.clone(), code, &reduced_query).await { - select_stmt = reduced_select_stmt; + + if let SetExpr::Select(select_stmt) = query.body.clone() { + let mut select_stmt = select_stmt.clone(); + let mut reduced_query = query.clone(); + // TODO: reduce expr + if select_stmt.selection.is_some() { + let mut reduced_select_stmt = select_stmt.clone(); + reduced_select_stmt.selection = None; + reduced_query.body = SetExpr::Select(reduced_select_stmt.clone()); + if self.execute_reduce_query(code, &reduced_query).await { + select_stmt = reduced_select_stmt; + } } - } - if select_stmt.window_list.is_none() && reduced_query.with.is_some() { - reduced_query.with = None; - if execute_reduce_query(conn.clone(), code, &reduced_query).await { - query = reduced_query.clone(); + if select_stmt.having.is_some() { + let mut reduced_select_stmt = select_stmt.clone(); + reduced_select_stmt.having = None; + reduced_query.body = SetExpr::Select(reduced_select_stmt.clone()); + if self.execute_reduce_query(code, &reduced_query).await { + select_stmt = reduced_select_stmt; + } } - } - let select_list = select_stmt.select_list.clone(); - let mut reduced_select_stmt = select_stmt.clone(); - for item in &select_list { - reduced_select_stmt.select_list = vec![item.clone()]; - reduced_query.body = SetExpr::Select(reduced_select_stmt.clone()); - if execute_reduce_query(conn.clone(), code, &reduced_query).await { - select_stmt = reduced_select_stmt; - break; + if select_stmt.window_list.is_none() && reduced_query.with.is_some() { + reduced_query.with = None; + if self.execute_reduce_query(code, &reduced_query).await { + query = reduced_query.clone(); + } + } + let select_list = select_stmt.select_list.clone(); + let mut reduced_select_stmt = select_stmt.clone(); + for item in &select_list { + reduced_select_stmt.select_list = vec![item.clone()]; + reduced_query.body = SetExpr::Select(reduced_select_stmt.clone()); + if self.execute_reduce_query(code, &reduced_query).await { + select_stmt = reduced_select_stmt; + break; + } } + query.body = SetExpr::Select(select_stmt); } - query.body = SetExpr::Select(select_stmt); - } - query -} + query + } -async fn execute_reduce_query(conn: Box, code: u16, query: &Query) -> bool { - let sql = query.to_string(); - if let Err(Error::Api(ClientError::InvalidResponse(err))) = conn.exec(&sql).await { - return err.code == code; + async fn execute_reduce_query(&mut self, err_code: u64, query: &Query) -> bool { + let query_sql = query.to_string(); + if let Ok(responses) = self.client.query(&query_sql).await { + if let Some(error) = &responses[0].error { + let value = error.as_object().unwrap(); + let code = value["code"].as_u64().unwrap(); + return err_code == code; + } + } + false } - false } diff --git a/src/tests/sqlsmith/src/runner.rs b/src/tests/sqlsmith/src/runner.rs index b224c83aceae..c03e348fb96a 100644 --- a/src/tests/sqlsmith/src/runner.rs +++ b/src/tests/sqlsmith/src/runner.rs @@ -15,26 +15,22 @@ use std::future::Future; use std::time::Duration; -use databend_client::error::Error as ClientError; use databend_common_ast::ast::AlterTableAction; use databend_common_ast::ast::CreateTableSource; use databend_common_ast::ast::CreateTableStmt; use databend_common_ast::ast::DropTableStmt; +use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; use databend_common_sql::resolve_type_name; -use databend_driver::Client; -use databend_driver::Connection; -use databend_driver::Error; -use databend_driver_core::value::Value; use rand::rngs::SmallRng; use rand::Rng; use rand::SeedableRng; -use tokio_stream::StreamExt; -use crate::reducer::try_reduce_query; +use crate::http_client::HttpClient; +use crate::http_client::QueryResponse; use crate::sql_gen::SqlGenerator; use crate::sql_gen::Table; @@ -74,25 +70,37 @@ const KNOWN_ERRORS: &[&str] = &[ "Expected Number, Date or Timestamp type, but got", "Unsupported data type for generate_series", "Having clause can't contain window functions", + "Cannot find common type for", + "null value in column", ]; pub struct Runner { count: usize, seed: Option, - client: Client, + pub(crate) client: HttpClient, + db: String, timeout: u64, } impl Runner { - pub fn new(dsn: String, count: usize, seed: Option, timeout: u64) -> Self { - let client = Client::new(dsn); + pub async fn try_new( + host: String, + username: String, + password: String, + db: String, + count: usize, + seed: Option, + timeout: u64, + ) -> Result { + let client = HttpClient::create(host, username, password).await?; - Self { + Ok(Self { count, seed, client, + db, timeout, - } + }) } fn generate_rng(seed: Option) -> impl Rng { @@ -103,20 +111,18 @@ impl Runner { } } - #[allow(clippy::borrowed_box)] async fn create_base_table( - &self, - conn: &Box, + &mut self, table_stmts: Vec<(DropTableStmt, CreateTableStmt)>, - ) -> Vec
{ + ) -> Result> { let mut tables = Vec::with_capacity(table_stmts.len()); for (drop_table_stmt, create_table_stmt) in table_stmts { let drop_table_sql = drop_table_stmt.to_string(); tracing::info!("drop_table_sql: {}", drop_table_sql); - Self::check_res(conn.exec(&drop_table_sql).await); + Self::check_res(self.client.query(&drop_table_sql).await); let create_table_sql = create_table_stmt.to_string(); tracing::info!("create_table_sql: {}", create_table_sql); - Self::check_res(conn.exec(&create_table_sql).await); + Self::check_res(self.client.query(&create_table_sql).await); let table_name = create_table_stmt.table.name.clone(); let mut fields = Vec::new(); @@ -132,75 +138,83 @@ impl Runner { let table = Table::new(table_name, schema); tables.push(table); } - tables + Ok(tables) } - #[allow(clippy::borrowed_box)] - async fn get_settings(&self, conn: &Box) -> Vec<(String, DataType)> { - let mut settings = vec![]; + async fn get_settings(&mut self) -> Result> { let show_settings = "show settings".to_string(); - let rows = conn.query_iter(&show_settings).await; - match rows { - Ok(mut rows) => { - while let Some(row) = rows.next().await { - if let Ok(row) = row { - match (row.values().first(), row.values().get(5)) { - (Some(Value::String(name)), Some(Value::String(ty))) => { - let data_type = match ty.as_str() { - "UInt64" => DataType::Number(NumberDataType::UInt64), - "String" => DataType::String, - _ => DataType::String, - }; - settings.push((name.clone(), data_type)); - } - (_, _) => { - continue; - } - } + let responses = self.client.query(&show_settings).await?; + + let mut settings = vec![]; + for response in responses { + if let Some(serde_json::Value::Array(arr)) = response.data { + for row in arr { + let name = row.get(0); + let ty = row.get(6); + if let ( + Some(serde_json::Value::String(name)), + Some(serde_json::Value::String(ty)), + ) = (name, ty) + { + let data_type = match ty.as_str() { + "UInt64" => DataType::Number(NumberDataType::UInt64), + "String" => DataType::String, + _ => DataType::String, + }; + settings.push((name.clone(), data_type)); } } } - Err(e) => { - let err = format!("show settings exec err: {}", e); - tracing::error!(err); - } } - settings + Ok(settings) } - async fn check_timeout(future: F, sql: String, sec: u64) + async fn check_timeout(future: F, sec: u64, timeout_err: &mut Option) where F: Future { if let Err(e) = tokio::time::timeout(Duration::from_secs(sec), future).await { - tracing::info!("sql: {}", sql); - let err = format!("sql timeout: {}", e); - tracing::error!(err); + *timeout_err = Some(format!("{}", e)); } } - fn check_res(res: databend_driver::Result) { - if let Err(Error::Api(ClientError::InvalidResponse(err))) = res { - if err.code == 1005 || err.code == 1065 { - return; + fn check_res(responses: Result>) { + match responses { + Ok(responses) => { + if let Some(error) = &responses[0].error { + let value = error.as_object().unwrap(); + let code = value["code"].as_u64().unwrap(); + let message = value["message"].as_str().unwrap(); + if code == 1005 || code == 1065 { + return; + } + if KNOWN_ERRORS + .iter() + .any(|known_err| message.starts_with(known_err)) + { + return; + } + let err = format!("sql exec err code: {} message: {}", code, message); + tracing::error!(err); + } } - if KNOWN_ERRORS - .iter() - .any(|known_err| err.message.starts_with(known_err)) - { - return; + Err(err) => { + let err = format!("http err: {}", err); + tracing::error!(err); } - let err = format!("sql exec err: {}", err.message); - tracing::error!(err); } } - pub async fn run(&self) { - let conn = self.client.get_conn().await.unwrap(); - let settings = self.get_settings(&conn).await; + pub async fn run(&mut self) -> Result<()> { + let create_db_sql = format!("CREATE OR REPLACE database {}", self.db); + let _ = self.client.query(&create_db_sql).await?; + let use_db_sql = format!("USE {}", self.db); + let _ = self.client.query(&use_db_sql).await?; + + let settings = self.get_settings().await?; let mut rng = Self::generate_rng(self.seed); let mut generator = SqlGenerator::new(&mut rng, settings); let table_stmts = generator.gen_base_tables(); - let tables = self.create_base_table(&conn, table_stmts).await; + let tables = self.create_base_table(table_stmts).await?; let row_count = 10; let mut new_tables = tables.clone(); @@ -208,7 +222,7 @@ impl Runner { let insert_stmt = generator.gen_insert(table, row_count); let insert_sql = insert_stmt.to_string(); tracing::info!("insert_sql: {}", insert_sql); - Self::check_res(conn.exec(&insert_sql).await); + Self::check_res(self.client.query(&insert_sql).await); let alter_stmt_opt = generator.gen_alter(table, row_count); if let Some((alter_stmt, new_table, insert_stmt_opt)) = alter_stmt_opt { @@ -222,24 +236,24 @@ impl Runner { }; let drop_table_sql = drop_table_stmt.to_string(); tracing::info!("drop_table_sql: {}", drop_table_sql); - Self::check_res(conn.exec(&drop_table_sql).await); + Self::check_res(self.client.query(&drop_table_sql).await); } let alter_sql = alter_stmt.to_string(); tracing::info!("alter_sql: {}", alter_sql); - Self::check_res(conn.exec(&alter_sql).await); + Self::check_res(self.client.query(&alter_sql).await); // save new table schema new_tables[i] = new_table; if let Some(insert_stmt) = insert_stmt_opt { let insert_sql = insert_stmt.to_string(); tracing::info!("after alter insert_sql: {}", insert_sql); - Self::check_res(conn.exec(&insert_sql).await); + Self::check_res(self.client.query(&insert_sql).await); } } } generator.tables = new_tables; let enable_merge = "set enable_experimental_merge_into = 1".to_string(); - Self::check_res(conn.exec(&enable_merge).await); + Self::check_res(self.client.query(&enable_merge).await); // generate merge, replace, update, delete for _ in 0..20 { let sql = match generator.rng.gen_range(0..=20) { @@ -249,56 +263,74 @@ impl Runner { 20 => generator.gen_delete().to_string(), _ => unreachable!(), }; - tracing::info!("sql: {}", sql); + let mut timeout_err = None; + tracing::info!("dml sql: {}", sql); Self::check_timeout( - async { Self::check_res(conn.exec(&sql.clone()).await) }, - sql.clone(), + async { Self::check_res(self.client.query(&sql).await) }, self.timeout, + &mut timeout_err, ) .await; + if let Some(timeout_err) = timeout_err { + tracing::error!("sql timeout: {}", timeout_err); + } } // generate query for _ in 0..self.count { let query = generator.gen_query(); let query_sql = query.to_string(); + let mut timeout_err = None; + let mut is_error = false; let mut try_reduce = false; let mut err_code = 0; - let mut err = String::new(); + let mut err_message = String::new(); Self::check_timeout( async { - if let Err(e) = conn.exec(&query_sql).await { - if let Error::Api(ClientError::InvalidResponse(err)) = &e { - // TODO: handle Syntax, Semantic and InvalidArgument errors - if err.code == 1005 - || err.code == 1065 - || err.code == 2004 - || err.code == 1010 - { - return; - } - if KNOWN_ERRORS - .iter() - .any(|known_err| err.message.starts_with(known_err)) - { - return; + match self.client.query(&query_sql).await { + Ok(responses) => { + if let Some(error) = &responses[0].error { + let value = error.as_object().unwrap(); + let code = value["code"].as_u64().unwrap(); + let message = value["message"].as_str().unwrap(); + if code == 1005 || code == 1065 || code == 2004 || code == 1010 { + return; + } + if KNOWN_ERRORS + .iter() + .any(|known_err| message.starts_with(known_err)) + { + return; + } + is_error = true; + err_code = code; + err_message = format!("error: {}", message); + try_reduce = true; } - err_code = err.code; } - err = format!("error: {}", e); - try_reduce = true; + Err(err) => { + is_error = true; + err_message = format!("http err: {}", err); + } } }, - query_sql.clone(), self.timeout, + &mut timeout_err, ) .await; - if try_reduce { + + if let Some(timeout_err) = timeout_err { tracing::info!("query_sql: {}", query_sql); - let reduced_query = try_reduce_query(conn.clone(), err_code, query).await; - tracing::info!("reduced query_sql: {}", reduced_query.to_string()); - tracing::error!(err); + tracing::error!("sql timeout: {}", timeout_err); + } else if is_error { + tracing::info!("query_sql: {}", query_sql); + if try_reduce { + let reduced_query = self.try_reduce_query(err_code, query).await; + tracing::info!("reduced query_sql: {}", reduced_query.to_string()); + } + tracing::error!(err_message); } } + Ok(()) } } diff --git a/src/tests/sqlsmith/src/sql_gen/expr.rs b/src/tests/sqlsmith/src/sql_gen/expr.rs index eb632b259ff0..bfe1a0dab2e9 100644 --- a/src/tests/sqlsmith/src/sql_gen/expr.rs +++ b/src/tests/sqlsmith/src/sql_gen/expr.rs @@ -267,11 +267,10 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { } } DataType::Geometry => { - let x: f64 = self.rng.gen_range(-1.7e10..=1.7e10); - let y: f64 = self.rng.gen_range(-1.7e10..=1.7e10); + let geo = self.gen_geometry(); let arg = Expr::Literal { span: None, - value: Literal::String(format!("POINT({} {})", x, y)), + value: Literal::String(geo), }; Expr::FunctionCall { span: None, @@ -634,19 +633,18 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { } } DataType::Geometry => { - let (func_name, args) = match self.rng.gen_range(0..=1) { + let (func_name, args) = match self.rng.gen_range(0..=10) { 0 => { let arg_ty = DataType::Number(NumberDataType::Float64); let x = self.gen_expr(&arg_ty); let y = self.gen_expr(&arg_ty); ("st_makegeompoint", vec![x, y]) } - 1 => { - let x: f64 = self.rng.gen_range(-1.7e10..=1.7e10); - let y: f64 = self.rng.gen_range(-1.7e10..=1.7e10); + _ => { + let geo = self.gen_geometry(); let arg0 = Expr::Literal { span: None, - value: Literal::String(format!("POINT({} {})", x, y)), + value: Literal::String(geo), }; let args = if self.rng.gen_bool(0.5) { let arg1_ty = DataType::Number(NumberDataType::Int32); @@ -657,7 +655,6 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { }; ("st_geometryfromwkt", args) } - _ => unreachable!(), }; Expr::FunctionCall { span: None, @@ -701,6 +698,106 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { } } + pub(crate) fn gen_point(&mut self) -> String { + let x: f64 = self.rng.gen_range(-1.7e10..=1.7e10); + let y: f64 = self.rng.gen_range(-1.7e10..=1.7e10); + format!("{} {}", x, y) + } + + pub(crate) fn gen_points(&mut self) -> String { + let mut points = vec![]; + for _ in 0..self.rng.gen_range(1..=6) { + let point = format!("{}", self.gen_point()); + points.push(point); + } + points.join(", ") + } + + pub(crate) fn gen_simple_geometry(&mut self) -> String { + match self.rng.gen_range(0..=5) { + 0 => match self.rng.gen_range(0..=10) { + 0 => "POINT EMPTY".to_string(), + _ => { + format!("POINT({})", self.gen_point()) + } + }, + 1 => match self.rng.gen_range(0..=10) { + 0 => "MULTIPOINT EMPTY".to_string(), + _ => { + let mut points = vec![]; + for _ in 0..self.rng.gen_range(1..=6) { + let point = format!("({})", self.gen_point()); + points.push(point); + } + format!("MULTIPOINT({})", points.join(", ")) + } + }, + 2 => match self.rng.gen_range(0..=10) { + 0 => "LINESTRING EMPTY".to_string(), + _ => { + let points = self.gen_points(); + format!("LINESTRING({})", points) + } + }, + 3 => match self.rng.gen_range(0..=10) { + 0 => "MULTILINESTRING EMPTY".to_string(), + _ => { + let mut lines = vec![]; + for _ in 0..self.rng.gen_range(1..=6) { + let points = self.gen_points(); + let line = format!("({})", points); + lines.push(line); + } + format!("MULTILINESTRING({})", lines.join(", ")) + } + }, + 4 => match self.rng.gen_range(0..=10) { + 0 => "POLYGON EMPTY".to_string(), + _ => { + let mut polygons = vec![]; + for _ in 0..self.rng.gen_range(1..=6) { + let points = self.gen_points(); + let polygon = format!("({})", points); + polygons.push(polygon); + } + format!("POLYGON({})", polygons.join(", ")) + } + }, + 5 => match self.rng.gen_range(0..=10) { + 0 => "MULTIPOLYGON EMPTY".to_string(), + _ => { + let mut polygons = vec![]; + for _ in 0..self.rng.gen_range(1..=6) { + let points = self.gen_points(); + let polygon = format!("(({}))", points); + polygons.push(polygon); + } + format!("MULTIPOLYGON({})", polygons.join(", ")) + } + }, + _ => unreachable!(), + } + } + + pub(crate) fn gen_geometry(&mut self) -> String { + let geo = match self.rng.gen_range(0..=8) { + 0 => { + let mut geos = vec![]; + for _ in 0..self.rng.gen_range(1..=4) { + geos.push(self.gen_simple_geometry()); + } + format!("GEOMETRYCOLLECTION({})", geos.join(", ")) + } + _ => self.gen_simple_geometry(), + }; + if self.rng.gen_bool(0.4) { + let srid = self.rng.gen_range(1..=10000); + format!("SRID={};{}", srid, geo) + } else { + geo + } + } + pub(crate) fn gen_binary_expr(&mut self) -> Expr { let (op, left, right) = match self.rng.gen_range(0..=3) { 0..=1 => { From 62e5f24bfa2648bed1d0505b42a73e063ab4888f Mon Sep 17 00:00:00 2001 From: zhya Date: Sat, 23 Nov 2024 17:50:11 +0800 Subject: [PATCH 81/92] chore(ci): flaky test (#16919) flaky test --- .../suites/base/09_fuse_engine/09_0011_change_tracking.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0011_change_tracking.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0011_change_tracking.test index 2263a860cd04..5d1c807d4c0a 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0011_change_tracking.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0011_change_tracking.test @@ -97,7 +97,7 @@ statement ok set enable_experimental_merge_into = 1 query TTT -merge into t using t2 on t.a = t2.a when matched and t2.a = 1 then update set t.a = 0 when matched and t2.a = 2 then delete when not matched then insert * +settings (max_threads = 8) merge into t using t2 on t.a = t2.a when matched and t2.a = 1 then update set t.a = 0 when matched and t2.a = 2 then delete when not matched then insert * ---- 1 1 1 From d09ffa5aba3b6684edab03f7abe318249eac9db5 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Sat, 23 Nov 2024 20:54:38 +0800 Subject: [PATCH 82/92] chore(query): add snapshot logs in read partitions (#16918) --- .../src/operations/read/parquet_rows_fetcher.rs | 14 ++++++++++++++ .../fuse/src/operations/read_partitions.rs | 9 +++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs b/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs index f08a192ba723..260fe64a36da 100644 --- a/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs +++ b/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs @@ -23,6 +23,7 @@ use databend_common_catalog::plan::split_row_id; use databend_common_catalog::plan::PartInfoPtr; use databend_common_catalog::plan::Projection; use databend_common_catalog::table::Table; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; @@ -142,6 +143,19 @@ impl RowsFetcher for ParquetRowsFetcher { }) .collect::>(); + // check if row index is in valid bounds cause we don't ensure rowid is valid + for (block_idx, row_idx, _) in indices.iter() { + if *block_idx as usize >= blocks.len() + || *row_idx as usize >= blocks[*block_idx as usize].num_rows() + { + return Err(ErrorCode::Internal(format!( + "RowID is invalid, block idx {block_idx}, row idx {row_idx}, blocks len {}, block idx len {:?}", + blocks.len(), + blocks.get(*block_idx as usize).map(|b| b.num_rows()), + ))); + } + } + Ok(DataBlock::take_blocks(&blocks, &indices, num_rows)) } diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 058ea80e573f..482bb42ee1ac 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -39,7 +39,6 @@ use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::table::ChangeType; -use log::debug; use log::info; use sha2::Digest; use sha2::Sha256; @@ -62,7 +61,6 @@ impl FuseTable { dry_run: bool, ) -> Result<(PartStatistics, Partitions)> { let distributed_pruning = ctx.get_settings().get_enable_distributed_pruning()?; - debug!("fuse table do read partitions, push downs:{:?}", push_downs); if let Some(changes_desc) = &self.changes_desc { // For "ANALYZE TABLE" statement, we need set the default change type to "Insert". let change_type = push_downs.as_ref().map_or(ChangeType::Insert, |v| { @@ -74,6 +72,13 @@ impl FuseTable { } let snapshot = self.read_table_snapshot().await?; + + info!( + "fuse table {} do read partitions, push downs:{:?}, snapshot id: {:?}", + self.name(), + push_downs, + snapshot.as_ref().map(|sn| sn.snapshot_id) + ); match snapshot { Some(snapshot) => { let snapshot_loc = self From c6b23aef4440e466607be8e71712a21871ac0ead Mon Sep 17 00:00:00 2001 From: Bohu Date: Sat, 23 Nov 2024 10:34:02 -0800 Subject: [PATCH 83/92] Update README.md --- src/common/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/README.md b/src/common/README.md index 25078fcb5c3f..1fa659066af2 100644 --- a/src/common/README.md +++ b/src/common/README.md @@ -1,6 +1,5 @@ # Databend Common -- [`arrow`](./arrow/), a simple wrapper for reading and writing parquet. - [`auth`](./auth/), a simple wrapper for authentication layer. - [`base`](./base/) contains runtime, pool, allocator and rangemap. - [`building`](./building/) sets up the environment for building components and internal use. From cf150d958ac5ad6041741bbde972218af955917d Mon Sep 17 00:00:00 2001 From: TCeason <33082201+TCeason@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:35:37 +0800 Subject: [PATCH 84/92] feat(query): add new function TO_TIMESTAMP (#16924) feat(query): add new function TO_TIMESTAMP(int, scale) scale support [0,6] select TO_TIMESTAMP(1, 0), TO_TIMESTAMP(1, 1), TO_TIMESTAMP(1, 2), TO_TIMESTAMP(1, 3), TO_TIMESTAMP(1, 4), TO_TIMESTAMP(1, 5), TO_TIMESTAMP(1, 6); ---- 1970-01-01 00:00:01.000000 1970-01-01 00:00:00.100000 1970-01-01 00:00:00.010000 1970-01-01 00:00:00.001000 1970-01-01 00:00:00.000100 1970-01-01 00:00:00.000010 1970-01-01 00:00:00.000001 --- src/query/functions/src/scalars/datetime.rs | 24 +++++++++++++++++++ .../it/scalars/testdata/function_list.txt | 4 ++++ .../functions/02_0012_function_datetimes.test | 15 ++++++++++++ 3 files changed, 43 insertions(+) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 87c29543af2f..23c7af7b8d48 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -541,6 +541,30 @@ fn register_number_to_timestamp(registry: &mut FunctionRegistry) { error_to_null(eval_number_to_timestamp), ); + registry.register_passthrough_nullable_2_arg::( + "to_timestamp", + |_, _, _| FunctionDomain::Full, + vectorize_with_builder_2_arg::( + |val, scale, output, _| { + let mut n = val * 10i64.pow(6 - scale.clamp(0, 6) as u32); + clamp_timestamp(&mut n); + output.push(n) + }, + ), + ); + + registry.register_passthrough_nullable_2_arg::( + "try_to_timestamp", + |_, _, _| FunctionDomain::Full, + vectorize_with_builder_2_arg::( + |val, scale, output, _| { + let mut n = val * 10i64.pow(6 - scale.clamp(0, 6) as u32); + clamp_timestamp(&mut n); + output.push(n); + }, + ), + ); + fn eval_number_to_timestamp( val: ValueRef, ctx: &mut EvalContext, diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index bb706257a921..afc725e13450 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -4164,6 +4164,8 @@ Functions overloads: 7 to_timestamp(Date NULL) :: Timestamp NULL 8 to_timestamp(Int64) :: Timestamp 9 to_timestamp(Int64 NULL) :: Timestamp NULL +10 to_timestamp(Int64, UInt64) :: Timestamp +11 to_timestamp(Int64 NULL, UInt64 NULL) :: Timestamp NULL 0 to_uint16(Variant) :: UInt16 1 to_uint16(Variant NULL) :: UInt16 NULL 2 to_uint16(String) :: UInt16 @@ -4614,6 +4616,8 @@ Functions overloads: 7 try_to_timestamp(Date NULL) :: Timestamp NULL 8 try_to_timestamp(Int64) :: Timestamp NULL 9 try_to_timestamp(Int64 NULL) :: Timestamp NULL +10 try_to_timestamp(Int64, UInt64) :: Timestamp +11 try_to_timestamp(Int64 NULL, UInt64 NULL) :: Timestamp NULL 0 try_to_uint16(Variant) :: UInt16 NULL 1 try_to_uint16(Variant NULL) :: UInt16 NULL 2 try_to_uint16(String) :: UInt16 NULL diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index a5220e9fa180..c38571b09a41 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -1451,3 +1451,18 @@ query T select to_timestamp('2022-03-27 07:54:31.12'); ---- 2022-03-27 07:54:31.120000 + +query T +select TO_TIMESTAMP(-7233803000000+1, 6), TO_TIMESTAMP(-7233803000, 3), TO_TIMESTAMP(-7233803000000-1, 6); +---- +1969-10-09 06:36:37.000001 1969-10-09 06:36:37.000000 1969-10-09 06:36:36.999999 + +query T +select TO_TIMESTAMP(1, 0), TO_TIMESTAMP(1, 1), TO_TIMESTAMP(1, 2), TO_TIMESTAMP(1, 3), TO_TIMESTAMP(1, 4), TO_TIMESTAMP(1, 5), TO_TIMESTAMP(1, 6); +---- +1970-01-01 00:00:01.000000 1970-01-01 00:00:00.100000 1970-01-01 00:00:00.010000 1970-01-01 00:00:00.001000 1970-01-01 00:00:00.000100 1970-01-01 00:00:00.000010 1970-01-01 00:00:00.000001 + +query T +select TRY_TO_TIMESTAMP(1, 0), TRY_TO_TIMESTAMP(1, null); +---- +1970-01-01 00:00:01.000000 NULL From 682039ffc942e5edec3f2a358e8b295622cb102b Mon Sep 17 00:00:00 2001 From: Yang Xiufeng Date: Mon, 25 Nov 2024 15:08:42 +0800 Subject: [PATCH 85/92] feat: disable specifying copy options when create stage. (#16925) feat disable specifying copy options when create stage. --- Cargo.lock | 2 + src/common/storage/Cargo.toml | 1 + src/common/storage/src/copy.rs | 2 +- src/meta/app/src/principal/user_stage.rs | 91 +--- src/query/ast/src/ast/format/syntax/dml.rs | 19 +- src/query/ast/src/ast/statements/copy.rs | 223 ++++++++-- src/query/ast/src/ast/statements/principal.rs | 60 --- src/query/ast/src/ast/statements/stage.rs | 15 - src/query/ast/src/parser/copy.rs | 24 +- src/query/ast/src/parser/statement.rs | 11 - src/query/ast/src/parser/token.rs | 2 - .../ast/tests/it/testdata/stmt-error.txt | 4 +- src/query/ast/tests/it/testdata/stmt.txt | 392 ++++++++++-------- .../plan/datasource/datasource_info/stage.rs | 4 +- .../interpreter_copy_into_location.rs | 1 + .../interpreter_copy_into_table.rs | 11 +- .../src/interpreters/interpreter_replace.rs | 13 +- .../builders/builder_copy_into_table.rs | 9 +- .../pipelines/builders/builder_on_finished.rs | 12 +- src/query/service/src/sessions/query_ctx.rs | 3 + .../service/src/sessions/query_ctx_shared.rs | 2 +- .../it/storages/testdata/columns_table.txt | 1 - .../physical_copy_into_table.rs | 1 - .../sql/src/planner/binder/copy_into_table.rs | 74 +--- src/query/sql/src/planner/binder/ddl/stage.rs | 16 - .../sql/src/planner/plans/copy_into_table.rs | 17 +- src/query/sql/src/planner/plans/insert.rs | 2 - src/query/storages/stage/Cargo.toml | 1 + .../storages/stage/src/read/error_handler.rs | 2 +- .../storages/stage/src/read/load_context.rs | 7 +- src/query/storages/system/src/stages_table.rs | 4 - 31 files changed, 502 insertions(+), 524 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc38613c7e9b..b25f028bcba7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4073,6 +4073,7 @@ dependencies = [ "async-backtrace", "chrono", "dashmap 6.1.0", + "databend-common-ast", "databend-common-auth", "databend-common-base", "databend-common-exception", @@ -4461,6 +4462,7 @@ dependencies = [ "async-trait", "bstr", "csv-core", + "databend-common-ast", "databend-common-base", "databend-common-building", "databend-common-catalog", diff --git a/src/common/storage/Cargo.toml b/src/common/storage/Cargo.toml index bc47fa73a375..f1990cdb5d73 100644 --- a/src/common/storage/Cargo.toml +++ b/src/common/storage/Cargo.toml @@ -15,6 +15,7 @@ arrow-schema = { workspace = true } async-backtrace = { workspace = true } chrono = { workspace = true } dashmap = { workspace = true, features = ["serde"] } +databend-common-ast = { workspace = true } databend-common-auth = { workspace = true } databend-common-base = { workspace = true } databend-common-exception = { workspace = true } diff --git a/src/common/storage/src/copy.rs b/src/common/storage/src/copy.rs index 95ea628b7a6f..97726787d452 100644 --- a/src/common/storage/src/copy.rs +++ b/src/common/storage/src/copy.rs @@ -14,8 +14,8 @@ use dashmap::mapref::entry::Entry; use dashmap::DashMap; +use databend_common_ast::ast::OnErrorMode; use databend_common_exception::ErrorCode; -use databend_common_meta_app::principal::OnErrorMode; use serde::Deserialize; use serde::Serialize; use thiserror::Error; diff --git a/src/meta/app/src/principal/user_stage.rs b/src/meta/app/src/principal/user_stage.rs index c116bb98aae8..ddeeca304de7 100644 --- a/src/meta/app/src/principal/user_stage.rs +++ b/src/meta/app/src/principal/user_stage.rs @@ -34,7 +34,6 @@ use crate::storage::StorageParams; // internalStageParams // directoryTableParams // [ FILE_FORMAT = ( { FORMAT_NAME = '' | TYPE = { CSV | JSON | AVRO | ORC | PARQUET | XML } [ formatTypeOptions ] ) } ] -// [ COPY_OPTIONS = ( copyOptions ) ] // [ COMMENT = '' ] // // -- External stage @@ -42,7 +41,6 @@ use crate::storage::StorageParams; // externalStageParams // directoryTableParams // [ FILE_FORMAT = ( { FORMAT_NAME = '' | TYPE = { CSV | JSON | AVRO | ORC | PARQUET | XML } [ formatTypeOptions ] ) } ] -// [ COPY_OPTIONS = ( copyOptions ) ] // [ COMMENT = '' ] // // @@ -52,7 +50,6 @@ use crate::storage::StorageParams; // 's3://[//]' // [ { CREDENTIALS = ( { { AWS_KEY_ID = '' AWS_SECRET_KEY = '' [ AWS_TOKEN = '' ] } | AWS_ROLE = '' } ) ) } ] // -// copyOptions ::= // ON_ERROR = { CONTINUE | SKIP_FILE | SKIP_FILE_ | SKIP_FILE_% | ABORT_STATEMENT } // SIZE_LIMIT = @@ -403,7 +400,7 @@ pub struct StageParams { pub storage: StorageParams, } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq, Copy)] pub enum OnErrorMode { Continue, SkipFileNum(u64), @@ -509,92 +506,6 @@ pub struct CopyOptions { pub detailed_output: bool, } -impl CopyOptions { - pub fn apply(&mut self, opts: &BTreeMap, ignore_unknown: bool) -> Result<()> { - if opts.is_empty() { - return Ok(()); - } - for (k, v) in opts.iter() { - match k.as_str() { - "on_error" => { - let on_error = OnErrorMode::from_str(v)?; - self.on_error = on_error; - } - "size_limit" => { - let size_limit = usize::from_str(v)?; - self.size_limit = size_limit; - } - "max_files" => { - let max_files = usize::from_str(v)?; - self.max_files = max_files; - } - "split_size" => { - let split_size = usize::from_str(v)?; - self.split_size = split_size; - } - "purge" => { - let purge = bool::from_str(v).map_err(|_| { - ErrorCode::StrParseError(format!("Cannot parse purge: {} as bool", v)) - })?; - self.purge = purge; - } - "single" => { - let single = bool::from_str(v).map_err(|_| { - ErrorCode::StrParseError(format!("Cannot parse single: {} as bool", v)) - })?; - self.single = single; - } - "max_file_size" => { - let max_file_size = usize::from_str(v)?; - self.max_file_size = max_file_size; - } - "disable_variant_check" => { - let disable_variant_check = bool::from_str(v).map_err(|_| { - ErrorCode::StrParseError(format!( - "Cannot parse disable_variant_check: {} as bool", - v - )) - })?; - self.disable_variant_check = disable_variant_check; - } - "return_failed_only" => { - let return_failed_only = bool::from_str(v).map_err(|_| { - ErrorCode::StrParseError(format!( - "Cannot parse return_failed_only: {} as bool", - v - )) - })?; - self.return_failed_only = return_failed_only; - } - _ => { - if !ignore_unknown { - return Err(ErrorCode::BadArguments(format!( - "Unknown stage copy option {}", - k - ))); - } - } - } - } - Ok(()) - } -} - -impl Display for CopyOptions { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "OnErrorMode {}", self.on_error)?; - write!(f, "SizeLimit {}", self.size_limit)?; - write!(f, "MaxFiles {}", self.max_files)?; - write!(f, "SplitSize {}", self.split_size)?; - write!(f, "Purge {}", self.purge)?; - write!(f, "DisableVariantCheck {}", self.disable_variant_check)?; - write!(f, "ReturnFailedOnly {}", self.return_failed_only)?; - write!(f, "MaxFileSize {}", self.max_file_size)?; - write!(f, "Single {}", self.single)?; - write!(f, "DetailedOutput {}", self.detailed_output) - } -} - #[derive(serde::Serialize, serde::Deserialize, Default, Clone, Debug, Eq, PartialEq)] #[serde(default)] pub struct StageInfo { diff --git a/src/query/ast/src/ast/format/syntax/dml.rs b/src/query/ast/src/ast/format/syntax/dml.rs index fc270b5e43b0..440368b25e7a 100644 --- a/src/query/ast/src/ast/format/syntax/dml.rs +++ b/src/query/ast/src/ast/format/syntax/dml.rs @@ -188,36 +188,39 @@ pub(crate) fn pretty_copy_into_table(copy_stmt: CopyIntoTableStmt) -> RcDoc<'sta } else { RcDoc::nil() }) - .append(if !copy_stmt.validation_mode.is_empty() { + .append(if !copy_stmt.options.validation_mode.is_empty() { RcDoc::line() .append(RcDoc::text("VALIDATION_MODE = ")) - .append(RcDoc::text(copy_stmt.validation_mode)) + .append(RcDoc::text(copy_stmt.options.validation_mode)) } else { RcDoc::nil() }) - .append(if copy_stmt.size_limit != 0 { + .append(if copy_stmt.options.size_limit != 0 { RcDoc::line() .append(RcDoc::text("SIZE_LIMIT = ")) - .append(RcDoc::text(format!("{}", copy_stmt.size_limit))) + .append(RcDoc::text(format!("{}", copy_stmt.options.size_limit))) } else { RcDoc::nil() }) - .append(if copy_stmt.max_files != 0 { + .append(if copy_stmt.options.max_files != 0 { RcDoc::line() .append(RcDoc::text("MAX_FILES = ")) - .append(RcDoc::text(format!("{}", copy_stmt.max_files))) + .append(RcDoc::text(format!("{}", copy_stmt.options.max_files))) } else { RcDoc::nil() }) .append( RcDoc::line() .append(RcDoc::text("PURGE = ")) - .append(RcDoc::text(format!("{}", copy_stmt.purge))), + .append(RcDoc::text(format!("{}", copy_stmt.options.purge))), ) .append( RcDoc::line() .append(RcDoc::text("DISABLE_VARIANT_CHECK = ")) - .append(RcDoc::text(format!("{}", copy_stmt.disable_variant_check))), + .append(RcDoc::text(format!( + "{}", + copy_stmt.options.disable_variant_check + ))), ) } diff --git a/src/query/ast/src/ast/statements/copy.rs b/src/query/ast/src/ast/statements/copy.rs index 3cfaac47e08f..241268dbd4dc 100644 --- a/src/query/ast/src/ast/statements/copy.rs +++ b/src/query/ast/src/ast/statements/copy.rs @@ -14,8 +14,10 @@ use std::collections::BTreeMap; use std::collections::HashSet; +use std::fmt; use std::fmt::Display; use std::fmt::Formatter; +use std::str::FromStr; use derive_visitor::Drive; use derive_visitor::DriveMut; @@ -55,36 +57,29 @@ pub struct CopyIntoTableStmt { // files to load pub files: Option>, pub pattern: Option, - pub force: bool, - // copy options - /// TODO(xuanwo): parse into validation_mode directly. - pub validation_mode: String, - pub size_limit: usize, - pub max_files: usize, - pub split_size: usize, - pub purge: bool, - pub disable_variant_check: bool, - pub return_failed_only: bool, - pub on_error: String, + pub options: CopyIntoTableOptions, } impl CopyIntoTableStmt { - pub fn apply_option(&mut self, opt: CopyIntoTableOption) { + pub fn apply_option( + &mut self, + opt: CopyIntoTableOption, + ) -> std::result::Result<(), &'static str> { match opt { CopyIntoTableOption::Files(v) => self.files = Some(v), CopyIntoTableOption::Pattern(v) => self.pattern = Some(v), CopyIntoTableOption::FileFormat(v) => self.file_format = v, - CopyIntoTableOption::ValidationMode(v) => self.validation_mode = v, - CopyIntoTableOption::SizeLimit(v) => self.size_limit = v, - CopyIntoTableOption::MaxFiles(v) => self.max_files = v, - CopyIntoTableOption::SplitSize(v) => self.split_size = v, - CopyIntoTableOption::Purge(v) => self.purge = v, - CopyIntoTableOption::Force(v) => self.force = v, - CopyIntoTableOption::DisableVariantCheck(v) => self.disable_variant_check = v, - CopyIntoTableOption::ReturnFailedOnly(v) => self.return_failed_only = v, - CopyIntoTableOption::OnError(v) => self.on_error = v, + CopyIntoTableOption::SizeLimit(v) => self.options.size_limit = v, + CopyIntoTableOption::MaxFiles(v) => self.options.max_files = v, + CopyIntoTableOption::SplitSize(v) => self.options.split_size = v, + CopyIntoTableOption::Purge(v) => self.options.purge = v, + CopyIntoTableOption::Force(v) => self.options.force = v, + CopyIntoTableOption::DisableVariantCheck(v) => self.options.disable_variant_check = v, + CopyIntoTableOption::ReturnFailedOnly(v) => self.options.return_failed_only = v, + CopyIntoTableOption::OnError(v) => self.options.on_error = OnErrorMode::from_str(&v)?, } + Ok(()) } } @@ -117,32 +112,119 @@ impl Display for CopyIntoTableStmt { write!(f, " FILE_FORMAT = ({})", self.file_format)?; } - if !self.validation_mode.is_empty() { - write!(f, "VALIDATION_MODE = {}", self.validation_mode)?; + if !self.options.validation_mode.is_empty() { + write!(f, "VALIDATION_MODE = {}", self.options.validation_mode)?; } - if self.size_limit != 0 { - write!(f, " SIZE_LIMIT = {}", self.size_limit)?; + if self.options.size_limit != 0 { + write!(f, " SIZE_LIMIT = {}", self.options.size_limit)?; } - if self.max_files != 0 { - write!(f, " MAX_FILES = {}", self.max_files)?; + if self.options.max_files != 0 { + write!(f, " MAX_FILES = {}", self.options.max_files)?; } - if self.split_size != 0 { - write!(f, " SPLIT_SIZE = {}", self.split_size)?; + if self.options.split_size != 0 { + write!(f, " SPLIT_SIZE = {}", self.options.split_size)?; } - write!(f, " PURGE = {}", self.purge)?; - write!(f, " FORCE = {}", self.force)?; - write!(f, " DISABLE_VARIANT_CHECK = {}", self.disable_variant_check)?; - write!(f, " ON_ERROR = {}", self.on_error)?; - write!(f, " RETURN_FAILED_ONLY = {}", self.return_failed_only)?; + write!(f, " PURGE = {}", self.options.purge)?; + write!(f, " FORCE = {}", self.options.force)?; + write!( + f, + " DISABLE_VARIANT_CHECK = {}", + self.options.disable_variant_check + )?; + write!(f, " ON_ERROR = {}", self.options.on_error)?; + write!( + f, + " RETURN_FAILED_ONLY = {}", + self.options.return_failed_only + )?; Ok(()) } } +#[derive( + serde::Serialize, serde::Deserialize, Debug, Clone, Default, PartialEq, Drive, DriveMut, Eq, +)] +pub struct CopyIntoTableOptions { + pub on_error: OnErrorMode, + pub size_limit: usize, + pub max_files: usize, + pub split_size: usize, + pub force: bool, + pub purge: bool, + pub disable_variant_check: bool, + pub return_failed_only: bool, + pub validation_mode: String, +} + +impl CopyIntoTableOptions { + fn parse_uint(k: &str, v: &String) -> std::result::Result { + usize::from_str(v).map_err(|e| format!("can not parse {}={} as uint: {}", k, v, e)) + } + fn parse_bool(k: &str, v: &String) -> std::result::Result { + bool::from_str(v).map_err(|e| format!("can not parse {}={} as bool: {}", k, v, e)) + } + + pub fn apply( + &mut self, + opts: &BTreeMap, + ignore_unknown: bool, + ) -> std::result::Result<(), String> { + if opts.is_empty() { + return Ok(()); + } + for (k, v) in opts.iter() { + match k.as_str() { + "on_error" => { + let on_error = OnErrorMode::from_str(v)?; + self.on_error = on_error; + } + "size_limit" => { + self.size_limit = Self::parse_uint(k, v)?; + } + "max_files" => { + self.max_files = Self::parse_uint(k, v)?; + } + "split_size" => { + self.split_size = Self::parse_uint(k, v)?; + } + "purge" => { + self.purge = Self::parse_bool(k, v)?; + } + "disable_variant_check" => { + self.disable_variant_check = Self::parse_bool(k, v)?; + } + "return_failed_only" => { + self.return_failed_only = Self::parse_bool(k, v)?; + } + _ => { + if !ignore_unknown { + return Err(format!("Unknown stage copy option {}", k)); + } + } + } + } + Ok(()) + } +} + +impl Display for CopyIntoTableOptions { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "OnErrorMode {}", self.on_error)?; + write!(f, "SizeLimit {}", self.size_limit)?; + write!(f, "MaxFiles {}", self.max_files)?; + write!(f, "SplitSize {}", self.split_size)?; + write!(f, "Purge {}", self.purge)?; + write!(f, "DisableVariantCheck {}", self.disable_variant_check)?; + write!(f, "ReturnFailedOnly {}", self.return_failed_only)?; + Ok(()) + } +} + #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Drive, DriveMut, Eq)] pub struct CopyIntoLocationOptions { pub single: bool, @@ -239,7 +321,7 @@ impl Display for CopyIntoTableSource { #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum CopyIntoLocationSource { Query(Box), - /// it will be rewrite as `(SELECT * FROM table)` + /// it will be rewritten as `(SELECT * FROM table)` Table(TableRef), } @@ -482,7 +564,6 @@ pub enum CopyIntoTableOption { Files(Vec), Pattern(LiteralStringOrVariable), FileFormat(FileFormatOptions), - ValidationMode(String), SizeLimit(usize), MaxFiles(usize), SplitSize(usize), @@ -563,3 +644,73 @@ impl Display for FileFormatValue { } } } + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Drive, DriveMut, Eq)] +pub enum OnErrorMode { + Continue, + SkipFileNum(u64), + AbortNum(u64), +} + +impl Default for OnErrorMode { + fn default() -> Self { + Self::AbortNum(1) + } +} + +impl Display for OnErrorMode { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + match self { + OnErrorMode::Continue => { + write!(f, "continue") + } + OnErrorMode::SkipFileNum(n) => { + if *n <= 1 { + write!(f, "skipfile") + } else { + write!(f, "skipfile_{}", n) + } + } + OnErrorMode::AbortNum(n) => { + if *n <= 1 { + write!(f, "abort") + } else { + write!(f, "abort_{}", n) + } + } + } + } +} + +const ERROR_MODE_MSG: &str = + "OnError must one of {{ CONTINUE | SKIP_FILE | SKIP_FILE_ | ABORT | ABORT_ }}"; +impl FromStr for OnErrorMode { + type Err = &'static str; + + fn from_str(s: &str) -> std::result::Result { + match s.to_uppercase().as_str() { + "" | "ABORT" => Ok(OnErrorMode::AbortNum(1)), + "CONTINUE" => Ok(OnErrorMode::Continue), + "SKIP_FILE" => Ok(OnErrorMode::SkipFileNum(1)), + v => { + if v.starts_with("ABORT_") { + let num_str = v.replace("ABORT_", ""); + let nums = num_str.parse::(); + match nums { + Ok(n) if n < 1 => Err(ERROR_MODE_MSG), + Ok(n) => Ok(OnErrorMode::AbortNum(n)), + Err(_) => Err(ERROR_MODE_MSG), + } + } else { + let num_str = v.replace("SKIP_FILE_", ""); + let nums = num_str.parse::(); + match nums { + Ok(n) if n < 1 => Err(ERROR_MODE_MSG), + Ok(n) => Ok(OnErrorMode::SkipFileNum(n)), + Err(_) => Err(ERROR_MODE_MSG), + } + } + } + } + } +} diff --git a/src/query/ast/src/ast/statements/principal.rs b/src/query/ast/src/ast/statements/principal.rs index accfcdcb695f..fe574ebb70c9 100644 --- a/src/query/ast/src/ast/statements/principal.rs +++ b/src/query/ast/src/ast/statements/principal.rs @@ -201,63 +201,3 @@ impl Display for ShareGrantObjectPrivilege { } } } - -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub struct CopyOptions { - pub on_error: OnErrorMode, - pub size_limit: usize, - pub max_files: usize, - pub split_size: usize, - pub purge: bool, - pub disable_variant_check: bool, - pub return_failed_only: bool, - pub max_file_size: usize, - pub single: bool, - pub detailed_output: bool, -} - -impl Display for CopyOptions { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "OnErrorMode {}", self.on_error)?; - write!(f, "SizeLimit {}", self.size_limit)?; - write!(f, "MaxFiles {}", self.max_files)?; - write!(f, "SplitSize {}", self.split_size)?; - write!(f, "Purge {}", self.purge)?; - write!(f, "DisableVariantCheck {}", self.disable_variant_check)?; - write!(f, "ReturnFailedOnly {}", self.return_failed_only)?; - write!(f, "MaxFileSize {}", self.max_file_size)?; - write!(f, "Single {}", self.single)?; - write!(f, "DetailedOutput {}", self.detailed_output) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub enum OnErrorMode { - Continue, - SkipFileNum(u64), - AbortNum(u64), -} - -impl Display for OnErrorMode { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - match self { - OnErrorMode::Continue => { - write!(f, "continue") - } - OnErrorMode::SkipFileNum(n) => { - if *n <= 1 { - write!(f, "skipfile") - } else { - write!(f, "skipfile_{}", n) - } - } - OnErrorMode::AbortNum(n) => { - if *n <= 1 { - write!(f, "abort") - } else { - write!(f, "abort_{}", n) - } - } - } - } -} diff --git a/src/query/ast/src/ast/statements/stage.rs b/src/query/ast/src/ast/statements/stage.rs index d8017c848198..428d40e4239c 100644 --- a/src/query/ast/src/ast/statements/stage.rs +++ b/src/query/ast/src/ast/statements/stage.rs @@ -35,9 +35,6 @@ pub struct CreateStageStmt { pub location: Option, pub file_format_options: FileFormatOptions, - pub on_error: String, - pub size_limit: usize, - pub validation_mode: String, pub comments: String, } @@ -61,18 +58,6 @@ impl Display for CreateStageStmt { write!(f, " FILE_FORMAT = ({})", self.file_format_options)?; } - if !self.on_error.is_empty() { - write!(f, " ON_ERROR = '{}'", self.on_error)?; - } - - if self.size_limit != 0 { - write!(f, " SIZE_LIMIT = {}", self.size_limit)?; - } - - if !self.validation_mode.is_empty() { - write!(f, " VALIDATION_MODE = {}", self.validation_mode)?; - } - if !self.comments.is_empty() { write!(f, " COMMENTS = '{}'", self.comments)?; } diff --git a/src/query/ast/src/parser/copy.rs b/src/query/ast/src/parser/copy.rs index e7a601a4eac3..7692e3b92b09 100644 --- a/src/query/ast/src/parser/copy.rs +++ b/src/query/ast/src/parser/copy.rs @@ -41,6 +41,7 @@ use crate::parser::stage::file_location; use crate::parser::statement::hint; use crate::parser::token::TokenKind::COPY; use crate::parser::token::TokenKind::*; +use crate::parser::ErrorKind; use crate::parser::Input; pub fn copy_into_table(i: Input) -> IResult { @@ -51,7 +52,7 @@ pub fn copy_into_table(i: Input) -> IResult { }), )); - map( + map_res( rule! { #with? ~ COPY ~ #hint? @@ -69,20 +70,15 @@ pub fn copy_into_table(i: Input) -> IResult { files: Default::default(), pattern: Default::default(), file_format: Default::default(), - validation_mode: Default::default(), - size_limit: Default::default(), - max_files: Default::default(), - split_size: Default::default(), - purge: Default::default(), - force: Default::default(), - disable_variant_check: Default::default(), - on_error: "abort".to_string(), - return_failed_only: Default::default(), + + options: Default::default(), }; for opt in opts { - copy_stmt.apply_option(opt); + copy_stmt + .apply_option(opt) + .map_err(|e| nom::Err::Failure(ErrorKind::Other(e)))?; } - Statement::CopyIntoTable(copy_stmt) + Ok(Statement::CopyIntoTable(copy_stmt)) }, )(i) } @@ -157,10 +153,6 @@ fn copy_into_table_option(i: Input) -> IResult { map(rule! { #file_format_clause }, |options| { CopyIntoTableOption::FileFormat(options) }), - map( - rule! { VALIDATION_MODE ~ "=" ~ #literal_string }, - |(_, _, validation_mode)| CopyIntoTableOption::ValidationMode(validation_mode), - ), map( rule! { SIZE_LIMIT ~ "=" ~ #literal_u64 }, |(_, _, size_limit)| CopyIntoTableOption::SizeLimit(size_limit as usize), diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 1705f94753fe..b92fabeb0516 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -1557,9 +1557,6 @@ pub fn statement_body(i: Input) -> IResult { ~ ( #stage_name ) ~ ( (URL ~ ^"=")? ~ #uri_location )? ~ ( #file_format_clause )? - ~ ( ON_ERROR ~ ^"=" ~ ^#ident )? - ~ ( SIZE_LIMIT ~ ^"=" ~ ^#literal_u64 )? - ~ ( VALIDATION_MODE ~ ^"=" ~ ^#ident )? ~ ( (COMMENT | COMMENTS) ~ ^"=" ~ ^#literal_string )? }, |( @@ -1570,9 +1567,6 @@ pub fn statement_body(i: Input) -> IResult { stage, url_opt, file_format_opt, - on_error_opt, - size_limit_opt, - validation_mode_opt, comment_opt, )| { let create_option = @@ -1582,11 +1576,6 @@ pub fn statement_body(i: Input) -> IResult { stage_name: stage.to_string(), location: url_opt.map(|(_, location)| location), file_format_options: file_format_opt.unwrap_or_default(), - on_error: on_error_opt.map(|v| v.2.to_string()).unwrap_or_default(), - size_limit: size_limit_opt.map(|v| v.2 as usize).unwrap_or_default(), - validation_mode: validation_mode_opt - .map(|v| v.2.to_string()) - .unwrap_or_default(), comments: comment_opt.map(|v| v.2).unwrap_or_default(), })) }, diff --git a/src/query/ast/src/parser/token.rs b/src/query/ast/src/parser/token.rs index 657ca0ccf0c7..232cd7210a23 100644 --- a/src/query/ast/src/parser/token.rs +++ b/src/query/ast/src/parser/token.rs @@ -1220,8 +1220,6 @@ pub enum TokenKind { VACUUM, #[token("VALUES", ignore(ascii_case))] VALUES, - #[token("VALIDATION_MODE", ignore(ascii_case))] - VALIDATION_MODE, #[token("VARBINARY", ignore(ascii_case))] VARBINARY, #[token("VARCHAR", ignore(ascii_case))] diff --git a/src/query/ast/tests/it/testdata/stmt-error.txt b/src/query/ast/tests/it/testdata/stmt-error.txt index 18ec23cfb979..7a5779795204 100644 --- a/src/query/ast/tests/it/testdata/stmt-error.txt +++ b/src/query/ast/tests/it/testdata/stmt-error.txt @@ -321,7 +321,7 @@ error: --> SQL:1:38 | 1 | COPY INTO mytable FROM 's3://bucket' CONECTION= (); - | ^^^^^^^^^ unexpected `CONECTION`, expecting `CONNECTION`, `ON_ERROR`, `RETURN_FAILED_ONLY`, `FORMAT`, `VALIDATION_MODE`, `FORCE`, `PATTERN`, `FILES`, `PURGE`, `SIZE_LIMIT`, `FILE_FORMAT`, `MAX_FILES`, `DISABLE_VARIANT_CHECK`, `SPLIT_SIZE`, or `;` + | ^^^^^^^^^ unexpected `CONECTION`, expecting `CONNECTION`, `ON_ERROR`, `RETURN_FAILED_ONLY`, `FORMAT`, `FORCE`, `PATTERN`, `FILES`, `PURGE`, `SIZE_LIMIT`, `FILE_FORMAT`, `MAX_FILES`, `DISABLE_VARIANT_CHECK`, `SPLIT_SIZE`, or `;` ---------- Input ---------- @@ -331,7 +331,7 @@ error: --> SQL:1:33 | 1 | COPY INTO mytable FROM @mystage CONNECTION = (); - | ^^^^^^^^^^ unexpected `CONNECTION`, expecting `ON_ERROR`, `RETURN_FAILED_ONLY`, `FORMAT`, `FORCE`, `FILES`, `PURGE`, `SIZE_LIMIT`, `FILE_FORMAT`, `VALIDATION_MODE`, `DISABLE_VARIANT_CHECK`, `PATTERN`, `MAX_FILES`, `SPLIT_SIZE`, or `;` + | ^^^^^^^^^^ unexpected `CONNECTION`, expecting `ON_ERROR`, `RETURN_FAILED_ONLY`, `FORMAT`, `FORCE`, `FILES`, `PURGE`, `SIZE_LIMIT`, `FILE_FORMAT`, `DISABLE_VARIANT_CHECK`, `PATTERN`, `MAX_FILES`, `SPLIT_SIZE`, or `;` ---------- Input ---------- diff --git a/src/query/ast/tests/it/testdata/stmt.txt b/src/query/ast/tests/it/testdata/stmt.txt index 8bd0d0d7a129..206022cca94e 100644 --- a/src/query/ast/tests/it/testdata/stmt.txt +++ b/src/query/ast/tests/it/testdata/stmt.txt @@ -11232,9 +11232,6 @@ CreateStage( file_format_options: FileFormatOptions { options: {}, }, - on_error: "", - size_limit: 0, - validation_mode: "", comments: "", }, ) @@ -11276,9 +11273,6 @@ CreateStage( ), }, }, - on_error: "", - size_limit: 0, - validation_mode: "", comments: "", }, ) @@ -11320,9 +11314,6 @@ CreateStage( ), }, }, - on_error: "", - size_limit: 0, - validation_mode: "", comments: "", }, ) @@ -11364,9 +11355,6 @@ CreateStage( ), }, }, - on_error: "", - size_limit: 0, - validation_mode: "", comments: "", }, ) @@ -11408,9 +11396,6 @@ CreateStage( ), }, }, - on_error: "", - size_limit: 0, - validation_mode: "", comments: "", }, ) @@ -13898,15 +13883,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -13965,15 +13954,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14041,15 +14034,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 10, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 10, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14117,15 +14114,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 3000, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 3000, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14197,15 +14198,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14277,15 +14282,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14332,15 +14341,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 0, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 0, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14387,15 +14400,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 0, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 0, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14458,15 +14475,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14715,15 +14736,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14782,15 +14807,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14849,15 +14878,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14916,15 +14949,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: true, - validation_mode: "", - size_limit: 0, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 0, + max_files: 0, + split_size: 0, + force: true, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -14992,15 +15029,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 10, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: true, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 10, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: true, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -15062,15 +15103,19 @@ CopyIntoTable( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 0, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 0, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, ) @@ -20887,15 +20932,19 @@ CreatePipe( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 0, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 0, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, }, ) @@ -20948,15 +20997,19 @@ CreatePipe( }, files: None, pattern: None, - force: false, - validation_mode: "", - size_limit: 0, - max_files: 0, - split_size: 0, - purge: false, - disable_variant_check: false, - return_failed_only: false, - on_error: "abort", + options: CopyIntoTableOptions { + on_error: AbortNum( + 1, + ), + size_limit: 0, + max_files: 0, + split_size: 0, + force: false, + purge: false, + disable_variant_check: false, + return_failed_only: false, + validation_mode: "", + }, }, }, ) @@ -22611,9 +22664,6 @@ CreateStage( ), }, }, - on_error: "", - size_limit: 0, - validation_mode: "", comments: "", }, ) diff --git a/src/query/catalog/src/plan/datasource/datasource_info/stage.rs b/src/query/catalog/src/plan/datasource/datasource_info/stage.rs index 06d7219925ab..c48ca6f63444 100644 --- a/src/query/catalog/src/plan/datasource/datasource_info/stage.rs +++ b/src/query/catalog/src/plan/datasource/datasource_info/stage.rs @@ -18,6 +18,7 @@ use std::fmt::Formatter; use std::sync::Arc; use databend_common_ast::ast::CopyIntoLocationOptions; +use databend_common_ast::ast::CopyIntoTableOptions; use databend_common_exception::Result; use databend_common_expression::RemoteExpr; use databend_common_expression::TableSchema; @@ -42,6 +43,7 @@ pub struct StageTableInfo { pub duplicated_files_detected: Vec, pub is_select: bool, pub copy_into_location_options: CopyIntoLocationOptions, + pub copy_into_table_options: CopyIntoTableOptions, } impl StageTableInfo { @@ -95,6 +97,6 @@ impl Display for StageTableInfo { write!(f, "StageParam {}", self.stage_info.stage_params.storage)?; write!(f, "IsTemporary {}", self.stage_info.is_temporary)?; write!(f, "FileFormatParams {}", self.stage_info.file_format_params)?; - write!(f, "CopyOption {}", self.stage_info.copy_options) + Ok(()) } } diff --git a/src/query/service/src/interpreters/interpreter_copy_into_location.rs b/src/query/service/src/interpreters/interpreter_copy_into_location.rs index c771f4d2eac0..7046f7ae3f17 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_location.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_location.rs @@ -112,6 +112,7 @@ impl CopyIntoLocationInterpreter { is_select: false, default_values: None, copy_into_location_options: options.clone(), + copy_into_table_options: Default::default(), }, })); diff --git a/src/query/service/src/interpreters/interpreter_copy_into_table.rs b/src/query/service/src/interpreters/interpreter_copy_into_table.rs index ff1894c9db06..1b2ecd128ebf 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_table.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_table.rs @@ -155,7 +155,6 @@ impl CopyIntoTableInterpreter { required_source_schema: plan.required_source_schema.clone(), stage_table_info: plan.stage_table_info.clone(), table_info: to_table.get_table_info().clone(), - force: plan.force, write_mode: plan.write_mode, validation_mode: plan.validation_mode.clone(), project_columns, @@ -184,8 +183,7 @@ impl CopyIntoTableInterpreter { let return_all = !self .plan .stage_table_info - .stage_info - .copy_options + .copy_into_table_options .return_failed_only; let cs = self.ctx.get_copy_status(); @@ -249,9 +247,8 @@ impl CopyIntoTableInterpreter { let copied_files_meta_req = PipelineBuilder::build_upsert_copied_files_to_meta_req( ctx.clone(), to_table.as_ref(), - &plan.stage_table_info.stage_info, &files_to_copy, - plan.force, + &plan.stage_table_info.copy_into_table_options, )?; to_table.commit_insertion( @@ -282,7 +279,7 @@ impl CopyIntoTableInterpreter { PipelineBuilder::set_purge_files_on_finished( ctx.clone(), files_to_be_deleted, - plan.stage_table_info.stage_info.copy_options.purge, + &plan.stage_table_info.copy_into_table_options, plan.stage_table_info.stage_info.clone(), main_pipeline, )?; @@ -300,7 +297,7 @@ impl CopyIntoTableInterpreter { // unfortunately, hooking the on_finished callback of a "blank" pipeline, // e.g. `PipelineBuildResult::create` leads to runtime error (during pipeline execution). - if self.plan.stage_table_info.stage_info.copy_options.purge + if self.plan.stage_table_info.copy_into_table_options.purge && !self .plan .stage_table_info diff --git a/src/query/service/src/interpreters/interpreter_replace.rs b/src/query/service/src/interpreters/interpreter_replace.rs index 0aea9d1f2e8a..6018939ae07a 100644 --- a/src/query/service/src/interpreters/interpreter_replace.rs +++ b/src/query/service/src/interpreters/interpreter_replace.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use databend_common_ast::ast::CopyIntoTableOptions; use databend_common_catalog::lock::LockTableOption; use databend_common_catalog::table::TableExt; use databend_common_catalog::table_context::TableContext; @@ -98,11 +99,11 @@ impl Interpreter for ReplaceInterpreter { .add_lock_guard(self.plan.lock_guard.clone()); // purge - if let Some((files, stage_info)) = purge_info { + if let Some((files, stage_info, options)) = purge_info { PipelineBuilder::set_purge_files_on_finished( self.ctx.clone(), files.into_iter().map(|v| v.path).collect(), - stage_info.copy_options.purge, + &options, stage_info, &mut pipeline.main_pipeline, )?; @@ -128,7 +129,10 @@ impl Interpreter for ReplaceInterpreter { impl ReplaceInterpreter { async fn build_physical_plan( &self, - ) -> Result<(Box, Option<(Vec, StageInfo)>)> { + ) -> Result<( + Box, + Option<(Vec, StageInfo, CopyIntoTableOptions)>, + )> { let plan = &self.plan; let table = self .ctx @@ -373,7 +377,7 @@ impl ReplaceInterpreter { ctx: Arc, source: &'a InsertInputSource, schema: DataSchemaRef, - purge_info: &mut Option<(Vec, StageInfo)>, + purge_info: &mut Option<(Vec, StageInfo, CopyIntoTableOptions)>, ) -> Result { match source { InsertInputSource::Values(source) => self @@ -399,6 +403,7 @@ impl ReplaceInterpreter { *purge_info = Some(( copy_plan.stage_table_info.files_to_copy.unwrap_or_default(), copy_plan.stage_table_info.stage_info.clone(), + copy_plan.stage_table_info.copy_into_table_options.clone(), )); Ok(ReplaceSourceCtx { root: Box::new(physical_plan), diff --git a/src/query/service/src/pipelines/builders/builder_copy_into_table.rs b/src/query/service/src/pipelines/builders/builder_copy_into_table.rs index 361cb573b406..a843cbba728b 100644 --- a/src/query/service/src/pipelines/builders/builder_copy_into_table.rs +++ b/src/query/service/src/pipelines/builders/builder_copy_into_table.rs @@ -16,6 +16,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; +use databend_common_ast::ast::CopyIntoTableOptions; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; @@ -26,7 +27,6 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::Scalar; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::ParquetFileFormatParams; -use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::schema::TableCopiedFileInfo; use databend_common_meta_app::schema::UpsertTableCopiedFileReq; use databend_common_pipeline_core::Pipeline; @@ -195,9 +195,8 @@ impl PipelineBuilder { pub(crate) fn build_upsert_copied_files_to_meta_req( ctx: Arc, to_table: &dyn Table, - stage_info: &StageInfo, copied_files: &[StageFileInfo], - force: bool, + options: &CopyIntoTableOptions, ) -> Result> { let mut copied_file_tree = BTreeMap::new(); for file in copied_files { @@ -216,7 +215,7 @@ impl PipelineBuilder { let expire_hours = ctx.get_settings().get_load_file_metadata_expire_hours()?; let upsert_copied_files_request = { - if stage_info.copy_options.purge && force { + if options.purge && options.force { // if `purge-after-copy` is enabled, and in `force` copy mode, // we do not need to upsert copied files into meta server info!( @@ -231,7 +230,7 @@ impl PipelineBuilder { let req = UpsertTableCopiedFileReq { file_info: copied_file_tree, ttl: Some(Duration::from_hours(expire_hours)), - insert_if_not_exists: !force, + insert_if_not_exists: !options.force, }; Some(req) } diff --git a/src/query/service/src/pipelines/builders/builder_on_finished.rs b/src/query/service/src/pipelines/builders/builder_on_finished.rs index 1c53f44a0556..a3b69a8a5a4d 100644 --- a/src/query/service/src/pipelines/builders/builder_on_finished.rs +++ b/src/query/service/src/pipelines/builders/builder_on_finished.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use std::time::Instant; +use databend_common_ast::ast::CopyIntoTableOptions; use databend_common_base::runtime::GlobalIORuntime; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; @@ -34,7 +35,7 @@ impl PipelineBuilder { pub fn set_purge_files_on_finished( ctx: Arc, files: Vec, - copy_purge_option: bool, + options: &CopyIntoTableOptions, stage_info: StageInfo, main_pipeline: &mut Pipeline, ) -> Result<()> { @@ -42,11 +43,14 @@ impl PipelineBuilder { let txn_mgr = ctx.txn_mgr(); let mut txn_mgr = txn_mgr.lock(); let is_active = txn_mgr.is_active(); - if is_active && copy_purge_option { + if is_active && options.purge { txn_mgr.add_need_purge_files(stage_info.clone(), files.clone()); } is_active }; + + let on_error_mode = options.on_error.clone(); + let purge = options.purge; // set on_finished callback. main_pipeline.set_on_finished(move |info: &ExecutionInfo| { match &info.res { @@ -58,7 +62,7 @@ impl PipelineBuilder { for (file_name, e) in error_map { error!( "copy(on_error={}): file {} encounter error {},", - stage_info.copy_options.on_error, + on_error_mode, file_name, e.to_string() ); @@ -67,7 +71,7 @@ impl PipelineBuilder { // 2. Try to purge copied files if purge option is true, if error will skip. // If a file is already copied(status with AlreadyCopied) we will try to purge them. - if !is_active && copy_purge_option { + if !is_active && purge { Self::try_purge_files(ctx.clone(), &stage_info, &files).await; } diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 9cd8430de768..85df6db7987e 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -1368,6 +1368,7 @@ impl TableContext for QueryContext { is_select: true, default_values: None, copy_into_location_options: Default::default(), + copy_into_table_options: Default::default(), }; OrcTable::try_create(info).await } @@ -1385,6 +1386,7 @@ impl TableContext for QueryContext { is_select: true, default_values: None, copy_into_location_options: Default::default(), + copy_into_table_options: Default::default(), }; StageTable::try_create(info) } @@ -1420,6 +1422,7 @@ impl TableContext for QueryContext { is_select: true, default_values: None, copy_into_location_options: Default::default(), + copy_into_table_options: Default::default(), }; StageTable::try_create(info) } diff --git a/src/query/service/src/sessions/query_ctx_shared.rs b/src/query/service/src/sessions/query_ctx_shared.rs index 145057beb63e..429a30cc4970 100644 --- a/src/query/service/src/sessions/query_ctx_shared.rs +++ b/src/query/service/src/sessions/query_ctx_shared.rs @@ -237,7 +237,7 @@ impl QueryContextShared { } pub fn get_on_error_mode(&self) -> Option { - self.on_error_mode.read().clone() + *self.on_error_mode.read() } pub fn set_on_error_mode(&self, mode: OnErrorMode) { diff --git a/src/query/service/tests/it/storages/testdata/columns_table.txt b/src/query/service/tests/it/storages/testdata/columns_table.txt index b18e7992fae0..745b091b3c1a 100644 --- a/src/query/service/tests/it/storages/testdata/columns_table.txt +++ b/src/query/service/tests/it/storages/testdata/columns_table.txt @@ -81,7 +81,6 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | 'constraint_catalog' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | | 'constraint_name' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | | 'constraint_schema' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | -| 'copy_options' | 'system' | 'stages' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'cpu_usage' | 'system' | 'query_log' | 'UInt32' | 'INT UNSIGNED' | '' | '' | 'NO' | '' | | 'create_time' | 'information_schema' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'created_on' | 'system' | 'background_jobs' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | diff --git a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs index eb8ebb1e4e4e..1fd92f5c3ce5 100644 --- a/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs +++ b/src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs @@ -34,7 +34,6 @@ pub struct CopyIntoTable { pub required_source_schema: DataSchemaRef, pub write_mode: CopyIntoTableMode, pub validation_mode: ValidationMode, - pub force: bool, pub stage_table_info: StageTableInfo, pub table_info: TableInfo, diff --git a/src/query/sql/src/planner/binder/copy_into_table.rs b/src/query/sql/src/planner/binder/copy_into_table.rs index 84adfb0b719d..de9dd5c660e7 100644 --- a/src/query/sql/src/planner/binder/copy_into_table.rs +++ b/src/query/sql/src/planner/binder/copy_into_table.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_ast::ast::ColumnID as AstColumnID; use databend_common_ast::ast::ColumnRef; +use databend_common_ast::ast::CopyIntoTableOptions; use databend_common_ast::ast::CopyIntoTableSource; use databend_common_ast::ast::CopyIntoTableStmt; use databend_common_ast::ast::Expr; @@ -52,7 +53,6 @@ use databend_common_meta_app::principal::EmptyFieldAs; use databend_common_meta_app::principal::FileFormatOptionsReader; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::NullAs; -use databend_common_meta_app::principal::OnErrorMode; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_storage::StageFilesInfo; @@ -150,12 +150,22 @@ impl<'a> Binder { .get_table(&catalog_name, &database_name, &table_name) .await?; - let validation_mode = ValidationMode::from_str(stmt.validation_mode.as_str()) + let validation_mode = ValidationMode::from_str(stmt.options.validation_mode.as_str()) .map_err(ErrorCode::SyntaxException)?; let (mut stage_info, path) = resolve_file_location(self.ctx.as_ref(), location).await?; - self.apply_copy_into_table_options(stmt, &mut stage_info) - .await?; + if !stmt.file_format.is_empty() { + stage_info.file_format_params = self.try_resolve_file_format(&stmt.file_format).await?; + } + + if !(stmt.options.purge && stmt.options.force) + && stmt.options.max_files > COPY_MAX_FILES_PER_COMMIT + { + return Err(ErrorCode::InvalidArgument(format!( + "max_files {} is too large, max_files should be less than {COPY_MAX_FILES_PER_COMMIT}", + stmt.options.max_files + ))); + } let pattern = match &stmt.pattern { None => None, Some(pattern) => Some(Self::resolve_copy_pattern(self.ctx.clone(), pattern)?), @@ -193,7 +203,6 @@ impl<'a> Binder { is_transform, no_file_to_copy: false, from_attachment: false, - force: stmt.force, stage_table_info: StageTableInfo { schema: stage_schema, files_info, @@ -203,13 +212,13 @@ impl<'a> Binder { is_select: false, default_values, copy_into_location_options: Default::default(), + copy_into_table_options: stmt.options.clone(), }, values_consts: vec![], required_source_schema: required_values_schema.clone(), required_values_schema: required_values_schema.clone(), write_mode: CopyIntoTableMode::Copy, query: None, - enable_distributed: false, }) } @@ -266,7 +275,7 @@ impl<'a> Binder { pub(crate) async fn bind_attachment( &mut self, attachment: StageAttachment, - ) -> Result<(StageInfo, StageFilesInfo)> { + ) -> Result<(StageInfo, StageFilesInfo, CopyIntoTableOptions)> { let (mut stage_info, path) = resolve_stage_location(self.ctx.as_ref(), &attachment.location[1..]).await?; @@ -287,16 +296,18 @@ impl<'a> Binder { } stage_info.file_format_params = params; } + let mut copy_options = CopyIntoTableOptions::default(); if let Some(ref options) = attachment.copy_options { - stage_info.copy_options.apply(options, true)?; + copy_options.apply(options, true)?; } + copy_options.force = true; let files_info = StageFilesInfo { path, files: None, pattern: None, }; - Ok((stage_info, files_info)) + Ok((stage_info, files_info, copy_options)) } /// Bind COPY INFO
FROM @@ -326,7 +337,7 @@ impl<'a> Binder { let thread_num = self.ctx.get_settings().get_max_threads()? as usize; - let (stage_info, files_info) = self.bind_attachment(attachment).await?; + let (stage_info, files_info, options) = self.bind_attachment(attachment).await?; // list the files to be copied in binding phase // note that, this method(`bind_copy_from_attachment`) are used by @@ -353,7 +364,6 @@ impl<'a> Binder { required_source_schema: data_schema.clone(), required_values_schema, values_consts: const_columns, - force: true, stage_table_info: StageTableInfo { schema: stage_schema, files_info, @@ -363,6 +373,7 @@ impl<'a> Binder { is_select: false, default_values: Some(default_values), copy_into_location_options: Default::default(), + copy_into_table_options: options, }, write_mode, query: None, @@ -441,8 +452,7 @@ impl<'a> Binder { // disable variant check to allow copy invalid JSON into tables let disable_variant_check = plan .stage_table_info - .stage_info - .copy_options + .copy_into_table_options .disable_variant_check; if disable_variant_check { let hints = Hint { @@ -474,44 +484,6 @@ impl<'a> Binder { Ok(Plan::CopyIntoTable(Box::new(plan))) } - #[async_backtrace::framed] - pub async fn apply_copy_into_table_options( - &mut self, - stmt: &CopyIntoTableStmt, - stage: &mut StageInfo, - ) -> Result<()> { - if !stmt.file_format.is_empty() { - stage.file_format_params = self.try_resolve_file_format(&stmt.file_format).await?; - } - - stage.copy_options.on_error = - OnErrorMode::from_str(&stmt.on_error).map_err(ErrorCode::SyntaxException)?; - - if stmt.size_limit != 0 { - stage.copy_options.size_limit = stmt.size_limit; - } - - stage.copy_options.split_size = stmt.split_size; - stage.copy_options.purge = stmt.purge; - stage.copy_options.disable_variant_check = stmt.disable_variant_check; - stage.copy_options.return_failed_only = stmt.return_failed_only; - - if stmt.max_files != 0 { - stage.copy_options.max_files = stmt.max_files; - } - - if !(stage.copy_options.purge && stmt.force) - && stage.copy_options.max_files > COPY_MAX_FILES_PER_COMMIT - { - return Err(ErrorCode::InvalidArgument(format!( - "max_files {} is too large, max_files should be less than {COPY_MAX_FILES_PER_COMMIT}", - stage.copy_options.max_files - ))); - } - - Ok(()) - } - #[async_backtrace::framed] pub(crate) async fn prepared_values( &self, diff --git a/src/query/sql/src/planner/binder/ddl/stage.rs b/src/query/sql/src/planner/binder/ddl/stage.rs index 2036ba4f8c2e..9e61e5904493 100644 --- a/src/query/sql/src/planner/binder/ddl/stage.rs +++ b/src/query/sql/src/planner/binder/ddl/stage.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::str::FromStr; - use databend_common_ast::ast::CreateStageStmt; use databend_common_ast::ast::FileFormatOptions; use databend_common_ast::ast::UriLocation; @@ -21,7 +19,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::FileFormatOptionsReader; use databend_common_meta_app::principal::FileFormatParams; -use databend_common_meta_app::principal::OnErrorMode; use databend_common_meta_app::principal::StageInfo; use databend_common_storage::init_operator; @@ -59,9 +56,6 @@ impl Binder { stage_name, location, file_format_options, - on_error, - size_limit, - validation_mode: _, comments: _, } = stmt; @@ -103,16 +97,6 @@ impl Binder { stage_info.file_format_params = self.try_resolve_file_format(file_format_options).await?; } - // Copy options. - { - // on_error. - if !on_error.is_empty() { - stage_info.copy_options.on_error = - OnErrorMode::from_str(on_error).map_err(ErrorCode::SyntaxException)?; - } - - stage_info.copy_options.size_limit = *size_limit; - } Ok(Plan::CreateStage(Box::new(CreateStagePlan { create_option: create_option.clone().into(), diff --git a/src/query/sql/src/planner/plans/copy_into_table.rs b/src/query/sql/src/planner/plans/copy_into_table.rs index d11050411cfd..75f30f612dce 100644 --- a/src/query/sql/src/planner/plans/copy_into_table.rs +++ b/src/query/sql/src/planner/plans/copy_into_table.rs @@ -130,7 +130,6 @@ pub struct CopyIntoTablePlan { pub write_mode: CopyIntoTableMode, pub validation_mode: ValidationMode, - pub force: bool, pub stage_table_info: StageTableInfo, pub query: Option>, @@ -146,7 +145,7 @@ impl CopyIntoTablePlan { let start = Instant::now(); let stage_table_info = &self.stage_table_info; - let max_files = stage_table_info.stage_info.copy_options.max_files; + let max_files = stage_table_info.copy_into_table_options.max_files; let max_files = if max_files == 0 { None } else { @@ -155,15 +154,16 @@ impl CopyIntoTablePlan { let thread_num = ctx.get_settings().get_max_threads()? as usize; let operator = init_stage_operator(&stage_table_info.stage_info)?; + let options = &stage_table_info.copy_into_table_options; let all_source_file_infos = if operator.info().native_capability().blocking { - if self.force { + if options.force { stage_table_info .files_info .blocking_list(&operator, max_files) } else { stage_table_info.files_info.blocking_list(&operator, None) } - } else if self.force { + } else if options.force { stage_table_info .files_info .list(&operator, thread_num, max_files) @@ -187,10 +187,8 @@ impl CopyIntoTablePlan { start.elapsed() )); - let (need_copy_file_infos, duplicated) = if self.force { - if !self.stage_table_info.stage_info.copy_options.purge - && all_source_file_infos.len() > COPY_MAX_FILES_PER_COMMIT - { + let (need_copy_file_infos, duplicated) = if options.force { + if !options.purge && all_source_file_infos.len() > COPY_MAX_FILES_PER_COMMIT { return Err(ErrorCode::Internal(COPY_MAX_FILES_COMMIT_MSG)); } info!( @@ -261,7 +259,6 @@ impl Debug for CopyIntoTablePlan { table_name, no_file_to_copy, validation_mode, - force, stage_table_info, query, .. @@ -274,8 +271,6 @@ impl Debug for CopyIntoTablePlan { write!(f, ", no_file_to_copy: {no_file_to_copy:?}")?; write!(f, ", validation_mode: {validation_mode:?}")?; write!(f, ", from: {stage_table_info:?}")?; - write!(f, " force: {force}")?; - write!(f, " is_from: {force}")?; write!(f, " query: {query:?}")?; Ok(()) } diff --git a/src/query/sql/src/planner/plans/insert.rs b/src/query/sql/src/planner/plans/insert.rs index f20a2c0830ee..7685f4c6c382 100644 --- a/src/query/sql/src/planner/plans/insert.rs +++ b/src/query/sql/src/planner/plans/insert.rs @@ -163,7 +163,6 @@ pub(crate) fn format_insert_source( required_source_schema, write_mode, validation_mode, - force, stage_table_info, enable_distributed, .. @@ -191,7 +190,6 @@ pub(crate) fn format_insert_source( )), FormatTreeNode::new(format!("write_mode: {write_mode}")), FormatTreeNode::new(format!("validation_mode: {validation_mode}")), - FormatTreeNode::new(format!("force: {force}")), FormatTreeNode::new(format!("stage_table_info: {stage_table_info}")), FormatTreeNode::new(format!("enable_distributed: {enable_distributed}")), ]; diff --git a/src/query/storages/stage/Cargo.toml b/src/query/storages/stage/Cargo.toml index 6988e16c3afb..e8f6e7044f1e 100644 --- a/src/query/storages/stage/Cargo.toml +++ b/src/query/storages/stage/Cargo.toml @@ -16,6 +16,7 @@ async-backtrace = { workspace = true } async-trait = { workspace = true } bstr = { workspace = true } csv-core = { workspace = true } +databend-common-ast = { workspace = true } databend-common-base = { workspace = true } databend-common-catalog = { workspace = true } databend-common-compress = { workspace = true } diff --git a/src/query/storages/stage/src/read/error_handler.rs b/src/query/storages/stage/src/read/error_handler.rs index f56453f96a39..3f9e22b3851a 100644 --- a/src/query/storages/stage/src/read/error_handler.rs +++ b/src/query/storages/stage/src/read/error_handler.rs @@ -15,9 +15,9 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; +use databend_common_ast::ast::OnErrorMode; use databend_common_exception::Result; use databend_common_expression::ColumnBuilder; -use databend_common_meta_app::principal::OnErrorMode; use databend_common_storage::FileParseError; use databend_common_storage::FileStatus; diff --git a/src/query/storages/stage/src/read/load_context.rs b/src/query/storages/stage/src/read/load_context.rs index 2f82bdd2c982..8088c771b466 100644 --- a/src/query/storages/stage/src/read/load_context.rs +++ b/src/query/storages/stage/src/read/load_context.rs @@ -57,14 +57,15 @@ impl LoadContext { pos_projection: Option>, block_compact_thresholds: BlockThresholds, ) -> Result { - let copy_options = &stage_table_info.stage_info.copy_options; let settings = ctx.get_settings(); let func_ctx = ctx.get_function_context()?; let is_select = stage_table_info.is_select; let mut file_format_options_ext = FileFormatOptionsExt::create_from_settings(&settings, is_select)?; - file_format_options_ext.disable_variant_check = copy_options.disable_variant_check; - let on_error_mode = copy_options.on_error.clone(); + file_format_options_ext.disable_variant_check = stage_table_info + .copy_into_table_options + .disable_variant_check; + let on_error_mode = stage_table_info.copy_into_table_options.on_error.clone(); let fields = stage_table_info .schema .fields() diff --git a/src/query/storages/system/src/stages_table.rs b/src/query/storages/system/src/stages_table.rs index 13a03d1ed9fa..d597a634db31 100644 --- a/src/query/storages/system/src/stages_table.rs +++ b/src/query/storages/system/src/stages_table.rs @@ -79,7 +79,6 @@ impl AsyncSystemTable for StagesTable { let mut name: Vec = Vec::with_capacity(stages.len()); let mut stage_type: Vec = Vec::with_capacity(stages.len()); let mut stage_params: Vec = Vec::with_capacity(stages.len()); - let mut copy_options: Vec = Vec::with_capacity(stages.len()); let mut file_format_options: Vec = Vec::with_capacity(stages.len()); let mut comment: Vec = Vec::with_capacity(stages.len()); let mut number_of_files: Vec> = Vec::with_capacity(stages.len()); @@ -97,7 +96,6 @@ impl AsyncSystemTable for StagesTable { ); stage_type.push(stage.stage_type.clone().to_string()); stage_params.push(format!("{:?}", stage.stage_params)); - copy_options.push(format!("{:?}", stage.copy_options)); file_format_options.push(format!("{:?}", stage.file_format_params)); // TODO(xuanwo): we will remove this line. match stage.stage_type { @@ -117,7 +115,6 @@ impl AsyncSystemTable for StagesTable { StringType::from_data(name), StringType::from_data(stage_type), StringType::from_data(stage_params), - StringType::from_data(copy_options), StringType::from_data(file_format_options), UInt64Type::from_opt_data(number_of_files), StringType::from_opt_data(creator), @@ -134,7 +131,6 @@ impl StagesTable { TableField::new("name", TableDataType::String), TableField::new("stage_type", TableDataType::String), TableField::new("stage_params", TableDataType::String), - TableField::new("copy_options", TableDataType::String), TableField::new("file_format_options", TableDataType::String), // NULL for external stage TableField::new( From 04df0949e9585ef9cfcc6d50aab8e52410068a74 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Mon, 25 Nov 2024 22:45:27 +0800 Subject: [PATCH 86/92] fix(query): join predict use cast_expr_to_non_null_boolean (#16937) * fix(query): join predict use cast_expr_to_non_null_boolean * update * update --- src/common/column/src/buffer/immutable.rs | 75 ++++++++----------- .../column/tests/it/buffer/immutable.rs | 3 + src/query/expression/benches/bench.rs | 17 +++++ .../processors/transforms/hash_join/desc.rs | 15 +++- .../sqllogictests/suites/query/join/join.test | 30 +++++--- 5 files changed, 82 insertions(+), 58 deletions(-) diff --git a/src/common/column/src/buffer/immutable.rs b/src/common/column/src/buffer/immutable.rs index 94724df55a73..5dca56248323 100644 --- a/src/common/column/src/buffer/immutable.rs +++ b/src/common/column/src/buffer/immutable.rs @@ -63,14 +63,20 @@ pub struct Buffer { /// the internal byte buffer. data: Arc>, - /// The offset into the buffer. - offset: usize, + /// Pointer into `data` valid + /// + /// We store a pointer instead of an offset to avoid pointer arithmetic + /// which causes LLVM to fail to vectorise code correctly + ptr: *const T, // the length of the buffer. Given a region `data` of N bytes, [offset..offset+length] is visible // to this buffer. length: usize, } +unsafe impl Send for Buffer {} +unsafe impl Sync for Buffer {} + impl PartialEq for Buffer { #[inline] fn eq(&self, other: &Self) -> bool { @@ -101,9 +107,10 @@ impl Buffer { /// Auxiliary method to create a new Buffer pub(crate) fn from_bytes(bytes: Bytes) -> Self { let length = bytes.len(); + let ptr = bytes.as_ptr(); Buffer { data: Arc::new(bytes), - offset: 0, + ptr, length, } } @@ -130,24 +137,7 @@ impl Buffer { /// Returns the byte slice stored in this buffer #[inline] pub fn as_slice(&self) -> &[T] { - // Safety: - // invariant of this struct `offset + length <= data.len()` - debug_assert!(self.offset + self.length <= self.data.len()); - unsafe { - self.data - .get_unchecked(self.offset..self.offset + self.length) - } - } - - /// Returns the byte slice stored in this buffer - /// # Safety - /// `index` must be smaller than `len` - #[inline] - pub(super) unsafe fn get_unchecked(&self, index: usize) -> &T { - // Safety: - // invariant of this function - debug_assert!(index < self.length); - unsafe { self.data.get_unchecked(self.offset + index) } + self } /// Returns a new [`Buffer`] that is a slice of this buffer starting at `offset`. @@ -193,20 +183,20 @@ impl Buffer { /// The caller must ensure `offset + length <= self.len()` #[inline] pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.offset += offset; + self.ptr = self.ptr.add(offset); self.length = length; } /// Returns a pointer to the start of this buffer. #[inline] pub(crate) fn data_ptr(&self) -> *const T { - self.data.deref().as_ptr() + self.data.as_ptr() } /// Returns the offset of this buffer. #[inline] pub fn offset(&self) -> usize { - self.offset + unsafe { self.ptr.offset_from(self.data_ptr()) as usize } } /// # Safety @@ -253,10 +243,11 @@ impl Buffer { /// * has not been imported from the c data interface (FFI) #[inline] pub fn get_mut_slice(&mut self) -> Option<&mut [T]> { + let offset = self.offset(); Arc::get_mut(&mut self.data) .and_then(|b| b.get_vec()) // Safety: the invariant of this struct - .map(|x| unsafe { x.get_unchecked_mut(self.offset..self.offset + self.length) }) + .map(|x| unsafe { x.get_unchecked_mut(offset..offset + self.length) }) } /// Get the strong count of underlying `Arc` data buffer. @@ -269,28 +260,14 @@ impl Buffer { Arc::weak_count(&self.data) } - /// Returns its internal representation - #[must_use] - pub fn into_inner(self) -> (Arc>, usize, usize) { - let Self { - data, - offset, - length, - } = self; - (data, offset, length) - } - /// Creates a `[Bitmap]` from its internal representation. /// This is the inverted from `[Bitmap::into_inner]` /// /// # Safety /// Callers must ensure all invariants of this struct are upheld. pub unsafe fn from_inner_unchecked(data: Arc>, offset: usize, length: usize) -> Self { - Self { - data, - offset, - length, - } + let ptr = data.as_ptr().add(offset); + Self { data, ptr, length } } } @@ -313,8 +290,9 @@ impl From> for Buffer { #[inline] fn from(p: Vec) -> Self { let bytes: Bytes = p.into(); + let ptr = bytes.as_ptr(); Self { - offset: 0, + ptr, length: bytes.len(), data: Arc::new(bytes), } @@ -326,7 +304,15 @@ impl std::ops::Deref for Buffer { #[inline] fn deref(&self) -> &[T] { - self.as_slice() + debug_assert!(self.offset() + self.length <= self.data.len()); + unsafe { std::slice::from_raw_parts(self.ptr, self.length) } + } +} + +impl AsRef<[T]> for Buffer { + #[inline] + fn as_ref(&self) -> &[T] { + self } } @@ -375,8 +361,9 @@ impl From for Buffer { impl From> for arrow_buffer::Buffer { fn from(value: Buffer) -> Self { + let offset = value.offset(); crate::buffer::to_buffer(value.data).slice_with_length( - value.offset * std::mem::size_of::(), + offset * std::mem::size_of::(), value.length * std::mem::size_of::(), ) } diff --git a/src/common/column/tests/it/buffer/immutable.rs b/src/common/column/tests/it/buffer/immutable.rs index 7cf5457260f6..7ab23c3f2ff2 100644 --- a/src/common/column/tests/it/buffer/immutable.rs +++ b/src/common/column/tests/it/buffer/immutable.rs @@ -27,6 +27,9 @@ fn from_slice() { let buffer = Buffer::::from(vec![0, 1, 2]); assert_eq!(buffer.len(), 3); assert_eq!(buffer.as_slice(), &[0, 1, 2]); + + assert_eq!(unsafe { *buffer.get_unchecked(1) }, 1); + assert_eq!(unsafe { *buffer.get_unchecked(2) }, 2); } #[test] diff --git a/src/query/expression/benches/bench.rs b/src/query/expression/benches/bench.rs index 8ff91219fa5c..bfc342bc322a 100644 --- a/src/query/expression/benches/bench.rs +++ b/src/query/expression/benches/bench.rs @@ -15,6 +15,7 @@ #[macro_use] extern crate criterion; +use arrow_buffer::ScalarBuffer; use criterion::Criterion; use databend_common_column::buffer::Buffer; use databend_common_expression::arrow::deserialize_column; @@ -135,6 +136,9 @@ fn bench(c: &mut Criterion) { for length in [10240, 102400] { let (left, right) = generate_random_int_data(&mut rng, length); + let left_scalar = ScalarBuffer::from_iter(left.iter().cloned()); + let right_scalar = ScalarBuffer::from_iter(right.iter().cloned()); + group.bench_function(format!("function_iterator_iterator_v1/{length}"), |b| { b.iter(|| { let left = left.clone(); @@ -170,6 +174,19 @@ fn bench(c: &mut Criterion) { }, ); + group.bench_function( + format!("function_buffer_scalar_index_unchecked_iterator/{length}"), + |b| { + b.iter(|| { + let _c = (0..length) + .map(|i| unsafe { + left_scalar.get_unchecked(i) + right_scalar.get_unchecked(i) + }) + .collect::>(); + }) + }, + ); + group.bench_function( format!("function_slice_index_unchecked_iterator/{length}"), |b| { diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/desc.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/desc.rs index c4d6a7ac7a4b..4abe752799c2 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/desc.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/desc.rs @@ -17,6 +17,7 @@ use databend_common_expression::type_check::check_function; use databend_common_expression::Expr; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_sql::executor::cast_expr_to_non_null_boolean; use databend_common_sql::executor::physical_plans::HashJoin; use databend_common_sql::IndexType; use parking_lot::RwLock; @@ -96,11 +97,21 @@ impl HashJoinDesc { } fn join_predicate(non_equi_conditions: &[RemoteExpr]) -> Result> { - non_equi_conditions + let expr = non_equi_conditions .iter() .map(|expr| expr.as_expr(&BUILTIN_FUNCTIONS)) .try_reduce(|lhs, rhs| { check_function(None, "and_filters", &[], &[lhs, rhs], &BUILTIN_FUNCTIONS) - }) + }); + // For RIGHT MARK join, we can't use is_true to cast filter into non_null boolean + match expr { + Ok(Some(expr)) => match expr { + Expr::Constant { ref scalar, .. } if !scalar.is_null() => { + Ok(Some(cast_expr_to_non_null_boolean(expr)?)) + } + _ => Ok(Some(expr)), + }, + other => other, + } } } diff --git a/tests/sqllogictests/suites/query/join/join.test b/tests/sqllogictests/suites/query/join/join.test index 31c0526253e1..f0f4bcb2d33f 100644 --- a/tests/sqllogictests/suites/query/join/join.test +++ b/tests/sqllogictests/suites/query/join/join.test @@ -2,7 +2,7 @@ statement ok drop table if exists t1; statement ok -create table t1 (a int); +create or replace table t1 (a int); # right join with empty build side query II @@ -82,7 +82,7 @@ statement ok drop table if exists t1; statement ok -create table t1(a int, b int) +create or replace table t1(a int, b int) statement ok insert into t1 values(1, 2), (2, 4), (3, 6), (4, 8), (5, 10) @@ -91,7 +91,7 @@ statement ok drop table if exists t2 statement ok -create table t2(a int, b int) +create or replace table t2(a int, b int) statement ok insert into t2 values(1, 2), (1, 4), (1, 6), (1, 8), (1, 10); @@ -124,10 +124,10 @@ statement ok drop table if exists t2; statement ok -create table t1 (id int, val bigint unsigned default 0); +create or replace table t1 (id int, val bigint unsigned default 0); statement ok -create table t2 (id int, val bigint unsigned default 0); +create or replace table t2 (id int, val bigint unsigned default 0); statement ok insert into t1 values(1, 1696549154011), (2, 1696549154013); @@ -153,13 +153,13 @@ statement ok drop table t2; statement ok -create table t(id int); +create or replace table t(id int); statement ok insert into t values(1), (2); statement ok -create table t1(id int, col1 varchar); +create or replace table t1(id int, col1 varchar); statement ok insert into t1 values(1, 'c'), (3, 'd'); @@ -203,13 +203,13 @@ statement ok drop table if exists t2; statement ok -create table t1(a int, b int); +create or replace table t1(a int, b int); statement ok insert into t1 values(1, 1),(2, 2),(3, 3); statement ok -create table t2(a int, b int); +create or replace table t2(a int, b int); statement ok insert into t2 values(1, 1),(2, 2); @@ -237,13 +237,13 @@ statement ok drop table if exists t2; statement ok -create table t1(a int, b int, c int, d int); +create or replace table t1(a int, b int, c int, d int); statement ok insert into t1 values(1, 2, 3, 4); statement ok -create table t2(a int, b int, c int, d int); +create or replace table t2(a int, b int, c int, d int); statement ok insert into t2 values(1, 2, 3, 4); @@ -255,7 +255,7 @@ statement ok drop table if exists t; statement ok -create table t(a int); +create or replace table t(a int); statement ok insert into t values(1),(2),(3); @@ -274,5 +274,11 @@ select * from (select number from numbers(5)) t2 full outer join (select a, 'A' 2 2 A 3 3 A +statement ok +select * from (select number from numbers(5)) t2 full outer join (select a, 'A' as name from t) t1 on t1.a = t2.number and 123; + +statement error +select * from (select number from numbers(5)) t2 full outer join (select a, 'A' as name from t) t1 on t1.a = t2.number and 11981933213501947393::DATE; + statement ok drop table if exists t; From 10a37b99fac05cd3db40f1a923010b7a5f59ee6e Mon Sep 17 00:00:00 2001 From: TCeason <33082201+TCeason@users.noreply.github.com> Date: Tue, 26 Nov 2024 11:01:58 +0800 Subject: [PATCH 87/92] chore: bump rustls version 0.23.18 (#16936) * chore: fix flaky ci 09_0001_json_response.sh add sleep, wait table drop success. bump rustls to 0.23.18 * fix typo err --- Cargo.lock | 80 ++++++------------- Cargo.toml | 6 +- .../src/kernels/group_by_hash/utils.rs | 2 +- .../src/servers/admin/admin_service.rs | 2 +- .../09_0001_json_response.result | 2 +- .../09_http_handler/09_0001_json_response.sh | 2 +- 6 files changed, 30 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b25f028bcba7..e3301c4633db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1406,16 +1406,14 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -5071,7 +5069,7 @@ dependencies = [ "regex", "reqwest", "rmp-serde", - "rustls 0.22.4", + "rustls 0.23.18", "rustls-pemfile 2.1.3", "rustls-pki-types", "rustyline", @@ -8272,7 +8270,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.12", + "rustls 0.23.18", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -9067,12 +9065,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128" version = "0.2.5" @@ -9757,14 +9749,13 @@ dependencies = [ [[package]] name = "msql-srv" version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b821d09e9a4ed6b61015a889597446b3b6c7721544d0f4b617bcfdacf6ee7877" +source = "git+https://github.com/databendlabs/msql-srv.git?rev=cd443dd#cd443dd062e2d64e029c82d7114ce5a6b324f73c" dependencies = [ "byteorder", "chrono", "mysql_common 0.31.0", "nom", - "rustls 0.22.4", + "rustls 0.23.18", ] [[package]] @@ -10431,18 +10422,18 @@ dependencies = [ [[package]] name = "opensrv-mysql" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148ab944991b0a33be74d2636a815268974578812a9e4cf7dc785325e858154" +version = "0.8.0" +source = "git+https://github.com/databendlabs/opensrv.git?rev=a1fb4da#a1fb4da215c8693c7e4f62be249a01b7fec52997" dependencies = [ "async-trait", "byteorder", + "bytes", "chrono", "mysql_common 0.32.4", "nom", "pin-project-lite", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", ] [[package]] @@ -11814,7 +11805,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.12", + "rustls 0.23.18", "socket2", "thiserror", "tokio", @@ -11831,7 +11822,7 @@ dependencies = [ "rand", "ring", "rustc-hash 2.0.0", - "rustls 0.23.12", + "rustls 0.23.18", "slab", "thiserror", "tinyvec", @@ -12227,7 +12218,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.12", + "rustls 0.23.18", "rustls-pemfile 2.1.3", "rustls-pki-types", "serde", @@ -12618,29 +12609,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki 0.102.6", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls" -version = "0.23.12" +version = "0.23.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" dependencies = [ "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.6", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] @@ -12691,9 +12668,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" @@ -12707,9 +12684,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -14501,24 +14478,13 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.12", + "rustls 0.23.18", "rustls-pki-types", "tokio", ] @@ -15073,7 +15039,7 @@ dependencies = [ "flate2", "log", "once_cell", - "rustls 0.23.12", + "rustls 0.23.18", "rustls-pki-types", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 0b82af877c9b..87be2fb47c36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -375,7 +375,7 @@ openraft = { version = "0.10.0", features = [ "tracing-log", "loosen-follower-log-revert", # allows removing all data from a follower and restoring from the leader. ] } -opensrv-mysql = { version = "0.7.0", features = ["tls"] } +opensrv-mysql = { git = "https://github.com/databendlabs/opensrv.git", rev = "a1fb4da", features = ["tls"] } orc-rust = "0.5.0" ordered-float = { version = "4.5.0", default-features = false } ordq = "0.2.0" @@ -424,7 +424,7 @@ roaring = { version = "0.10.1", features = ["serde"] } rotbl = { version = "0.1.2", features = [] } rust_decimal = "1.26" rustix = "0.38.37" -rustls = "0.22" # FIXME: we should bump to 0.23 +rustls = { version = "0.23.18", features = ["ring", "tls12"], default-features = false } rustls-pemfile = "2" rustls-pki-types = "1" rustyline = "14" @@ -536,7 +536,7 @@ tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json", "valuable"] } # Databend Integration Test -msql-srv = "0.11.0" +msql-srv = { git = "https://github.com/databendlabs/msql-srv.git", rev = "cd443dd" } mysql_common = "0.32.4" quickcheck = "1.0" sqllogictest = "0.21.0" diff --git a/src/query/expression/src/kernels/group_by_hash/utils.rs b/src/query/expression/src/kernels/group_by_hash/utils.rs index 280edbe6e850..d682067e373d 100644 --- a/src/query/expression/src/kernels/group_by_hash/utils.rs +++ b/src/query/expression/src/kernels/group_by_hash/utils.rs @@ -40,7 +40,7 @@ pub fn serialize_group_columns( } builder.commit_row(); } - // For nulllable column it will only serialize valid row data + // For nullable column it will only serialize valid row data debug_assert!(builder.data.len() <= serialize_size); builder.build() } diff --git a/src/query/service/src/servers/admin/admin_service.rs b/src/query/service/src/servers/admin/admin_service.rs index c6454ea0af21..61b3f6731f9a 100644 --- a/src/query/service/src/servers/admin/admin_service.rs +++ b/src/query/service/src/servers/admin/admin_service.rs @@ -184,7 +184,7 @@ impl Server for AdminService { async fn shutdown(&mut self, _graceful: bool) { // intendfully do nothing: sometimes we hope to diagnose the backtraces or metrics after // the process got the sigterm signal, we can still leave the admin service port open until - // the process exited. it's not an user facing service, it's allowed to shutdown forcely. + // the process exited. it's not an user facing service, it's allowed to force shutdown. } #[async_backtrace::framed] diff --git a/tests/suites/1_stateful/09_http_handler/09_0001_json_response.result b/tests/suites/1_stateful/09_http_handler/09_0001_json_response.result index d7cb376029cd..0c22e50db4d7 100755 --- a/tests/suites/1_stateful/09_http_handler/09_0001_json_response.result +++ b/tests/suites/1_stateful/09_http_handler/09_0001_json_response.result @@ -1,3 +1,3 @@ -{"code":1025,"message":"error: \n --> SQL:1:15\n |\n1 | select * from t1\n | ^^ Unknown table \"default\".\"default\".t1 .\n\n"} +{"code":1025,"message":"error: \n --> SQL:1:15\n |\n1 | select * from system.t1\n | ^^^^^^^^^ Unknown table \"default\".system.t1 .\n\n"} {"error":{"code":400,"message":"parse error: key must be a string at line 1 column 2"}} {"error":{"code":404,"message":"not found"}} diff --git a/tests/suites/1_stateful/09_http_handler/09_0001_json_response.sh b/tests/suites/1_stateful/09_http_handler/09_0001_json_response.sh index 0bb648ddc668..ef9bf8fcb098 100755 --- a/tests/suites/1_stateful/09_http_handler/09_0001_json_response.sh +++ b/tests/suites/1_stateful/09_http_handler/09_0001_json_response.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo "drop table if exists t1;" | $BENDSQL_CLIENT_CONNECT -curl -s --header 'Content-Type: application/json' --request POST '127.0.0.1:8000/v1/query/' --data-raw '{"sql": "select * from t1", "pagination": { "wait_time_secs": 2}}' -u root: | jq -c ".error" +curl -s --header 'Content-Type: application/json' --request POST '127.0.0.1:8000/v1/query/' --data-raw '{"sql": "select * from system.t1", "pagination": { "wait_time_secs": 2}}' -u root: | jq -c ".error" curl -s --header 'Content-Type: application/json' --request POST '127.0.0.1:8000/v1/query/' --data-raw '{sql": "select * from tx", "pagination": { "wait_time_secs": 2}}' -u root: echo "" From c9b1a82a15f810d585e99a0db3b82cc7fa78fbf3 Mon Sep 17 00:00:00 2001 From: zhya Date: Tue, 26 Nov 2024 12:08:23 +0800 Subject: [PATCH 88/92] fix(ci): flaky test (#16933) * flaky test * fix * fix test --- .../pipelines/builders/builder_mutation.rs | 23 ++++++++++++++++++- .../09_0011_change_tracking.test | 12 +++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_mutation.rs b/src/query/service/src/pipelines/builders/builder_mutation.rs index 18b64ffe59f0..3cd64c367fc5 100644 --- a/src/query/service/src/pipelines/builders/builder_mutation.rs +++ b/src/query/service/src/pipelines/builders/builder_mutation.rs @@ -20,6 +20,7 @@ use databend_common_exception::Result; use databend_common_expression::BlockThresholds; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; +use databend_common_pipeline_core::processors::create_resize_item; use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::OutputPort; use databend_common_pipeline_core::processors::ProcessorPtr; @@ -239,6 +240,18 @@ impl PipelineBuilder { ) -> Result<()> { // we should avoid too much little block write, because for s3 write, there are too many // little blocks, it will cause high latency. + let mut origin_len = transform_len; + let mut resize_len = 1; + let mut pipe_items = Vec::with_capacity(2); + if need_match { + origin_len += 1; + resize_len += 1; + pipe_items.push(create_dummy_item()); + } + pipe_items.push(create_resize_item(transform_len, 1)); + self.main_pipeline + .add_pipe(Pipe::create(origin_len, resize_len, pipe_items)); + let mut builder = self.main_pipeline.add_transform_with_specified_len( |transform_input_port, transform_output_port| { Ok(ProcessorPtr::create(AccumulatingTransformer::create( @@ -247,13 +260,21 @@ impl PipelineBuilder { BlockCompactBuilder::new(block_thresholds), ))) }, - transform_len, + 1, )?; if need_match { builder.add_items_prepend(vec![create_dummy_item()]); } self.main_pipeline.add_pipe(builder.finalize()); + let mut pipe_items = Vec::with_capacity(2); + if need_match { + pipe_items.push(create_dummy_item()); + } + pipe_items.push(create_resize_item(1, transform_len)); + self.main_pipeline + .add_pipe(Pipe::create(resize_len, origin_len, pipe_items)); + let mut builder = self.main_pipeline.add_transform_with_specified_len( |transform_input_port, transform_output_port| { Ok(ProcessorPtr::create(BlockMetaTransformer::create( diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0011_change_tracking.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0011_change_tracking.test index 5d1c807d4c0a..6025b15c33f4 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0011_change_tracking.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0011_change_tracking.test @@ -91,25 +91,25 @@ statement ok create table t2(a int) statement ok -insert into t2 values(1),(2),(8) +insert into t2 values(0),(2),(1) statement ok set enable_experimental_merge_into = 1 query TTT -settings (max_threads = 8) merge into t using t2 on t.a = t2.a when matched and t2.a = 1 then update set t.a = 0 when matched and t2.a = 2 then delete when not matched then insert * +merge into t using t2 on t.a = t2.a when matched and t2.a = 1 then update set t.a = 8 when matched and t2.a = 2 then delete when not matched then insert * ---- 1 1 1 query IBBII select a, _origin_version is null, _origin_block_id is null, _origin_block_row_num, _row_version from t order by a ---- -0 0 0 0 1 +0 0 0 0 0 3 0 0 1 1 5 0 0 0 0 6 0 0 0 0 7 0 0 1 0 -8 0 0 0 0 +8 0 0 0 1 statement ok create table t1(a int) change_tracking = true @@ -131,12 +131,12 @@ merge into t using t1 on t.a = t1.a when matched and t1.a = 0 then update set t. query IBBII select a, _origin_version is null, _origin_block_id is null, _origin_block_row_num, _row_version from t order by a ---- -1 0 0 0 2 +1 0 0 0 1 2 0 0 1 2 5 0 0 0 0 6 0 0 0 0 7 0 0 1 0 -8 0 0 0 0 +8 0 0 0 1 ############### # issue 14955 # From 7b98a1353b09a08e8faa49588529e75039241de6 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Tue, 26 Nov 2024 12:33:21 +0800 Subject: [PATCH 89/92] chore(query): spilt binary symbol (#16861) * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * Update action.yml * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol --- .github/actions/build_linux/action.yml | 8 ++ .github/actions/pack_binaries/action.yml | 95 ++++++++++++++++++++ .github/actions/pack_distribution/action.yml | 94 +++++++++++++++++++ .github/actions/publish_binary/action.yml | 8 +- .github/workflows/release.yml | 87 ++---------------- scripts/distribution/nfpm-dbg.yaml | 17 ++++ scripts/setup/dev_setup.sh | 23 +++++ 7 files changed, 252 insertions(+), 80 deletions(-) create mode 100644 .github/actions/pack_binaries/action.yml create mode 100644 .github/actions/pack_distribution/action.yml create mode 100644 scripts/distribution/nfpm-dbg.yaml diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml index c974a814354b..3e5daae8778e 100644 --- a/.github/actions/build_linux/action.yml +++ b/.github/actions/build_linux/action.yml @@ -92,6 +92,14 @@ runs: readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true + - name: Spilt Binary Symbols + shell: bash + run: | + objcopy --only-keep-debug ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug + chmod 0644 ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug + strip --strip-debug --remove-section=.comment --remove-section=.note ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query + pushd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }} && objcopy --add-gnu-debuglink databend-query.debug databend-query && popd + # - name: Compress Binaries with UPX # if: env.BUILD_PROFILE == 'debug' # uses: crazy-max/ghaction-upx@v2 diff --git a/.github/actions/pack_binaries/action.yml b/.github/actions/pack_binaries/action.yml new file mode 100644 index 000000000000..64d3283b129b --- /dev/null +++ b/.github/actions/pack_binaries/action.yml @@ -0,0 +1,95 @@ +name: "Pack Binary" +description: "Pack releases binaries" +inputs: + target: + description: "Release target" + required: true + category: + description: "Release default/hdfs/udf/testsuite" + required: false + default: default + version: + description: "Release version" + required: true + +runs: + using: "composite" + steps: + - name: Download artifact + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: ${{ inputs.target }} + category: ${{ inputs.category }} + path: distro/bin + artifacts: metactl,meta,query,query.debug + - name: Pack Binaries + id: pack_binaries + shell: bash + run: | + target=${{ inputs.target }} + version=${{ inputs.version }} + case ${{ inputs.category }} in + default) + pkg_name="databend-${version}-${target}" + ;; + *) + pkg_name="databend-${{ inputs.category }}-${version}-${target}" + ;; + esac + mkdir -p distro/{bin,configs,systemd,scripts} + cp ./scripts/distribution/systemd/databend-* distro/systemd/ + cp ./scripts/distribution/configs/databend-* distro/configs/ + cp ./scripts/distribution/release-readme.txt distro/readme.txt + cp -r ./scripts/distribution/local-scripts/* distro/scripts/ + cp -r ./scripts/distribution/package-scripts/* distro/scripts/ + tar -C ./distro --exclude='*.debug' -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt + sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt + echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT + - name: post sha256 + uses: actions/upload-artifact@v4 + with: + name: sha256sums-${{ inputs.category }}-${{ inputs.target }} + path: sha256-${{ steps.pack_binaries.outputs.pkg_name }}.txt + retention-days: 1 + - name: post binaries + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz + path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz + retention-days: 1 + - name: Pack DBG Binaries + id: pack_dbg_binaries + shell: bash + run: | + target=${{ inputs.target }} + version=${{ inputs.version }} + case ${{ inputs.category }} in + default) + pkg_name="databend-${version}-${target}-dbg" + ;; + *) + pkg_name="databend-${{ inputs.category }}-${version}-${target}-dbg" + ;; + esac + mkdir -p distro/{bin,configs,systemd,scripts} + cp ./scripts/distribution/systemd/databend-* distro/systemd/ + cp ./scripts/distribution/configs/databend-* distro/configs/ + cp ./scripts/distribution/release-readme.txt distro/readme.txt + cp -r ./scripts/distribution/local-scripts/* distro/scripts/ + cp -r ./scripts/distribution/package-scripts/* distro/scripts/ + tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt + sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt + echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT + - name: post dbg sha256 + uses: actions/upload-artifact@v4 + with: + name: sha256sums-${{ inputs.category }}-${{ inputs.target }}-gdb + path: sha256-${{ steps.pack_dbg_binaries.outputs.pkg_name }}.txt + retention-days: 1 + - name: post dbg binaries + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz + path: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz + retention-days: 1 diff --git a/.github/actions/pack_distribution/action.yml b/.github/actions/pack_distribution/action.yml new file mode 100644 index 000000000000..57eab18853e2 --- /dev/null +++ b/.github/actions/pack_distribution/action.yml @@ -0,0 +1,94 @@ +name: "Pack Deb" +description: "Pack releases deb" +inputs: + arch: + description: "Release arch" + required: true + packager: + description: "Release default/hdfs/udf/testsuite" + required: false + default: default + version: + description: "Release version" + required: true + +runs: + using: "composite" + steps: + - name: Install nfpm@latest + shell: bash + run: | + curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz + tar xf nfpm.tar.gz + sudo mv nfpm /usr/local/bin + sudo chmod a+x /usr/local/bin/nfpm + rm nfpm.tar.gz + - name: Get target + id: target + shell: bash + run: | + echo 'target=${{ inputs.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT + - name: Download artifacts + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: ${{ steps.target.outputs.target }} + category: default + artifacts: metactl,meta,query,query.debug + path: distro/bin + - name: Build Packages + shell: bash + id: build_packages + run: | + export name="databend" + export version="${{ inputs.version }}" + export path="distro" + case "${{ inputs.arch }}" in + x86_64) + export arch="amd64" + ;; + aarch64) + export arch="arm64" + ;; + esac + + deb_version=${version/-/.} + pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}.${{ inputs.packager }}" + mkdir -p distro/{bin,configs,systemd,scripts} + cp ./scripts/distribution/systemd/databend-* distro/systemd/ + cp ./scripts/distribution/configs/databend-* distro/configs/ + cp ./scripts/distribution/release-readme.txt distro/readme.txt + cp -r ./scripts/distribution/local-scripts/* distro/scripts/ + cp -r ./scripts/distribution/package-scripts/* distro/scripts/ + nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml) + echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT + - name: Build dbg Packages + shell: bash + id: build_dbg_packages + run: | + export name="databend_dbg" + export version="${{ inputs.version }}" + export path="distro" + case "${{ inputs.arch }}" in + x86_64) + export arch="amd64" + ;; + aarch64) + export arch="arm64" + ;; + esac + + deb_version=${version/-/.} + pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}_dbg.${{ inputs.packager }}" + nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm-dbg.yaml) + echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT + - name: Update release to github + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Reference: https://cli.github.com/manual/gh_release_upload + run: | + version="${{ inputs.version }}" + # name looks like: `databend_0.8.144~nightly_amd64.deb` + gh release upload ${version} ${{ steps.build_packages.outputs.pkg_name }} --clobber + gh release upload ${version} ${{ steps.build_dbg_packages.outputs.pkg_name }} --clobber diff --git a/.github/actions/publish_binary/action.yml b/.github/actions/publish_binary/action.yml index f73fd02f419d..b9e3d470df9a 100644 --- a/.github/actions/publish_binary/action.yml +++ b/.github/actions/publish_binary/action.yml @@ -37,13 +37,19 @@ runs: shell: bash # Reference: https://cli.github.com/manual/gh_release_upload run: | - gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.* --clobber + gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.tar.gz --clobber + if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then + gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}-dbg.* --clobber + fi - name: Sync normal release to R2 shell: bash if: inputs.category == 'default' run: | aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress + if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then + aws s3 cp ${{ steps.name.outputs.name }}-dbg.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}-dbg.tar.gz --no-progress + fi gh api /repos/databendlabs/databend/tags > tags.json aws s3 cp ./tags.json s3://repo/databend/tags.json gh api /repos/databendlabs/databend/releases > releases.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 366a7c7a3d13..2382725e1278 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -203,40 +203,12 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ needs.create_release.outputs.sha }} - - name: Download artifact - uses: ./.github/actions/artifact_download + - name: Pack Binaries + uses: ./.github/actions/pack_binaries with: - sha: ${{ github.sha }} + version: ${{ needs.create_release.outputs.version }} target: ${{ matrix.target }} category: ${{ matrix.category }} - path: distro/bin - artifacts: metactl,meta,query - - name: Pack Binaries - run: | - target=${{ matrix.target }} - version=${{ needs.create_release.outputs.version }} - case ${{ matrix.category }} in - default) - pkg_name="databend-${version}-${target}" - ;; - *) - pkg_name="databend-${{ matrix.category }}-${version}-${target}" - ;; - esac - mkdir -p distro/{bin,configs,systemd,scripts} - cp ./scripts/distribution/systemd/databend-* distro/systemd/ - cp ./scripts/distribution/configs/databend-* distro/configs/ - cp ./scripts/distribution/release-readme.txt distro/readme.txt - cp -r ./scripts/distribution/local-scripts/* distro/scripts/ - cp -r ./scripts/distribution/package-scripts/* distro/scripts/ - tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt - sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt - - name: post sha256 - uses: actions/upload-artifact@v4 - with: - name: sha256sums-${{ matrix.category }}-${{ matrix.target }} - path: sha256-*.txt - retention-days: 1 - name: Publish Binaries uses: ./.github/actions/publish_binary env: @@ -455,55 +427,12 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ needs.create_release.outputs.sha }} - - name: Install nfpm@latest - run: | - curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz - tar xf nfpm.tar.gz - sudo mv nfpm /usr/local/bin - sudo chmod a+x /usr/local/bin/nfpm - rm nfpm.tar.gz - - name: Get target - id: target - run: | - echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT - - name: Download artifacts - uses: ./.github/actions/artifact_download + - name: Pack distribution + uses: ./.github/actions/pack_distribution with: - sha: ${{ github.sha }} - target: ${{ steps.target.outputs.target }} - category: default - artifacts: metactl,meta,query - path: distro/bin - - name: Build Packages - id: build_packages - run: | - export name="databend" - export version="${{ needs.create_release.outputs.version }}" - export path="distro" - case "${{ matrix.arch }}" in - x86_64) - export arch="amd64" - ;; - aarch64) - export arch="arm64" - ;; - esac - mkdir -p distro/{bin,configs,systemd,scripts} - cp ./scripts/distribution/systemd/databend-* distro/systemd/ - cp ./scripts/distribution/configs/databend-* distro/configs/ - cp ./scripts/distribution/release-readme.txt distro/readme.txt - cp -r ./scripts/distribution/local-scripts/* distro/scripts/ - cp -r ./scripts/distribution/package-scripts/* distro/scripts/ - nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml) - - name: Update release to github - shell: bash - env: - GH_TOKEN: ${{ github.token }} - # Reference: https://cli.github.com/manual/gh_release_upload - run: | - version="${{ needs.create_release.outputs.version }}" - # name looks like: `databend_0.8.144~nightly_amd64.deb` - gh release upload ${version} databend_*.${{ matrix.packager }} --clobber + version: ${{ needs.create_release.outputs.version }} + arch: ${{ matrix.arch }} + packager: ${{ matrix.packager }} # bindings_python: # if: inputs.stable diff --git a/scripts/distribution/nfpm-dbg.yaml b/scripts/distribution/nfpm-dbg.yaml new file mode 100644 index 000000000000..811f607cf171 --- /dev/null +++ b/scripts/distribution/nfpm-dbg.yaml @@ -0,0 +1,17 @@ +name: "${name}" +arch: "${arch}" +platform: "linux" +version: "${version}" +section: "database" +priority: "extra" +maintainer: "Databend Authors " +description: | + Databend is a powerful cloud data warehouse. Built for elasticity and efficiency. + Free and open. Also available in the cloud: https://app.databend.com +vendor: "Datafuse Labs" +homepage: "https://databend.com" +license: "Apache-2.0" +contents: + # Binaries + - src: ${path}/bin/databend-query.debug + dst: /usr/bin/databend-query.debug diff --git a/scripts/setup/dev_setup.sh b/scripts/setup/dev_setup.sh index 24a2ace0f21d..7f7b8b95d0dc 100755 --- a/scripts/setup/dev_setup.sh +++ b/scripts/setup/dev_setup.sh @@ -393,6 +393,28 @@ function install_libtiff { esac } +function install_binutils { + PACKAGE_MANAGER=$1 + + echo "==> installing binutils..." + + case "$PACKAGE_MANAGER" in + apt-get) + install_pkg binutils "$PACKAGE_MANAGER" + ;; + yum | dnf) + install_pkg binutils "$PACKAGE_MANAGER" + ;; + brew) + # skip + ;; + *) + echo "Unable to install binutils with package manager: $PACKAGE_MANAGER" + exit 1 + ;; + esac +} + function install_rustup { RUST_TOOLCHAIN=$1 @@ -624,6 +646,7 @@ if [[ "$INSTALL_BUILD_TOOLS" == "true" ]]; then install_python3 "$PACKAGE_MANAGER" install_sqlite3 "$PACKAGE_MANAGER" install_libtiff "$PACKAGE_MANAGER" + install_binutils "$PACKAGE_MANAGER" # Any call to cargo will make rustup install the correct toolchain cargo version From 794ec0e4cb417d1cc6f16bb77b7b0be5b8e75338 Mon Sep 17 00:00:00 2001 From: coldWater Date: Tue, 26 Nov 2024 15:47:00 +0800 Subject: [PATCH 90/92] chore: `k way merge sort` add boundary check to prevent process crash (#16934) assert Signed-off-by: coldWater --- .../transforms/sort/k_way_merge_sort_partition.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/k_way_merge_sort_partition.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/k_way_merge_sort_partition.rs index 08baf135365e..0719ebf2d458 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/k_way_merge_sort_partition.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/k_way_merge_sort_partition.rs @@ -223,11 +223,15 @@ impl List for Option { } fn cmp_value<'a>(&'a self, i: usize, target: &R::Item<'a>) -> Ordering { - self.as_ref().unwrap().row(i).cmp(target) + let rows = self.as_ref().unwrap(); + assert!(i < rows.len(), "len {}, index {}", rows.len(), i); + rows.row(i).cmp(target) } fn index(&self, i: usize) -> R::Item<'_> { - self.as_ref().unwrap().row(i) + let rows = self.as_ref().unwrap(); + assert!(i < rows.len(), "len {}, index {}", rows.len(), i); + rows.row(i) } } From 105809bcee0366afe21fcd211130a4a6dfc52d6c Mon Sep 17 00:00:00 2001 From: TCeason <33082201+TCeason@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:47:57 +0800 Subject: [PATCH 91/92] fix(query): add_hours function may panic if the argument is too big (#16929) fix(query): add_hours function may panic if the argument is too big --- src/query/expression/src/utils/date_helper.rs | 7 ++++--- .../functions/02_0012_function_datetimes.test | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index a52a7ea6f136..462934c3fdbe 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -293,7 +293,7 @@ impl EvalDaysImpl { } pub fn eval_timestamp(date: i64, delta: impl AsPrimitive) -> i64 { - let mut value = date.wrapping_add(delta.as_() * MICROSECS_PER_DAY); + let mut value = date.wrapping_add(delta.as_().wrapping_mul(MICROSECS_PER_DAY)); clamp_timestamp(&mut value); value } @@ -311,12 +311,13 @@ pub struct EvalTimesImpl; impl EvalTimesImpl { pub fn eval_date(date: i32, delta: impl AsPrimitive, factor: i64) -> i32 { clamp_date( - (date as i64 * MICROSECS_PER_DAY).wrapping_add(delta.as_() * factor * MICROS_PER_SEC), + (date as i64 * MICROSECS_PER_DAY) + .wrapping_add(delta.as_().wrapping_mul(factor * MICROS_PER_SEC)), ) } pub fn eval_timestamp(us: i64, delta: impl AsPrimitive, factor: i64) -> i64 { - let mut ts = us.wrapping_add(delta.as_() * factor * MICROS_PER_SEC); + let mut ts = us.wrapping_add(delta.as_().wrapping_mul(factor * MICROS_PER_SEC)); clamp_timestamp(&mut ts); ts } diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index c38571b09a41..e76537179a32 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -1466,3 +1466,23 @@ query T select TRY_TO_TIMESTAMP(1, 0), TRY_TO_TIMESTAMP(1, null); ---- 1970-01-01 00:00:01.000000 NULL + +query T +SELECT add_hours(to_date(710455), 1512263452497496403); +---- +1000-01-01 00:00:00.000000 + +query T +SELECT add_hours(to_timestamp(710455), 1512263452497496403); +---- +1000-01-01 00:00:00.000000 + +query T +SELECT add_hours(to_date(710455), 2147483647); +---- +1000-01-01 00:00:00.000000 + +query T +SELECT add_hours(to_timestamp(710455), 2147483647); +---- +1000-01-01 00:00:00.000000 From 519f9146ea2d51c1fc0eb9ba12046e7d2ee79ec6 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Tue, 26 Nov 2024 18:46:58 +0800 Subject: [PATCH 92/92] chore(ci): remove aarch64 musl in release (#16946) --- .github/workflows/release.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2382725e1278..14b554639d82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,6 @@ jobs: matrix: target: - x86_64-unknown-linux-musl - - aarch64-unknown-linux-musl steps: - name: Checkout uses: actions/checkout@v4 @@ -196,8 +195,6 @@ jobs: target: aarch64-unknown-linux-gnu - category: default target: x86_64-unknown-linux-musl - - category: default - target: aarch64-unknown-linux-musl steps: - name: Checkout uses: actions/checkout@v4

(page).map(State::Required), - _ => Err(utils::not_implemented(page)), - } - } - - /// Initializes a new state - fn with_capacity(&self, capacity: usize) -> Self::DecodedState { - ( - Vec::::with_capacity(capacity), - MutableBitmap::with_capacity(capacity), - ) - } - - fn push_valid(&self, state: &mut Self::State, decoded: &mut Self::DecodedState) -> Result<()> { - let (values, validity) = decoded; - match state { - State::Optional(page_values) => { - let value = page_values.values.by_ref().next().map(decode).map(self.op); - // convert unwrap to error - values.push(value.unwrap_or_default()); - validity.push(true); - } - State::Required(page_values) => { - let value = page_values.values.by_ref().next().map(decode).map(self.op); - // convert unwrap to error - values.push(value.unwrap_or_default()); - } - State::RequiredDictionary(page) => { - let value = page - .values - .next() - .map(|index| page.dict[index.unwrap() as usize]); - - values.push(value.unwrap_or_default()); - } - State::OptionalDictionary(page) => { - let value = page - .values - .next() - .map(|index| page.dict[index.unwrap() as usize]); - - values.push(value.unwrap_or_default()); - validity.push(true); - } - } - Ok(()) - } - - fn push_null(&self, decoded: &mut Self::DecodedState) { - let (values, validity) = decoded; - values.push(T::default()); - validity.push(false) - } - - fn deserialize_dict(&self, page: &DictPage) -> Self::Dictionary { - deserialize_plain(&page.buffer, self.op) - } -} - -fn finish( - data_type: &DataType, - values: Vec, - validity: MutableBitmap, -) -> PrimitiveArray { - PrimitiveArray::new(data_type.clone(), values.into(), validity.into()) -} - -/// An iterator adapter over [`Pages`] assumed to be encoded as boolean arrays -#[derive(Debug)] -pub struct NestedIter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - iter: I, - init: Vec, - data_type: DataType, - items: VecDeque<(NestedState, (Vec, MutableBitmap))>, - dict: Option>, - remaining: usize, - chunk_size: Option, - decoder: PrimitiveDecoder, -} - -impl NestedIter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - pub fn new( - iter: I, - init: Vec, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - op: F, - ) -> Self { - Self { - iter, - init, - data_type, - items: VecDeque::new(), - dict: None, - chunk_size, - remaining: num_rows, - decoder: PrimitiveDecoder::new(op), - } - } -} - -impl Iterator for NestedIter -where - I: Pages, - T: NativeType, - P: ParquetNativeType, - F: Copy + Fn(P) -> T, -{ - type Item = Result<(NestedState, PrimitiveArray)>; - - fn next(&mut self) -> Option { - let maybe_state = next( - &mut self.iter, - &mut self.items, - &mut self.dict, - &mut self.remaining, - &self.init, - self.chunk_size, - &self.decoder, - ); - match maybe_state { - utils::MaybeNext::Some(Ok((nested, state))) => { - Some(Ok((nested, finish(&self.data_type, state.0, state.1)))) - } - utils::MaybeNext::Some(Err(e)) => Some(Err(e)), - utils::MaybeNext::None => None, - utils::MaybeNext::More => self.next(), - } - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/simple.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/simple.rs deleted file mode 100644 index 91ec5b8db03a..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/simple.rs +++ /dev/null @@ -1,687 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use ethnum::I256; -use parquet2::schema::types::PhysicalType; -use parquet2::schema::types::PrimitiveLogicalType; -use parquet2::schema::types::PrimitiveType; -use parquet2::schema::types::TimeUnit as ParquetTimeUnit; -use parquet2::types::int96_to_i64_ns; - -use super::super::ArrayIter; -use super::super::Pages; -use super::binary; -use super::binview; -use super::boolean; -use super::fixed_size_binary; -use super::null; -use super::primitive; -use crate::arrow::array::Array; -use crate::arrow::array::DictionaryKey; -use crate::arrow::array::MutablePrimitiveArray; -use crate::arrow::array::PrimitiveArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::IntervalUnit; -use crate::arrow::datatypes::TimeUnit; -use crate::arrow::error::Error; -use crate::arrow::error::Result; -use crate::arrow::types::days_ms; -use crate::arrow::types::i256; -use crate::arrow::types::NativeType; - -/// Converts an iterator of arrays to a trait object returning trait objects -#[inline] -fn dyn_iter<'a, A, I>(iter: I) -> ArrayIter<'a> -where - A: Array, - I: Iterator> + Send + Sync + 'a, -{ - Box::new(iter.map(|x| x.map(|x| Box::new(x) as Box))) -} - -/// Converts an iterator of [MutablePrimitiveArray] into an iterator of [PrimitiveArray] -#[inline] -fn iden(iter: I) -> impl Iterator>> -where - T: NativeType, - I: Iterator>>, -{ - iter.map(|x| x.map(|x| x.into())) -} - -#[inline] -fn op(iter: I, op: F) -> impl Iterator>> -where - T: NativeType, - I: Iterator>>, - F: Fn(T) -> T + Copy, -{ - iter.map(move |x| { - x.map(move |mut x| { - x.values_mut_slice().iter_mut().for_each(|x| *x = op(*x)); - x.into() - }) - }) -} - -/// An iterator adapter that maps an iterator of Pages into an iterator of Arrays -/// of [`DataType`] `data_type` and length `chunk_size`. -pub fn page_iter_to_arrays<'a, I: Pages + 'a>( - pages: I, - type_: &PrimitiveType, - data_type: DataType, - chunk_size: Option, - num_rows: usize, -) -> Result> { - use DataType::*; - - let physical_type = &type_.physical_type; - let logical_type = &type_.logical_type; - - Ok(match (physical_type, data_type.to_logical_type()) { - (_, Null) => null::iter_to_arrays(pages, data_type, chunk_size, num_rows), - (PhysicalType::Boolean, Boolean) => { - dyn_iter(boolean::Iter::new(pages, data_type, chunk_size, num_rows)) - } - (PhysicalType::Int32, UInt8) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i32| x as u8, - ))), - (PhysicalType::Int32, UInt16) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i32| x as u16, - ))), - (PhysicalType::Int32, UInt32) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i32| x as u32, - ))), - (PhysicalType::Int64, UInt32) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i64| x as u32, - ))), - (PhysicalType::Int32, Int8) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i32| x as i8, - ))), - (PhysicalType::Int32, Int16) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i32| x as i16, - ))), - (PhysicalType::Int32, Int32 | Date32 | Time32(_)) => dyn_iter(iden( - primitive::IntegerIter::new(pages, data_type, num_rows, chunk_size, |x: i32| x), - )), - (PhysicalType::Int64 | PhysicalType::Int96, Timestamp(time_unit, _)) => { - let time_unit = *time_unit; - return timestamp( - pages, - physical_type, - logical_type, - data_type, - num_rows, - chunk_size, - time_unit, - ); - } - (PhysicalType::FixedLenByteArray(_), FixedSizeBinary(_)) => dyn_iter( - fixed_size_binary::Iter::new(pages, data_type, num_rows, chunk_size), - ), - (PhysicalType::FixedLenByteArray(12), Interval(IntervalUnit::YearMonth)) => { - let n = 12; - let pages = fixed_size_binary::Iter::new( - pages, - DataType::FixedSizeBinary(n), - num_rows, - chunk_size, - ); - - let pages = pages.map(move |maybe_array| { - let array = maybe_array?; - let values = array - .values() - .chunks_exact(n) - .map(|value: &[u8]| i32::from_le_bytes(value[..4].try_into().unwrap())) - .collect::>(); - let validity = array.validity().cloned(); - - PrimitiveArray::::try_new(data_type.clone(), values.into(), validity) - }); - - let arrays = pages.map(|x| x.map(|x| x.boxed())); - - Box::new(arrays) as _ - } - (PhysicalType::FixedLenByteArray(12), Interval(IntervalUnit::DayTime)) => { - let n = 12; - let pages = fixed_size_binary::Iter::new( - pages, - DataType::FixedSizeBinary(n), - num_rows, - chunk_size, - ); - - let pages = pages.map(move |maybe_array| { - let array = maybe_array?; - let values = array - .values() - .chunks_exact(n) - .map(super::super::convert_days_ms) - .collect::>(); - let validity = array.validity().cloned(); - - PrimitiveArray::::try_new(data_type.clone(), values.into(), validity) - }); - - let arrays = pages.map(|x| x.map(|x| x.boxed())); - - Box::new(arrays) as _ - } - (PhysicalType::Int32, Decimal(_, _)) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i32| x as i128, - ))), - (PhysicalType::Int64, Decimal(_, _)) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i64| x as i128, - ))), - (PhysicalType::FixedLenByteArray(n), Decimal(_, _)) if *n > 16 => { - return Err(Error::NotYetImplemented(format!( - "Can't decode Decimal128 type from Fixed Size Byte Array of len {n:?}" - ))); - } - (PhysicalType::FixedLenByteArray(n), Decimal(_, _)) => { - let n = *n; - - let pages = fixed_size_binary::Iter::new( - pages, - DataType::FixedSizeBinary(n), - num_rows, - chunk_size, - ); - - let pages = pages.map(move |maybe_array| { - let array = maybe_array?; - let values = array - .values() - .chunks_exact(n) - .map(|value: &[u8]| super::super::convert_i128(value, n)) - .collect::>(); - let validity = array.validity().cloned(); - - PrimitiveArray::::try_new(data_type.clone(), values.into(), validity) - }); - - let arrays = pages.map(|x| x.map(|x| x.boxed())); - - Box::new(arrays) as _ - } - (PhysicalType::Int32, Decimal256(_, _)) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i32| i256(I256::new(x as i128)), - ))), - (PhysicalType::Int64, Decimal256(_, _)) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i64| i256(I256::new(x as i128)), - ))), - (PhysicalType::FixedLenByteArray(n), Decimal256(_, _)) if *n <= 16 => { - let n = *n; - - let pages = fixed_size_binary::Iter::new( - pages, - DataType::FixedSizeBinary(n), - num_rows, - chunk_size, - ); - - let pages = pages.map(move |maybe_array| { - let array = maybe_array?; - let values = array - .values() - .chunks_exact(n) - .map(|value: &[u8]| i256(I256::new(super::super::convert_i128(value, n)))) - .collect::>(); - let validity = array.validity().cloned(); - - PrimitiveArray::::try_new(data_type.clone(), values.into(), validity) - }); - - let arrays = pages.map(|x| x.map(|x| x.boxed())); - - Box::new(arrays) as _ - } - (PhysicalType::FixedLenByteArray(n), Decimal256(_, _)) if *n <= 32 => { - let n = *n; - - let pages = fixed_size_binary::Iter::new( - pages, - DataType::FixedSizeBinary(n), - num_rows, - chunk_size, - ); - - let pages = pages.map(move |maybe_array| { - let array = maybe_array?; - let values = array - .values() - .chunks_exact(n) - .map(super::super::convert_i256) - .collect::>(); - let validity = array.validity().cloned(); - - PrimitiveArray::::try_new(data_type.clone(), values.into(), validity) - }); - - let arrays = pages.map(|x| x.map(|x| x.boxed())); - - Box::new(arrays) as _ - } - (PhysicalType::FixedLenByteArray(n), Decimal256(_, _)) if *n > 32 => { - return Err(Error::NotYetImplemented(format!( - "Can't decode Decimal256 type from Fixed Size Byte Array of len {n:?}" - ))); - } - (PhysicalType::Int32, Date64) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i32| x as i64 * 86400000, - ))), - (PhysicalType::Int64, Date64) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i64| x, - ))), - (PhysicalType::Int64, Int64 | Time64(_) | Duration(_)) => dyn_iter(iden( - primitive::IntegerIter::new(pages, data_type, num_rows, chunk_size, |x: i64| x), - )), - (PhysicalType::Int64, UInt64) => dyn_iter(iden(primitive::IntegerIter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: i64| x as u64, - ))), - (PhysicalType::Float, Float32) => dyn_iter(iden(primitive::Iter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: f32| x, - ))), - (PhysicalType::Double, Float64) => dyn_iter(iden(primitive::Iter::new( - pages, - data_type, - num_rows, - chunk_size, - |x: f64| x, - ))), - - (PhysicalType::ByteArray, Utf8 | Binary) => Box::new(binary::Iter::::new( - pages, data_type, chunk_size, num_rows, - )), - (PhysicalType::ByteArray, LargeBinary | LargeUtf8) => Box::new( - binary::Iter::::new(pages, data_type, chunk_size, num_rows), - ), - (PhysicalType::ByteArray, BinaryView | Utf8View) => Box::new( - binview::BinaryViewArrayIter::new(pages, data_type, chunk_size, num_rows), - ), - - (_, Dictionary(key_type, _, _)) => { - return match_integer_type!(key_type, |$K| { - dict_read::<$K, _>(pages, physical_type, logical_type, data_type, num_rows, chunk_size) - }); - } - (from, to) => { - return Err(Error::NotYetImplemented(format!( - "Reading parquet type {from:?} to {to:?} still not implemented" - ))); - } - }) -} - -/// Unify the timestamp unit from parquet TimeUnit into arrow's TimeUnit -/// Returns (a int64 factor, is_multiplier) -fn unify_timestamp_unit( - logical_type: &Option, - time_unit: TimeUnit, -) -> (i64, bool) { - if let Some(PrimitiveLogicalType::Timestamp { unit, .. }) = logical_type { - match (*unit, time_unit) { - (ParquetTimeUnit::Milliseconds, TimeUnit::Millisecond) - | (ParquetTimeUnit::Microseconds, TimeUnit::Microsecond) - | (ParquetTimeUnit::Nanoseconds, TimeUnit::Nanosecond) => (1, true), - - (ParquetTimeUnit::Milliseconds, TimeUnit::Second) - | (ParquetTimeUnit::Microseconds, TimeUnit::Millisecond) - | (ParquetTimeUnit::Nanoseconds, TimeUnit::Microsecond) => (1000, false), - - (ParquetTimeUnit::Microseconds, TimeUnit::Second) - | (ParquetTimeUnit::Nanoseconds, TimeUnit::Millisecond) => (1_000_000, false), - - (ParquetTimeUnit::Nanoseconds, TimeUnit::Second) => (1_000_000_000, false), - - (ParquetTimeUnit::Milliseconds, TimeUnit::Microsecond) - | (ParquetTimeUnit::Microseconds, TimeUnit::Nanosecond) => (1_000, true), - - (ParquetTimeUnit::Milliseconds, TimeUnit::Nanosecond) => (1_000_000, true), - } - } else { - (1, true) - } -} - -#[inline] -pub fn int96_to_i64_us(value: [u32; 3]) -> i64 { - const JULIAN_DAY_OF_EPOCH: i64 = 2_440_588; - const SECONDS_PER_DAY: i64 = 86_400; - const MICROS_PER_SECOND: i64 = 1_000_000; - - let day = value[2] as i64; - let microseconds = (((value[1] as i64) << 32) + value[0] as i64) / 1_000; - let seconds = (day - JULIAN_DAY_OF_EPOCH) * SECONDS_PER_DAY; - - seconds * MICROS_PER_SECOND + microseconds -} - -#[inline] -pub fn int96_to_i64_ms(value: [u32; 3]) -> i64 { - const JULIAN_DAY_OF_EPOCH: i64 = 2_440_588; - const SECONDS_PER_DAY: i64 = 86_400; - const MILLIS_PER_SECOND: i64 = 1_000; - - let day = value[2] as i64; - let milliseconds = (((value[1] as i64) << 32) + value[0] as i64) / 1_000_000; - let seconds = (day - JULIAN_DAY_OF_EPOCH) * SECONDS_PER_DAY; - - seconds * MILLIS_PER_SECOND + milliseconds -} - -#[inline] -pub fn int96_to_i64_s(value: [u32; 3]) -> i64 { - const JULIAN_DAY_OF_EPOCH: i64 = 2_440_588; - const SECONDS_PER_DAY: i64 = 86_400; - - let day = value[2] as i64; - let seconds = (((value[1] as i64) << 32) + value[0] as i64) / 1_000_000_000; - let day_seconds = (day - JULIAN_DAY_OF_EPOCH) * SECONDS_PER_DAY; - - day_seconds + seconds -} - -fn timestamp<'a, I: Pages + 'a>( - pages: I, - physical_type: &PhysicalType, - logical_type: &Option, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - time_unit: TimeUnit, -) -> Result> { - if physical_type == &PhysicalType::Int96 { - return match time_unit { - TimeUnit::Nanosecond => Ok(dyn_iter(iden(primitive::Iter::new( - pages, - data_type, - num_rows, - chunk_size, - int96_to_i64_ns, - )))), - TimeUnit::Microsecond => Ok(dyn_iter(iden(primitive::Iter::new( - pages, - data_type, - num_rows, - chunk_size, - int96_to_i64_us, - )))), - TimeUnit::Millisecond => Ok(dyn_iter(iden(primitive::Iter::new( - pages, - data_type, - num_rows, - chunk_size, - int96_to_i64_ms, - )))), - TimeUnit::Second => Ok(dyn_iter(iden(primitive::Iter::new( - pages, - data_type, - num_rows, - chunk_size, - int96_to_i64_s, - )))), - }; - }; - - if physical_type != &PhysicalType::Int64 { - return Err(Error::nyi( - "Can't decode a timestamp from a non-int64 parquet type", - )); - } - - let iter = primitive::IntegerIter::new(pages, data_type, num_rows, chunk_size, |x: i64| x); - let (factor, is_multiplier) = unify_timestamp_unit(logical_type, time_unit); - match (factor, is_multiplier) { - (1, _) => Ok(dyn_iter(iden(iter))), - (a, true) => Ok(dyn_iter(op(iter, move |x| x * a))), - (a, false) => Ok(dyn_iter(op(iter, move |x| x / a))), - } -} - -fn timestamp_dict<'a, K: DictionaryKey, I: Pages + 'a>( - pages: I, - physical_type: &PhysicalType, - logical_type: &Option, - data_type: DataType, - num_rows: usize, - chunk_size: Option, - time_unit: TimeUnit, -) -> Result> { - if physical_type == &PhysicalType::Int96 { - let logical_type = PrimitiveLogicalType::Timestamp { - unit: ParquetTimeUnit::Nanoseconds, - is_adjusted_to_utc: false, - }; - let (factor, is_multiplier) = unify_timestamp_unit(&Some(logical_type), time_unit); - return match (factor, is_multiplier) { - (a, true) => Ok(dyn_iter(primitive::DictIter::::new( - pages, - DataType::Timestamp(TimeUnit::Nanosecond, None), - num_rows, - chunk_size, - move |x| int96_to_i64_ns(x) * a, - ))), - (a, false) => Ok(dyn_iter(primitive::DictIter::::new( - pages, - DataType::Timestamp(TimeUnit::Nanosecond, None), - num_rows, - chunk_size, - move |x| int96_to_i64_ns(x) / a, - ))), - }; - }; - - let (factor, is_multiplier) = unify_timestamp_unit(logical_type, time_unit); - match (factor, is_multiplier) { - (a, true) => Ok(dyn_iter(primitive::DictIter::::new( - pages, - data_type, - num_rows, - chunk_size, - move |x: i64| x * a, - ))), - (a, false) => Ok(dyn_iter(primitive::DictIter::::new( - pages, - data_type, - num_rows, - chunk_size, - move |x: i64| x / a, - ))), - } -} - -fn dict_read<'a, K: DictionaryKey, I: Pages + 'a>( - iter: I, - physical_type: &PhysicalType, - logical_type: &Option, - data_type: DataType, - num_rows: usize, - chunk_size: Option, -) -> Result> { - use DataType::*; - let values_data_type = if let Dictionary(_, v, _) = &data_type { - v.as_ref() - } else { - panic!() - }; - - Ok(match (physical_type, values_data_type.to_logical_type()) { - (PhysicalType::Int32, UInt8) => dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: i32| x as u8, - )), - (PhysicalType::Int32, UInt16) => dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: i32| x as u16, - )), - (PhysicalType::Int32, UInt32) => dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: i32| x as u32, - )), - (PhysicalType::Int64, UInt64) => dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: i64| x as u64, - )), - (PhysicalType::Int32, Int8) => dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: i32| x as i8, - )), - (PhysicalType::Int32, Int16) => dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: i32| x as i16, - )), - (PhysicalType::Int32, Int32 | Date32 | Time32(_) | Interval(IntervalUnit::YearMonth)) => { - dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: i32| x, - )) - } - - (PhysicalType::Int64, Timestamp(time_unit, _)) => { - let time_unit = *time_unit; - return timestamp_dict::( - iter, - physical_type, - logical_type, - data_type, - num_rows, - chunk_size, - time_unit, - ); - } - - (PhysicalType::Int64, Int64 | Date64 | Time64(_) | Duration(_)) => { - dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: i64| x, - )) - } - (PhysicalType::Float, Float32) => dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: f32| x, - )), - (PhysicalType::Double, Float64) => dyn_iter(primitive::DictIter::::new( - iter, - data_type, - num_rows, - chunk_size, - |x: f64| x, - )), - - (PhysicalType::ByteArray, Utf8 | Binary) => dyn_iter(binary::DictIter::::new( - iter, data_type, num_rows, chunk_size, - )), - (PhysicalType::ByteArray, LargeUtf8 | LargeBinary) => dyn_iter( - binary::DictIter::::new(iter, data_type, num_rows, chunk_size), - ), - (PhysicalType::ByteArray, Utf8View | BinaryView) => dyn_iter( - binview::DictIter::::new(iter, data_type, num_rows, chunk_size), - ), - (PhysicalType::FixedLenByteArray(_), FixedSizeBinary(_)) => dyn_iter( - fixed_size_binary::DictIter::::new(iter, data_type, num_rows, chunk_size), - ), - other => { - return Err(Error::nyi(format!( - "Reading dictionaries of type {other:?}" - ))); - } - }) -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/struct_.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/struct_.rs deleted file mode 100644 index a862e8f826fa..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/struct_.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::nested_utils::NestedArrayIter; -use super::nested_utils::NestedState; -use crate::arrow::array::Array; -use crate::arrow::array::StructArray; -use crate::arrow::datatypes::DataType; -use crate::arrow::datatypes::Field; -use crate::arrow::error::Error; - -/// An iterator adapter over [`NestedArrayIter`] assumed to be encoded as Struct arrays -pub struct StructIterator<'a> { - iters: Vec>, - fields: Vec, -} - -impl<'a> StructIterator<'a> { - /// Creates a new [`StructIterator`] with `iters` and `fields`. - pub fn new(iters: Vec>, fields: Vec) -> Self { - assert_eq!(iters.len(), fields.len()); - Self { iters, fields } - } -} - -impl<'a> Iterator for StructIterator<'a> { - type Item = Result<(NestedState, Box), Error>; - - fn next(&mut self) -> Option { - let values = self - .iters - .iter_mut() - .map(|iter| iter.next()) - .collect::>(); - - if values.iter().any(|x| x.is_none()) { - return None; - } - - // todo: unzip of Result not yet supported in stable Rust - let mut nested = vec![]; - let mut new_values = vec![]; - for x in values { - match x.unwrap() { - Ok((nest, values)) => { - new_values.push(values); - nested.push(nest); - } - Err(e) => return Some(Err(e)), - } - } - let mut nested = nested.pop().unwrap(); - let (_, validity) = nested.nested.pop().unwrap().inner(); - - Some(Ok(( - nested, - Box::new(StructArray::new( - DataType::Struct(self.fields.clone()), - new_values, - validity.and_then(|x| x.into()), - )), - ))) - } -} diff --git a/src/common/arrow/src/arrow/io/parquet/read/deserialize/utils.rs b/src/common/arrow/src/arrow/io/parquet/read/deserialize/utils.rs deleted file mode 100644 index 14ca2ca7bb74..000000000000 --- a/src/common/arrow/src/arrow/io/parquet/read/deserialize/utils.rs +++ /dev/null @@ -1,584 +0,0 @@ -// Copyright 2020-2022 Jorge C. Leitão -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::VecDeque; - -use parquet2::deserialize::FilteredHybridEncoded; -use parquet2::deserialize::FilteredHybridRleDecoderIter; -use parquet2::deserialize::HybridDecoderBitmapIter; -use parquet2::deserialize::HybridEncoded; -use parquet2::encoding::hybrid_rle; -use parquet2::indexes::Interval; -use parquet2::page::split_buffer; -use parquet2::page::DataPage; -use parquet2::page::DictPage; -use parquet2::page::Page; -use parquet2::schema::Repetition; - -use super::super::Pages; -use crate::arrow::array::MutableBinaryViewArray; -use crate::arrow::array::ViewType; -use crate::arrow::bitmap::utils::BitmapIter; -use crate::arrow::bitmap::MutableBitmap; -use crate::arrow::error::Error; - -pub fn not_implemented(page: &DataPage) -> Error { - let is_optional = page.descriptor.primitive_type.field_info.repetition == Repetition::Optional; - let is_filtered = page.selected_rows().is_some(); - let required = if is_optional { "optional" } else { "required" }; - let is_filtered = if is_filtered { ", index-filtered" } else { "" }; - Error::NotYetImplemented(format!( - "Decoding {:?} \"{:?}\"-encoded {} {} parquet pages", - page.descriptor.primitive_type.physical_type, - page.encoding(), - required, - is_filtered, - )) -} - -/// A private trait representing structs that can receive elements. -pub(super) trait Pushable: Sized { - fn reserve(&mut self, additional: usize); - fn push(&mut self, value: T); - fn len(&self) -> usize; - fn push_null(&mut self); - fn extend_constant(&mut self, additional: usize, value: T); -} - -impl Pushable for MutableBitmap { - #[inline] - fn reserve(&mut self, additional: usize) { - MutableBitmap::reserve(self, additional) - } - #[inline] - fn len(&self) -> usize { - self.len() - } - - #[inline] - fn push(&mut self, value: bool) { - self.push(value) - } - - #[inline] - fn push_null(&mut self) { - self.push(false) - } - - #[inline] - fn extend_constant(&mut self, additional: usize, value: bool) { - self.extend_constant(additional, value) - } -} - -impl Pushable<&T> for MutableBinaryViewArray { - #[inline] - fn reserve(&mut self, additional: usize) { - MutableBinaryViewArray::reserve(self, additional) - } - - #[inline] - fn push(&mut self, value: &T) { - MutableBinaryViewArray::push_value(self, value) - } - - #[inline] - fn len(&self) -> usize { - MutableBinaryViewArray::len(self) - } - - fn push_null(&mut self) { - MutableBinaryViewArray::push_null(self) - } - - fn extend_constant(&mut self, additional: usize, value: &T) { - // First push a value to get the View - MutableBinaryViewArray::push_value(self, value); - - // And then use that new view to extend - let views = self.views_mut(); - let view = *views.last().unwrap(); - - let remaining = additional - 1; - for _ in 0..remaining { - views.push(view); - } - - if let Some(bitmap) = self.validity_mut() { - bitmap.extend_constant(remaining, true) - } - } -} - -impl Pushable for Vec { - #[inline] - fn reserve(&mut self, additional: usize) { - Vec::reserve(self, additional) - } - #[inline] - fn len(&self) -> usize { - self.len() - } - - #[inline] - fn push_null(&mut self) { - self.push(A::default()) - } - - #[inline] - fn push(&mut self, value: A) { - self.push(value) - } - - #[inline] - fn extend_constant(&mut self, additional: usize, value: A) { - self.resize(self.len() + additional, value); - } -} - -/// The state of a partially deserialized page -pub(super) trait PageValidity<'a> { - fn next_limited(&mut self, limit: usize) -> Option>; -} - -#[derive(Debug, Clone)] -pub struct FilteredOptionalPageValidity<'a> { - iter: FilteredHybridRleDecoderIter<'a>, - current: Option<(FilteredHybridEncoded<'a>, usize)>, -} - -impl<'a> FilteredOptionalPageValidity<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, validity, _) = split_buffer(page)?; - - let iter = hybrid_rle::Decoder::new(validity, 1); - let iter = HybridDecoderBitmapIter::new(iter, page.num_values()); - let selected_rows = get_selected_rows(page); - let iter = FilteredHybridRleDecoderIter::new(iter, selected_rows); - - Ok(Self { - iter, - current: None, - }) - } - - pub fn len(&self) -> usize { - self.iter.len() - } -} - -pub fn get_selected_rows(page: &DataPage) -> VecDeque { - page.selected_rows() - .unwrap_or(&[Interval::new(0, page.num_values())]) - .iter() - .copied() - .collect() -} - -impl<'a> PageValidity<'a> for FilteredOptionalPageValidity<'a> { - fn next_limited(&mut self, limit: usize) -> Option> { - let (run, own_offset) = if let Some((run, offset)) = self.current { - (run, offset) - } else { - // a new run - let run = self.iter.next()?.unwrap(); // no run -> None - self.current = Some((run, 0)); - return self.next_limited(limit); - }; - - match run { - FilteredHybridEncoded::Bitmap { - values, - offset, - length, - } => { - let run_length = length - own_offset; - - let length = limit.min(run_length); - - if length == run_length { - self.current = None; - } else { - self.current = Some((run, own_offset + length)); - } - - Some(FilteredHybridEncoded::Bitmap { - values, - offset, - length, - }) - } - FilteredHybridEncoded::Repeated { is_set, length } => { - let run_length = length - own_offset; - - let length = limit.min(run_length); - - if length == run_length { - self.current = None; - } else { - self.current = Some((run, own_offset + length)); - } - - Some(FilteredHybridEncoded::Repeated { is_set, length }) - } - FilteredHybridEncoded::Skipped(set) => { - self.current = None; - Some(FilteredHybridEncoded::Skipped(set)) - } - } - } -} - -pub struct Zip { - validity: V, - values: I, -} - -impl Zip { - pub fn new(validity: V, values: I) -> Self { - Self { validity, values } - } -} - -impl, I: Iterator> Iterator for Zip { - type Item = Option; - - #[inline] - fn next(&mut self) -> Option { - self.validity - .next() - .map(|x| if x { self.values.next() } else { None }) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.validity.size_hint() - } -} - -#[derive(Debug, Clone)] -pub struct OptionalPageValidity<'a> { - iter: HybridDecoderBitmapIter<'a>, - current: Option<(HybridEncoded<'a>, usize)>, -} - -impl<'a> OptionalPageValidity<'a> { - pub fn try_new(page: &'a DataPage) -> Result { - let (_, validity, _) = split_buffer(page)?; - - let iter = hybrid_rle::Decoder::new(validity, 1); - let iter = HybridDecoderBitmapIter::new(iter, page.num_values()); - Ok(Self { - iter, - current: None, - }) - } - - /// Number of items remaining - pub fn len(&self) -> usize { - self.iter.len() - + self - .current - .as_ref() - .map(|(run, offset)| run.len() - offset) - .unwrap_or_default() - } - - fn next_limited(&mut self, limit: usize) -> Option> { - let (run, offset) = if let Some((run, offset)) = self.current { - (run, offset) - } else { - // a new run - let run = self.iter.next()?.unwrap(); // no run -> None - self.current = Some((run, 0)); - return self.next_limited(limit); - }; - - match run { - HybridEncoded::Bitmap(values, length) => { - let run_length = length - offset; - - let length = limit.min(run_length); - - if length == run_length { - self.current = None; - } else { - self.current = Some((run, offset + length)); - } - - Some(FilteredHybridEncoded::Bitmap { - values, - offset, - length, - }) - } - HybridEncoded::Repeated(is_set, run_length) => { - let run_length = run_length - offset; - - let length = limit.min(run_length); - - if length == run_length { - self.current = None; - } else { - self.current = Some((run, offset + length)); - } - - Some(FilteredHybridEncoded::Repeated { is_set, length }) - } - } - } -} - -impl<'a> PageValidity<'a> for OptionalPageValidity<'a> { - fn next_limited(&mut self, limit: usize) -> Option> { - self.next_limited(limit) - } -} - -/// Extends a [`Pushable`] from an iterator of non-null values and an hybrid-rle decoder -pub(super) fn extend_from_decoder, I: Iterator>( - validity: &mut MutableBitmap, - page_validity: &mut dyn PageValidity, - limit: Option, - pushable: &mut P, - mut values_iter: I, -) { - let limit = limit.unwrap_or(usize::MAX); - - let mut runs = vec![]; - let mut remaining = limit; - let mut reserve_pushable = 0; - - // first do a scan so that we know how much to reserve up front - while remaining > 0 { - let run = page_validity.next_limited(remaining); - let run = if let Some(run) = run { run } else { break }; - - match run { - FilteredHybridEncoded::Bitmap { length, .. } => { - reserve_pushable += length; - remaining -= length; - } - FilteredHybridEncoded::Repeated { length, .. } => { - reserve_pushable += length; - remaining -= length; - } - _ => {} - }; - runs.push(run) - } - pushable.reserve(reserve_pushable); - validity.reserve(reserve_pushable); - - // then a second loop to really fill the buffers - for run in runs { - match run { - FilteredHybridEncoded::Bitmap { - values, - offset, - length, - } => { - // consume `length` items - let iter = BitmapIter::new(values, offset, length); - let iter = Zip::new(iter, &mut values_iter); - - for item in iter { - if let Some(item) = item { - pushable.push(item) - } else { - pushable.push_null() - } - } - validity.extend_from_slice(values, offset, length); - } - FilteredHybridEncoded::Repeated { is_set, length } => { - validity.extend_constant(length, is_set); - if is_set { - for v in (&mut values_iter).take(length) { - pushable.push(v) - } - } else { - pushable.extend_constant(length, T::default()); - } - } - FilteredHybridEncoded::Skipped(valids) => for _ in values_iter.by_ref().take(valids) {}, - }; - } -} - -/// The state of a partially deserialized page -pub(super) trait PageState<'a>: std::fmt::Debug { - fn len(&self) -> usize; -} - -/// The state of a partially deserialized page -pub(super) trait DecodedState: std::fmt::Debug { - // the number of values that the state already has - fn len(&self) -> usize; -} - -/// A decoder that knows how to map `State` -> Array -pub(super) trait Decoder<'a> { - /// The state that this decoder derives from a [`DataPage`]. This is bound to the page. - type State: PageState<'a>; - /// The dictionary representation that the decoder uses - type Dict; - /// The target state that this Decoder decodes into. - type DecodedState: DecodedState; - - /// Creates a new `Self::State` - fn build_state( - &self, - page: &'a DataPage, - dict: Option<&'a Self::Dict>, - ) -> Result; - - /// Initializes a new [`Self::DecodedState`]. - fn with_capacity(&self, capacity: usize) -> Self::DecodedState; - - /// extends [`Self::DecodedState`] by deserializing items in [`Self::State`]. - /// It guarantees that the length of `decoded` is at most `decoded.len() + remaining`. - fn extend_from_state( - &self, - page: &mut Self::State, - decoded: &mut Self::DecodedState, - additional: usize, - ); - - /// Deserializes a [`DictPage`] into [`Self::Dict`]. - fn deserialize_dict(&self, page: &DictPage) -> Self::Dict; -} - -pub(super) fn extend_from_new_page<'a, T: Decoder<'a>>( - mut page: T::State, - chunk_size: Option, - items: &mut VecDeque, - remaining: &mut usize, - decoder: &T, -) { - let capacity = chunk_size.unwrap_or(0); - let chunk_size = chunk_size.unwrap_or(usize::MAX); - - let mut decoded = if let Some(decoded) = items.pop_back() { - decoded - } else { - // there is no state => initialize it - decoder.with_capacity(capacity) - }; - let existing = decoded.len(); - - let additional = (chunk_size - existing).min(*remaining); - - decoder.extend_from_state(&mut page, &mut decoded, additional); - *remaining -= decoded.len() - existing; - items.push_back(decoded); - - while page.len() > 0 && *remaining > 0 { - let additional = chunk_size.min(*remaining); - - let mut decoded = decoder.with_capacity(additional); - decoder.extend_from_state(&mut page, &mut decoded, additional); - *remaining -= decoded.len(); - items.push_back(decoded) - } -} - -/// Represents what happened when a new page was consumed -#[derive(Debug)] -pub enum MaybeNext