Skip to content

Commit

Permalink
Rename partitions to partition_summaries in Iceberg manifests table
Browse files Browse the repository at this point in the history
  • Loading branch information
ebyhr committed Dec 11, 2024
1 parent 6dbbfd9 commit d6ce464
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 7 deletions.
4 changes: 2 additions & 2 deletions docs/src/main/sphinx/connector/iceberg.md
Original file line number Diff line number Diff line change
Expand Up @@ -1118,7 +1118,7 @@ SELECT * FROM "test_table$manifests"
```

```text
path | length | partition_spec_id | added_snapshot_id | added_data_files_count | added_rows_count | existing_data_files_count | existing_rows_count | deleted_data_files_count | deleted_rows_count | partitions
path | length | partition_spec_id | added_snapshot_id | added_data_files_count | added_rows_count | existing_data_files_count | existing_rows_count | deleted_data_files_count | deleted_rows_count | partition_summaries
----------------------------------------------------------------------------------------------------------------+-----------------+----------------------+-----------------------+-------------------------+------------------+-----------------------------+---------------------+-----------------------------+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------
hdfs://hadoop-master:9000/user/hive/warehouse/test_table/metadata/faa19903-1455-4bb8-855a-61a1bbafbaa7-m0.avro | 6277 | 0 | 7860805980949777961 | 1 | 100 | 0 | 0 | 0 | 0 | {{contains_null=false, contains_nan= false, lower_bound=1, upper_bound=1},{contains_null=false, contains_nan= false, lower_bound=2021-01-12, upper_bound=2021-01-12}}
```
Expand Down Expand Up @@ -1167,7 +1167,7 @@ The output of the query has the following columns:
- `BIGINT`
- The total number of rows in all data files with status `DELETED` in the
manifest file.
* - `partitions`
* - `partition_summaries`
- `ARRAY(row(contains_null BOOLEAN, contains_nan BOOLEAN, lower_bound VARCHAR, upper_bound VARCHAR))`
- Partition range metadata.
:::
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public ManifestsTable(SchemaTableName tableName, Table icebergTable, Optional<Lo
.add(new ColumnMetadata("existing_rows_count", BIGINT))
.add(new ColumnMetadata("deleted_data_files_count", INTEGER))
.add(new ColumnMetadata("deleted_rows_count", BIGINT))
.add(new ColumnMetadata("partitions", new ArrayType(RowType.rowType(
.add(new ColumnMetadata("partition_summaries", new ArrayType(RowType.rowType(
RowType.field("contains_null", BOOLEAN),
RowType.field("contains_nan", BOOLEAN),
RowType.field("lower_bound", VARCHAR),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -347,22 +347,22 @@ public void testManifestsTable()
"('existing_rows_count', 'bigint', '', '')," +
"('deleted_data_files_count', 'integer', '', '')," +
"('deleted_rows_count', 'bigint', '', '')," +
"('partitions', 'array(row(contains_null boolean, contains_nan boolean, lower_bound varchar, upper_bound varchar))', '', '')");
"('partition_summaries', 'array(row(contains_null boolean, contains_nan boolean, lower_bound varchar, upper_bound varchar))', '', '')");
assertQuerySucceeds("SELECT * FROM test_schema.\"test_table$manifests\"");
assertThat(query("SELECT added_data_files_count, existing_rows_count, added_rows_count, deleted_data_files_count, deleted_rows_count, partitions FROM test_schema.\"test_table$manifests\""))
assertThat(query("SELECT added_data_files_count, existing_rows_count, added_rows_count, deleted_data_files_count, deleted_rows_count, partition_summaries FROM test_schema.\"test_table$manifests\""))
.matches(
"VALUES " +
" (2, BIGINT '0', BIGINT '3', 0, BIGINT '0', CAST(ARRAY[ROW(false, false, '2019-09-08', '2019-09-09')] AS array(row(contains_null boolean, contains_nan boolean, lower_bound varchar, upper_bound varchar)))) , " +
" (2, BIGINT '0', BIGINT '3', 0, BIGINT '0', CAST(ARRAY[ROW(false, false, '2019-09-09', '2019-09-10')] AS array(row(contains_null boolean, contains_nan boolean, lower_bound varchar, upper_bound varchar))))");

assertQuerySucceeds("SELECT * FROM test_schema.\"test_table_multilevel_partitions$manifests\"");
assertThat(query("SELECT added_data_files_count, existing_rows_count, added_rows_count, deleted_data_files_count, deleted_rows_count, partitions FROM test_schema.\"test_table_multilevel_partitions$manifests\""))
assertThat(query("SELECT added_data_files_count, existing_rows_count, added_rows_count, deleted_data_files_count, deleted_rows_count, partition_summaries FROM test_schema.\"test_table_multilevel_partitions$manifests\""))
.matches(
"VALUES " +
"(3, BIGINT '0', BIGINT '3', 0, BIGINT '0', CAST(ARRAY[ROW(false, false, '0', '1'), ROW(false, false, '2019-09-08', '2019-09-09')] AS array(row(contains_null boolean, contains_nan boolean, lower_bound varchar, upper_bound varchar))))");

assertQuerySucceeds("SELECT * FROM test_schema.\"test_table_with_dml$manifests\"");
assertThat(query("SELECT added_data_files_count, existing_rows_count, added_rows_count, deleted_data_files_count, deleted_rows_count, partitions FROM test_schema.\"test_table_with_dml$manifests\""))
assertThat(query("SELECT added_data_files_count, existing_rows_count, added_rows_count, deleted_data_files_count, deleted_rows_count, partition_summaries FROM test_schema.\"test_table_with_dml$manifests\""))
.matches(
"VALUES " +
// INSERT on '2022-01-01', '2022-02-02', '2022-03-03' partitions
Expand Down

0 comments on commit d6ce464

Please sign in to comment.