Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: purge dropped view metadata during vacuum #16819

Merged
merged 2 commits into from
Nov 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ use databend_common_meta_app::schema::DroppedId;
use databend_common_meta_app::schema::GcDroppedTableReq;
use databend_common_meta_app::schema::ListDroppedTableReq;
use databend_common_sql::plans::VacuumDropTablePlan;
use databend_common_storages_view::view_table::VIEW_ENGINE;
use databend_enterprise_vacuum_handler::get_vacuum_handler;
use log::info;

Expand Down Expand Up @@ -156,13 +157,17 @@ impl Interpreter for VacuumDropTablesInterpreter {
drop_ids
);

// TODO buggy, table as catalog obj should be allowed to drop
// also drop ids
// filter out read-only tables
let tables = tables
// Filter out read-only tables and views.
// Note: The drop_ids list still includes view IDs
let (views, tables): (Vec<_>, Vec<_>) = tables
.into_iter()
.filter(|tbl| !tbl.as_ref().is_read_only())
.collect::<Vec<_>>();
.partition(|tbl| tbl.get_table_info().meta.engine == VIEW_ENGINE);

{
let view_ids = views.into_iter().map(|v| v.get_id()).collect::<Vec<_>>();
info!("view ids excluded from purging data: {:?}", view_ids);
}

let handler = get_vacuum_handler();
let threads_nums = self.ctx.get_settings().get_max_threads()? as usize;
Expand All @@ -187,6 +192,8 @@ impl Interpreter for VacuumDropTablesInterpreter {
// gc metadata only when not dry run
if self.plan.option.dry_run.is_none() {
let mut success_dropped_ids = vec![];
// Since drop_ids contains view IDs, any views (if present) will be added to
// the success_dropped_id list, with removal from the meta-server attempted later.
for drop_id in drop_ids {
match &drop_id {
DroppedId::Db { db_id, db_name: _ } => {
Expand All @@ -205,6 +212,7 @@ impl Interpreter for VacuumDropTablesInterpreter {
"failed dbs:{:?}, failed_tables:{:?}, success_drop_ids:{:?}",
failed_db_ids, failed_tables, success_dropped_ids
);

self.gc_drop_tables(catalog, success_dropped_ids).await?;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
## Copyright 2023 Databend Cloud
##
## Licensed under the Elastic License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
## https://www.elastic.co/licensing/elastic-license
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.

statement ok
drop database if exists vacuum_view_test;

statement ok
create database vacuum_view_test;

statement ok
use vacuum_view_test;

statement ok
create table t as select * from numbers(1);

statement ok
create view v as select * from t;

statement ok
drop view v;

# the dropped view vacuum_view_test.v should be there
query I
select count() from system.tables_with_history where database = 'vacuum_view_test' and name = 'v';
----
1

statement ok
set data_retention_time_in_days = 0;

statement ok
vacuum drop table from vacuum_view_test;

# the dropped view vacuum_view_test.v should be vacuumed
query I
select count() from system.tables_with_history where database = 'vacuum_view_test' and name = 'v';
----
0

statement ok
drop database vacuum_view_test;
Loading