Skip to content

Commit

Permalink
[Fix](Nereids) fix insert into table of random distribution for nerei…
Browse files Browse the repository at this point in the history
…ds (#22831)

currently insert into a table of random distribution info is not supported, we fix it by set physical properties to Any.
  • Loading branch information
sohardforaname authored Aug 11, 2023
1 parent a2fd488 commit 0458439
Show file tree
Hide file tree
Showing 14 changed files with 234 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@

import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.HashDistributionInfo;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType;
import org.apache.doris.nereids.properties.LogicalProperties;
Expand Down Expand Up @@ -190,22 +193,30 @@ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalPr
*/
public PhysicalProperties getRequirePhysicalProperties() {
if (targetTable.isPartitioned()) {
HashDistributionInfo distributionInfo = ((HashDistributionInfo) targetTable.getDefaultDistributionInfo());
List<Column> distributedColumns = distributionInfo.getDistributionColumns();
List<Integer> columnIndexes = Lists.newArrayList();
int idx = 0;
for (int i = 0; i < targetTable.getFullSchema().size(); ++i) {
if (targetTable.getFullSchema().get(i).equals(distributedColumns.get(idx))) {
columnIndexes.add(i);
idx++;
if (idx == distributedColumns.size()) {
break;
DistributionInfo distributionInfo = targetTable.getDefaultDistributionInfo();
if (distributionInfo instanceof HashDistributionInfo) {
HashDistributionInfo hashDistributionInfo
= ((HashDistributionInfo) targetTable.getDefaultDistributionInfo());
List<Column> distributedColumns = hashDistributionInfo.getDistributionColumns();
List<Integer> columnIndexes = Lists.newArrayList();
int idx = 0;
for (int i = 0; i < targetTable.getFullSchema().size(); ++i) {
if (targetTable.getFullSchema().get(i).equals(distributedColumns.get(idx))) {
columnIndexes.add(i);
idx++;
if (idx == distributedColumns.size()) {
break;
}
}
}
return PhysicalProperties.createHash(columnIndexes.stream()
.map(colIdx -> child().getOutput().get(colIdx).getExprId())
.collect(Collectors.toList()), ShuffleType.NATURAL);
} else if (distributionInfo instanceof RandomDistributionInfo) {
return PhysicalProperties.ANY;
} else {
throw new AnalysisException("Unknown distributionInfo for Nereids to calculate physical properties");
}
return PhysicalProperties.createHash(columnIndexes.stream()
.map(colIdx -> child().getOutput().get(colIdx).getExprId())
.collect(Collectors.toList()), ShuffleType.NATURAL);
} else {
return PhysicalProperties.GATHER;
}
Expand Down
137 changes: 137 additions & 0 deletions regression-test/data/nereids_p0/insert_into_table/random.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !11 --
\N \N \N \N \N \N \N \N
1 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
1 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
2 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
2 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
3 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
4 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
4 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
5 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
5 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
6 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
6 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
7 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
7 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634
8 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
8 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
9 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
9 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
10 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
10 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
11 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
11 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
12 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
12 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
13 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634

-- !12 --
\N \N \N \N \N \N \N \N
\N \N \N \N \N \N \N \N
1 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
1 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
1 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
1 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
2 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
2 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
2 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
2 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
3 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
3 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
4 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
4 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
4 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
4 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
5 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
5 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
5 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
5 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
6 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
6 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
6 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
6 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
7 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
7 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
7 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634
7 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634
8 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
8 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
8 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
8 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
9 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
9 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
9 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
9 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
10 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
10 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
10 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
10 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
11 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
11 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
11 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
11 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
12 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
12 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
12 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
12 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
13 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634
13 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634

-- !13 --
\N \N \N \N \N \N \N \N
\N \N \N \N \N \N \N \N
1 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
1 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
1 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
1 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
1 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
1 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
2 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
2 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
2 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
2 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
2 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
2 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
3 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
3 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
3 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
4 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
4 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
4 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
4 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
5 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
5 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
5 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
5 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
6 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
6 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
6 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
6 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
7 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
7 1 10.0 23795.000000 5354529 2012-03-01 2012-03-01T01:00:01 0.524
7 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634
7 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634
8 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
8 2 20.0 47545.000000 10698279 2012-03-02 2012-03-02T02:01:02 0.742
8 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
8 3 40.0 95045.000000 21385779 2012-03-03 2012-03-03T03:02:03 1.037
9 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
9 4 80.0 190045.000000 42760779 2012-03-04 2012-03-04T04:03:04 1.449
9 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
9 5 160.0 380045.000000 85510779 2012-03-05 2012-03-05T05:04:05 2.031
10 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
10 6 320.0 760045.000000 171010779 2012-03-06 2012-03-06T06:05:06 2.855
10 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
10 7 640.0 1520045.000000 342010779 2012-03-07 2012-03-07T07:06:07 4.022
11 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
11 8 1280.0 3040045.000000 684010779 2012-03-08 2012-03-08T08:07:08 5.675
11 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
11 9 2560.0 6080045.000000 1368010779 2012-03-09 2012-03-09T09:08:09 8.014
12 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
12 10 5120.0 12160045.000000 2736010779 2012-03-10 2012-03-10T10:09:10 11.325
12 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
12 11 10240.0 24320045.000000 5472010779 2012-03-11 2012-03-11T11:10:11 16.009
13 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634
13 12 20480.0 48640045.000000 10944010779 2012-03-12 2012-03-12T12:11:12 22.634

Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ suite("nereids_insert_aggregate") {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

sql '''insert into nereids_insert_into_table_test.agg_t
select * except(kaint) from src'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ suite('complex_insert') {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

sql 'use nereids_insert_into_table_test'

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
create table dup_t_type_cast_rd (
`id` int null,
`kint` int(11) null,
`kdbl` double null,
`kdcml` decimal(20, 6) null,
`kvchr` varchar(20) null,
`kdt` date null,
`kdtmv2` datetimev2(0) null,
`kdcml32v3` decimalv3(7, 3) null
) engine=OLAP
duplicate key(id)
partition by range(id) (
partition p1 values less than ("3"),
partition p2 values less than ("5"),
partition p3 values less than ("7"),
partition p4 values less than ("15")
)
distributed by random buckets 4
properties (
"replication_num"="1"
);
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ suite("nereids_insert_duplicate") {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

sql '''insert into dup_t
select * except(kaint) from src'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ suite("load") {
'agg_nop_t', 'agg_t', 'agg_type_cast',
'dup_nop_t', 'dup_t', 'dup_type_cast',
'uni_nop_t', 'uni_t', 'uni_type_cast',
'arr_t'
'arr_t', 'random_t'
]

for (String file in files) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ suite('nereids_insert_no_partition') {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

explain {
// TODO: test turn off pipeline when dml, remove it if pipeline sink is ok
Expand Down
42 changes: 42 additions & 0 deletions regression-test/suites/nereids_p0/insert_into_table/random.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite('nereids_insert_random') {
sql 'use nereids_insert_into_table_test'
sql 'clean label from nereids_insert_into_table_test'

sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set enable_strict_consistency_dml=true'

sql '''insert into dup_t_type_cast_rd
select id, ktint, ksint, kint, kbint, kdtv2, kdtm, kdbl from src'''
sql 'sync'
qt_11 'select * from dup_t_type_cast_rd order by id, kint'

sql '''insert into dup_t_type_cast_rd with label label_dup_type_cast_cte_rd
with cte as (select id, ktint, ksint, kint, kbint, kdtv2, kdtm, kdbl from src)
select * from cte'''
sql 'sync'
qt_12 'select * from dup_t_type_cast_rd order by id, kint'

sql '''insert into dup_t_type_cast_rd partition (p1, p2) with label label_dup_type_cast_rd
select id, ktint, ksint, kint, kbint, kdtv2, kdtm, kdbl from src where id < 4'''
sql 'sync'
qt_13 'select * from dup_t_type_cast_rd order by id, kint'
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ suite("nereids_insert_aggregate_type_cast") {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

sql '''insert into agg_t_type_cast
select id, ktint, ksint, kint, kbint, kdtv2, kdtm, kdbl from src'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ suite("nereids_insert_duplicate") {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

sql '''insert into dup_t_type_cast
select id, ktint, ksint, kint, kbint, kdtv2, kdtm, kdbl from src'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ suite("nereids_insert_unique_type_cast") {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

sql '''insert into uni_t_type_cast
select id, ktint, ksint, kint, kbint, kdtv2, kdtm, kdbl from src'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ suite("nereids_insert_unique") {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

sql '''insert into uni_t
select * except(kaint) from src'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ suite("nereids_insert_array_type") {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_nereids_dml=true'
sql 'set parallel_fragment_exec_instance_num=13'
sql 'set enable_strict_consistency_dml=true'

test {
sql 'insert into arr_t select id, kaint from src'
Expand Down

0 comments on commit 0458439

Please sign in to comment.