Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -625,9 +625,13 @@ public Void visitArraySortBy(ArraySortBy arraySortBy, CollectorContext context)
@Override
public Void visitIsNull(IsNull isNull, CollectorContext context) {
Expression arg = isNull.child();
// Skip variant sub-column paths (v['k'] IS NULL): the sub-column path is already baked
// into the SlotReference, so null-only access doesn't apply the same way.
if (arg instanceof SlotReference && ((SlotReference) arg).hasSubColPath()) {
// SlotReferences that must not be pruned to NULL-only access:
// 1. variant sub-column path (v['k'] IS NULL) — the path is already baked into the slot
// 2. physical column is NOT NULL, only made nullable by an outer join — no physical null
// map at the scan side, so the regular data path is sufficient to evaluate IS NULL
if (arg instanceof SlotReference && (((SlotReference) arg).hasSubColPath()
|| (((SlotReference) arg).getOriginalColumn().isPresent()
&& !((SlotReference) arg).getOriginalColumn().get().isAllowNull()))) {
return visit(isNull, context);
}
// Optimize IS NULL on nullable expressions: create a context with NULL suffix to indicate
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,20 @@ public void createTable() throws Exception {
+ " map_col map<string, string>\n"
+ ") properties ('replication_num'='1')");

createTable("create table nn_join_dim(\n"
+ " id int not null,\n"
+ " segment varchar(32) not null\n"
+ ") unique key(id)\n"
+ "distributed by hash(id) buckets 1\n"
+ "properties ('replication_num'='1', 'enable_unique_key_merge_on_write'='true')");

createTable("create table nn_join_fact(\n"
+ " fact_id int not null,\n"
+ " dim_id int not null\n"
+ ") duplicate key(fact_id)\n"
+ "distributed by hash(fact_id) buckets 1\n"
+ "properties ('replication_num'='1')");

createTable("create table nested_container_tbl(\n"
+ " id int,\n"
+ " s struct<\n"
Expand Down Expand Up @@ -624,6 +638,24 @@ public void testFilter() throws Throwable {
);
}

@Test
public void testIsNullOnNotNullColumnAfterLeftJoin() throws Exception {
Pair<PhysicalPlan, List<SlotDescriptor>> result = collectAllSlots(
"select sum(if(d.segment is null, 1, 0)) "
+ "from nn_join_fact f left join nn_join_dim d on f.dim_id = d.id");
SlotDescriptor segmentSlot = result.second.stream()
.filter(slot -> slot.getColumn() != null
&& "segment".equalsIgnoreCase(slot.getColumn().getName()))
.findFirst()
.orElseThrow();

// Physical column segment is NOT NULL and only made nullable by the LEFT JOIN: no
// NULL-only access path may be generated. Only the full-column [segment] path exists,
// which shouldSkipAccessInfo drops (whole column read needs no access info for BE).
Assertions.assertTrue(segmentSlot.getAllAccessPaths().isEmpty());
Assertions.assertFalse(segmentSlot.getAllAccessPaths().contains(path("segment", "NULL")));
}

@Test
public void testMapKeysAndValuesFunctionNullCheckUseParentMapNullPath() throws Exception {
// map_keys/map_values are PropagateNullable functions: the returned array is NULL only
Expand Down Expand Up @@ -1611,22 +1643,28 @@ private void assertNoAccessPaths(SlotReference slot) {
}

private Pair<PhysicalPlan, List<SlotDescriptor>> collectComplexSlots(String sql) throws Exception {
NereidsPlanner planner = (NereidsPlanner) executeNereidsSql(sql).planner();
Pair<PhysicalPlan, List<SlotDescriptor>> result = collectAllSlots(sql);
List<SlotDescriptor> complexSlots = new ArrayList<>();
for (SlotDescriptor slot : result.second) {
Type type = slot.getType();
if (type.isComplexType() || type.isVariantType()) {
complexSlots.add(slot);
}
}
return Pair.of(result.first, complexSlots);
}

private Pair<PhysicalPlan, List<SlotDescriptor>> collectAllSlots(String sql) throws Exception {
NereidsPlanner planner = (NereidsPlanner) executeNereidsSql(sql).planner();
List<SlotDescriptor> allSlots = new ArrayList<>();
PhysicalPlan physicalPlan = planner.getPhysicalPlan();
for (PlanFragment fragment : planner.getFragments()) {
List<OlapScanNode> olapScanNodes = fragment.getPlanRoot().collectInCurrentFragment(OlapScanNode.class::isInstance);
for (OlapScanNode olapScanNode : olapScanNodes) {
List<SlotDescriptor> slots = olapScanNode.getTupleDesc().getSlots();
for (SlotDescriptor slot : slots) {
Type type = slot.getType();
if (type.isComplexType() || type.isVariantType()) {
complexSlots.add(slot);
}
}
allSlots.addAll(olapScanNode.getTupleDesc().getSlots());
}
}
return Pair.of(physicalPlan, complexSlots);
return Pair.of(physicalPlan, allSlots);
}

private ColumnAccessPath path(String... path) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !left_join_not_null_column --
3 6 60 600.00 1

Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("left_join_not_null_column") {
sql "set enable_prune_nested_column = true"

sql "drop table if exists left_join_not_null_dim"
sql "drop table if exists left_join_not_null_fact"

sql """
create table left_join_not_null_dim (
id int not null,
segment varchar(32) not null
)
unique key(id)
distributed by hash(id) buckets 1
properties (
"replication_num" = "1",
"enable_unique_key_merge_on_write" = "true"
)
"""

sql """
create table left_join_not_null_fact (
fact_id int not null,
dim_id int not null,
quantity int not null,
amount decimal(10, 2) not null
)
unique key(fact_id)
distributed by hash(fact_id) buckets 1
properties (
"replication_num" = "1",
"enable_unique_key_merge_on_write" = "true"
)
"""

sql "insert into left_join_not_null_dim values (1, 'segment-a'), (2, 'segment-b')"
sql "insert into left_join_not_null_fact values (1, 1, 10, 100.00), (2, 2, 20, 200.00), (3, 3, 30, 300.00)"

explain {
sql """
select count(*), sum(f.fact_id), sum(f.quantity), sum(f.amount),
sum(case when d.segment is null then 1 else 0 end)
from left_join_not_null_fact f
left join left_join_not_null_dim d on f.dim_id = d.id
"""
notContains "segment.NULL"
}

qt_left_join_not_null_column """
select count(*), sum(f.fact_id), sum(f.quantity), sum(f.amount),
sum(case when d.segment is null then 1 else 0 end)
from left_join_not_null_fact f
left join left_join_not_null_dim d on f.dim_id = d.id
"""
}
Loading