From 32d15f0e56d6fb89672132f4bde2e6b54a3c1ad1 Mon Sep 17 00:00:00 2001 From: coldWater Date: Wed, 30 Sep 2026 16:28:40 +0800 Subject: [PATCH 1/4] refactor(query): prepare mutation inputs before physical planning --- src/query/sql/src/planner/optimizer/mod.rs | 1 + .../sql/src/planner/optimizer/mutation.rs | 213 +++ .../sql/src/planner/optimizer/optimizer.rs | 206 +-- src/query/sql/tests/it/optimizer/mod.rs | 1 + src/query/sql/tests/it/optimizer/mutation.rs | 270 ++++ src/query/sql/tests/it/optimizer/mutation.txt | 1297 +++++++++++++++++ 6 files changed, 1808 insertions(+), 180 deletions(-) create mode 100644 src/query/sql/src/planner/optimizer/mutation.rs create mode 100644 src/query/sql/tests/it/optimizer/mutation.rs create mode 100644 src/query/sql/tests/it/optimizer/mutation.txt diff --git a/src/query/sql/src/planner/optimizer/mod.rs b/src/query/sql/src/planner/optimizer/mod.rs index dcd9c81608f..cb7966450a1 100644 --- a/src/query/sql/src/planner/optimizer/mod.rs +++ b/src/query/sql/src/planner/optimizer/mod.rs @@ -16,6 +16,7 @@ databend_common_tracing::register_module_tag!("[OPTIMIZER]"); mod cost; pub mod ir; +mod mutation; #[allow(clippy::module_inception)] mod optimizer; mod optimizer_api; diff --git a/src/query/sql/src/planner/optimizer/mutation.rs b/src/query/sql/src/planner/optimizer/mutation.rs new file mode 100644 index 00000000000..8751ba52b40 --- /dev/null +++ b/src/query/sql/src/planner/optimizer/mutation.rs @@ -0,0 +1,213 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::Result; +use databend_common_expression::DataSchemaRef; + +use super::optimizer::query_logical_pipeline; +use super::optimizer::query_planning_pipeline; +use crate::binder::MutationStrategy; +use crate::binder::MutationType; +use crate::binder::target_probe; +use crate::optimizer::OptimizerContext; +use crate::optimizer::ir::SExpr; +use crate::optimizer::optimizers::distributed::BroadcastToShuffleOptimizer; +use crate::optimizer::optimizers::recursive::RecursiveRuleOptimizer; +use crate::optimizer::optimizers::rule::RuleID; +use crate::optimizer::pipeline::OptimizerPipeline; +use crate::plans::Join; +use crate::plans::JoinType; +use crate::plans::MatchedEvaluator; +use crate::plans::Mutation; +use crate::plans::Operator; +use crate::plans::Plan; +use crate::plans::RelOp; +use crate::plans::RelOperator; + +/// Mutation-specific logical state, ready for input-plan selection. Keep preparation +/// separate from decisions that depend on the selected join/distribution implementation. +struct PreparedMutation { + mutation: Mutation, + input: SExpr, + schema: DataSchemaRef, +} + +impl PreparedMutation { + async fn prepare(opt_ctx: Arc, s_expr: &SExpr) -> Result { + let mut mutation: Mutation = s_expr.plan().clone().try_into()?; + // Logical simplification must not change the statement's result columns. + let schema = mutation.schema(); + let mut pipeline = + query_logical_pipeline(opt_ctx.clone(), s_expr.child(0)?.clone(), None).await?; + let input = pipeline.execute().await?; + // Mutation preparation is required even when optional recursive rewrites are + // disabled. Keep this outside the pipeline's optimizer skip-list mechanism. + let mut input = RecursiveRuleOptimizer::new(opt_ctx, &[RuleID::MergeFilterIntoMutation]) + .optimize_sync(input)?; + prepare_empty_input(&input, &mut mutation); + if mutation.strategy == MutationStrategy::Direct + && let Some(prepared) = prepare_direct_source(&input, &mut mutation)? + { + input = prepared; + } + #[cfg(debug_assertions)] + { + input.validate_types(&mutation.metadata)?; + input.validate_column_scope(&mutation.metadata)?; + if let Some(index) = mutation.predicate_column_index { + debug_assert!(mutation.required_columns.contains(&index)); + // MutationSource materializes this execution-only predicate column; + // it is not part of the source's logical output-column set. + } + } + Ok(Self { + mutation, + input, + schema, + }) + } +} + +/// Select and finalize the input using the legacy mutation distribution policy. A local +/// retry reuses the prepared logical input, rather than rerunning preparation on raw SQL. +async fn plan_input(opt_ctx: Arc, input: SExpr, local: bool) -> Result { + let pipeline = OptimizerPipeline::new(opt_ctx.clone(), input).await?; + if local { + // Pipeline configuration must not re-enable distribution for warehouse tables. + opt_ctx.set_enable_distributed_optimization(false); + } + let mut pipeline = query_planning_pipeline(opt_ctx, pipeline)?; + pipeline.execute().await +} + +pub(super) async fn optimize_mutation( + opt_ctx: Arc, + s_expr: SExpr, +) -> Result { + let PreparedMutation { + mut mutation, + input, + schema, + } = PreparedMutation::prepare(opt_ctx.clone(), &s_expr).await?; + let mut input_s_expr = plan_input(opt_ctx.clone(), input.clone(), false).await?; + + // Preserve the legacy mutation consumer policy until requirements are part of + // physical search: discard the query-root Exchange and retry locally if necessary. + if matches!(input_s_expr.plan(), RelOperator::Exchange(_)) { + input_s_expr = input_s_expr.child(0)?.clone(); + } + if input_s_expr.has_merge_exchange() { + input_s_expr = plan_input(opt_ctx.clone(), input, true).await?; + } + mutation.distributed = opt_ctx.get_enable_distributed_optimization(); + let inner_rel_op = input_s_expr.plan.rel_op(); + input_s_expr = match mutation.mutation_type { + MutationType::Merge => { + if mutation.distributed && inner_rel_op == RelOp::Join { + let join = Join::try_from(input_s_expr.plan().clone())?; + let broadcast_to_shuffle = BroadcastToShuffleOptimizer::create(); + let is_broadcast = broadcast_to_shuffle.matcher.matches(&input_s_expr) + && broadcast_to_shuffle.is_broadcast(&input_s_expr)?; + + // If the mutation strategy is matched only, the join type is inner join, if it is a broadcast + // join and the target table on the probe side, we can avoid row id shuffle after the join. + let target_probe = target_probe(&input_s_expr, mutation.target_table_index)?; + if is_broadcast + && target_probe + && mutation.strategy == MutationStrategy::MatchedOnly + { + mutation.row_id_shuffle = false; + } + + // Change broadcast join to shuffle join if the join type is left or left-anti join, because + // broadcast join can not deduplicate row ids. + if is_broadcast && matches!(join.join_type, JoinType::Left | JoinType::LeftAnti) { + broadcast_to_shuffle.optimize(&input_s_expr)? + } else { + input_s_expr + } + } else { + input_s_expr + } + } + MutationType::Update | MutationType::Delete => input_s_expr, + }; + + Ok(Plan::DataMutation { + schema, + s_expr: Box::new(SExpr::create_unary( + Arc::new(RelOperator::Mutation(mutation)), + Arc::new(input_s_expr), + )), + metadata: opt_ctx.get_metadata(), + }) +} + +fn prepare_empty_input(input: &SExpr, mutation: &mut Mutation) { + if mutation.matched_evaluators.is_empty() { + return; + } + match input.plan() { + RelOperator::ConstantTableScan(scan) if scan.num_rows == 0 => mutation.no_effect = true, + RelOperator::Join(_) => { + // Logical join rewrites may commute the target. Its row-id symbol survives + // an empty-scan rewrite and identifies it without relying on child position. + for child in input.children() { + if let RelOperator::ConstantTableScan(scan) = child.plan() + && scan.num_rows == 0 + && scan.columns.contains(&mutation.row_id_index) + { + mutation.matched_evaluators = vec![MatchedEvaluator { + condition: None, + update: None, + }]; + mutation.can_try_update_column_only = false; + break; + } + } + } + _ => {} + } +} + +fn prepare_direct_source(s_expr: &SExpr, mutation: &mut Mutation) -> Result> { + match s_expr.plan() { + RelOperator::MutationSource(rel) => { + let mut rel = rel.clone(); + rel.refresh_read_partition_columns(); + let is_truncate = rel.mutation_type == MutationType::Delete && !rel.has_predicates(); + let direct_filter = rel.all_predicates_cloned(); + let predicate_column_index = + rel.ensure_mutation_predicate_column_if_needed(&mutation.metadata); + let new_s_expr = SExpr::create_leaf(Arc::new(RelOperator::MutationSource(rel))); + mutation.truncate_table = is_truncate; + mutation.direct_filter = direct_filter; + if let Some(index) = predicate_column_index { + mutation.required_columns.insert(index); + mutation.predicate_column_index = Some(index); + } + Ok(Some(new_s_expr)) + } + RelOperator::Udf(_) | RelOperator::EvalScalar(_) if s_expr.arity() == 1 => { + if let Some(child) = prepare_direct_source(s_expr.unary_child(), mutation)? { + Ok(Some(s_expr.replace_children([Arc::new(child)]))) + } else { + Ok(None) + } + } + _ => Ok(None), + } +} diff --git a/src/query/sql/src/planner/optimizer/optimizer.rs b/src/query/sql/src/planner/optimizer/optimizer.rs index fa8717ba870..380af4394d7 100644 --- a/src/query/sql/src/planner/optimizer/optimizer.rs +++ b/src/query/sql/src/planner/optimizer/optimizer.rs @@ -22,20 +22,15 @@ use databend_common_expression::Symbol; use log::info; use crate::InsertInputSource; -use crate::MetadataRef; -use crate::ScalarExpr; -use crate::binder::MutationStrategy; -use crate::binder::MutationType; -use crate::binder::target_probe; use crate::optimizer::OptimizerContext; use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; +use crate::optimizer::mutation::optimize_mutation; use crate::optimizer::optimizers::CTEFilterPushdownOptimizer; use crate::optimizer::optimizers::CascadesOptimizer; use crate::optimizer::optimizers::CommonSubexpressionOptimizer; use crate::optimizer::optimizers::DPhpyOptimizer; use crate::optimizer::optimizers::EliminateSelfJoinOptimizer; -use crate::optimizer::optimizers::distributed::BroadcastToShuffleOptimizer; use crate::optimizer::optimizers::operator::CleanupUnusedCTEOptimizer; use crate::optimizer::optimizers::operator::DeduplicateJoinConditionOptimizer; use crate::optimizer::optimizers::operator::FinalizeSpatialJoinOptimizer; @@ -50,15 +45,8 @@ use crate::optimizer::optimizers::rule::RuleEagerAggregation; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::pipeline::OptimizerPipeline; use crate::optimizer::statistics::CollectStatisticsOptimizer; -use crate::plans::ConstantTableScan; use crate::plans::EvalScalar; -use crate::plans::Join; -use crate::plans::JoinType; -use crate::plans::MatchedEvaluator; -use crate::plans::Mutation; -use crate::plans::Operator; use crate::plans::Plan; -use crate::plans::RelOp; use crate::plans::RelOperator; use crate::plans::ScalarItem; use crate::plans::SetScalarsOrQuery; @@ -278,8 +266,20 @@ async fn optimize_query_inner( s_expr: SExpr, output_columns: Option>, ) -> Result { + let pipeline = query_logical_pipeline(opt_ctx.clone(), s_expr, output_columns).await?; + let mut pipeline = query_planning_pipeline(opt_ctx, pipeline)?; + pipeline.execute().await +} + +/// Build the common logical passes without selecting distributions or execution plans. +/// Mutation preparation runs after these passes and before query planning. +pub(super) async fn query_logical_pipeline( + opt_ctx: Arc, + s_expr: SExpr, + output_columns: Option>, +) -> Result { let settings = opt_ctx.get_table_ctx().get_settings(); - let mut pipeline = OptimizerPipeline::new(opt_ctx.clone(), s_expr.clone()) + let pipeline = OptimizerPipeline::new(opt_ctx.clone(), s_expr) .await? // Eliminate subqueries by rewriting them into more efficient form .add(SubqueryDecorrelatorOptimizer::new(opt_ctx.clone(), None)) @@ -328,7 +328,17 @@ async fn optimize_query_inner( .add_if( settings.get_force_eager_aggregate()?, RuleEagerAggregation::new(opt_ctx.get_metadata()), - ) + ); + Ok(pipeline) +} + +/// Append the existing planning and cleanup passes. Ordinary queries keep a single +/// pipeline; mutation inputs enter here only after their logical preparation. +pub(super) fn query_planning_pipeline( + opt_ctx: Arc, + pipeline: OptimizerPipeline, +) -> Result { + Ok(pipeline // Cascades optimizer may fail due to timeout, fallback to heuristic optimizer in this case. .add(CascadesOptimizer::new(opt_ctx.clone())?) // Eliminate unnecessary scalar calculations to clean up the final plan @@ -339,12 +349,7 @@ async fn optimize_query_inner( // Clean up unused CTEs .add(CleanupUnusedCTEOptimizer) // Finalize derived join annotations after all logical rewrites. - .add(FinalizeSpatialJoinOptimizer::new(opt_ctx.clone())); - - // 17. Execute the pipeline - let s_expr = pipeline.execute().await?; - - Ok(s_expr) + .add(FinalizeSpatialJoinOptimizer::new(opt_ctx.clone()))) } fn rewrite_insert_multi_table_whens( @@ -427,162 +432,3 @@ async fn get_optimized_memo(opt_ctx: Arc, s_expr: SExpr) -> Re Ok(pipeline.memo()) } - -async fn optimize_mutation(opt_ctx: Arc, s_expr: SExpr) -> Result { - // Optimize the input plan. - let mut input_s_expr = optimize_query(opt_ctx.clone(), s_expr.child(0)?.clone()).await?; - input_s_expr = RecursiveRuleOptimizer::new(opt_ctx.clone(), &[RuleID::MergeFilterIntoMutation]) - .optimize_sync(input_s_expr)?; - - // For distributed query optimization, we need to remove the Exchange operator at the top of the plan. - if let &RelOperator::Exchange(_) = input_s_expr.plan() { - input_s_expr = input_s_expr.child(0)?.clone(); - } - // If there still exists an Exchange::Merge operator, we should disable distributed optimization and - // optimize the input plan again. - if input_s_expr.has_merge_exchange() { - opt_ctx.set_enable_distributed_optimization(false); - input_s_expr = optimize_query(opt_ctx.clone(), s_expr.child(0)?.clone()).await?; - } - - let mut mutation: Mutation = s_expr.plan().clone().try_into()?; - mutation.distributed = opt_ctx.get_enable_distributed_optimization(); - - let schema = mutation.schema(); - // To fix issue #16588, if target table is rewritten as an empty scan, that means - // the condition is false and the match branch can never be executed. - // Therefore, the match evaluators can be reset. - let inner_rel_op = input_s_expr.plan.rel_op(); - if !mutation.matched_evaluators.is_empty() { - match inner_rel_op { - RelOp::ConstantTableScan => { - let constant_table_scan = ConstantTableScan::try_from(input_s_expr.plan().clone())?; - if constant_table_scan.num_rows == 0 { - mutation.no_effect = true; - } - } - RelOp::Join => { - let mut right_child = input_s_expr.child(1)?; - let mut right_child_rel = right_child.plan.rel_op(); - if right_child_rel == RelOp::Exchange { - right_child_rel = right_child.child(0)?.plan.rel_op(); - right_child = right_child.child(0)?; - } - if right_child_rel == RelOp::ConstantTableScan { - let constant_table_scan = - ConstantTableScan::try_from(right_child.plan().clone())?; - if constant_table_scan.num_rows == 0 { - mutation.matched_evaluators = vec![MatchedEvaluator { - condition: None, - update: None, - }]; - mutation.can_try_update_column_only = false; - } - } - } - _ => (), - } - } - - input_s_expr = match mutation.mutation_type { - MutationType::Merge => { - if mutation.distributed && inner_rel_op == RelOp::Join { - let join = Join::try_from(input_s_expr.plan().clone())?; - let broadcast_to_shuffle = BroadcastToShuffleOptimizer::create(); - let is_broadcast = broadcast_to_shuffle.matcher.matches(&input_s_expr) - && broadcast_to_shuffle.is_broadcast(&input_s_expr)?; - - // If the mutation strategy is matched only, the join type is inner join, if it is a broadcast - // join and the target table on the probe side, we can avoid row id shuffle after the join. - let target_probe = target_probe(&input_s_expr, mutation.target_table_index)?; - if is_broadcast - && target_probe - && mutation.strategy == MutationStrategy::MatchedOnly - { - mutation.row_id_shuffle = false; - } - - // Change broadcast join to shuffle join if the join type is left or left-anti join, because - // broadcast join can not deduplicate row ids. - if is_broadcast && matches!(join.join_type, JoinType::Left | JoinType::LeftAnti) { - broadcast_to_shuffle.optimize(&input_s_expr)? - } else { - input_s_expr - } - } else { - input_s_expr - } - } - MutationType::Update | MutationType::Delete => { - #[allow(clippy::type_complexity)] - fn finalize_mutation_source( - s_expr: &SExpr, - metadata: &MetadataRef, - ) -> Result, Option)>> { - match s_expr.plan() { - RelOperator::MutationSource(rel) => { - let mut rel = rel.clone(); - rel.refresh_read_partition_columns(); - let is_truncate = - rel.mutation_type == MutationType::Delete && !rel.has_predicates(); - let direct_filter = rel.all_predicates_cloned(); - let predicate_column_index = - rel.ensure_mutation_predicate_column_if_needed(metadata); - let new_s_expr = - SExpr::create_leaf(Arc::new(RelOperator::MutationSource(rel))); - Ok(Some(( - new_s_expr, - is_truncate, - direct_filter, - predicate_column_index, - ))) - } - RelOperator::Udf(_) | RelOperator::EvalScalar(_) if s_expr.arity() == 1 => { - if let Some((child, is_truncate, direct_filter, pred_idx)) = - finalize_mutation_source(s_expr.unary_child(), metadata)? - { - Ok(Some(( - s_expr.replace_children(vec![Arc::new(child)]), - is_truncate, - direct_filter, - pred_idx, - ))) - } else { - Ok(None) - } - } - _ => Ok(None), - } - } - - // finalize_mutation_source only applies to Direct strategy where the - // plan tree contains a MutationSource leaf. Non-direct mutations - // (e.g., UPDATE ... FROM, subquery cases) have Join/Filter roots - // with no MutationSource node. - if mutation.strategy == MutationStrategy::Direct { - let metadata = opt_ctx.get_metadata(); - if let Some((new_s_expr, is_truncate, direct_filter, pred_idx)) = - finalize_mutation_source(&input_s_expr, &metadata)? - { - input_s_expr = new_s_expr; - mutation.truncate_table = is_truncate; - mutation.direct_filter = direct_filter; - if let Some(index) = pred_idx { - mutation.required_columns.insert(index); - mutation.predicate_column_index = Some(index); - } - } - } - input_s_expr - } - }; - - Ok(Plan::DataMutation { - schema, - s_expr: Box::new(SExpr::create_unary( - Arc::new(RelOperator::Mutation(mutation)), - Arc::new(input_s_expr), - )), - metadata: opt_ctx.get_metadata(), - }) -} diff --git a/src/query/sql/tests/it/optimizer/mod.rs b/src/query/sql/tests/it/optimizer/mod.rs index 2696336c04f..43a2117ed77 100644 --- a/src/query/sql/tests/it/optimizer/mod.rs +++ b/src/query/sql/tests/it/optimizer/mod.rs @@ -28,6 +28,7 @@ mod eager_aggregation; mod hierarchical_grouping_sets; mod join_cardinality; mod materialized_cte_distribution; +mod mutation; mod normalize_scalar; mod outer_join_to_anti; mod planning_context; diff --git a/src/query/sql/tests/it/optimizer/mutation.rs b/src/query/sql/tests/it/optimizer/mutation.rs new file mode 100644 index 00000000000..f9c53cb492a --- /dev/null +++ b/src/query/sql/tests/it/optimizer/mutation.rs @@ -0,0 +1,270 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Write; + +use databend_common_catalog::table_context::TableContextSettings; +use databend_common_exception::Result; +use databend_common_sql::binder::MutationStrategy; +use databend_common_sql::optimizer::OptimizerContext; +use databend_common_sql::optimizer::ir::StatContext; +use databend_common_sql::optimizer::optimize; +use databend_common_sql::plans::Plan; + +use crate::framework::LiteTableContext; +use crate::framework::golden::SqlTestCase; +use crate::framework::golden::open_golden_file; +use crate::framework::golden::write_case_header; + +#[derive(Clone, Copy, Default)] +struct ExpectedMutation { + direct: bool, + truncate: bool, + no_effect: bool, + empty_target: bool, + local_retry: bool, + predicate_column: bool, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_mutation_preparation() -> Result<()> { + let mut file = open_golden_file("optimizer", "mutation.txt")?; + let cases = [ + ( + "direct_update", + "UPDATE target SET v = v + 1 WHERE k > 10", + ExpectedMutation { + direct: true, + predicate_column: true, + ..Default::default() + }, + ), + ( + "direct_update_false", + "UPDATE target SET v = v + 1 WHERE false", + ExpectedMutation { + direct: true, + no_effect: true, + ..Default::default() + }, + ), + ( + "direct_delete_filter", + "DELETE FROM target WHERE k > 10", + ExpectedMutation { + direct: true, + ..Default::default() + }, + ), + ( + "direct_delete_all", + "DELETE FROM target", + ExpectedMutation { + direct: true, + truncate: true, + ..Default::default() + }, + ), + ( + "direct_delete_true", + "DELETE FROM target WHERE true", + ExpectedMutation { + direct: true, + truncate: true, + ..Default::default() + }, + ), + ( + "direct_delete_false", + "DELETE FROM target WHERE false", + ExpectedMutation { + direct: true, + no_effect: true, + ..Default::default() + }, + ), + ( + "subquery_update", + "UPDATE target SET v = v + 1 WHERE k IN (SELECT k FROM source)", + ExpectedMutation::default(), + ), + ( + "subquery_delete", + "DELETE FROM target WHERE k IN (SELECT k FROM source)", + ExpectedMutation::default(), + ), + ( + "matched_merge", + "MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v", + ExpectedMutation::default(), + ), + ( + "insert_only_merge", + "MERGE INTO target t USING source s ON t.k = s.k WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v)", + ExpectedMutation::default(), + ), + ( + "mixed_merge", + "MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v)", + ExpectedMutation::default(), + ), + ( + "aggregate_source_local_retry", + "MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v)", + ExpectedMutation { + local_retry: true, + ..Default::default() + }, + ), + ( + "empty_target_local_retry", + "MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v)", + ExpectedMutation { + empty_target: true, + local_retry: true, + ..Default::default() + }, + ), + ( + "empty_target_merge", + "MERGE INTO target t USING source s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v)", + ExpectedMutation { + empty_target: true, + ..Default::default() + }, + ), + ( + "empty_source_merge", + "MERGE INTO target t USING (SELECT * FROM source WHERE false) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v)", + ExpectedMutation { + no_effect: true, + ..Default::default() + }, + ), + ]; + for distributed in [false, true] { + for (name, sql, expected) in cases { + let case = SqlTestCase { + name, + description: "Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy.", + setup_sqls: &[ + "CREATE TABLE target(k Int64 NOT NULL, v Int64 NOT NULL)", + "CREATE TABLE source(k Int64 NOT NULL, v Int64 NOT NULL)", + ], + sql, + }; + let ctx = LiteTableContext::create().await?; + ctx.set_cluster_node_num(if distributed { 2 } else { 1 }); + ctx.set_table_warehouse_distribution(distributed); + for setup in case.setup_sqls { + ctx.register_setup_sql(setup).await?; + } + let raw = ctx.bind_sql(sql).await?; + let Plan::DataMutation { + metadata, schema, .. + } = &raw + else { + unreachable!() + }; + let opt_ctx = + OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?) + .with_settings(&ctx.get_settings())?; + opt_ctx.set_enable_distributed_optimization(distributed); + let optimized = optimize(opt_ctx, raw.clone()).await?; + let Plan::DataMutation { + s_expr, + schema: optimized_schema, + .. + } = &optimized + else { + unreachable!() + }; + let mutation = s_expr.plan().as_mutation().unwrap(); + assert_eq!( + schema, optimized_schema, + "bound result schema changed: {name}" + ); + assert_eq!( + mutation.strategy == MutationStrategy::Direct, + expected.direct, + "{name}" + ); + assert_eq!(mutation.truncate_table, expected.truncate, "{name}"); + assert_eq!(mutation.no_effect, expected.no_effect, "{name}"); + assert_eq!( + mutation.predicate_column_index.is_some(), + expected.predicate_column, + "{name}" + ); + if expected.empty_target { + assert!( + !mutation.no_effect, + "unmatched inserts must survive an empty target" + ); + assert!( + mutation + .matched_evaluators + .iter() + .all(|action| action.update.is_none()) + ); + assert!(!mutation.can_try_update_column_only); + } + if expected.local_retry && distributed { + assert!( + !mutation.distributed, + "an internal Merge requires local retry" + ); + assert!(!s_expr.child(0)?.has_merge_exchange()); + } + if expected.direct && !expected.truncate && !expected.no_effect { + assert!(!mutation.direct_filter.is_empty(), "{name}"); + } + if let Some(predicate) = mutation.predicate_column_index { + assert!(mutation.required_columns.contains(&predicate), "{name}"); + } + s_expr.child(0)?.validate_types(metadata)?; + s_expr.child(0)?.validate_column_scope(metadata)?; + write_case_header(&mut file, &case)?; + writeln!(file, "requested_distributed: {distributed}")?; + writeln!( + file, + "raw_plan:\n{}", + raw.format_indent(Default::default(), &StatContext::default())? + )?; + writeln!( + file, + "optimized_plan:\n{}", + optimized.format_indent(Default::default(), &StatContext::default())? + )?; + writeln!( + file, + "mutation_state: strategy={:?}, distributed={}, row_id_shuffle={}, no_effect={}, truncate={}, predicate_column={:?}, direct_filter_count={}, matched_update_count={}", + mutation.strategy, + mutation.distributed, + mutation.row_id_shuffle, + mutation.no_effect, + mutation.truncate_table, + mutation.predicate_column_index, + mutation.direct_filter.len(), + mutation + .matched_evaluators + .iter() + .filter(|action| action.update.is_some()) + .count() + )?; + writeln!(file)?; + } + } + Ok(()) +} diff --git a/src/query/sql/tests/it/optimizer/mutation.txt b/src/query/sql/tests/it/optimizer/mutation.txt new file mode 100644 index 00000000000..1ca9a55852e --- /dev/null +++ b/src/query/sql/tests/it/optimizer/mutation.txt @@ -0,0 +1,1297 @@ +=== direct_update === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: UPDATE target SET v = v + 1 WHERE k > 10 +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Filter + ├── filters: [gt(target.k (#0), 10)] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [FunctionCall(FunctionCall { span: None, func_name: "gt", params: [], arguments: [BoundColumnRef(BoundColumnRef { span: Some(34..35), column: ColumnBinding { database_name: Some("default"), table_name: Some("target"), column_position: Some(1), table_index: Some(0), column_name: "k", column_name_lower: None, index: 0, data_type: Number(Int64), visibility: Visible, virtual_expr: None, is_srf: false } }), ConstantExpr(ConstantExpr { span: None, value: Number(10_i64) })] })], predicate_column_index: Some(2), read_partition_columns: {0} }) + +mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=Some(2), direct_filter_count=1, matched_update_count=1 + +=== direct_update_false === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: UPDATE target SET v = v + 1 WHERE false +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Filter + ├── filters: [false] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── EmptyResultScan + +mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== direct_delete_filter === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target WHERE k > 10 +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Filter + ├── filters: [gt(target.k (#0), 10)] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [FunctionCall(FunctionCall { span: None, func_name: "gt", params: [], arguments: [BoundColumnRef(BoundColumnRef { span: Some(25..26), column: ColumnBinding { database_name: Some("default"), table_name: Some("target"), column_position: Some(1), table_index: Some(0), column_name: "k", column_name_lower: None, index: 0, data_type: Number(Int64), visibility: Visible, virtual_expr: None, is_srf: false } }), ConstantExpr(ConstantExpr { span: None, value: Number(10_i64) })] })], predicate_column_index: None, read_partition_columns: {0} }) + +mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=1, matched_update_count=0 + +=== direct_delete_all === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== direct_delete_true === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target WHERE true +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Filter + ├── filters: [true] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== direct_delete_false === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target WHERE false +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Filter + ├── filters: [false] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── EmptyResultScan + +mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== subquery_update === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: UPDATE target SET v = v + 1 WHERE k IN (SELECT k FROM source) +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Filter + ├── filters: [marker (#5)] + └── Join(RightMark) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [source.k (#2) AS (#2)] + │ └── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Join(LeftSemi) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== subquery_delete === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target WHERE k IN (SELECT k FROM source) +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Filter + ├── filters: [marker (#5)] + └── Join(RightMark) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [source.k (#2) AS (#2)] + │ └── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Join(LeftSemi) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== matched_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +└── Join(Inner) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +└── Join(Inner) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== insert_only_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING source s ON t.k = s.k WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(RightAnti) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(LeftAnti) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=NotMatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== mixed_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Right) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== aggregate_source_local_retry === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = max(v) (#3)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Right) + ├── build keys: [max(k) (#2)] + ├── probe keys: [target.k (#4)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [max(k) (#2) AS (#2), max(v) (#3) AS (#3)] + │ └── Aggregate(Initial) + │ ├── group items: [] + │ ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + │ └── EvalScalar + │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] + │ └── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = max(v) (#3)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#4)] + ├── probe keys: [max(k) (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== empty_target_local_retry === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = max(v) (#3)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Right) + ├── build keys: [max(k) (#2)] + ├── probe keys: [target.k (#4)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [max(k) (#2) AS (#2), max(v) (#3) AS (#3)] + │ └── Aggregate(Initial) + │ ├── group items: [] + │ ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + │ └── EvalScalar + │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] + │ └── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Filter + ├── filters: [gt(target.k (#4), 10), lt(target.k (#4), 0)] + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#4)] + ├── probe keys: [max(k) (#2)] + ├── other filters: [] + ├── EmptyResultScan + └── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== empty_target_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING source s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Right) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Filter + ├── filters: [gt(target.k (#2), 10), lt(target.k (#2), 0)] + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── EmptyResultScan + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== empty_source_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING (SELECT * FROM source WHERE false) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: false +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Right) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] + │ └── Filter + │ ├── filters: [false] + │ └── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── EmptyResultScan + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== direct_update === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: UPDATE target SET v = v + 1 WHERE k > 10 +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Filter + ├── filters: [gt(target.k (#0), 10)] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [FunctionCall(FunctionCall { span: None, func_name: "gt", params: [], arguments: [BoundColumnRef(BoundColumnRef { span: Some(34..35), column: ColumnBinding { database_name: Some("default"), table_name: Some("target"), column_position: Some(1), table_index: Some(0), column_name: "k", column_name_lower: None, index: 0, data_type: Number(Int64), visibility: Visible, virtual_expr: None, is_srf: false } }), ConstantExpr(ConstantExpr { span: None, value: Number(10_i64) })] })], predicate_column_index: Some(2), read_partition_columns: {0} }) + +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=Some(2), direct_filter_count=1, matched_update_count=1 + +=== direct_update_false === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: UPDATE target SET v = v + 1 WHERE false +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Filter + ├── filters: [false] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── EmptyResultScan + +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== direct_delete_filter === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target WHERE k > 10 +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Filter + ├── filters: [gt(target.k (#0), 10)] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [FunctionCall(FunctionCall { span: None, func_name: "gt", params: [], arguments: [BoundColumnRef(BoundColumnRef { span: Some(25..26), column: ColumnBinding { database_name: Some("default"), table_name: Some("target"), column_position: Some(1), table_index: Some(0), column_name: "k", column_name_lower: None, index: 0, data_type: Number(Int64), visibility: Visible, virtual_expr: None, is_srf: false } }), ConstantExpr(ConstantExpr { span: None, value: Number(10_i64) })] })], predicate_column_index: None, read_partition_columns: {0} }) + +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=1, matched_update_count=0 + +=== direct_delete_all === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== direct_delete_true === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target WHERE true +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Filter + ├── filters: [true] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== direct_delete_false === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target WHERE false +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Filter + ├── filters: [false] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── EmptyResultScan + +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== subquery_update === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: UPDATE target SET v = v + 1 WHERE k IN (SELECT k FROM source) +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Filter + ├── filters: [marker (#5)] + └── Join(RightMark) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [source.k (#2) AS (#2)] + │ └── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Join(LeftSemi) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── Exchange(Broadcast) + │ └── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Exchange(Hash) + ├── Exchange(Hash): keys: [target.k (#0)] + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== subquery_delete === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: DELETE FROM target WHERE k IN (SELECT k FROM source) +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Filter + ├── filters: [marker (#5)] + └── Join(RightMark) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [source.k (#2) AS (#2)] + │ └── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Join(LeftSemi) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── Exchange(Broadcast) + │ └── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Exchange(Hash) + ├── Exchange(Hash): keys: [target.k (#0)] + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== matched_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +└── Join(Inner) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +└── Join(Inner) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Exchange(Broadcast) + │ └── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Exchange(Hash) + ├── Exchange(Hash): keys: [target.k (#2)] + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=true, row_id_shuffle=false, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== insert_only_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING source s ON t.k = s.k WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(RightAnti) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(LeftAnti) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── Exchange(Hash) + │ ├── Exchange(Hash): keys: [target.k (#2)] + │ └── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Exchange(Hash) + ├── Exchange(Hash): keys: [source.k (#0)] + └── Exchange(Hash) + ├── Exchange(Hash): keys: [source.k (#0)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=NotMatchedOnly, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== mixed_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Right) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── Exchange(Hash) + │ ├── Exchange(Hash): keys: [target.k (#2)] + │ └── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Exchange(Hash) + ├── Exchange(Hash): keys: [source.k (#0)] + └── Exchange(Hash) + ├── Exchange(Hash): keys: [source.k (#0)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== aggregate_source_local_retry === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = max(v) (#3)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Right) + ├── build keys: [max(k) (#2)] + ├── probe keys: [target.k (#4)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [max(k) (#2) AS (#2), max(v) (#3) AS (#3)] + │ └── Aggregate(Initial) + │ ├── group items: [] + │ ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + │ └── EvalScalar + │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] + │ └── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = max(v) (#3)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#4)] + ├── probe keys: [max(k) (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +=== empty_target_local_retry === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = max(v) (#3)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Right) + ├── build keys: [max(k) (#2)] + ├── probe keys: [target.k (#4)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [max(k) (#2) AS (#2), max(v) (#3) AS (#3)] + │ └── Aggregate(Initial) + │ ├── group items: [] + │ ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + │ └── EvalScalar + │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] + │ └── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Filter + ├── filters: [gt(target.k (#4), 10), lt(target.k (#4), 0)] + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#4)] + ├── probe keys: [max(k) (#2)] + ├── other filters: [] + ├── EmptyResultScan + └── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== empty_target_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING source s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Right) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Filter + ├── filters: [gt(target.k (#2), 10), lt(target.k (#2), 0)] + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── Exchange(Hash) + │ ├── Exchange(Hash): keys: [target.k (#2)] + │ └── EmptyResultScan + └── Exchange(Hash) + ├── Exchange(Hash): keys: [source.k (#0)] + └── Exchange(Hash) + ├── Exchange(Hash): keys: [source.k (#0)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +=== empty_source_merge === +description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. +sql: MERGE INTO target t USING (SELECT * FROM source WHERE false) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) +requested_distributed: true +raw_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Right) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── EvalScalar + │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] + │ └── Filter + │ ├── filters: [false] + │ └── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── EmptyResultScan + +mutation_state: strategy=MixedMatched, distributed=true, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + From e3182316fcb2c21e4523e57f358093a81c737ab5 Mon Sep 17 00:00:00 2001 From: coldWater Date: Thu, 1 Oct 2026 13:40:48 +0800 Subject: [PATCH 2/4] fix(query): decorrelate INSERT WHEN before source planning --- .../tests/it/sql/exec/multi_table_insert.rs | 25 + .../sql/src/planner/optimizer/optimizer.rs | 25 +- .../tests/it/optimizer/insert_multi_table.rs | 183 +++ .../tests/it/optimizer/insert_multi_table.txt | 1032 +++++++++++++++++ src/query/sql/tests/it/optimizer/mod.rs | 1 + 5 files changed, 1264 insertions(+), 2 deletions(-) create mode 100644 src/query/sql/tests/it/optimizer/insert_multi_table.rs create mode 100644 src/query/sql/tests/it/optimizer/insert_multi_table.txt diff --git a/src/query/service/tests/it/sql/exec/multi_table_insert.rs b/src/query/service/tests/it/sql/exec/multi_table_insert.rs index 2e428a494aa..35a910929b5 100644 --- a/src/query/service/tests/it/sql/exec/multi_table_insert.rs +++ b/src/query/service/tests/it/sql/exec/multi_table_insert.rs @@ -49,6 +49,31 @@ fn test_explain_fragments_insert_multi_table_does_not_record_txn_timestamp() -> .execute_command("create table mti_even(a int)") .await?; + // WHEN decorrelation must run before source planning: the scalar + // aggregate needs Partial/Final lowering, and the mark-join symbol + // must survive source projection for branch evaluation. + fixture + .execute_command("create table mti_lookup(a int)") + .await?; + fixture + .execute_command("insert into mti_lookup values (1)") + .await?; + fixture + .execute_command("create table mti_subquery(a int, b int)") + .await?; + fixture + .execute_command( + "INSERT FIRST + WHEN a IN (SELECT a FROM mti_lookup) THEN INTO mti_subquery + WHEN a > (SELECT max(a) FROM mti_lookup) THEN INTO mti_subquery + SELECT a, b FROM mti_src", + ) + .await?; + let count = fixture + .execute_query("SELECT count(*) FROM mti_subquery") + .await?; + assert_eq!(databend_query::test_kits::query_count(count).await?, 2); + let session = fixture.default_session(); let begin_ctx = fixture.new_query_ctx().await?; execute_command(begin_ctx, "begin transaction").await?; diff --git a/src/query/sql/src/planner/optimizer/optimizer.rs b/src/query/sql/src/planner/optimizer/optimizer.rs index 380af4394d7..4f61243d14f 100644 --- a/src/query/sql/src/planner/optimizer/optimizer.rs +++ b/src/query/sql/src/planner/optimizer/optimizer.rs @@ -191,8 +191,29 @@ pub async fn optimize(opt_ctx: Arc, plan: Plan) -> Result { - plan.input_source = optimize(opt_ctx.clone(), plan.input_source.clone()).await?; - rewrite_insert_multi_table_whens(opt_ctx, plan.as_mut())?; + // WHEN subqueries introduce logical joins/aggregates. Rewrite them before + // selecting the source implementation so those nodes participate in CBO. + rewrite_insert_multi_table_whens(opt_ctx.clone(), plan.as_mut())?; + if let Plan::Query { + s_expr, + bind_context, + .. + } = &mut plan.input_source + { + let mut output_columns = bind_context + .column_set() + .into_iter() + .collect::>(); + for when in &plan.whens { + output_columns.extend(when.condition.used_columns()); + } + let input = s_expr.as_ref().clone(); + let planned = + optimize_query_with_output_columns(opt_ctx, input, output_columns).await?; + *s_expr = Box::new(planned); + } else { + plan.input_source = optimize(opt_ctx, plan.input_source.clone()).await?; + } Ok(Plan::InsertMultiTable(plan)) } Plan::Replace(mut plan) => { diff --git a/src/query/sql/tests/it/optimizer/insert_multi_table.rs b/src/query/sql/tests/it/optimizer/insert_multi_table.rs new file mode 100644 index 00000000000..09a1237b8e7 --- /dev/null +++ b/src/query/sql/tests/it/optimizer/insert_multi_table.rs @@ -0,0 +1,183 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Write; + +use databend_common_catalog::table_context::TableContextSettings; +use databend_common_exception::Result; +use databend_common_sql::optimizer::OptimizerContext; +use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::SExprVisitor; +use databend_common_sql::optimizer::ir::StatContext; +use databend_common_sql::optimizer::ir::VisitAction; +use databend_common_sql::optimizer::optimize; +use databend_common_sql::plans::AggregateMode; +use databend_common_sql::plans::Plan; + +use crate::framework::golden::SqlTestCase; +use crate::framework::golden::open_golden_file; +use crate::framework::golden::setup_context; +use crate::framework::golden::write_case_header; + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_when_subqueries_are_planned_with_source() -> Result<()> { + let mut file = open_golden_file("optimizer", "insert_multi_table.txt")?; + for (name, sql, has_join) in [ + ( + "no_subquery", + "INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t", + false, + ), + ( + "constant_subquery", + "INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t", + false, + ), + ( + "in_subquery", + "INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t", + true, + ), + ( + "correlated_exists", + "INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s", + true, + ), + ( + "scalar_aggregate", + "INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t", + true, + ), + ( + "multiple_first", + "INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t", + true, + ), + ( + "nullable_not_in", + "INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t", + true, + ), + ( + "source_limit", + "INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2", + true, + ), + ] { + for cbo in [false, true] { + for distributed in [false, true] { + let case = SqlTestCase { + name, + description: "WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization.", + setup_sqls: &[ + "CREATE TABLE t(k Int64 NULL, v Int64 NULL)", + "CREATE TABLE lookup(k Int64 NULL)", + "CREATE TABLE dst(k Int64 NULL, v Int64 NULL)", + "CREATE TABLE dst2(k Int64 NULL, v Int64 NULL)", + ], + sql, + }; + let ctx = setup_context(&case).await?; + ctx.set_cluster_node_num(if distributed { 2 } else { 1 }); + ctx.get_settings() + .set_setting("enable_cbo".to_string(), u8::from(cbo).to_string())?; + let raw = ctx.bind_sql(sql).await?; + let Plan::InsertMultiTable(bound) = &raw else { + unreachable!() + }; + let Plan::Query { bind_context, .. } = &bound.input_source else { + unreachable!() + }; + let source_columns = bind_context.result_columns(); + let context = OptimizerContext::new( + ctx.clone(), + bound.meta_data.clone(), + ctx.get_function_context()?, + ) + .with_settings(&ctx.get_settings())?; + context.set_enable_distributed_optimization(distributed); + let planned = optimize(context, raw.clone()).await?; + let Plan::InsertMultiTable(insert) = &planned else { + unreachable!() + }; + let Plan::Query { + s_expr, + bind_context, + .. + } = &insert.input_source + else { + unreachable!() + }; + assert_eq!(source_columns, bind_context.result_columns()); + let source = s_expr.as_ref(); + source.validate_types(&insert.meta_data)?; + source.validate_column_scope(&insert.meta_data)?; + let property = source.derive_relational_prop()?; + let output = &property.output_columns; + for when in &insert.whens { + assert!(!when.condition.has_subquery()); + assert!( + when.condition + .used_columns() + .iter() + .all(|column| output.contains(column)) + ); + } + struct Check { + joins: usize, + } + impl SExprVisitor for Check { + fn visit(&mut self, expr: &SExpr) -> Result { + self.joins += usize::from(expr.plan().as_join().is_some()); + if let Some(aggregate) = expr.plan().as_aggregate() { + assert_ne!( + aggregate.mode, + AggregateMode::Initial, + "WHEN aggregate skipped source lowering" + ); + } + Ok(VisitAction::Continue) + } + } + let mut check = Check { joins: 0 }; + source.accept(&mut check)?; + assert_eq!(check.joins > 0, has_join, "{name}"); + write_case_header(&mut file, &case)?; + writeln!(file, "cbo: {cbo}, distributed: {distributed}")?; + writeln!( + file, + "raw_source:\n{}", + bound + .input_source + .format_indent(Default::default(), &StatContext::default())? + )?; + writeln!( + file, + "planned_source:\n{}", + insert + .input_source + .format_indent(Default::default(), &StatContext::default())? + )?; + for (index, when) in insert.whens.iter().enumerate() { + writeln!( + file, + "when_{index}: {}", + when.condition.as_expr()?.sql_display() + )?; + } + } + } + } + Ok(()) +} diff --git a/src/query/sql/tests/it/optimizer/insert_multi_table.txt b/src/query/sql/tests/it/optimizer/insert_multi_table.txt new file mode 100644 index 00000000000..fb1d0179baf --- /dev/null +++ b/src/query/sql/tests/it/optimizer/insert_multi_table.txt @@ -0,0 +1,1032 @@ +=== no_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +when_0: t.k (#0) > CAST(0 AS Int64 NULL) +=== no_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +when_0: t.k (#0) > CAST(0 AS Int64 NULL) +=== no_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +when_0: t.k (#0) > CAST(0 AS Int64 NULL) +=== no_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +when_0: t.k (#0) > CAST(0 AS Int64 NULL) +=== constant_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +when_0: t.k (#0) = CAST(1 AS Int64 NULL) +=== constant_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +when_0: t.k (#0) = CAST(1 AS Int64 NULL) +=== constant_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +when_0: t.k (#0) = CAST(1 AS Int64 NULL) +=== constant_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +when_0: t.k (#0) = CAST(1 AS Int64 NULL) +=== in_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: 4 (#4) +=== in_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: 4 (#4) +=== in_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: 4 (#4) +=== in_subquery === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: 4 (#4) +=== correlated_exists === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s +cbo: false, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── EvalScalar +│ ├── scalars: [lookup.k (#2) AS (#2), 1 AS (#3)] +│ └── Filter +│ ├── filters: [eq(lookup.k (#2), lookup.k (#2))] +│ └── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [eq(lookup.k (#2), lookup.k (#2))] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: is_true(5 (#5)) +=== correlated_exists === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s +cbo: false, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── EvalScalar +│ ├── scalars: [lookup.k (#2) AS (#2), 1 AS (#3)] +│ └── Filter +│ ├── filters: [eq(lookup.k (#2), lookup.k (#2))] +│ └── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [eq(lookup.k (#2), lookup.k (#2))] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: is_true(5 (#5)) +=== correlated_exists === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s +cbo: true, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── EvalScalar +│ ├── scalars: [lookup.k (#2) AS (#2), 1 AS (#3)] +│ └── Filter +│ ├── filters: [eq(lookup.k (#2), lookup.k (#2))] +│ └── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [eq(lookup.k (#2), lookup.k (#2))] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: is_true(5 (#5)) +=== correlated_exists === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s +cbo: true, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── EvalScalar +│ ├── scalars: [lookup.k (#2) AS (#2), 1 AS (#3)] +│ └── Filter +│ ├── filters: [eq(lookup.k (#2), lookup.k (#2))] +│ └── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [eq(lookup.k (#2), lookup.k (#2))] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: is_true(5 (#5)) +=== scalar_aggregate === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightSingle) +├── build keys: [] +├── probe keys: [] +├── other filters: [] +├── Scan +│ ├── table: default.t (#0) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(lookup.k (#2)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(lookup.k (#2)) AS (#3)] + └── Scan + ├── table: default.lookup (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: t.k (#0) > scalar_subquery_3 (#3) +=== scalar_aggregate === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightSingle) +├── build keys: [] +├── probe keys: [] +├── other filters: [] +├── Scan +│ ├── table: default.t (#0) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(lookup.k (#2)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(lookup.k (#2)) AS (#3)] + └── Scan + ├── table: default.lookup (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: t.k (#0) > scalar_subquery_3 (#3) +=== scalar_aggregate === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightSingle) +├── build keys: [] +├── probe keys: [] +├── other filters: [] +├── Scan +│ ├── table: default.t (#0) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(lookup.k (#2)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(lookup.k (#2)) AS (#3)] + └── Scan + ├── table: default.lookup (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: t.k (#0) > scalar_subquery_3 (#3) +=== scalar_aggregate === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightSingle) +├── build keys: [] +├── probe keys: [] +├── other filters: [] +├── Scan +│ ├── table: default.t (#0) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(lookup.k (#2)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(lookup.k (#2)) AS (#3)] + └── Scan + ├── table: default.lookup (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: t.k (#0) > scalar_subquery_3 (#3) +=== multiple_first === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t +cbo: false, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightSingle) +├── build keys: [] +├── probe keys: [] +├── other filters: [] +├── Join(RightMark) +│ ├── build keys: [lookup.k (#2)] +│ ├── probe keys: [t.k (#0)] +│ ├── other filters: [] +│ ├── Scan +│ │ ├── table: default.lookup (#1) +│ │ ├── filters: [] +│ │ ├── order by: [] +│ │ └── limit: NONE +│ └── Scan +│ ├── table: default.t (#0) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [min(lookup.k (#3)) AS (#4)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [min(lookup.k (#3)) AS (#4)] + └── Scan + ├── table: default.lookup (#2) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: 6 (#6) +when_1: t.k (#0) > scalar_subquery_4 (#4) +=== multiple_first === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t +cbo: false, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightSingle) +├── build keys: [] +├── probe keys: [] +├── other filters: [] +├── Join(RightMark) +│ ├── build keys: [lookup.k (#2)] +│ ├── probe keys: [t.k (#0)] +│ ├── other filters: [] +│ ├── Scan +│ │ ├── table: default.lookup (#1) +│ │ ├── filters: [] +│ │ ├── order by: [] +│ │ └── limit: NONE +│ └── Scan +│ ├── table: default.t (#0) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [min(lookup.k (#3)) AS (#4)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [min(lookup.k (#3)) AS (#4)] + └── Scan + ├── table: default.lookup (#2) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: 6 (#6) +when_1: t.k (#0) > scalar_subquery_4 (#4) +=== multiple_first === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t +cbo: true, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightSingle) +├── build keys: [] +├── probe keys: [] +├── other filters: [] +├── Join(RightMark) +│ ├── build keys: [lookup.k (#2)] +│ ├── probe keys: [t.k (#0)] +│ ├── other filters: [] +│ ├── Scan +│ │ ├── table: default.lookup (#1) +│ │ ├── filters: [] +│ │ ├── order by: [] +│ │ └── limit: NONE +│ └── Scan +│ ├── table: default.t (#0) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [min(lookup.k (#3)) AS (#4)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [min(lookup.k (#3)) AS (#4)] + └── Scan + ├── table: default.lookup (#2) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: 6 (#6) +when_1: t.k (#0) > scalar_subquery_4 (#4) +=== multiple_first === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t +cbo: true, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightSingle) +├── build keys: [] +├── probe keys: [] +├── other filters: [] +├── Join(RightMark) +│ ├── build keys: [lookup.k (#2)] +│ ├── probe keys: [t.k (#0)] +│ ├── other filters: [] +│ ├── Scan +│ │ ├── table: default.lookup (#1) +│ │ ├── filters: [] +│ │ ├── order by: [] +│ │ └── limit: NONE +│ └── Scan +│ ├── table: default.t (#0) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [min(lookup.k (#3)) AS (#4)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [min(lookup.k (#3)) AS (#4)] + └── Scan + ├── table: default.lookup (#2) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: 6 (#6) +when_1: t.k (#0) > scalar_subquery_4 (#4) +=== nullable_not_in === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: NOT 4 (#4) +=== nullable_not_in === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: false, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: NOT 4 (#4) +=== nullable_not_in === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: false +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: NOT 4 (#4) +=== nullable_not_in === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t +cbo: true, distributed: true +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +when_0: NOT 4 (#4) +=== source_limit === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2 +cbo: false, distributed: false +raw_source: +Limit +├── limit: [2] +├── offset: [0] +└── Sort + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [NONE] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── TopN + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [2] + ├── offset: [0] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [t.k (#0) ASC] + └── limit: 2 + +when_0: 4 (#4) +=== source_limit === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2 +cbo: false, distributed: true +raw_source: +Limit +├── limit: [2] +├── offset: [0] +└── Sort + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [NONE] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── TopN + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [2] + ├── offset: [0] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [t.k (#0) ASC] + └── limit: 2 + +when_0: 4 (#4) +=== source_limit === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2 +cbo: true, distributed: false +raw_source: +Limit +├── limit: [2] +├── offset: [0] +└── Sort + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [NONE] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── TopN + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [2] + ├── offset: [0] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [t.k (#0) ASC] + └── limit: 2 + +when_0: 4 (#4) +=== source_limit === +description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. +sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2 +cbo: true, distributed: true +raw_source: +Limit +├── limit: [2] +├── offset: [0] +└── Sort + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [NONE] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [lookup.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.lookup (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── TopN + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [2] + ├── offset: [0] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [t.k (#0) ASC] + └── limit: 2 + +when_0: 4 (#4) diff --git a/src/query/sql/tests/it/optimizer/mod.rs b/src/query/sql/tests/it/optimizer/mod.rs index 43a2117ed77..bb67214e5f4 100644 --- a/src/query/sql/tests/it/optimizer/mod.rs +++ b/src/query/sql/tests/it/optimizer/mod.rs @@ -26,6 +26,7 @@ mod decorrelate_correlated_aliases; mod distributed_join; mod eager_aggregation; mod hierarchical_grouping_sets; +mod insert_multi_table; mod join_cardinality; mod materialized_cte_distribution; mod mutation; From 2c10f0e54b291a89460bc4a12bb466f0e2081117 Mon Sep 17 00:00:00 2001 From: coldWater Date: Thu, 1 Oct 2026 13:50:15 +0800 Subject: [PATCH 3/4] refactor(query): separate physical planning with a PExpr fork Keep logical preparation on SExpr and extract selected implementations as an independent PExpr tree. Migrate physical passes and recursive builders in place without changing their algorithms or execution order. Separate bound and planned query/mutation lifecycles, apply physical EvalScalar cleanup directly, and retain mutation distribution policy on the physical tree. Remove reverse expression conversion and obsolete logical-pipeline Memo capture. Include lifecycle, expression-fork, cleanup, mutation and plan-golden regression coverage with the final interfaces, without intermediate wrapper/pass designs. --- src/query/ee/src/materialized_view/refresh.rs | 5 +- .../interpreters/access/privilege_access.rs | 6 +- .../interpreter_copy_into_location.rs | 2 +- .../interpreter_copy_into_table.rs | 2 +- .../src/interpreters/interpreter_explain.rs | 48 +- .../src/interpreters/interpreter_factory.rs | 7 +- .../src/interpreters/interpreter_insert.rs | 4 +- .../interpreter_insert_multi_table.rs | 2 +- .../src/interpreters/interpreter_mutation.rs | 8 +- .../interpreter_optimize_compact_block.rs | 8 +- .../src/interpreters/interpreter_replace.rs | 2 +- .../src/interpreters/interpreter_select.rs | 10 +- .../src/interpreters/interpreter_set.rs | 2 +- .../interpreter_table_add_column.rs | 4 +- .../interpreters/interpreter_table_analyze.rs | 4 +- .../interpreter_table_modify_column.rs | 4 +- .../physical_aggregate_final.rs | 6 +- .../src/physical_plans/physical_async_func.rs | 4 +- .../physical_plans/physical_eval_scalar.rs | 10 +- .../src/physical_plans/physical_exchange.rs | 4 +- .../physical_expression_scan.rs | 4 +- .../src/physical_plans/physical_filter.rs | 4 +- .../src/physical_plans/physical_hash_join.rs | 6 +- .../src/physical_plans/physical_join.rs | 16 +- .../src/physical_plans/physical_limit.rs | 4 +- .../physical_materialized_cte.rs | 4 +- .../src/physical_plans/physical_mutation.rs | 4 +- .../physical_plans/physical_plan_builder.rs | 40 +- .../physical_plans/physical_project_set.rs | 4 +- .../src/physical_plans/physical_range_join.rs | 8 +- .../src/physical_plans/physical_sequence.rs | 4 +- .../src/physical_plans/physical_sort.rs | 4 +- .../physical_plans/physical_spatial_join.rs | 12 +- .../src/physical_plans/physical_top_n.rs | 4 +- .../src/physical_plans/physical_udf.rs | 4 +- .../src/physical_plans/physical_union_all.rs | 4 +- .../src/physical_plans/physical_window.rs | 6 +- .../physical_plans/runtime_filter/builder.rs | 10 +- src/query/service/src/test_kits/fuse.rs | 4 +- .../service/tests/it/parquet_rs/utils.rs | 4 +- src/query/service/tests/it/sql/exec/window.rs | 2 +- .../sql/planner/optimizer/optimizer_test.rs | 4 +- .../sql/planner/optimizer/statistics_trace.rs | 2 +- .../it/sql/planner/semantic/type_check.rs | 4 +- .../storages/fuse/operations/table_analyze.rs | 4 +- .../src/planner/binder/bind_mutation/bind.rs | 2 +- .../src/planner/binder/bind_mutation/mod.rs | 1 - .../bind_mutation/mutation_expression.rs | 18 - .../planner/binder/bind_mutation/update.rs | 3 +- .../bind_table_function.rs | 2 +- src/query/sql/src/planner/binder/binder.rs | 2 +- .../src/planner/binder/copy_into_location.rs | 2 +- .../sql/src/planner/binder/copy_into_table.rs | 2 +- .../planner/binder/ddl/materialized_view.rs | 2 +- .../src/planner/binder/insert_multi_table.rs | 2 +- src/query/sql/src/planner/binder/mod.rs | 1 - src/query/sql/src/planner/format/display.rs | 15 +- .../sql/src/planner/format/display_plan.rs | 25 +- src/query/sql/src/planner/lineage.rs | 5 +- .../ir/expr/column_scope_validator.rs | 19 +- .../planner/optimizer/ir/expr/conversion.rs | 129 +++ .../src/planner/optimizer/ir/expr/extract.rs | 6 +- .../sql/src/planner/optimizer/ir/expr/mod.rs | 20 +- .../sql/src/planner/optimizer/ir/expr/node.rs | 456 +++++++++ .../src/planner/optimizer/ir/expr/p_expr.rs | 177 ++++ .../src/planner/optimizer/ir/expr/s_expr.rs | 406 ++------ .../optimizer/ir/expr/type_validator.rs | 70 +- .../src/planner/optimizer/ir/expr/visitor.rs | 141 +-- .../sql/src/planner/optimizer/ir/memo.rs | 2 +- src/query/sql/src/planner/optimizer/ir/mod.rs | 15 + .../src/planner/optimizer/ir/mutation_plan.rs | 84 ++ .../planner/optimizer/ir/property/builder.rs | 35 +- .../planner/optimizer/ir/property/enforcer.rs | 26 +- .../src/planner/optimizer/ir/query_plan.rs | 108 ++ src/query/sql/src/planner/optimizer/mod.rs | 2 + .../sql/src/planner/optimizer/mutation.rs | 82 +- .../sql/src/planner/optimizer/optimizer.rs | 79 +- .../src/planner/optimizer/optimizer_api.rs | 14 +- .../optimizer/optimizers/cascades/cascade.rs | 37 +- .../optimizers/distributed/distributed.rs | 8 +- .../distributed/distributed_merge.rs | 10 +- .../distributed/materialized_cte.rs | 28 +- .../optimizers/distributed/sort_and_limit.rs | 24 +- .../operator/cte/cleanup_unused_cte.rs | 42 +- .../operator/eliminate_eval_scalar.rs | 162 +++ .../operator/join/finalize_spatial_join.rs | 14 +- .../optimizer/optimizers/operator/mod.rs | 3 + .../optimizer/optimizers/rule/factory.rs | 6 +- .../optimizers/rule/scalar_rules/mod.rs | 2 - .../rule_eliminate_eval_scalar.rs | 116 --- .../src/planner/optimizer/physical_planner.rs | 207 ++++ .../sql/src/planner/optimizer/pipeline/mod.rs | 1 + .../planner/optimizer/pipeline/pipeline.rs | 76 +- .../optimizer/pipeline/trace/expr_diff.rs | 7 +- .../planner/optimizer/pipeline/trace/trace.rs | 13 +- src/query/sql/src/planner/planner.rs | 2 +- src/query/sql/src/planner/plans/plan.rs | 13 +- .../sql/test-support/src/optimizer/mod.rs | 9 +- .../tests/it/optimizer/collect_statistics.rs | 10 +- .../sql/tests/it/optimizer/column_scope.rs | 17 +- .../tests/it/optimizer/eager_aggregation.rs | 12 +- .../it/optimizer/eliminate_eval_scalar.rs | 134 +++ .../it/optimizer/eliminate_eval_scalar.txt | 130 +++ .../optimizer/hierarchical_grouping_sets.rs | 17 +- .../tests/it/optimizer/insert_multi_table.rs | 30 +- .../tests/it/optimizer/insert_multi_table.txt | 322 +----- .../it/optimizer/join_cardinality/common.rs | 6 +- .../optimizer/join_cardinality/histogram.rs | 9 +- .../join_cardinality/histogram/decimal.rs | 8 +- .../join_cardinality/histogram/single.rs | 50 +- .../optimizer/join_cardinality/statistics.rs | 9 +- src/query/sql/tests/it/optimizer/mod.rs | 2 + src/query/sql/tests/it/optimizer/mutation.rs | 46 +- src/query/sql/tests/it/optimizer/mutation.txt | 920 ++++++------------ .../tests/it/optimizer/physical_planner.rs | 341 +++++++ .../tests/it/optimizer/physical_planner.txt | 170 ++++ .../it/optimizer/physical_planner_nested.txt | 77 ++ .../sql/tests/it/optimizer/stat_derivation.rs | 12 +- src/query/sql/tests/it/optimizer/union_all.rs | 31 +- src/query/sql/tests/it/planner.rs | 6 +- src/query/sql/tests/it/semantic/binder.rs | 8 +- src/tests/planner_replay/src/main.rs | 6 +- 122 files changed, 3343 insertions(+), 2071 deletions(-) create mode 100644 src/query/sql/src/planner/optimizer/ir/expr/conversion.rs create mode 100644 src/query/sql/src/planner/optimizer/ir/expr/node.rs create mode 100644 src/query/sql/src/planner/optimizer/ir/expr/p_expr.rs create mode 100644 src/query/sql/src/planner/optimizer/ir/mutation_plan.rs create mode 100644 src/query/sql/src/planner/optimizer/ir/query_plan.rs create mode 100644 src/query/sql/src/planner/optimizer/optimizers/operator/eliminate_eval_scalar.rs delete mode 100644 src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs create mode 100644 src/query/sql/src/planner/optimizer/physical_planner.rs create mode 100644 src/query/sql/tests/it/optimizer/eliminate_eval_scalar.rs create mode 100644 src/query/sql/tests/it/optimizer/eliminate_eval_scalar.txt create mode 100644 src/query/sql/tests/it/optimizer/physical_planner.rs create mode 100644 src/query/sql/tests/it/optimizer/physical_planner.txt create mode 100644 src/query/sql/tests/it/optimizer/physical_planner_nested.txt diff --git a/src/query/ee/src/materialized_view/refresh.rs b/src/query/ee/src/materialized_view/refresh.rs index 07befe21631..abc42842edd 100644 --- a/src/query/ee/src/materialized_view/refresh.rs +++ b/src/query/ee/src/materialized_view/refresh.rs @@ -49,7 +49,6 @@ use databend_common_meta_app::schema::UpsertTableOptionReq; use databend_common_sql::MaterializedViewChecker; use databend_common_sql::Planner; use databend_common_sql::parse_materialized_view_query; -use databend_common_sql::plans::Mutation; use databend_common_sql::plans::Plan; use databend_common_sql::validate_materialized_view_source; use databend_common_storages_fuse::FuseTable; @@ -366,10 +365,10 @@ impl<'a> MaterializedViewRefresh<'a> { self.mv_table.get_id(), )?, Plan::DataMutation { s_expr, schema, .. } => { - let mutation: Mutation = s_expr.plan().clone().try_into()?; + let mutation = s_expr.mutation()?.clone(); Arc::new(MutationInterpreter::try_create_materialized_view_refresh( self.ctx.clone(), - *s_expr, + (*s_expr).into_planned()?, schema, mutation.metadata, self.mv_table.get_id(), diff --git a/src/query/service/src/interpreters/access/privilege_access.rs b/src/query/service/src/interpreters/access/privilege_access.rs index d92597e56c7..18a228de0ff 100644 --- a/src/query/service/src/interpreters/access/privilege_access.rs +++ b/src/query/service/src/interpreters/access/privilege_access.rs @@ -49,7 +49,6 @@ use databend_common_sql::plans::AlterSharePlanAction; use databend_common_sql::plans::InsertInputSource; use databend_common_sql::plans::MaintenanceTarget; use databend_common_sql::plans::ModifyColumnAction; -use databend_common_sql::plans::Mutation; use databend_common_sql::plans::OptimizeCompactBlock; use databend_common_sql::plans::PresignAction; use databend_common_sql::plans::RewriteKind; @@ -2245,10 +2244,9 @@ impl AccessChecker for PrivilegeAccess { self.validate_insert_source(ctx, &plan.source).await?; } Plan::DataMutation { s_expr, .. } => { - let plan: Mutation = s_expr.plan().clone().try_into()?; + let plan = s_expr.mutation()?; if enable_experimental_rbac_check { - let s_expr = s_expr.child(0)?; - match s_expr.get_udfs() { + match s_expr.input_udfs() { Ok(udfs) => { if !udfs.is_empty() { self.validate_udf_access(udfs).await?; diff --git a/src/query/service/src/interpreters/interpreter_copy_into_location.rs b/src/query/service/src/interpreters/interpreter_copy_into_location.rs index 2327fe65e3a..11fd3433501 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_location.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_location.rs @@ -67,7 +67,7 @@ impl CopyIntoLocationInterpreter { let select_interpreter = SelectInterpreter::try_create( self.ctx.clone(), *(bind_context.clone()), - *s_expr.clone(), + s_expr.planned()?.clone(), metadata.clone(), formatted_ast.clone(), false, diff --git a/src/query/service/src/interpreters/interpreter_copy_into_table.rs b/src/query/service/src/interpreters/interpreter_copy_into_table.rs index a75e338cd7b..be90f70cc6b 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_table.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_table.rs @@ -221,7 +221,7 @@ impl CopyIntoTableInterpreter { let select_interpreter = SelectInterpreter::try_create( self.ctx.clone(), *(bind_context.clone()), - *s_expr.clone(), + s_expr.planned()?.clone(), metadata.clone(), formatted_ast.clone(), false, diff --git a/src/query/service/src/interpreters/interpreter_explain.rs b/src/query/service/src/interpreters/interpreter_explain.rs index 3fbf46a6ec5..c989695fd04 100644 --- a/src/query/service/src/interpreters/interpreter_explain.rs +++ b/src/query/service/src/interpreters/interpreter_explain.rs @@ -36,6 +36,8 @@ use databend_common_sql::ColumnSet; use databend_common_sql::FormatOptions; use databend_common_sql::MetadataRef; use databend_common_sql::binder::ExplainConfig; +use databend_common_sql::optimizer::ir::PExpr; +use databend_common_sql::optimizer::ir::PlannedQuery; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::plans::Mutation; use databend_common_storages_basic::ResultCacheReader; @@ -71,7 +73,6 @@ use crate::sessions::TableContextQueryIdentity; use crate::sessions::TableContextQueryProfile; use crate::sessions::TableContextRuntimeFilter; use crate::sessions::TableContextSettings; -use crate::sql::optimizer::ir::SExpr; use crate::sql::plans::Plan; pub struct ExplainInterpreter { @@ -116,7 +117,7 @@ impl Interpreter for ExplainInterpreter { formatted_ast, .. } => { - self.explain_query(s_expr, metadata, bind_context, formatted_ast) + self.explain_query(s_expr.planned()?, metadata, bind_context, formatted_ast) .await? } Plan::Insert(insert_plan) => { @@ -142,8 +143,13 @@ impl Interpreter for ExplainInterpreter { vec!["CreateTableAsSelect:", ""], )])]; res.extend( - self.explain_query(s_expr, metadata, bind_context, formatted_ast) - .await?, + self.explain_query( + s_expr.planned()?, + metadata, + bind_context, + formatted_ast, + ) + .await?, ); vec![DataBlock::concat(&res)?] } @@ -164,10 +170,10 @@ impl Interpreter for ExplainInterpreter { schema, metadata, } => { - let mutation: Mutation = s_expr.plan().clone().try_into()?; + let mutation: Mutation = s_expr.mutation()?.clone(); let interpreter = MutationInterpreter::try_create( self.ctx.clone(), - *s_expr.clone(), + s_expr.planned()?.clone(), schema.clone(), metadata.clone(), )?; @@ -187,7 +193,9 @@ impl Interpreter for ExplainInterpreter { } => { let ctx = self.ctx.clone(); let mut builder = PhysicalPlanBuilder::new(metadata.clone(), ctx, true); - let plan = builder.build(s_expr, bind_context.column_set()).await?; + let plan = builder + .build_query(s_expr.planned()?, bind_context.column_set()) + .await?; let metadata = metadata.read(); let mut context = FormatContext { @@ -218,7 +226,7 @@ impl Interpreter for ExplainInterpreter { .. } => { self.explain_analyze( - s_expr, + s_expr.planned()?.expr(), metadata, bind_context.column_set(), None, @@ -227,11 +235,11 @@ impl Interpreter for ExplainInterpreter { .await? } Plan::DataMutation { s_expr, .. } => { - let plan: Mutation = s_expr.plan().clone().try_into()?; + let plan: Mutation = s_expr.mutation()?.clone(); let mutation_build_info = build_mutation_info(self.ctx.clone(), &plan, true, None).await?; self.explain_analyze( - s_expr.child(0)?, + s_expr.planned()?.child(0)?, &plan.metadata, *plan.required_columns.clone(), Some(mutation_build_info), @@ -289,14 +297,14 @@ impl Interpreter for ExplainInterpreter { .. } => { self.explain_fragments( - *s_expr.clone(), + s_expr.planned()?.clone(), metadata.clone(), bind_context.column_set(), ) .await? } Plan::DataMutation { s_expr, schema, .. } => { - self.explain_merge_fragments(*s_expr.clone(), schema.clone()) + self.explain_merge_fragments(s_expr.planned()?.clone(), schema.clone()) .await? } Plan::InsertMultiTable(plan) => { @@ -449,13 +457,13 @@ impl ExplainInterpreter { #[async_backtrace::framed] async fn explain_fragments( &self, - s_expr: SExpr, + query: PlannedQuery, metadata: MetadataRef, required: ColumnSet, ) -> Result> { let ctx = self.ctx.clone(); let plan = PhysicalPlanBuilder::new(metadata.clone(), self.ctx.clone(), true) - .build(&s_expr, required) + .build_query(&query, required) .await?; let fragments = Fragmenter::try_create(ctx.clone())?.build_fragment(&plan)?; @@ -502,7 +510,7 @@ impl ExplainInterpreter { #[async_backtrace::framed] async fn explain_analyze_graphical( &self, - s_expr: &SExpr, + s_expr: &PExpr, metadata: &MetadataRef, required: ColumnSet, ignore_result: bool, @@ -525,7 +533,7 @@ impl ExplainInterpreter { #[async_backtrace::framed] async fn explain_analyze( &self, - s_expr: &SExpr, + s_expr: &PExpr, metadata: &MetadataRef, required: ColumnSet, mutation_build_info: Option, @@ -625,7 +633,7 @@ impl ExplainInterpreter { async fn explain_query( &self, - s_expr: &SExpr, + query: &PlannedQuery, metadata: &MetadataRef, bind_context: &BindContext, formatted_ast: &Option, @@ -636,7 +644,9 @@ impl ExplainInterpreter { // we should not use `dry_run` mode to build the physical plan. // It's because we need to get the same partitions as the original selecting plan. let mut builder = PhysicalPlanBuilder::new(metadata.clone(), ctx, formatted_ast.is_none()); - let mut plan = builder.build(s_expr, bind_context.column_set()).await?; + let mut plan = builder + .build_query(query, bind_context.column_set()) + .await?; self.inject_pruned_partitions_stats(&mut plan, metadata)?; self.explain_physical_plan(&plan, metadata, formatted_ast) .await @@ -644,7 +654,7 @@ impl ExplainInterpreter { async fn explain_merge_fragments( &self, - s_expr: SExpr, + s_expr: PExpr, schema: DataSchemaRef, ) -> Result> { let mutation: Mutation = s_expr.plan().clone().try_into()?; diff --git a/src/query/service/src/interpreters/interpreter_factory.rs b/src/query/service/src/interpreters/interpreter_factory.rs index 7bea708f9f3..18812bfbe86 100644 --- a/src/query/service/src/interpreters/interpreter_factory.rs +++ b/src/query/service/src/interpreters/interpreter_factory.rs @@ -20,7 +20,6 @@ use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::binder::ExplainConfig; -use databend_common_sql::plans::Mutation; use log::error; use log::warn; @@ -282,7 +281,7 @@ impl InterpreterFactory { } => Ok(Arc::new(SelectInterpreter::try_create( ctx, *bind_context.clone(), - *s_expr.clone(), + s_expr.planned()?.clone(), metadata.clone(), formatted_ast.clone(), *ignore_result, @@ -654,10 +653,10 @@ impl InterpreterFactory { Plan::Replace(replace) => ReplaceInterpreter::try_create(ctx, *replace.clone()), Plan::DataMutation { s_expr, schema, .. } => { - let mutation: Mutation = s_expr.plan().clone().try_into()?; + let mutation = s_expr.mutation()?; Ok(Arc::new(MutationInterpreter::try_create( ctx, - *s_expr.clone(), + s_expr.planned()?.clone(), schema.clone(), mutation.metadata.clone(), )?)) diff --git a/src/query/service/src/interpreters/interpreter_insert.rs b/src/query/service/src/interpreters/interpreter_insert.rs index 669fc318a81..f96f1ae3b41 100644 --- a/src/query/service/src/interpreters/interpreter_insert.rs +++ b/src/query/service/src/interpreters/interpreter_insert.rs @@ -572,7 +572,9 @@ impl Interpreter for InsertInterpreter { let mut builder1 = PhysicalPlanBuilder::new(metadata.clone(), self.ctx.clone(), false); ( - builder1.build(s_expr, bind_context.column_set()).await?, + builder1 + .build_query(s_expr.planned()?, bind_context.column_set()) + .await?, bind_context.columns.clone(), metadata, ) diff --git a/src/query/service/src/interpreters/interpreter_insert_multi_table.rs b/src/query/service/src/interpreters/interpreter_insert_multi_table.rs index bc676f8f38b..076440bc671 100644 --- a/src/query/service/src/interpreters/interpreter_insert_multi_table.rs +++ b/src/query/service/src/interpreters/interpreter_insert_multi_table.rs @@ -325,7 +325,7 @@ impl InsertMultiTableInterpreter { ordered_source_projection_columns(&result_columns, &source_required); let mut required = bind_context.column_set(); required.extend(source_required); - let input_source = builder1.build(s_expr, required).await?; + let input_source = builder1.build_query(s_expr.planned()?, required).await?; // Lazy materialization (triggered by WHERE + LIMIT) may reorder physical output // columns and inject internal columns like _row_id. Add a reorder EvalScalar diff --git a/src/query/service/src/interpreters/interpreter_mutation.rs b/src/query/service/src/interpreters/interpreter_mutation.rs index dc52b10d13d..1daa7b0661b 100644 --- a/src/query/service/src/interpreters/interpreter_mutation.rs +++ b/src/query/service/src/interpreters/interpreter_mutation.rs @@ -31,7 +31,7 @@ use databend_common_pipeline::sinks::EmptySink; use databend_common_sql::binder::MutationStrategy; use databend_common_sql::binder::MutationType; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::planner::MetadataRef; use databend_common_sql::plans; use databend_common_sql::plans::Mutation; @@ -60,7 +60,7 @@ use crate::stream::DataBlockStream; pub struct MutationInterpreter { ctx: Arc, - s_expr: SExpr, + s_expr: PExpr, schema: DataSchemaRef, metadata: MetadataRef, materialized_view_refresh_target: Option, @@ -69,7 +69,7 @@ pub struct MutationInterpreter { impl MutationInterpreter { pub fn try_create( ctx: Arc, - s_expr: SExpr, + s_expr: PExpr, schema: DataSchemaRef, metadata: MetadataRef, ) -> Result { @@ -84,7 +84,7 @@ impl MutationInterpreter { pub fn try_create_materialized_view_refresh( ctx: Arc, - s_expr: SExpr, + s_expr: PExpr, schema: DataSchemaRef, metadata: MetadataRef, target_table_id: u64, diff --git a/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs b/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs index 79c86c5f438..7911de05a9f 100644 --- a/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs +++ b/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs @@ -79,7 +79,13 @@ impl Interpreter for OptimizeCompactBlockInterpreter { let mut build_res = PipelineBuildResult::create(); let mut builder = PhysicalPlanBuilder::new(MetadataRef::default(), self.ctx.clone(), false); - match builder.build(&self.s_expr, ColumnSet::new()).await { + match builder + .build( + &databend_common_sql::optimizer::ir::PExpr::from(self.s_expr.clone()), + ColumnSet::new(), + ) + .await + { Ok(physical_plan) => { build_res = build_query_pipeline_without_render_result_set(&self.ctx, &physical_plan) diff --git a/src/query/service/src/interpreters/interpreter_replace.rs b/src/query/service/src/interpreters/interpreter_replace.rs index 8c61f2f79a2..2babf6f24f1 100644 --- a/src/query/service/src/interpreters/interpreter_replace.rs +++ b/src/query/service/src/interpreters/interpreter_replace.rs @@ -486,7 +486,7 @@ impl ReplaceInterpreter { let select_interpreter = SelectInterpreter::try_create( ctx.clone(), *(bind_context.clone()), - *s_expr.clone(), + s_expr.planned()?.clone(), metadata.clone(), formatted_ast.clone(), false, diff --git a/src/query/service/src/interpreters/interpreter_select.rs b/src/query/service/src/interpreters/interpreter_select.rs index 5805b7bd69f..0e30bbfbb97 100644 --- a/src/query/service/src/interpreters/interpreter_select.rs +++ b/src/query/service/src/interpreters/interpreter_select.rs @@ -62,12 +62,12 @@ use crate::sessions::TableContextSettings; use crate::sessions::TableContextTableAccess; use crate::sessions::TableContextTelemetry; use crate::sql::BindContext; -use crate::sql::optimizer::ir::SExpr; +use crate::sql::optimizer::ir::PlannedQuery; /// Interpret SQL query with new SQL planner pub struct SelectInterpreter { ctx: Arc, - s_expr: SExpr, + query: PlannedQuery, bind_context: BindContext, metadata: MetadataRef, formatted_ast: Option, @@ -78,14 +78,14 @@ impl SelectInterpreter { pub fn try_create( ctx: Arc, bind_context: BindContext, - s_expr: SExpr, + query: PlannedQuery, metadata: MetadataRef, formatted_ast: Option, ignore_result: bool, ) -> Result { Ok(SelectInterpreter { ctx, - s_expr, + query, bind_context, metadata, formatted_ast, @@ -128,7 +128,7 @@ impl SelectInterpreter { let mut builder = PhysicalPlanBuilder::new(self.metadata.clone(), self.ctx.clone(), false); self.ctx.set_status_info("Building physical plan"); builder - .build(&self.s_expr, self.bind_context.column_set()) + .build_query(&self.query, self.bind_context.column_set()) .await } diff --git a/src/query/service/src/interpreters/interpreter_set.rs b/src/query/service/src/interpreters/interpreter_set.rs index d7e8d7b480e..04fbd9c5924 100644 --- a/src/query/service/src/interpreters/interpreter_set.rs +++ b/src/query/service/src/interpreters/interpreter_set.rs @@ -200,7 +200,7 @@ impl Interpreter for SetInterpreter { let select_interpreter = SelectInterpreter::try_create( self.ctx.clone(), *(bind_context.clone()), - *s_expr.clone(), + s_expr.planned()?.clone(), metadata.clone(), formatted_ast.clone(), false, diff --git a/src/query/service/src/interpreters/interpreter_table_add_column.rs b/src/query/service/src/interpreters/interpreter_table_add_column.rs index 131eee762c2..1e762a691c0 100644 --- a/src/query/service/src/interpreters/interpreter_table_add_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_add_column.rs @@ -237,10 +237,10 @@ impl Interpreter for AddTableColumnInterpreter { let mut planner = Planner::new(self.ctx.clone()); let (plan, _) = planner.plan_sql(&query).await?; if let Plan::DataMutation { s_expr, schema, .. } = plan { - let mutation: Mutation = s_expr.plan().clone().try_into()?; + let mutation: Mutation = s_expr.mutation()?.clone(); let interpreter = MutationInterpreter::try_create( self.ctx.clone(), - *s_expr, + (*s_expr).into_planned()?, schema, mutation.metadata.clone(), )?; diff --git a/src/query/service/src/interpreters/interpreter_table_analyze.rs b/src/query/service/src/interpreters/interpreter_table_analyze.rs index 9df77457284..92f4f4c4b54 100644 --- a/src/query/service/src/interpreters/interpreter_table_analyze.rs +++ b/src/query/service/src/interpreters/interpreter_table_analyze.rs @@ -137,7 +137,9 @@ impl AnalyzeTableInterpreter { let mut builder = PhysicalPlanBuilder::new(metadata.clone(), self.ctx.clone(), false); ( - builder.build(s_expr, bind_context.column_set()).await?, + builder + .build_query(s_expr.planned()?, bind_context.column_set()) + .await?, (**bind_context).clone(), ) } diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index d3e349c177b..e69ea0266d6 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -1332,7 +1332,9 @@ pub async fn build_modify_column_physical_plan( } => { let mut builder = PhysicalPlanBuilder::new(metadata.clone(), ctx.clone(), false); ( - builder.build(&s_expr, bind_context.column_set()).await?, + builder + .build_query(s_expr.planned()?, bind_context.column_set()) + .await?, bind_context.columns.clone(), ) } diff --git a/src/query/service/src/physical_plans/physical_aggregate_final.rs b/src/query/service/src/physical_plans/physical_aggregate_final.rs index 8e4afdaf07a..5781719c3d2 100644 --- a/src/query/service/src/physical_plans/physical_aggregate_final.rs +++ b/src/query/service/src/physical_plans/physical_aggregate_final.rs @@ -29,7 +29,7 @@ use databend_common_sql::Symbol; use databend_common_sql::executor::physical_plans::AggregateFunctionDesc; use databend_common_sql::executor::physical_plans::AggregateFunctionSignature; use databend_common_sql::executor::physical_plans::SortDesc; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::Aggregate; use databend_common_sql::plans::AggregateMode; use databend_common_sql::plans::ConstantTableScan; @@ -210,7 +210,7 @@ impl IPhysicalPlan for AggregateFinal { impl PhysicalPlanBuilder { pub async fn build_aggregate( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, agg: &Aggregate, mut required: ColumnSet, stat_info: PlanStatsInfo, @@ -235,7 +235,7 @@ impl PhysicalPlanBuilder { let mut s = ConstantTableScan::new_empty_scan(DataSchemaRef::default(), ColumnSet::new()); s.num_rows = 1; - let expr = SExpr::create_leaf(s); + let expr = PExpr::create_leaf(s); return self.build(&expr, required).await; } diff --git a/src/query/service/src/physical_plans/physical_async_func.rs b/src/query/service/src/physical_plans/physical_async_func.rs index 046843f3015..77bafe227c9 100644 --- a/src/query/service/src/physical_plans/physical_async_func.rs +++ b/src/query/service/src/physical_plans/physical_async_func.rs @@ -24,7 +24,7 @@ use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_sql::ColumnSet; use databend_common_sql::ScalarExpr; use databend_common_sql::binder::AsyncFunctionDesc; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use itertools::Itertools; use crate::physical_plans::explain::PlanStatsInfo; @@ -130,7 +130,7 @@ impl IPhysicalPlan for AsyncFunction { impl PhysicalPlanBuilder { pub async fn build_async_func( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, async_func_plan: &databend_common_sql::plans::AsyncFunction, mut required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_eval_scalar.rs b/src/query/service/src/physical_plans/physical_eval_scalar.rs index fc6fd347067..6074329c990 100644 --- a/src/query/service/src/physical_plans/physical_eval_scalar.rs +++ b/src/query/service/src/physical_plans/physical_eval_scalar.rs @@ -35,7 +35,7 @@ use databend_common_sql::Symbol; use databend_common_sql::TypeCheck; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::optimizer::ir::Matcher; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::BoundColumnRef; use databend_common_sql::plans::Filter; use databend_common_sql::plans::FunctionCall; @@ -209,7 +209,7 @@ impl IPhysicalPlan for EvalScalar { impl PhysicalPlanBuilder { pub async fn build_eval_scalar( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, eval_scalar: &databend_common_sql::plans::EvalScalar, mut required: ColumnSet, stat_info: PlanStatsInfo, @@ -305,8 +305,8 @@ impl PhysicalPlanBuilder { fn try_eliminate_flatten_columns( &mut self, scalar_items: &Vec, - s_expr: &SExpr, - ) -> Result> { + s_expr: &PExpr, + ) -> Result> { // (1) ProjectSet // \ // * @@ -352,7 +352,7 @@ impl PhysicalPlanBuilder { let mut new_child = child.clone(); new_child.plan = Arc::new(new_project_set.into()); let new_filter = - SExpr::create_unary(Arc::new(s_expr.plan().clone()), Arc::new(new_child)); + PExpr::create_unary(Arc::new(s_expr.plan().clone()), Arc::new(new_child)); Ok(Some(new_filter)) } else { let project_set: ProjectSet = s_expr.plan().clone().try_into()?; diff --git a/src/query/service/src/physical_plans/physical_exchange.rs b/src/query/service/src/physical_plans/physical_exchange.rs index a906b3fd62a..69b327c5830 100644 --- a/src/query/service/src/physical_plans/physical_exchange.rs +++ b/src/query/service/src/physical_plans/physical_exchange.rs @@ -23,7 +23,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::ColumnSet; use databend_common_sql::TypeCheck; use databend_common_sql::executor::physical_plans::FragmentKind; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::format::ExchangeFormatter; @@ -96,7 +96,7 @@ impl IPhysicalPlan for Exchange { impl PhysicalPlanBuilder { pub async fn build_exchange( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, exchange: &databend_common_sql::plans::Exchange, mut required: ColumnSet, ) -> Result { diff --git a/src/query/service/src/physical_plans/physical_expression_scan.rs b/src/query/service/src/physical_plans/physical_expression_scan.rs index 26f2603ca70..0100f25d199 100644 --- a/src/query/service/src/physical_plans/physical_expression_scan.rs +++ b/src/query/service/src/physical_plans/physical_expression_scan.rs @@ -23,7 +23,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::ProcessorPtr; use databend_common_sql::ColumnSet; use databend_common_sql::TypeCheck; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::format::ExpressionScanFormatter; @@ -114,7 +114,7 @@ impl IPhysicalPlan for ExpressionScan { impl PhysicalPlanBuilder { pub async fn build_expression_scan( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, scan: &databend_common_sql::plans::ExpressionScan, required: ColumnSet, ) -> Result { diff --git a/src/query/service/src/physical_plans/physical_filter.rs b/src/query/service/src/physical_plans/physical_filter.rs index 75aff4c1065..63b0f5b70b5 100644 --- a/src/query/service/src/physical_plans/physical_filter.rs +++ b/src/query/service/src/physical_plans/physical_filter.rs @@ -27,7 +27,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::ColumnSet; use databend_common_sql::TypeCheck; use databend_common_sql::executor::cast_expr_to_non_null_boolean; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; @@ -147,7 +147,7 @@ impl IPhysicalPlan for Filter { impl PhysicalPlanBuilder { pub async fn build_filter( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, filter: &databend_common_sql::plans::Filter, mut required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_hash_join.rs b/src/query/service/src/physical_plans/physical_hash_join.rs index 15629eef09a..f41417a4132 100644 --- a/src/query/service/src/physical_plans/physical_hash_join.rs +++ b/src/query/service/src/physical_plans/physical_hash_join.rs @@ -43,7 +43,7 @@ use databend_common_sql::IndexType; use databend_common_sql::ScalarExpr; use databend_common_sql::Symbol; use databend_common_sql::TypeCheck; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::Join; use databend_common_sql::plans::JoinType; use tokio::sync::Barrier; @@ -540,7 +540,7 @@ impl PhysicalPlanBuilder { /// left-child then right-child order. pub async fn build_join_sides( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, left_required: ColumnSet, right_required: ColumnSet, ) -> Result<(PhysicalPlan, PhysicalPlan)> { @@ -1283,7 +1283,7 @@ impl PhysicalPlanBuilder { pub async fn build_hash_join( &mut self, join: &Join, - s_expr: &SExpr, + s_expr: &PExpr, mut required: ColumnSet, mut others_required: ColumnSet, left_required: ColumnSet, diff --git a/src/query/service/src/physical_plans/physical_join.rs b/src/query/service/src/physical_plans/physical_join.rs index 100bf77f269..f1cbcf17c39 100644 --- a/src/query/service/src/physical_plans/physical_join.rs +++ b/src/query/service/src/physical_plans/physical_join.rs @@ -18,8 +18,8 @@ use databend_common_expression::types::DataType; use databend_common_sql::ColumnSet; use databend_common_sql::ScalarExpr; use databend_common_sql::binder::is_range_join_condition; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::RelExpr; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::plans::FunctionCall; use databend_common_sql::plans::Join; @@ -39,17 +39,17 @@ fn is_precise_single_row(stat_info: &databend_common_sql::optimizer::ir::StatInf fn single_join_scalar_side_is_precise_single_row( join: &Join, - s_expr: &SExpr, + s_expr: &PExpr, stat_context: &StatContext, ) -> Result { match join.single_to_inner { Some(JoinType::LeftSingle) => { - let right_rel_expr = RelExpr::with_s_expr(s_expr.right_child()); + let right_rel_expr = RelExpr::with_p_expr(s_expr.right_child()); let right_stat_info = right_rel_expr.derive_cardinality(stat_context)?; Ok(is_precise_single_row(&right_stat_info)) } Some(JoinType::RightSingle) => { - let left_rel_expr = RelExpr::with_s_expr(s_expr.left_child()); + let left_rel_expr = RelExpr::with_p_expr(s_expr.left_child()); let left_stat_info = left_rel_expr.derive_cardinality(stat_context)?; Ok(is_precise_single_row(&left_stat_info)) } @@ -86,7 +86,7 @@ fn asof_hash_join_type(join_type: JoinType) -> JoinType { // Choose physical join type by join conditions fn physical_join( join: &Join, - s_expr: &SExpr, + s_expr: &PExpr, stat_context: &StatContext, ) -> Result { if join.equi_conditions.is_empty() && join.join_type.is_any_join() { @@ -95,8 +95,8 @@ fn physical_join( )); } - let left_rel_expr = RelExpr::with_s_expr(s_expr.left_child()); - let right_rel_expr = RelExpr::with_s_expr(s_expr.right_child()); + let left_rel_expr = RelExpr::with_p_expr(s_expr.left_child()); + let right_rel_expr = RelExpr::with_p_expr(s_expr.right_child()); let left_stat_info = left_rel_expr.derive_cardinality(stat_context)?; let right_stat_info = right_rel_expr.derive_cardinality(stat_context)?; @@ -140,7 +140,7 @@ fn physical_join( impl PhysicalPlanBuilder { pub async fn build_join( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, join: &databend_common_sql::plans::Join, required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_limit.rs b/src/query/service/src/physical_plans/physical_limit.rs index 0211afdea0a..e6722d4b026 100644 --- a/src/query/service/src/physical_plans/physical_limit.rs +++ b/src/query/service/src/physical_plans/physical_limit.rs @@ -26,7 +26,7 @@ use databend_common_sql::ColumnEntry; use databend_common_sql::ColumnSet; use databend_common_sql::IndexType; use databend_common_sql::Symbol; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; @@ -137,7 +137,7 @@ impl IPhysicalPlan for Limit { impl PhysicalPlanBuilder { pub async fn build_limit( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, limit: &databend_common_sql::plans::Limit, mut required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_materialized_cte.rs b/src/query/service/src/physical_plans/physical_materialized_cte.rs index d6052adeb7f..d63a7d111b5 100644 --- a/src/query/service/src/physical_plans/physical_materialized_cte.rs +++ b/src/query/service/src/physical_plans/physical_materialized_cte.rs @@ -20,7 +20,7 @@ use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_sql::Symbol; use databend_common_sql::evaluator::BlockOperator; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlan; @@ -140,7 +140,7 @@ impl IPhysicalPlan for MaterializedCTE { impl PhysicalPlanBuilder { pub(crate) async fn build_materialized_cte( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, materialized_cte: &databend_common_sql::plans::MaterializedCTE, stat_info: PlanStatsInfo, ) -> Result { diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index 588328b5f84..350ce4c4d0e 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -60,7 +60,7 @@ use databend_common_sql::binder::MutationType; use databend_common_sql::binder::wrap_cast; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::parse_computed_field_index_expr; use databend_common_sql::plans::BoundColumnRef; use databend_common_sql::plans::ConstantExpr; @@ -271,7 +271,7 @@ fn add_matched_mutation_workers( impl PhysicalPlanBuilder { pub async fn build_mutation( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, mutation: &databend_common_sql::plans::Mutation, mut required: ColumnSet, ) -> Result { diff --git a/src/query/service/src/physical_plans/physical_plan_builder.rs b/src/query/service/src/physical_plans/physical_plan_builder.rs index 5063ab527fe..3f36e1fa6b6 100644 --- a/src/query/service/src/physical_plans/physical_plan_builder.rs +++ b/src/query/service/src/physical_plans/physical_plan_builder.rs @@ -25,8 +25,9 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_common_sql::ColumnSet; use databend_common_sql::MetadataRef; +use databend_common_sql::optimizer::ir::PExpr; +use databend_common_sql::optimizer::ir::PlannedQuery; use databend_common_sql::optimizer::ir::RelExpr; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::plans::RelOperator; use databend_storages_common_table_meta::meta::TableMetaTimestamps; @@ -68,8 +69,8 @@ impl PhysicalPlanBuilder { } } - pub fn build_plan_stat_info(&self, s_expr: &SExpr) -> Result { - let rel_expr = RelExpr::with_s_expr(s_expr); + pub fn build_plan_stat_info(&self, s_expr: &PExpr) -> Result { + let rel_expr = RelExpr::with_p_expr(s_expr); let stat_context = StatContext::new(self.func_ctx.clone()); let stat_info = rel_expr.derive_cardinality(&stat_context)?; @@ -78,7 +79,16 @@ impl PhysicalPlanBuilder { }) } - pub async fn build(&mut self, s_expr: &SExpr, required: ColumnSet) -> Result { + /// Build a planned query; recursive operator builders consume its physical expression. + pub async fn build_query( + &mut self, + query: &PlannedQuery, + required: ColumnSet, + ) -> Result { + self.build(query.expr(), required).await + } + + pub async fn build(&mut self, s_expr: &PExpr, required: ColumnSet) -> Result { let is_root_build = self.build_depth == 0; if is_root_build { self.ctx.clear_pruned_partitions_stats(); @@ -121,7 +131,7 @@ impl PhysicalPlanBuilder { #[async_recursion::async_recursion(#[recursive::recursive])] pub async fn build_physical_plan( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, required: ColumnSet, ) -> Result { // Build stat info. @@ -209,7 +219,7 @@ impl PhysicalPlanBuilder { pub(crate) fn derive_children_required_columns( &self, - s_expr: &SExpr, + s_expr: &PExpr, parent_required: &ColumnSet, ) -> Result> { let arity = s_expr.arity(); @@ -406,14 +416,14 @@ impl PhysicalPlanBuilder { } #[recursive::recursive] - fn requires_cte_column_collection(s_expr: &SExpr) -> bool { + fn requires_cte_column_collection(s_expr: &PExpr) -> bool { matches!( s_expr.plan(), RelOperator::Sequence(_) | RelOperator::MaterializedCTERef(_) ) || s_expr.children().any(Self::requires_cte_column_collection) } - fn collect_cte_required_columns(&mut self, s_expr: &SExpr, required: ColumnSet) -> Result<()> { + fn collect_cte_required_columns(&mut self, s_expr: &PExpr, required: ColumnSet) -> Result<()> { match s_expr.plan() { RelOperator::MaterializedCTERef(cte_ref) => { let mut required_mapped = ColumnSet::new(); @@ -478,7 +488,7 @@ pub struct MutationBuildInfo { mod tests { use std::collections::HashMap; - use databend_common_sql::optimizer::ir::SExpr; + use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::DummyTableScan; use databend_common_sql::plans::Limit; use databend_common_sql::plans::MaterializedCTERef; @@ -488,28 +498,28 @@ mod tests { #[test] fn test_requires_cte_column_collection() { - let leaf = SExpr::create_leaf(DummyTableScan::new()); + let leaf = PExpr::create_leaf(DummyTableScan::new()); let limit = Limit { before_exchange: false, limit: Some(1), offset: 0, lazy_columns: Default::default(), }; - let ordinary_plan = SExpr::create_unary(limit.clone(), leaf.clone()); + let ordinary_plan = PExpr::create_unary(limit.clone(), leaf.clone()); assert!(!PhysicalPlanBuilder::requires_cte_column_collection( &ordinary_plan )); - let sequence = SExpr::create_binary(Sequence, leaf.clone(), leaf.clone()); - let nested_sequence = SExpr::create_unary(limit, sequence); + let sequence = PExpr::create_binary(Sequence, leaf.clone(), leaf.clone()); + let nested_sequence = PExpr::create_unary(limit, sequence); assert!(PhysicalPlanBuilder::requires_cte_column_collection( &nested_sequence )); - let cte_ref = SExpr::create_leaf(MaterializedCTERef { + let cte_ref = PExpr::create_leaf(MaterializedCTERef { cte_name: "cte".to_string(), output_columns: vec![], - def: leaf, + def: databend_common_sql::optimizer::ir::SExpr::create_leaf(DummyTableScan::new()), column_mapping: HashMap::new(), stat_info: None, }); diff --git a/src/query/service/src/physical_plans/physical_project_set.rs b/src/query/service/src/physical_plans/physical_project_set.rs index 68d7b3c2470..57884053fae 100644 --- a/src/query/service/src/physical_plans/physical_project_set.rs +++ b/src/query/service/src/physical_plans/physical_project_set.rs @@ -28,7 +28,7 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_sql::ColumnSet; use databend_common_sql::Symbol; use databend_common_sql::TypeCheck; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use itertools::Itertools; use crate::physical_plans::PhysicalPlanBuilder; @@ -148,7 +148,7 @@ impl IPhysicalPlan for ProjectSet { impl PhysicalPlanBuilder { pub async fn build_project_set( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, project_set: &databend_common_sql::plans::ProjectSet, mut required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_range_join.rs b/src/query/service/src/physical_plans/physical_range_join.rs index 44a8800ce6c..d750f5d8e57 100644 --- a/src/query/service/src/physical_plans/physical_range_join.rs +++ b/src/query/service/src/physical_plans/physical_range_join.rs @@ -29,9 +29,9 @@ use databend_common_sql::ScalarExpr; use databend_common_sql::TypeCheck; use databend_common_sql::binder::JoinPredicate; use databend_common_sql::binder::wrap_cast; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::RelExpr; use databend_common_sql::optimizer::ir::RelationalProperty; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::JoinType; use crate::physical_plans::PhysicalPlanBuilder; @@ -209,14 +209,14 @@ impl PhysicalPlanBuilder { pub async fn build_range_join( &mut self, join_type: JoinType, - s_expr: &SExpr, + s_expr: &PExpr, left_required: ColumnSet, right_required: ColumnSet, mut range_conditions: Vec, mut other_conditions: Vec, ) -> Result { - let left_prop = RelExpr::with_s_expr(s_expr.right_child()).derive_relational_prop()?; - let right_prop = RelExpr::with_s_expr(s_expr.left_child()).derive_relational_prop()?; + let left_prop = RelExpr::with_p_expr(s_expr.right_child()).derive_relational_prop()?; + let right_prop = RelExpr::with_p_expr(s_expr.left_child()).derive_relational_prop()?; debug_assert!(!range_conditions.is_empty()); diff --git a/src/query/service/src/physical_plans/physical_sequence.rs b/src/query/service/src/physical_plans/physical_sequence.rs index 4642019c961..0ba7722fa2c 100644 --- a/src/query/service/src/physical_plans/physical_sequence.rs +++ b/src/query/service/src/physical_plans/physical_sequence.rs @@ -17,7 +17,7 @@ use std::any::Any; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_sql::ColumnSet; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlan; @@ -103,7 +103,7 @@ impl IPhysicalPlan for Sequence { impl PhysicalPlanBuilder { pub(crate) async fn build_sequence( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, _sequence: &databend_common_sql::plans::Sequence, stat_info: PlanStatsInfo, required: ColumnSet, diff --git a/src/query/service/src/physical_plans/physical_sort.rs b/src/query/service/src/physical_plans/physical_sort.rs index 60b4a04a44d..ba5cc47c006 100644 --- a/src/query/service/src/physical_plans/physical_sort.rs +++ b/src/query/service/src/physical_plans/physical_sort.rs @@ -30,7 +30,7 @@ use databend_common_sql::IndexType; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::SortDesc; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::WindowFuncType; use itertools::Itertools; @@ -366,7 +366,7 @@ impl Sort { impl PhysicalPlanBuilder { pub async fn build_sort( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, sort: &databend_common_sql::plans::Sort, mut required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_spatial_join.rs b/src/query/service/src/physical_plans/physical_spatial_join.rs index ea6ea3a17ef..7440b417565 100644 --- a/src/query/service/src/physical_plans/physical_spatial_join.rs +++ b/src/query/service/src/physical_plans/physical_spatial_join.rs @@ -29,8 +29,8 @@ use databend_common_sql::ScalarExpr; use databend_common_sql::Symbol; use databend_common_sql::TypeCheck; use databend_common_sql::optimizer::ir::Distribution; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::RelExpr; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::plans::Exchange; use databend_common_sql::plans::SpatialJoinCandidate; @@ -177,17 +177,17 @@ impl PhysicalPlanBuilder { pub async fn try_build_spatial_join( &mut self, candidate: SpatialJoinCandidate, - s_expr: &SExpr, + s_expr: &PExpr, required: ColumnSet, left_required: ColumnSet, right_required: ColumnSet, ) -> Result> { let max_build_rows = self.ctx.get_settings().get_spatial_join_max_build_rows()? as f64; let stat_ctx = StatContext::new(self.func_ctx.clone()); - let left_card = RelExpr::with_s_expr(s_expr.left_child()) + let left_card = RelExpr::with_p_expr(s_expr.left_child()) .derive_cardinality(&stat_ctx)? .cardinality; - let right_card = RelExpr::with_s_expr(s_expr.right_child()) + let right_card = RelExpr::with_p_expr(s_expr.right_child()) .derive_cardinality(&stat_ctx)? .cardinality; let smaller_side = if left_card < right_card { @@ -201,11 +201,11 @@ impl PhysicalPlanBuilder { let build_side = if is_cluster { let left_exchange = s_expr.left_child().get_data_distribution()?; let right_exchange = s_expr.right_child().get_data_distribution()?; - let left_distribution = RelExpr::with_s_expr(s_expr.left_child()) + let left_distribution = RelExpr::with_p_expr(s_expr.left_child()) .derive_physical_prop()? .distribution .clone(); - let right_distribution = RelExpr::with_s_expr(s_expr.right_child()) + let right_distribution = RelExpr::with_p_expr(s_expr.right_child()) .derive_physical_prop()? .distribution .clone(); diff --git a/src/query/service/src/physical_plans/physical_top_n.rs b/src/query/service/src/physical_plans/physical_top_n.rs index 3fa0e1ebcc5..734b163b119 100644 --- a/src/query/service/src/physical_plans/physical_top_n.rs +++ b/src/query/service/src/physical_plans/physical_top_n.rs @@ -34,7 +34,7 @@ use databend_common_pipeline_transforms::sorts::core::SortKeyDescription; use databend_common_pipeline_transforms::sorts::core::select_row_type; use databend_common_sql::ColumnSet; use databend_common_sql::executor::physical_plans::SortDesc; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_storages_parquet::ReadSettings; use itertools::Itertools; @@ -466,7 +466,7 @@ impl FinalTopNParams { impl PhysicalPlanBuilder { pub async fn build_top_n( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, top_n: &databend_common_sql::plans::TopN, mut required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_udf.rs b/src/query/service/src/physical_plans/physical_udf.rs index 88a665edc49..c8691b75b04 100644 --- a/src/query/service/src/physical_plans/physical_udf.rs +++ b/src/query/service/src/physical_plans/physical_udf.rs @@ -28,7 +28,7 @@ use databend_common_sql::ColumnSet; use databend_common_sql::IndexType; use databend_common_sql::ScalarExpr; use databend_common_sql::Symbol; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::UDFType; use databend_query_script_udf_support::ScriptUdfFunctionDesc; use itertools::Itertools; @@ -181,7 +181,7 @@ impl From for ScriptUdfFunctionDesc { impl PhysicalPlanBuilder { pub async fn build_udf( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, udf_plan: &databend_common_sql::plans::Udf, mut required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_union_all.rs b/src/query/service/src/physical_plans/physical_union_all.rs index 1965739f66f..4c1af376d27 100644 --- a/src/query/service/src/physical_plans/physical_union_all.rs +++ b/src/query/service/src/physical_plans/physical_union_all.rs @@ -28,7 +28,7 @@ use databend_common_sql::ScalarExpr; use databend_common_sql::Symbol; use databend_common_sql::TypeCheck; use databend_common_sql::evaluator::BlockOperator; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use itertools::Itertools; use crate::physical_plans::Exchange; @@ -209,7 +209,7 @@ impl UnionAll { impl PhysicalPlanBuilder { pub async fn build_union_all( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, union_all: &databend_common_sql::plans::UnionAll, required: ColumnSet, stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/physical_window.rs b/src/query/service/src/physical_plans/physical_window.rs index 426e7484fe7..6af97e64af1 100644 --- a/src/query/service/src/physical_plans/physical_window.rs +++ b/src/query/service/src/physical_plans/physical_window.rs @@ -42,7 +42,7 @@ use databend_common_sql::binder::wrap_cast; use databend_common_sql::executor::physical_plans::AggregateFunctionDesc; use databend_common_sql::executor::physical_plans::AggregateFunctionSignature; use databend_common_sql::executor::physical_plans::SortDesc; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::WindowFuncFrame; use databend_common_sql::plans::WindowFuncFrameBound; use databend_common_sql::plans::WindowFuncType; @@ -515,7 +515,7 @@ fn window_top_n(window: &WindowSpec) -> Option<(usize, WindowPartitionTopNFunc)> impl PhysicalPlanBuilder { pub async fn build_window_group( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, window_group: &databend_common_sql::plans::WindowGroup, mut required: ColumnSet, stat_info: PlanStatsInfo, @@ -611,7 +611,7 @@ impl PhysicalPlanBuilder { pub async fn build_window( &mut self, - s_expr: &SExpr, + s_expr: &PExpr, window: &databend_common_sql::plans::Window, mut required: ColumnSet, _stat_info: PlanStatsInfo, diff --git a/src/query/service/src/physical_plans/runtime_filter/builder.rs b/src/query/service/src/physical_plans/runtime_filter/builder.rs index 0b02fad8b73..3c4c2f80177 100644 --- a/src/query/service/src/physical_plans/runtime_filter/builder.rs +++ b/src/query/service/src/physical_plans/runtime_filter/builder.rs @@ -31,7 +31,7 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::IndexType; use databend_common_sql::MetadataRef; use databend_common_sql::Symbol; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::Exchange; use databend_common_sql::plans::Join; use databend_common_sql::plans::JoinEquiCondition; @@ -85,7 +85,7 @@ pub fn supported_join_type_for_runtime_filter(join_type: &JoinType) -> bool { /// * `ctx` - Table context /// * `metadata` - Metadata reference /// * `join` - Join plan -/// * `s_expr` - SExpr for the join +/// * `s_expr` - PExpr for the join /// * `build_keys` - Build side keys /// * `probe_keys` - Probe keys with their scan, source column, connector, and null-equality metadata /// @@ -95,7 +95,7 @@ pub async fn build_runtime_filter( ctx: Arc, metadata: &MetadataRef, join: &Join, - s_expr: &SExpr, + s_expr: &PExpr, build_keys: &[RemoteExpr], probe_keys: Vec>>>, build_table_indexes: Vec>, @@ -210,7 +210,7 @@ async fn get_build_table_rows( fn find_probe_targets( metadata: &MetadataRef, - s_expr: &SExpr, + s_expr: &PExpr, probe_key: RemoteExpr, probe_scan_id: usize, probe_key_col_idx: Symbol, @@ -319,7 +319,7 @@ fn propagate_probe_targets( Ok(result) } -fn collect_equi_conditions(s_expr: &SExpr) -> Result> { +fn collect_equi_conditions(s_expr: &PExpr) -> Result> { let mut conditions = Vec::new(); if let RelOperator::Join(join) = s_expr.plan() { diff --git a/src/query/service/src/test_kits/fuse.rs b/src/query/service/src/test_kits/fuse.rs index 41ac35c08c7..cdf36d3d3d7 100644 --- a/src/query/service/src/test_kits/fuse.rs +++ b/src/query/service/src/test_kits/fuse.rs @@ -28,7 +28,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::ScalarRef; use databend_common_expression::SendableDataBlockStream; use databend_common_expression::types::NumberScalar; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::plans::Mutation; use databend_common_storages_factory::Table; use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; @@ -355,7 +355,7 @@ pub async fn append_sample_data(num_blocks: usize, fixture: &TestFixture) -> Res pub async fn do_mutation( ctx: Arc, - s_expr: SExpr, + s_expr: PExpr, schema: DataSchemaRef, ) -> Result<()> { let mutation: Mutation = s_expr.plan().clone().try_into()?; diff --git a/src/query/service/tests/it/parquet_rs/utils.rs b/src/query/service/tests/it/parquet_rs/utils.rs index 41f0cd2bbc0..d8fd7d28897 100644 --- a/src/query/service/tests/it/parquet_rs/utils.rs +++ b/src/query/service/tests/it/parquet_rs/utils.rs @@ -40,7 +40,9 @@ pub async fn get_data_source_plan(ctx: Arc, sql: &str) -> Resu } = plan { let mut builder = PhysicalPlanBuilder::new(metadata, ctx, false); - let physcail_plan = builder.build(&s_expr, bind_context.column_set()).await?; + let physcail_plan = builder + .build_query(s_expr.planned()?, bind_context.column_set()) + .await?; physcail_plan.try_find_single_data_source().unwrap().clone() } else { unreachable!() diff --git a/src/query/service/tests/it/sql/exec/window.rs b/src/query/service/tests/it/sql/exec/window.rs index bc91f943771..68036aae366 100644 --- a/src/query/service/tests/it/sql/exec/window.rs +++ b/src/query/service/tests/it/sql/exec/window.rs @@ -124,7 +124,7 @@ async fn test_window_inputs_prune_json_after_evaluation() -> Result<()> { panic!("expected query plan"); }; let plan = PhysicalPlanBuilder::new(metadata, ctx, false) - .build(&s_expr, bind_context.column_set()) + .build_query(s_expr.planned()?, bind_context.column_set()) .await?; assert!( check_window_input_json(&plan, keep_json, &format!("{nodes} nodes: {sql}"))? > 0 diff --git a/src/query/service/tests/it/sql/planner/optimizer/optimizer_test.rs b/src/query/service/tests/it/sql/planner/optimizer/optimizer_test.rs index 9dfb7d1bf8e..b4a5e428382 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/optimizer_test.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/optimizer_test.rs @@ -73,7 +73,9 @@ impl TestCaseRunner for ServiceRunner { } = optimized { let mut builder = PhysicalPlanBuilder::new(metadata.clone(), self.0.clone(), false); - let physical = builder.build(s_expr, bind_context.column_set()).await?; + let physical = builder + .build_query(s_expr.planned()?, bind_context.column_set()) + .await?; let metadata = metadata.read(); Ok(Some( physical diff --git a/src/query/service/tests/it/sql/planner/optimizer/statistics_trace.rs b/src/query/service/tests/it/sql/planner/optimizer/statistics_trace.rs index 18e4a04f0b1..82236cc6012 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/statistics_trace.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/statistics_trace.rs @@ -188,7 +188,7 @@ async fn collect_statistics_trace(ctx: &Arc, sql: &str) -> Result< let mut optimizer = CollectStatisticsOptimizer::new(opt_ctx).with_trace_collector(trace_collector.clone()); - let _ = optimizer.optimize(*s_expr).await?; + let _ = optimizer.optimize((*s_expr).into_logical()?).await?; trace_collector .take() .ok_or_else(|| ErrorCode::Internal("statistics trace was not collected")) diff --git a/src/query/service/tests/it/sql/planner/semantic/type_check.rs b/src/query/service/tests/it/sql/planner/semantic/type_check.rs index a33e56a698c..87f8278a6b9 100644 --- a/src/query/service/tests/it/sql/planner/semantic/type_check.rs +++ b/src/query/service/tests/it/sql/planner/semantic/type_check.rs @@ -198,7 +198,9 @@ async fn test_grouping_sets_to_union_keeps_grouping_id_for_qualify_windows() -> }; let mut builder = PhysicalPlanBuilder::new(metadata, ctx, false); - builder.build(&s_expr, bind_context.column_set()).await?; + builder + .build_query(s_expr.planned()?, bind_context.column_set()) + .await?; Ok(()) } diff --git a/src/query/service/tests/it/storages/fuse/operations/table_analyze.rs b/src/query/service/tests/it/storages/fuse/operations/table_analyze.rs index 2ad3e2b88d0..3e1af52ace2 100644 --- a/src/query/service/tests/it/storages/fuse/operations/table_analyze.rs +++ b/src/query/service/tests/it/storages/fuse/operations/table_analyze.rs @@ -104,7 +104,7 @@ async fn test_table_modify_column_ndv_statistics() -> anyhow::Result<()> { let mut planner = Planner::new(ctx.clone()); let (plan, _) = planner.plan_sql(query).await?; if let Plan::DataMutation { s_expr, schema, .. } = plan { - do_mutation(ctx.clone(), *s_expr.clone(), schema.clone()).await?; + do_mutation(ctx.clone(), (*s_expr).into_planned()?, schema.clone()).await?; } ctx.evict_table_from_cache("default", "default", "t")?; fixture.execute_command(statistics_sql).await?; @@ -136,7 +136,7 @@ async fn test_table_update_analyze_statistics() -> anyhow::Result<()> { let mut planner = Planner::new(ctx.clone()); let (plan, _) = planner.plan_sql(&query).await?; if let Plan::DataMutation { s_expr, schema, .. } = plan { - do_mutation(ctx.clone(), *s_expr.clone(), schema.clone()).await?; + do_mutation(ctx.clone(), (*s_expr).into_planned()?, schema.clone()).await?; } // check summary after update diff --git a/src/query/sql/src/planner/binder/bind_mutation/bind.rs b/src/query/sql/src/planner/binder/bind_mutation/bind.rs index f6b48d48112..31052e5752d 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/bind.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/bind.rs @@ -316,7 +316,7 @@ impl Binder { s_expr = self.rewrite_udf(&mut bind_context, s_expr)?; Ok(Plan::DataMutation { - s_expr: Box::new(s_expr), + s_expr: Box::new(s_expr.into()), schema, metadata: self.metadata.clone(), }) diff --git a/src/query/sql/src/planner/binder/bind_mutation/mod.rs b/src/query/sql/src/planner/binder/bind_mutation/mod.rs index e1e6b1e2649..986809c734f 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/mod.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/mod.rs @@ -20,4 +20,3 @@ mod update; pub use bind::MutationStrategy; pub use bind::MutationType; -pub use mutation_expression::target_probe; diff --git a/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs b/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs index f0c8885b682..d14fec9bfe2 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs @@ -713,21 +713,3 @@ pub struct MutationExpressionBindResult { pub source: Option, pub predicates: Vec, } - -pub fn target_probe(s_expr: &SExpr, target_table_index: usize) -> Result { - if !matches!(s_expr.plan(), RelOperator::Join(_)) { - return Ok(false); - } - - fn contains_target_table(s_expr: &SExpr, target_table_index: usize) -> bool { - if let RelOperator::Scan(scan) = s_expr.plan() { - scan.table_index == target_table_index - } else { - s_expr - .children() - .any(|child| contains_target_table(child, target_table_index)) - } - } - - Ok(contains_target_table(s_expr.child(0)?, target_table_index)) -} diff --git a/src/query/sql/src/planner/binder/bind_mutation/update.rs b/src/query/sql/src/planner/binder/bind_mutation/update.rs index c096f272145..c96dd6726d5 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/update.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/update.rs @@ -131,6 +131,7 @@ impl Binder { else { return Ok(plan); }; + let s_expr = s_expr.logical()?; let RelOperator::Mutation(mutation) = s_expr.plan() else { return Ok(plan); }; @@ -345,7 +346,7 @@ impl Binder { unreachable!() }; Ok(Plan::DataMutation { - s_expr, + s_expr: Box::new((*s_expr).into()), schema, metadata, }) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index 38c698873aa..7df39318a47 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -252,7 +252,7 @@ impl Binder { } .into(), ), - s_expr, + (*s_expr).into_logical()?, ); return Ok(Some((s_expr, *bind_context))); diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index 3233dcab9e6..cfbeca51bad 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -188,7 +188,7 @@ impl Binder { None }; Plan::Query { - s_expr: Box::new(s_expr), + s_expr: Box::new(s_expr.into()), metadata: self.metadata.clone(), bind_context: Box::new(bind_context), rewrite_kind: None, diff --git a/src/query/sql/src/planner/binder/copy_into_location.rs b/src/query/sql/src/planner/binder/copy_into_location.rs index 408f86f11b5..16683fbb5ad 100644 --- a/src/query/sql/src/planner/binder/copy_into_location.rs +++ b/src/query/sql/src/planner/binder/copy_into_location.rs @@ -104,7 +104,7 @@ impl Binder { }?; let mut is_ordered = false; if let Plan::Query { s_expr, .. } = &query { - let p = s_expr.derive_relational_prop()?; + let p = s_expr.logical()?.derive_relational_prop()?; if !p.orderings.is_empty() { is_ordered = true; } diff --git a/src/query/sql/src/planner/binder/copy_into_table.rs b/src/query/sql/src/planner/binder/copy_into_table.rs index 4d6087b2c8e..4971dc613c7 100644 --- a/src/query/sql/src/planner/binder/copy_into_table.rs +++ b/src/query/sql/src/planner/binder/copy_into_table.rs @@ -559,7 +559,7 @@ impl Binder { output_context.columns = from_context.columns; plan.query = Some(Box::new(Plan::Query { - s_expr: Box::new(s_expr), + s_expr: Box::new(s_expr.into()), metadata: self.metadata.clone(), bind_context: Box::new(output_context), rewrite_kind: None, diff --git a/src/query/sql/src/planner/binder/ddl/materialized_view.rs b/src/query/sql/src/planner/binder/ddl/materialized_view.rs index 23516e48d29..46900667bdb 100644 --- a/src/query/sql/src/planner/binder/ddl/materialized_view.rs +++ b/src/query/sql/src/planner/binder/ddl/materialized_view.rs @@ -658,7 +658,7 @@ impl Binder { )); }; let physical_schema = normalize_null_fields(Self::materialized_view_physical_schema( - storage_expr, + storage_expr.logical()?, physical_bind_context, physical_metadata.clone(), physical_rewriter.physical_names(), diff --git a/src/query/sql/src/planner/binder/insert_multi_table.rs b/src/query/sql/src/planner/binder/insert_multi_table.rs index 59517f14903..0f9cfca4998 100644 --- a/src/query/sql/src/planner/binder/insert_multi_table.rs +++ b/src/query/sql/src/planner/binder/insert_multi_table.rs @@ -67,7 +67,7 @@ impl Binder { let (s_expr, bind_context) = self.bind_table_reference(bind_context, &table_ref)?; let select_plan = Plan::Query { - s_expr: Box::new(s_expr), + s_expr: Box::new(s_expr.into()), metadata: self.metadata.clone(), bind_context: Box::new(bind_context.clone()), rewrite_kind: None, diff --git a/src/query/sql/src/planner/binder/mod.rs b/src/query/sql/src/planner/binder/mod.rs index 6dc1a3410fc..81610dd0259 100644 --- a/src/query/sql/src/planner/binder/mod.rs +++ b/src/query/sql/src/planner/binder/mod.rs @@ -68,7 +68,6 @@ pub use async_function_desc::AsyncFunctionDesc; pub use bind_context::*; pub use bind_mutation::MutationStrategy; pub use bind_mutation::MutationType; -pub use bind_mutation::target_probe; pub use bind_query::bind_values; pub use bind_table_reference::is_range_join_condition; pub use bind_table_reference::parse_result_scan_args; diff --git a/src/query/sql/src/planner/format/display.rs b/src/query/sql/src/planner/format/display.rs index 9c6ff1674b7..f2f59e3ccad 100644 --- a/src/query/sql/src/planner/format/display.rs +++ b/src/query/sql/src/planner/format/display.rs @@ -21,9 +21,7 @@ use crate::IndexType; use crate::Metadata; use crate::ScalarExpr; use crate::Symbol; -use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; -use crate::optimizer::ir::SExpr; use crate::optimizer::ir::StatContext; use crate::optimizer::ir::StatInfo; use crate::plans::RelOperator; @@ -158,14 +156,17 @@ where } } - pub fn humanize_s_expr(&self, s_expr: &SExpr) -> Result { + pub fn humanize_expr( + &self, + s_expr: &crate::optimizer::ir::Expr, + ) -> Result { let op = s_expr.plan(); let mut tree = self .operator_humanizer .humanize_operator(self.id_humanizer, op); if self.id_humanizer.options().verbose { - let rel_expr = RelExpr::with_s_expr(s_expr); + let rel_expr = K::rel_expr(s_expr); let prop = rel_expr.derive_relational_prop()?; let stat = rel_expr.derive_cardinality(self.id_humanizer.stat_context())?; let properties = self.humanize_property(&prop); @@ -213,13 +214,13 @@ where // in the surrounding Sequence. } else if s_expr.plan.is_join() { tree.children - .push(self.humanize_s_expr(s_expr.build_side_child())?); + .push(self.humanize_expr(s_expr.build_side_child())?); tree.children - .push(self.humanize_s_expr(s_expr.probe_side_child())?); + .push(self.humanize_expr(s_expr.probe_side_child())?); } else { let children = s_expr .children() - .map(|s_expr| self.humanize_s_expr(s_expr)) + .map(|s_expr| self.humanize_expr(s_expr)) .collect::>>()?; tree.children.extend(children); }; diff --git a/src/query/sql/src/planner/format/display_plan.rs b/src/query/sql/src/planner/format/display_plan.rs index 187fbedf79a..50ef5a649e7 100644 --- a/src/query/sql/src/planner/format/display_plan.rs +++ b/src/query/sql/src/planner/format/display_plan.rs @@ -21,19 +21,38 @@ use super::display::FormatOptions; use super::display::IdHumanizer; use super::display::MetadataIdHumanizer; use super::display::TreeHumanizer; -use crate::optimizer::ir::SExpr; +use crate::optimizer::ir::Expr; +use crate::optimizer::ir::RelExprKind; use crate::optimizer::ir::StatContext; use crate::plans::CreateTablePlan; use crate::plans::Plan; -impl SExpr { +impl Expr { pub(crate) fn to_format_tree( &self, id_humanizer: &I, ) -> Result { let operator_humanizer = DefaultOperatorHumanizer; let tree_humanizer = TreeHumanizer::new(id_humanizer, &operator_humanizer); - tree_humanizer.humanize_s_expr(self) + tree_humanizer.humanize_expr(self) + } +} + +impl crate::optimizer::ir::QueryPlan { + pub(crate) fn to_format_tree(&self, id: &I) -> Result { + match self { + Self::Logical(expr) => expr.to_format_tree(id), + Self::Planned(plan) => plan.expr().to_format_tree(id), + } + } +} + +impl crate::optimizer::ir::MutationPlan { + pub(crate) fn to_format_tree(&self, id: &I) -> Result { + match self { + Self::Logical(expr) => expr.to_format_tree(id), + Self::Planned(plan) => plan.to_format_tree(id), + } } } diff --git a/src/query/sql/src/planner/lineage.rs b/src/query/sql/src/planner/lineage.rs index d4bdf81b445..668fae39d6c 100644 --- a/src/query/sql/src/planner/lineage.rs +++ b/src/query/sql/src/planner/lineage.rs @@ -602,6 +602,7 @@ impl<'a> RelationExtractor<'a> { Plan::DataMutation { s_expr, metadata, .. } => { + let s_expr = s_expr.logical()?; let Some(mutation) = find_mutation(s_expr) else { return Ok(None); }; @@ -1234,7 +1235,7 @@ fn query_parts(plan: &Plan) -> Result<(&SExpr, &MetadataRef, &BindContext)> { metadata, bind_context, .. - } => Ok((s_expr, metadata, bind_context)), + } => Ok((s_expr.logical()?, metadata, bind_context)), _ => Err(ErrorCode::Internal( "Lineage extraction expects a query plan".to_string(), )), @@ -2192,7 +2193,7 @@ mod tests { ..Default::default() }; Plan::Query { - s_expr: Box::new(s_expr), + s_expr: Box::new(s_expr.into()), metadata, bind_context: Box::new(bind_context), rewrite_kind: None, diff --git a/src/query/sql/src/planner/optimizer/ir/expr/column_scope_validator.rs b/src/query/sql/src/planner/optimizer/ir/expr/column_scope_validator.rs index 350b0df8fb7..c91854da42d 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/column_scope_validator.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/column_scope_validator.rs @@ -15,7 +15,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use super::SExpr; +use super::node::Expr; +use super::node::RelExprKind; use crate::ColumnSet; use crate::MetadataRef; use crate::Symbol; @@ -26,7 +27,7 @@ use crate::plans::ScalarExpr; use crate::plans::SubqueryExpr; use crate::plans::Visitor as ScalarExprVisitor; -impl SExpr { +impl Expr { /// Validate that every column an operator references is produced by its inputs. /// /// For each node, the columns referenced by its scalar expressions must be a subset of @@ -50,7 +51,7 @@ struct ColumnScopeValidator<'a> { impl ColumnScopeValidator<'_> { #[recursive::recursive] - fn validate(&self, s_expr: &SExpr, ambient: &ColumnSet) -> Result<()> { + fn validate(&self, s_expr: &Expr, ambient: &ColumnSet) -> Result<()> { let plan = s_expr.plan(); let mut available = ambient.clone(); @@ -86,14 +87,14 @@ impl ColumnScopeValidator<'_> { if let RelOperator::WindowGroup(group) = plan { // The physical builder evaluates scalar items over the child before evaluating // windows. An item cannot read its own output or another item's output. - self.validate_scalars( + self.validate_scalars::( plan, group.scalar_items.iter().map(|item| &item.scalar), &available, )?; let mut window_available = available.clone(); window_available.extend(group.scalar_items.iter().map(|item| item.index)); - self.validate_scalars( + self.validate_scalars::( plan, group .windows @@ -102,7 +103,7 @@ impl ColumnScopeValidator<'_> { &window_available, )?; } else { - self.validate_scalars(plan, plan.scalar_expr_iter(), &available)?; + self.validate_scalars::(plan, plan.scalar_expr_iter(), &available)?; } // The right side of a LATERAL join is correlated to the left side without a // `SubqueryExpr`, so the left outputs become its ambient scope. @@ -126,7 +127,7 @@ impl ColumnScopeValidator<'_> { Ok(()) } - fn validate_scalars<'a>( + fn validate_scalars<'a, K: RelExprKind>( &self, plan: &RelOperator, scalars: impl IntoIterator, @@ -151,7 +152,8 @@ impl ColumnScopeValidator<'_> { drop(metadata); if !unresolved.is_empty() { return Err(ErrorCode::Internal(format!( - "SExpr column scope violation in {:?}: references {} which no input produces; available columns: {}", + "{} column scope violation in {:?}: references {} which no input produces; available columns: {}", + K::NAME, plan.rel_op(), self.describe(&unresolved), self.describe(&available.iter().copied().collect::>()), @@ -217,6 +219,7 @@ mod tests { use crate::ColumnBindingBuilder; use crate::Metadata; use crate::Visibility; + use crate::optimizer::ir::SExpr; use crate::plans::BoundColumnRef; use crate::plans::EvalScalar; use crate::plans::ExpressionScan; diff --git a/src/query/sql/src/planner/optimizer/ir/expr/conversion.rs b/src/query/sql/src/planner/optimizer/ir/expr/conversion.rs new file mode 100644 index 00000000000..7ac09633f03 --- /dev/null +++ b/src/query/sql/src/planner/optimizer/ir/expr/conversion.rs @@ -0,0 +1,129 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use super::PExpr; +use super::p_expr::PhysicalState; +use crate::optimizer::ir::SExpr; + +impl From for PExpr { + #[recursive::recursive] + fn from(expr: SExpr) -> Self { + Self { + plan: expr.plan, + children: expr + .children + .into_iter() + .map(|child| Arc::new(Self::from(Arc::unwrap_or_clone(child)))) + .collect(), + state: PhysicalState { + original_group: expr.state.original_group, + rel_prop: expr.state.rel_prop, + stat_info: expr.state.stat_info, + applied_rules: expr.state.applied_rules, + }, + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hash; + use std::hash::Hasher; + use std::sync::OnceLock; + + use super::*; + use crate::optimizer::ir::RelExpr; + use crate::optimizer::ir::StatContext; + use crate::optimizer::optimizers::rule::RuleID; + use crate::plans::DummyTableScan; + + #[test] + fn conversion_preserves_identity_and_replacement_invalidates_caches() + -> databend_common_exception::Result<()> { + // Local node/cache invariant; SQL-driven integration tests cover real trees. + let mut logical = SExpr::create(DummyTableScan::new(), vec![], Some(7), None, None); + logical.set_applied_rule(&RuleID::EliminateEvalScalar); + logical.derive_relational_prop()?; + RelExpr::with_s_expr(&logical).derive_cardinality(&StatContext::default())?; + let physical = PExpr::from(logical.clone()); + assert_eq!(physical.original_group(), Some(7)); + assert!(physical.applied_rule(&RuleID::EliminateEvalScalar)); + assert!(Arc::ptr_eq( + &logical.state.rel_prop, + &physical.state.rel_prop + )); + assert!(Arc::ptr_eq( + &logical.state.stat_info, + &physical.state.stat_info + )); + fn hash(value: &impl Hash) -> u64 { + let mut hash = DefaultHasher::new(); + value.hash(&mut hash); + hash.finish() + } + assert_eq!(hash(&logical), hash(&physical)); + let logical_clone = logical.clone(); + let physical_clone = physical.clone(); + assert!(Arc::ptr_eq( + &logical.state.rel_prop, + &logical_clone.state.rel_prop + )); + assert!(Arc::ptr_eq( + &physical.state.stat_info, + &physical_clone.state.stat_info + )); + assert_eq!(logical, logical_clone); + assert_eq!(physical, physical_clone); + // Cache contents are not identity, but group and rule state still are. + let fresh = PExpr::create(DummyTableScan::new(), vec![], Some(7), None, None); + let mut same_identity = fresh.clone(); + same_identity.set_applied_rule(&RuleID::EliminateEvalScalar); + assert_eq!(physical, same_identity); + assert_eq!(hash(&physical), hash(&same_identity)); + assert_ne!(physical, fresh); + let mut different_group = PExpr::create(DummyTableScan::new(), vec![], Some(8), None, None); + different_group.set_applied_rule(&RuleID::EliminateEvalScalar); + assert_ne!(physical, different_group); + for replaced in [ + logical.replace_plan(logical.plan.clone()), + logical.replace_children([]), + ] { + assert_eq!(replaced.original_group(), None); + assert!(OnceLock::get(&replaced.state.rel_prop).is_none()); + assert!(OnceLock::get(&replaced.state.stat_info).is_none()); + assert!(replaced.applied_rule(&RuleID::EliminateEvalScalar)); + assert!(!Arc::ptr_eq( + &logical.state.rel_prop, + &replaced.state.rel_prop + )); + } + for replaced in [ + physical.replace_plan(physical.plan.clone()), + physical.replace_children([]), + ] { + assert_eq!(replaced.original_group(), None); + assert!(OnceLock::get(&replaced.state.rel_prop).is_none()); + assert!(OnceLock::get(&replaced.state.stat_info).is_none()); + assert!(replaced.applied_rule(&RuleID::EliminateEvalScalar)); + assert!(!Arc::ptr_eq( + &physical.state.rel_prop, + &replaced.state.rel_prop + )); + } + Ok(()) + } +} diff --git a/src/query/sql/src/planner/optimizer/ir/expr/extract.rs b/src/query/sql/src/planner/optimizer/ir/expr/extract.rs index adbe467a460..0dbb652198d 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/extract.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/extract.rs @@ -17,6 +17,8 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::optimizer::ir::Expr; +use crate::optimizer::ir::ExprKind; use crate::optimizer::ir::Group; use crate::optimizer::ir::Memo; use crate::optimizer::ir::expr::MExpr; @@ -80,9 +82,9 @@ macro_rules! match_op { } impl Matcher { - /// Check if the `SExpr` can be matched by the `Matcher`. + /// Check whether a relational expression matches this pattern. #[recursive::recursive] - pub fn matches(&self, s_expr: &SExpr) -> bool { + pub fn matches>(&self, s_expr: &Expr) -> bool { match self { Matcher::MatchOp { op_type, children } => { if s_expr.plan().rel_op() != *op_type { diff --git a/src/query/sql/src/planner/optimizer/ir/expr/mod.rs b/src/query/sql/src/planner/optimizer/ir/expr/mod.rs index 4813a9ec47a..05f256565f4 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/mod.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/mod.rs @@ -13,8 +13,11 @@ // limitations under the License. mod column_scope_validator; +mod conversion; mod extract; mod m_expr; +mod node; +mod p_expr; mod s_expr; mod type_validator; mod visitor; @@ -22,9 +25,18 @@ mod visitor; pub use extract::Matcher; pub use extract::PatternExtractor; pub use m_expr::MExpr; +pub use node::Expr; +pub use node::ExprKind; +pub use node::RelExprKind; +pub use node::RewriteExprKind; +pub use node::ScanRequiredColumns; +pub use node::Side; +pub use p_expr::PExpr; +pub use p_expr::Physical; +pub use s_expr::Logical; pub use s_expr::SExpr; -pub use s_expr::ScanRequiredColumns; -pub use s_expr::Side; -pub use visitor::AsyncSExprVisitor; -pub use visitor::SExprVisitor; +pub use visitor::AsyncExprVisitor; +pub use visitor::AsyncExprVisitor as AsyncSExprVisitor; +pub use visitor::ExprVisitor; +pub use visitor::ExprVisitor as SExprVisitor; pub use visitor::VisitAction; diff --git a/src/query/sql/src/planner/optimizer/ir/expr/node.rs b/src/query/sql/src/planner/optimizer/ir/expr/node.rs new file mode 100644 index 00000000000..d77e02e659e --- /dev/null +++ b/src/query/sql/src/planner/optimizer/ir/expr/node.rs @@ -0,0 +1,456 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::HashSet; +use std::fmt::Debug; +use std::hash::Hash; +use std::sync::Arc; + +use databend_common_catalog::plan::InvertedIndexInfo; +use databend_common_catalog::plan::VectorIndexInfo; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use educe::Educe; + +use crate::IndexType; +use crate::Symbol; +use crate::plans::Exchange; +use crate::plans::Operator; +use crate::plans::RelOperator; + +/// Stage-specific operator and state types for a shared recursive expression. +/// Constructors and invalidation remain stage-owned; relational utilities can be +/// shared by opting into `RelExprKind`. +pub trait ExprKind { + type Operator: Clone + Debug + Eq + Hash; + type State: Clone + Eq + Hash; +} + +/// Opt-in support for rewriting the recursive shape; invalidation stays stage-owned. +pub trait RewriteExprKind: ExprKind { + fn rewritten_state(state: &Self::State) -> Self::State; +} + +/// Shared relational utilities apply only while a stage uses RelOperator. A future +/// physical operator can provide its own utilities without changing the tree skeleton. +pub trait RelExprKind: RewriteExprKind + Sized { + const NAME: &'static str; + fn rel_expr(expr: &Expr) -> crate::optimizer::ir::RelExpr<'_>; + fn relational_cache( + state: &Self::State, + ) -> &Arc>>; + fn statistics_cache( + state: &Self::State, + ) -> &Arc>>; +} + +#[derive(Educe)] +#[educe( + PartialEq(bound = false, attrs = "#[recursive::recursive]"), + Eq, + Hash(bound = false, attrs = "#[recursive::recursive]"), + Clone(bound = false, attrs = "#[recursive::recursive]") +)] +pub struct Expr { + pub plan: Arc, + pub children: Vec>, + pub(crate) state: K::State, +} + +impl Expr { + pub fn plan(&self) -> &K::Operator { + &self.plan + } + + pub fn children(&self) -> impl Iterator { + self.children.iter().map(|v| v.as_ref()) + } + + pub fn child(&self, n: usize) -> Result<&Self> { + self.children + .get(n) + .map(|v| v.as_ref()) + .ok_or_else(|| ErrorCode::Internal(format!("Invalid children index: {}", n))) + } + + pub fn unary_child(&self) -> &Self { + debug_assert_eq!(self.children.len(), 1); + &self.children[0] + } + + pub fn unary_child_arc(&self) -> Arc { + assert_eq!(self.children.len(), 1); + self.children[0].clone() + } + + pub fn left_child(&self) -> &Self { + debug_assert_eq!(self.children.len(), 2); + &self.children[0] + } + + pub fn left_child_arc(&self) -> Arc { + assert_eq!(self.children.len(), 2); + self.children[0].clone() + } + + pub fn right_child(&self) -> &Self { + debug_assert_eq!(self.children.len(), 2); + &self.children[1] + } + + pub fn right_child_arc(&self) -> Arc { + assert_eq!(self.children.len(), 2); + self.children[1].clone() + } + + pub fn arity(&self) -> usize { + self.children.len() + } +} + +impl Expr { + pub fn replace_children(&self, children: impl IntoIterator>) -> Self { + Self { + plan: self.plan.clone(), + children: children.into_iter().collect(), + state: K::rewritten_state(&self.state), + } + } +} + +impl Expr { + pub fn derive_relational_prop(&self) -> Result> { + use crate::plans::Operator; + let prop = K::relational_cache(&self.state) + .get_or_try_init(|| self.plan.derive_relational_prop(&K::rel_expr(self)))?; + Ok(prop.clone()) + } + + pub fn derive_cardinality( + &self, + ctx: &crate::optimizer::ir::StatContext, + ) -> Result> { + use crate::plans::Operator; + let stats = K::statistics_cache(&self.state) + .get_or_try_init(|| self.plan.derive_stats(&K::rel_expr(self), ctx))?; + Ok(stats.clone()) + } +} + +#[derive(Clone, Default)] +pub struct ScanRequiredColumns { + pub columns: BTreeSet, + pub inverted_index: Option, + pub vector_index: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Side { + Left, + Right, +} + +impl Side { + pub fn opposite(self) -> Self { + match self { + Side::Left => Side::Right, + Side::Right => Side::Left, + } + } + + pub fn child(self, s_expr: &Expr) -> Arc> { + match self { + Side::Left => s_expr.left_child_arc(), + Side::Right => s_expr.right_child_arc(), + } + } +} + +impl Expr { + pub fn build_side_child(&self) -> &Expr { + debug_assert_eq!(self.plan.rel_op(), crate::plans::RelOp::Join); + &self.children[1] + } + + pub fn probe_side_child(&self) -> &Expr { + debug_assert_eq!(self.plan.rel_op(), crate::plans::RelOp::Join); + &self.children[0] + } + + #[recursive::recursive] + pub fn support_lazy_materialize(&self) -> bool { + self.plan.support_lazy_materialize() + && self + .children + .iter() + .all(|child| child.support_lazy_materialize()) + } + + #[recursive::recursive] + pub fn get_udfs(&self) -> Result> { + let mut udfs = HashSet::new(); + let iter = self.plan.scalar_expr_iter(); + for scalar in iter { + for udf in scalar.get_udf_names()? { + udfs.insert(udf); + } + } + + for child in &self.children { + let udf = child.get_udfs()?; + udf.iter().for_each(|udf| { + udfs.insert(*udf); + }) + } + Ok(udfs) + } + + #[recursive::recursive] + pub fn get_udfs_col_ids(&self) -> Result> { + let mut udf_ids = BTreeSet::new(); + if let RelOperator::Udf(udf) = self.plan.as_ref() { + for item in udf.items.iter() { + udf_ids.insert(item.index); + } + } + for child in &self.children { + let udfs = child.get_udfs_col_ids()?; + udf_ids.extend(udfs); + } + Ok(udf_ids) + } + + // Add column index to Scan nodes that match the given table index + pub fn add_column_index_to_scans(&self, table_index: IndexType, column_index: Symbol) -> Self { + let mut required_columns = BTreeMap::new(); + required_columns.insert(table_index, ScanRequiredColumns { + columns: BTreeSet::from([column_index]), + inverted_index: None, + vector_index: None, + }); + self.add_column_indexes_to_scans(&required_columns) + } + + // Add column indexes to Scan nodes that match the given table indexes. + pub fn add_column_indexes_to_scans( + &self, + required_columns: &BTreeMap, + ) -> Self { + struct Visitor<'a> { + required_columns: &'a BTreeMap, + } + + impl super::visitor::ExprVisitor for Visitor<'_> { + fn visit(&mut self, expr: &Expr) -> Result> { + if let Some(p) = expr.plan.as_ref().as_scan() { + if let Some(required_columns) = self.required_columns.get(&p.table_index) { + let mut p = p.clone(); + p.columns.extend(required_columns.columns.iter().copied()); + if required_columns.inverted_index.is_some() { + p.inverted_index = required_columns.inverted_index.clone(); + } + if required_columns.vector_index.is_some() { + p.vector_index = required_columns.vector_index.clone(); + } + let expr = expr.replace_plan(p); + return Ok(super::visitor::VisitAction::Replace(expr)); + } else { + return Ok(super::visitor::VisitAction::SkipChildren); + } + } + Ok(super::visitor::VisitAction::Continue) + } + } + + let mut visitor = Visitor { required_columns }; + let expr = self.accept(&mut visitor); + if let Ok(Some(expr)) = expr { + return expr; + } + self.clone() + } + + #[recursive::recursive] + pub fn has_merge_exchange(&self) -> bool { + if let RelOperator::Exchange(Exchange::Merge) = self.plan.as_ref() { + return true; + } + self.children.iter().any(|child| child.has_merge_exchange()) + } + + pub fn get_data_distribution(&self) -> Result> { + struct DataDistributionVisitor { + result: Option, + } + impl super::visitor::ExprVisitor for DataDistributionVisitor { + fn visit(&mut self, expr: &Expr) -> Result> { + match expr.plan.as_ref() { + RelOperator::Exchange(exchange) => { + self.result = Some(exchange.clone()); + Ok(super::visitor::VisitAction::Stop) + } + + RelOperator::Join(_) => { + let child = expr.probe_side_child(); + self.result = child.get_data_distribution()?; + Ok(super::visitor::VisitAction::Stop) + } + _ => { + if expr.arity() > 0 { + Ok(super::visitor::VisitAction::Continue) + } else { + Ok(super::visitor::VisitAction::Stop) + } + } + } + } + } + + let mut visitor = DataDistributionVisitor { result: None }; + let _ = self.accept(&mut visitor); + Ok(visitor.result) + } +} + +impl> Expr { + pub fn replace_left_child(&self, left: impl Into>) -> Self { + assert_eq!(self.children.len(), 2); + Self { + plan: self.plan.clone(), + state: K::rewritten_state(&self.state), + children: vec![left.into(), self.children[1].clone()], + } + } + + pub fn replace_right_child(&self, right: impl Into>) -> Self { + assert_eq!(self.children.len(), 2); + Self { + plan: self.plan.clone(), + state: K::rewritten_state(&self.state), + children: vec![self.children[0].clone(), right.into()], + } + } + + pub fn replace_side_child(&self, side: Side, child: impl Into>) -> Self { + match side { + Side::Left => self.replace_left_child(child), + Side::Right => self.replace_right_child(child), + } + } + + pub fn replace_plan(&self, plan: impl Into>) -> Self { + Self { + plan: plan.into(), + state: K::rewritten_state(&self.state), + children: self.children.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The skeleton does not require the physical stage to retain logical cache/rule + // fields, or even the same operator type. + struct IndependentStage; + + impl ExprKind for IndependentStage { + type Operator = &'static str; + type State = u64; + } + + impl RewriteExprKind for IndependentStage { + fn rewritten_state(_: &Self::State) -> Self::State { + 99 + } + } + + struct ReplaceScan; + + impl crate::optimizer::ir::ExprVisitor for ReplaceScan { + fn visit( + &mut self, + expr: &Expr, + ) -> Result> { + use crate::optimizer::ir::VisitAction; + if *expr.plan() == "scan" { + Ok(VisitAction::Replace(Expr { + plan: Arc::new("new_scan"), + children: vec![], + state: 7, + })) + } else { + Ok(VisitAction::Continue) + } + } + } + + #[async_trait::async_trait] + impl crate::optimizer::ir::AsyncExprVisitor for ReplaceScan { + async fn visit( + &mut self, + expr: &Expr, + ) -> Result> { + crate::optimizer::ir::ExprVisitor::visit(self, expr) + } + } + + #[tokio::test] + async fn shared_traversal_uses_stage_invalidation() -> Result<()> { + let tree = Expr:: { + plan: Arc::new("filter"), + children: vec![Arc::new(Expr { + plan: Arc::new("scan"), + children: vec![], + state: 11, + })], + state: 23, + }; + let replaced = tree.accept(&mut ReplaceScan)?.unwrap(); + assert_eq!(replaced.state, 99); + assert_eq!(*replaced.child(0)?.plan(), "new_scan"); + assert_eq!(replaced.child(0)?.state, 7); + let asynchronous = tree.accept_async(&mut ReplaceScan).await?.unwrap(); + assert!(replaced == asynchronous); + assert_eq!(tree.state, 23); + assert_eq!(*tree.child(0)?.plan(), "scan"); + Ok(()) + } + + #[test] + fn tree_access_does_not_depend_on_stage_state() -> Result<()> { + let leaf = Arc::new(Expr:: { + plan: Arc::new("scan"), + children: vec![], + state: 11, + }); + let tree = Expr:: { + plan: Arc::new("filter"), + children: vec![leaf.clone()], + state: 23, + }; + assert_eq!(*tree.plan(), "filter"); + assert_eq!(tree.arity(), 1); + assert_eq!(tree.child(0)?.state, 11); + assert!(tree.child(1).is_err()); + assert!(Arc::ptr_eq(&leaf, &tree.unary_child_arc())); + let cloned = tree.clone(); + assert!(tree == cloned); + assert!(Arc::ptr_eq(&tree.children[0], &cloned.children[0])); + Ok(()) + } +} diff --git a/src/query/sql/src/planner/optimizer/ir/expr/p_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/p_expr.rs new file mode 100644 index 00000000000..c0c80b2ba5a --- /dev/null +++ b/src/query/sql/src/planner/optimizer/ir/expr/p_expr.rs @@ -0,0 +1,177 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; +use std::sync::OnceLock; + +use educe::Educe; + +use crate::IndexType; +use crate::optimizer::ir::Expr; +use crate::optimizer::ir::ExprKind; +use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::RelExprKind; +use crate::optimizer::ir::RelationalProperty; +use crate::optimizer::ir::RewriteExprKind; +use crate::optimizer::ir::StatInfo; +use crate::optimizer::optimizers::rule::AppliedRules; +use crate::optimizer::optimizers::rule::RuleID; +use crate::plans::RelOperator; + +/// Physical expression, with stage-specific state on a shared recursive tree. +pub type PExpr = Expr; + +pub struct Physical; + +impl ExprKind for Physical { + type Operator = RelOperator; + type State = PhysicalState; +} + +#[derive(Educe)] +#[educe(PartialEq, Eq, Hash, Clone)] +pub struct PhysicalState { + pub(crate) original_group: Option, + /// Shared, lazily populated caches; excluded from expression identity. + #[educe(Hash(ignore), PartialEq(ignore))] + pub(crate) rel_prop: Arc>>, + #[educe(Hash(ignore), PartialEq(ignore))] + pub(crate) stat_info: Arc>>, + pub(crate) applied_rules: AppliedRules, +} + +impl RewriteExprKind for Physical { + fn rewritten_state(state: &Self::State) -> Self::State { + Self::State { + original_group: None, + rel_prop: Default::default(), + stat_info: Default::default(), + applied_rules: state.applied_rules.clone(), + } + } +} + +impl RelExprKind for Physical { + const NAME: &'static str = "PExpr"; + fn rel_expr(expr: &Expr) -> RelExpr<'_> { + RelExpr::with_p_expr(expr) + } + fn relational_cache(state: &Self::State) -> &Arc>> { + &state.rel_prop + } + fn statistics_cache(state: &Self::State) -> &Arc>> { + &state.stat_info + } +} + +// Preserve diagnostic output rather than exposing the implementation's state wrapper. +impl std::fmt::Debug for Expr { + #[recursive::recursive] + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PExpr") + .field("plan", &self.plan) + .field("children", &self.children) + .field("original_group", &self.state.original_group) + .field("rel_prop", &self.state.rel_prop) + .field("stat_info", &self.state.stat_info) + .field("applied_rules", &self.state.applied_rules) + .finish() + } +} + +impl PExpr { + pub fn create( + plan: impl Into>, + children: Vec>, + original_group: Option, + rel_prop: Option>, + stat_info: Option>, + ) -> Self { + PExpr { + plan: plan.into(), + children, + state: PhysicalState { + original_group, + rel_prop: Arc::new(match rel_prop { + Some(rel_prop) => OnceLock::from(rel_prop), + None => OnceLock::new(), + }), + stat_info: Arc::new(match stat_info { + Some(stat_info) => OnceLock::from(stat_info), + None => OnceLock::new(), + }), + applied_rules: AppliedRules::default(), + }, + } + } + + pub fn create_unary(plan: impl Into>, child: impl Into>) -> Self { + Self::create(plan.into(), vec![child.into()], None, None, None) + } + + pub fn create_binary( + plan: impl Into>, + left_child: impl Into>, + right_child: impl Into>, + ) -> Self { + Self::create( + plan, + vec![left_child.into(), right_child.into()], + None, + None, + None, + ) + } + + pub fn create_leaf(plan: impl Into>) -> Self { + Self::create(plan, vec![], None, None, None) + } + + pub fn build_unary(self, plan: impl Into>) -> Self { + Self::create(plan, vec![self.into()], None, None, None) + } + + pub fn ref_build_unary(self: &Arc, plan: impl Into>) -> Self { + Self::create(plan, vec![self.clone()], None, None, None) + } + + pub fn original_group(&self) -> Option { + self.state.original_group + } + + /// Record the applied rule id in current PExpr + pub(crate) fn set_applied_rule(&mut self, rule_id: &RuleID) { + self.state.applied_rules.set(rule_id, true); + } + + /// Check if a rule is applied for current PExpr + pub(crate) fn applied_rule(&self, rule_id: &RuleID) -> bool { + self.state.applied_rules.get(rule_id) + } + + // The method will clear the applied rules of current PExpr and its children. + #[recursive::recursive] + pub fn clear_applied_rules(&mut self) { + self.state.applied_rules.clear(); + let children = self + .children() + .map(|child| { + let mut child = child.clone(); + child.clear_applied_rules(); + Arc::new(child) + }) + .collect::>(); + self.children = children; + } +} diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index 62cd2480c84..d5a27f7b4c1 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -12,68 +12,82 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; -use std::collections::BTreeSet; -use std::collections::HashSet; use std::sync::Arc; use std::sync::OnceLock; -use databend_common_catalog::plan::InvertedIndexInfo; -use databend_common_catalog::plan::VectorIndexInfo; -use databend_common_exception::ErrorCode; -use databend_common_exception::Result; use educe::Educe; use crate::IndexType; -use crate::Symbol; -use crate::optimizer::ir::SExprVisitor; +use crate::optimizer::ir::Expr; +use crate::optimizer::ir::ExprKind; +use crate::optimizer::ir::RelExprKind; +use crate::optimizer::ir::RewriteExprKind; use crate::optimizer::ir::StatInfo; -use crate::optimizer::ir::VisitAction; use crate::optimizer::ir::property::RelExpr; use crate::optimizer::ir::property::RelationalProperty; use crate::optimizer::optimizers::rule::AppliedRules; use crate::optimizer::optimizers::rule::RuleID; -use crate::plans::Exchange; -use crate::plans::Operator; use crate::plans::RelOperator; -/// `SExpr` is abbreviation of single expression, which is a tree of relational operators. -#[derive(Educe)] -#[educe( - PartialEq(bound = false, attrs = "#[recursive::recursive]"), - Eq, - Hash(bound = false, attrs = "#[recursive::recursive]"), - Clone(bound = false, attrs = "#[recursive::recursive]"), - Debug(bound = false, attrs = "#[recursive::recursive]") -)] -pub struct SExpr { - pub plan: Arc, - pub children: Vec>, +/// Logical expression, with stage-specific state on a shared recursive tree. +pub type SExpr = Expr; - original_group: Option, +pub struct Logical; - /// A cache of relational property of current `SExpr`, will - /// be lazily computed as soon as `RelExpr::derive_relational_prop` - /// is invoked on current `SExpr`. - /// - /// Since `SExpr` is `Send + Sync`, we use `OnceLock` to protect - /// the cache. - #[educe(Hash(ignore), PartialEq(ignore))] - rel_prop: Arc>>, +impl ExprKind for Logical { + type Operator = RelOperator; + type State = LogicalState; +} +#[derive(Educe)] +#[educe(PartialEq, Eq, Hash, Clone)] +pub struct LogicalState { + pub(crate) original_group: Option, + /// Shared, lazily populated caches; excluded from expression identity. + #[educe(Hash(ignore), PartialEq(ignore))] + pub(crate) rel_prop: Arc>>, #[educe(Hash(ignore), PartialEq(ignore))] pub(crate) stat_info: Arc>>, - - /// A bitmap to record applied rules on current SExpr, to prevent - /// redundant transformations. pub(crate) applied_rules: AppliedRules, } -#[derive(Clone, Default)] -pub struct ScanRequiredColumns { - pub columns: BTreeSet, - pub inverted_index: Option, - pub vector_index: Option, +impl RewriteExprKind for Logical { + fn rewritten_state(state: &Self::State) -> Self::State { + Self::State { + original_group: None, + rel_prop: Default::default(), + stat_info: Default::default(), + applied_rules: state.applied_rules.clone(), + } + } +} + +impl RelExprKind for Logical { + const NAME: &'static str = "SExpr"; + fn rel_expr(expr: &Expr) -> RelExpr<'_> { + RelExpr::with_s_expr(expr) + } + fn relational_cache(state: &Self::State) -> &Arc>> { + &state.rel_prop + } + fn statistics_cache(state: &Self::State) -> &Arc>> { + &state.stat_info + } +} + +// Preserve diagnostic output rather than exposing the implementation's state wrapper. +impl std::fmt::Debug for Expr { + #[recursive::recursive] + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SExpr") + .field("plan", &self.plan) + .field("children", &self.children) + .field("original_group", &self.state.original_group) + .field("rel_prop", &self.state.rel_prop) + .field("stat_info", &self.state.stat_info) + .field("applied_rules", &self.state.applied_rules) + .finish() + } } impl SExpr { @@ -87,16 +101,18 @@ impl SExpr { SExpr { plan: plan.into(), children, - original_group, - rel_prop: Arc::new(match rel_prop { - Some(rel_prop) => OnceLock::from(rel_prop), - None => OnceLock::new(), - }), - stat_info: Arc::new(match stat_info { - Some(stat_info) => OnceLock::from(stat_info), - None => OnceLock::new(), - }), - applied_rules: AppliedRules::default(), + state: LogicalState { + original_group, + rel_prop: Arc::new(match rel_prop { + Some(rel_prop) => OnceLock::from(rel_prop), + None => OnceLock::new(), + }), + stat_info: Arc::new(match stat_info { + Some(stat_info) => OnceLock::from(stat_info), + None => OnceLock::new(), + }), + applied_rules: AppliedRules::default(), + }, } } @@ -130,232 +146,24 @@ impl SExpr { Self::create(plan, vec![self.clone()], None, None, None) } - pub fn plan(&self) -> &RelOperator { - &self.plan - } - - pub fn children(&self) -> impl Iterator { - self.children.iter().map(|v| v.as_ref()) - } - - pub fn child(&self, n: usize) -> Result<&SExpr> { - self.children - .get(n) - .map(|v| v.as_ref()) - .ok_or_else(|| ErrorCode::Internal(format!("Invalid children index: {}", n))) - } - - pub fn unary_child(&self) -> &SExpr { - debug_assert_eq!(self.children.len(), 1); - &self.children[0] - } - - pub fn unary_child_arc(&self) -> Arc { - assert_eq!(self.children.len(), 1); - self.children[0].clone() - } - - pub fn left_child(&self) -> &SExpr { - debug_assert_eq!(self.children.len(), 2); - &self.children[0] - } - - pub fn left_child_arc(&self) -> Arc { - assert_eq!(self.children.len(), 2); - self.children[0].clone() - } - - pub fn right_child(&self) -> &SExpr { - debug_assert_eq!(self.children.len(), 2); - &self.children[1] - } - - pub fn right_child_arc(&self) -> Arc { - assert_eq!(self.children.len(), 2); - self.children[1].clone() - } - - pub fn build_side_child(&self) -> &SExpr { - debug_assert_eq!(self.plan.rel_op(), crate::plans::RelOp::Join); - &self.children[1] - } - - pub fn probe_side_child(&self) -> &SExpr { - debug_assert_eq!(self.plan.rel_op(), crate::plans::RelOp::Join); - &self.children[0] - } - - pub fn arity(&self) -> usize { - self.children.len() - } - pub fn original_group(&self) -> Option { - self.original_group - } - - /// Replace children with given new `children`. - /// Note that this method will keep the `applied_rules` of - /// current `SExpr` unchanged. - pub fn replace_children(&self, children: impl IntoIterator>) -> Self { - Self { - plan: self.plan.clone(), - original_group: None, - rel_prop: Default::default(), - stat_info: Default::default(), - applied_rules: self.applied_rules.clone(), - children: children.into_iter().collect(), - } - } - - pub fn replace_left_child(&self, left: impl Into>) -> Self { - assert_eq!(self.children.len(), 2); - Self { - plan: self.plan.clone(), - original_group: None, - rel_prop: Default::default(), - stat_info: Default::default(), - applied_rules: self.applied_rules.clone(), - children: vec![left.into(), self.children[1].clone()], - } - } - - pub fn replace_right_child(&self, right: impl Into>) -> Self { - assert_eq!(self.children.len(), 2); - Self { - plan: self.plan.clone(), - original_group: None, - rel_prop: Default::default(), - stat_info: Default::default(), - applied_rules: self.applied_rules.clone(), - children: vec![self.children[0].clone(), right.into()], - } - } - - pub fn replace_side_child(&self, side: Side, child: impl Into>) -> SExpr { - match side { - Side::Left => self.replace_left_child(child), - Side::Right => self.replace_right_child(child), - } - } - - pub fn replace_plan(&self, plan: impl Into>) -> Self { - Self { - plan: plan.into(), - original_group: None, - rel_prop: Default::default(), - stat_info: Default::default(), - applied_rules: self.applied_rules.clone(), - children: self.children.clone(), - } + self.state.original_group } /// Record the applied rule id in current SExpr pub(crate) fn set_applied_rule(&mut self, rule_id: &RuleID) { - self.applied_rules.set(rule_id, true); + self.state.applied_rules.set(rule_id, true); } /// Check if a rule is applied for current SExpr pub(crate) fn applied_rule(&self, rule_id: &RuleID) -> bool { - self.applied_rules.get(rule_id) - } - - #[recursive::recursive] - pub fn support_lazy_materialize(&self) -> bool { - self.plan.support_lazy_materialize() - && self - .children - .iter() - .all(|child| child.support_lazy_materialize()) - } - - #[recursive::recursive] - pub fn get_udfs(&self) -> Result> { - let mut udfs = HashSet::new(); - let iter = self.plan.scalar_expr_iter(); - for scalar in iter { - for udf in scalar.get_udf_names()? { - udfs.insert(udf); - } - } - - for child in &self.children { - let udf = child.get_udfs()?; - udf.iter().for_each(|udf| { - udfs.insert(*udf); - }) - } - Ok(udfs) - } - - #[recursive::recursive] - pub fn get_udfs_col_ids(&self) -> Result> { - let mut udf_ids = BTreeSet::new(); - if let RelOperator::Udf(udf) = self.plan.as_ref() { - for item in udf.items.iter() { - udf_ids.insert(item.index); - } - } - for child in &self.children { - let udfs = child.get_udfs_col_ids()?; - udf_ids.extend(udfs); - } - Ok(udf_ids) - } - - // Add column index to Scan nodes that match the given table index - pub fn add_column_index_to_scans(&self, table_index: IndexType, column_index: Symbol) -> SExpr { - let mut required_columns = BTreeMap::new(); - required_columns.insert(table_index, ScanRequiredColumns { - columns: BTreeSet::from([column_index]), - inverted_index: None, - vector_index: None, - }); - self.add_column_indexes_to_scans(&required_columns) - } - - // Add column indexes to Scan nodes that match the given table indexes. - pub fn add_column_indexes_to_scans( - &self, - required_columns: &BTreeMap, - ) -> SExpr { - struct Visitor<'a> { - required_columns: &'a BTreeMap, - } - - impl<'a> SExprVisitor for Visitor<'a> { - fn visit(&mut self, expr: &SExpr) -> Result { - if let Some(p) = expr.plan.as_ref().as_scan() { - if let Some(required_columns) = self.required_columns.get(&p.table_index) { - let mut p = p.clone(); - p.columns.extend(required_columns.columns.iter().copied()); - if required_columns.inverted_index.is_some() { - p.inverted_index = required_columns.inverted_index.clone(); - } - if required_columns.vector_index.is_some() { - p.vector_index = required_columns.vector_index.clone(); - } - let expr = expr.replace_plan(p); - return Ok(VisitAction::Replace(expr)); - } else { - return Ok(VisitAction::SkipChildren); - } - } - Ok(VisitAction::Continue) - } - } - - let mut visitor = Visitor { required_columns }; - let expr = self.accept(&mut visitor); - if let Ok(Some(expr)) = expr { - return expr; - } - self.clone() + self.state.applied_rules.get(rule_id) } // The method will clear the applied rules of current SExpr and its children. #[recursive::recursive] pub fn clear_applied_rules(&mut self) { - self.applied_rules.clear(); + self.state.applied_rules.clear(); let children = self .children() .map(|child| { @@ -366,76 +174,4 @@ impl SExpr { .collect::>(); self.children = children; } - - #[recursive::recursive] - pub fn has_merge_exchange(&self) -> bool { - if let RelOperator::Exchange(Exchange::Merge) = self.plan.as_ref() { - return true; - } - self.children.iter().any(|child| child.has_merge_exchange()) - } - - pub fn derive_relational_prop(&self) -> Result> { - let rel_prop = self.rel_prop.get_or_try_init(|| { - self.plan - .derive_relational_prop(&RelExpr::SExpr { expr: self }) - })?; - - Ok(rel_prop.clone()) - } - - pub fn get_data_distribution(&self) -> Result> { - struct DataDistributionVisitor { - result: Option, - } - impl SExprVisitor for DataDistributionVisitor { - fn visit(&mut self, expr: &SExpr) -> Result { - match expr.plan.as_ref() { - RelOperator::Exchange(exchange) => { - self.result = Some(exchange.clone()); - Ok(VisitAction::Stop) - } - - RelOperator::Join(_) => { - let child = expr.probe_side_child(); - self.result = child.get_data_distribution()?; - Ok(VisitAction::Stop) - } - _ => { - if expr.arity() > 0 { - Ok(VisitAction::Continue) - } else { - Ok(VisitAction::Stop) - } - } - } - } - } - - let mut visitor = DataDistributionVisitor { result: None }; - let _ = self.accept(&mut visitor); - Ok(visitor.result) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Side { - Left, - Right, -} - -impl Side { - pub fn opposite(self) -> Self { - match self { - Side::Left => Side::Right, - Side::Right => Side::Left, - } - } - - pub fn child(self, s_expr: &SExpr) -> Arc { - match self { - Side::Left => s_expr.left_child_arc(), - Side::Right => s_expr.right_child_arc(), - } - } } diff --git a/src/query/sql/src/planner/optimizer/ir/expr/type_validator.rs b/src/query/sql/src/planner/optimizer/ir/expr/type_validator.rs index 62a8009e594..81196cb1d3b 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/type_validator.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/type_validator.rs @@ -20,9 +20,10 @@ use databend_common_expression::aggregate::aggregate_function::RawAggregateCall; use databend_common_expression::types::DataType; use databend_common_functions::aggregates::AGGR_REGISTRY; -use super::SExpr; -use super::SExprVisitor; -use super::VisitAction; +use super::node::Expr; +use super::node::RelExprKind; +use super::visitor::ExprVisitor; +use super::visitor::VisitAction; use crate::MetadataRef; use crate::Symbol; use crate::plans::AggregateFunction; @@ -36,29 +37,33 @@ use crate::plans::ScalarItem; use crate::plans::Visitor as ScalarExprVisitor; use crate::plans::WindowFuncType; -impl SExpr { +impl Expr { /// Validate embedded type declarations against the types that can be inferred /// from scalar expressions and aggregate function signatures. /// /// Nullability is intentionally ignored when comparing a symbol with metadata: /// outer joins may add a nullable wrapper without changing the global symbol. pub fn validate_types(&self, metadata: &MetadataRef) -> Result<()> { - self.accept(&mut SExprTypeValidator { metadata }) - .map(|_| ()) + self.accept(&mut ExprTypeValidator { + metadata, + name: K::NAME, + }) + .map(|_| ()) } } -struct SExprTypeValidator<'a> { +struct ExprTypeValidator<'a> { + name: &'static str, metadata: &'a MetadataRef, } -impl SExprVisitor for SExprTypeValidator<'_> { - fn visit(&mut self, s_expr: &SExpr) -> Result { +impl ExprVisitor for ExprTypeValidator<'_> { + fn visit(&mut self, s_expr: &Expr) -> Result> { if let RelOperator::Scan(scan) = s_expr.plan() { self.validate_scan_scalars(scan)?; } else { for scalar in s_expr.plan().scalar_expr_iter() { - self.validate_scalar(scalar, SymbolTypeSource::create(self.metadata))?; + self.validate_scalar(scalar, SymbolTypeSource::create(self.metadata, self.name))?; } } @@ -93,7 +98,8 @@ impl SExprVisitor for SExprTypeValidator<'_> { || union.left_outputs.len() != union.output_indexes.len() { return Err(ErrorCode::Internal(format!( - "SExpr union output length mismatch: left {}, right {}, output {}", + "{} union output length mismatch: left {}, right {}, output {}", + self.name, union.left_outputs.len(), union.right_outputs.len(), union.output_indexes.len() @@ -124,7 +130,7 @@ impl SExprVisitor for SExprTypeValidator<'_> { } } -impl SExprTypeValidator<'_> { +impl ExprTypeValidator<'_> { fn validate_items(&self, items: &[ScalarItem]) -> Result<()> { for item in items { self.validate_symbol_type( @@ -140,7 +146,8 @@ impl SExprTypeValidator<'_> { let metadata_type = self.metadata_type(index)?; if metadata_type.remove_nullable() != actual.remove_nullable() { return Err(ErrorCode::Internal(format!( - "SExpr type mismatch for {source} {index}: metadata declares {metadata_type:?}, expression declares {actual:?}" + "{} type mismatch for {source} {index}: metadata declares {metadata_type:?}, expression declares {actual:?}", + self.name ))); } Ok(()) @@ -153,14 +160,17 @@ impl SExprTypeValidator<'_> { .get(index.as_usize()) .map(|column| column.data_type()) .ok_or_else(|| { - ErrorCode::Internal(format!("SExpr references unknown metadata symbol {index}")) + ErrorCode::Internal(format!( + "{} references unknown metadata symbol {index}", + self.name + )) }) } fn validate_window_function(&self, function: &WindowFuncType) -> Result<()> { if let WindowFuncType::Aggregate(aggregate) = function { ScalarTypeValidator { - symbol_types: SymbolTypeSource::create(self.metadata), + symbol_types: SymbolTypeSource::create(self.metadata, self.name), } .validate_aggregate_function(aggregate)?; } @@ -168,7 +178,7 @@ impl SExprTypeValidator<'_> { } fn validate_scan_scalars(&self, scan: &crate::plans::Scan) -> Result<()> { - let metadata_symbols = SymbolTypeSource::create(self.metadata); + let metadata_symbols = SymbolTypeSource::create(self.metadata, self.name); for scalar in scan .push_down_predicates .iter() @@ -193,12 +203,13 @@ impl SExprTypeValidator<'_> { #[derive(Clone, Copy)] struct SymbolTypeSource<'a> { + name: &'static str, metadata: &'a MetadataRef, } impl<'a> SymbolTypeSource<'a> { - fn create(metadata: &'a MetadataRef) -> SymbolTypeSource<'a> { - Self { metadata } + fn create(metadata: &'a MetadataRef, name: &'static str) -> SymbolTypeSource<'a> { + Self { metadata, name } } fn validate(&self, index: Symbol, actual: &DataType) -> Result<()> { @@ -208,13 +219,17 @@ impl<'a> SymbolTypeSource<'a> { .get(index.as_usize()) .map(|column| column.data_type()) .ok_or_else(|| { - ErrorCode::Internal(format!("SExpr references unknown metadata symbol {index}")) + ErrorCode::Internal(format!( + "{} references unknown metadata symbol {index}", + self.name + )) })?; let types_match = expected.remove_nullable() == actual.remove_nullable(); if !types_match { return Err(ErrorCode::Internal(format!( - "SExpr bound column type mismatch for {index}: source declares {expected:?}, expression declares {actual:?}" + "{} bound column type mismatch for {index}: source declares {expected:?}, expression declares {actual:?}", + self.name ))); } Ok(()) @@ -230,8 +245,8 @@ impl ScalarTypeValidator<'_> { let inferred = function.infer_return_type()?; if inferred != *function.return_type { return Err(ErrorCode::Internal(format!( - "SExpr function return type mismatch for {}: stored {:?}, inferred {inferred:?}", - function.func_name, function.return_type + "{} function return type mismatch for {}: stored {:?}, inferred {inferred:?}", + self.symbol_types.name, function.func_name, function.return_type ))); } Ok(()) @@ -242,8 +257,11 @@ impl ScalarTypeValidator<'_> { refreshed.refresh_return_type()?; if refreshed.return_type != function.return_type { return Err(ErrorCode::Internal(format!( - "SExpr lambda return type mismatch for {}: stored {:?}, expected {:?}", - function.func_name, function.return_type, refreshed.return_type + "{} lambda return type mismatch for {}: stored {:?}, expected {:?}", + self.symbol_types.name, + function.func_name, + function.return_type, + refreshed.return_type ))); } Ok(()) @@ -269,8 +287,8 @@ impl ScalarTypeValidator<'_> { .clone(); if inferred != *aggregate.return_type { return Err(ErrorCode::Internal(format!( - "SExpr aggregate return type mismatch for {}: stored {:?}, inferred {inferred:?}", - aggregate.display_name, aggregate.return_type + "{} aggregate return type mismatch for {}: stored {:?}, inferred {inferred:?}", + self.symbol_types.name, aggregate.display_name, aggregate.return_type ))); } Ok(()) diff --git a/src/query/sql/src/planner/optimizer/ir/expr/visitor.rs b/src/query/sql/src/planner/optimizer/ir/expr/visitor.rs index 8261213c146..eadb92ba98a 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/visitor.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/visitor.rs @@ -12,89 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # Expression Visitor Pattern -//! -//! This module implements the visitor pattern for traversing and transforming expression trees -//! in both synchronous and asynchronous contexts. The visitor pattern allows for separation of -//! algorithms from the structure they operate on, making it easier to add new operations without -//! modifying the expression classes. -//! -//! ## Examples -//! -//! ### Synchronous Visitor Example -//! -//! ```rust -//! use databend_common_exception::Result; -//! -//! use crate::optimizer::ir::expr::SExpr; -//! use crate::optimizer::ir::expr::SExprVisitor; -//! use crate::optimizer::ir::expr::VisitAction; -//! -//! // A visitor that counts the number of nodes in an expression tree -//! struct NodeCounter { -//! count: usize, -//! } -//! -//! impl SExprVisitor for NodeCounter { -//! fn visit(&mut self, _expr: &SExpr) -> Result { -//! self.count += 1; -//! Ok(VisitAction::Continue) // Continue traversing children -//! } -//! } -//! -//! // Usage: -//! // let expr = create_some_expression(); -//! // let mut counter = NodeCounter { count: 0 }; -//! // expr.accept(&mut counter)?; -//! // println!("Expression has {} nodes", counter.count); -//! ``` -//! -//! ### Asynchronous Visitor Example -//! -//! ```rust -//! use std::sync::Arc; -//! -//! use databend_common_exception::Result; -//! -//! use crate::optimizer::ir::expr::AsyncSExprVisitor; -//! use crate::optimizer::ir::expr::SExpr; -//! use crate::optimizer::ir::expr::VisitAction; -//! use crate::plans::RelOperator; -//! -//! // Visitor that eliminates unnecessary Sort operations with empty sort keys -//! struct EmptySortEliminator; -//! -//! #[async_trait::async_trait] -//! impl AsyncSExprVisitor for EmptySortEliminator { -//! async fn visit(&mut self, expr: &SExpr) -> Result { -//! // Check if this is a Sort operator with empty sort keys -//! if let RelOperator::Sort(sort) = expr.plan.as_ref() { -//! if sort.sort_keys.is_empty() { -//! // If sort has no keys, it's unnecessary - replace with its child -//! if expr.arity() == 1 { -//! return Ok(VisitAction::Replace(expr.unary_child().clone())); -//! } -//! } -//! } -//! Ok(VisitAction::Continue) -//! } -//! } -//! -//! // Usage example: -//! // let expr = create_some_expression(); -//! // let mut eliminator = EmptySortEliminator; -//! // let optimized_expr = expr.accept_async(&mut eliminator).await?; -//! ``` +//! Shared recursive traversal with stage-owned invalidation. use std::sync::Arc; use databend_common_exception::Result; -use crate::optimizer::ir::expr::SExpr; +use super::node::Expr; +use super::node::ExprKind; +use super::node::RewriteExprKind; +use crate::optimizer::ir::Logical; /// Action to take after visiting a node -#[derive(Clone)] -pub enum VisitAction { +#[derive(educe::Educe)] +#[educe(Clone(bound = false))] +pub enum VisitAction { /// Continue traversing the children Continue, /// Skip the children of the current node @@ -102,26 +34,29 @@ pub enum VisitAction { /// Stop the entire traversal Stop, /// Replace the current node with a new expression - Replace(SExpr), + Replace(Expr), } // Synchronous Visitor Implementation // -/// Visitor for SExpr nodes (synchronous version) -pub trait SExprVisitor { +/// Visitor for Expr nodes (synchronous version) +pub trait ExprVisitor { /// Visit an expression node - fn visit(&mut self, expr: &SExpr) -> Result; + fn visit(&mut self, expr: &Expr) -> Result>; /// Post-visit an expression node after its children have been visited - fn post_visit(&mut self, _expr: &SExpr) -> Result { + fn post_visit(&mut self, _expr: &Expr) -> Result> { Ok(VisitAction::Continue) } } /// Traverse an expression tree using a synchronous visitor #[recursive::recursive] -pub fn visit_sexpr(visitor: &mut V, expr: &SExpr) -> Result> { +pub fn visit_expr>( + visitor: &mut V, + expr: &Expr, +) -> Result>> { // Pre-order visit match visitor.visit(expr)? { VisitAction::Continue => {} @@ -140,7 +75,7 @@ pub fn visit_sexpr(visitor: &mut V, expr: &SExpr) -> Result(visitor: &mut V, expr: &SExpr) -> Result nodes (asynchronous version) #[async_trait::async_trait] -pub trait AsyncSExprVisitor { +pub trait AsyncExprVisitor { /// Visit an expression node - async fn visit(&mut self, expr: &SExpr) -> Result; + async fn visit(&mut self, expr: &Expr) -> Result>; /// Post-visit an expression node after its children have been visited - async fn post_visit(&mut self, _expr: &SExpr) -> Result { + async fn post_visit(&mut self, _expr: &Expr) -> Result> { Ok(VisitAction::Continue) } } @@ -186,10 +121,13 @@ pub trait AsyncSExprVisitor { /// Traverse an expression tree using an async visitor #[allow(clippy::multiple_bound_locations)] #[async_recursion::async_recursion(# [recursive::recursive])] -pub async fn visit_sexpr_async( - visitor: &mut T, - expr: &SExpr, -) -> Result> { +pub async fn visit_expr_async(visitor: &mut T, expr: &Expr) -> Result>> +where + K: RewriteExprKind + Send + Sync, + K::Operator: Send + Sync, + K::State: Send + Sync, + T: AsyncExprVisitor + Send, +{ // Pre-order visit match visitor.visit(expr).await? { VisitAction::Continue => {} @@ -208,7 +146,7 @@ pub async fn visit_sexpr_async( let mut children_changed = false; for child in expr.children.iter() { - if let Some(new_child) = visit_sexpr_async(visitor, child).await? { + if let Some(new_child) = visit_expr_async(visitor, child).await? { children.push(Arc::new(new_child)); children_changed = true; } else { @@ -236,18 +174,23 @@ pub async fn visit_sexpr_async( } } -/// Extension methods for SExpr -impl SExpr { +/// Extension methods for Expr +impl Expr { /// Apply a synchronous visitor to this expression - pub fn accept(&self, visitor: &mut V) -> Result> { - visit_sexpr(visitor, self) + pub fn accept>(&self, visitor: &mut V) -> Result>> { + visit_expr(visitor, self) } /// Apply an asynchronous visitor to this expression - pub async fn accept_async( + pub async fn accept_async + Send>( &self, visitor: &mut T, - ) -> Result> { - visit_sexpr_async(visitor, self).await + ) -> Result>> + where + K: Send + Sync, + K::Operator: Send + Sync, + K::State: Send + Sync, + { + visit_expr_async(visitor, self).await } } diff --git a/src/query/sql/src/planner/optimizer/ir/memo.rs b/src/query/sql/src/planner/optimizer/ir/memo.rs index 091a394a23e..80414760069 100644 --- a/src/query/sql/src/planner/optimizer/ir/memo.rs +++ b/src/query/sql/src/planner/optimizer/ir/memo.rs @@ -116,7 +116,7 @@ impl Memo { self.group(group_index)?.num_exprs(), s_expr.plan, children_group, - s_expr.applied_rules, + s_expr.state.applied_rules, ); self.insert_m_expr(group_index, m_expr)?; diff --git a/src/query/sql/src/planner/optimizer/ir/mod.rs b/src/query/sql/src/planner/optimizer/ir/mod.rs index a00aae20fbf..f7032888df3 100644 --- a/src/query/sql/src/planner/optimizer/ir/mod.rs +++ b/src/query/sql/src/planner/optimizer/ir/mod.rs @@ -16,14 +16,25 @@ mod expr; mod format; mod group; mod memo; +mod mutation_plan; mod property; +mod query_plan; mod stats; pub use databend_common_statistics::HistogramBuilder; +pub use expr::AsyncExprVisitor; pub use expr::AsyncSExprVisitor; +pub use expr::Expr; +pub use expr::ExprKind; +pub use expr::ExprVisitor; +pub use expr::Logical; pub use expr::MExpr; pub use expr::Matcher; +pub use expr::PExpr; pub use expr::PatternExtractor; +pub use expr::Physical; +pub use expr::RelExprKind; +pub use expr::RewriteExprKind; pub use expr::SExpr; pub use expr::SExprVisitor; pub use expr::ScanRequiredColumns; @@ -32,6 +43,7 @@ pub use expr::VisitAction; pub use group::Group; pub use group::GroupState; pub use memo::Memo; +pub use mutation_plan::MutationPlan; pub use property::Distribution; pub use property::DistributionEnforcer; pub use property::Enforcer; @@ -42,6 +54,8 @@ pub use property::RelationalProperty; pub use property::RequiredProperty; pub use property::StatInfo; pub use property::Statistics; +pub use query_plan::PlannedQuery; +pub use query_plan::QueryPlan; pub use stats::ColumnStat; pub use stats::ColumnStatSet; pub use stats::CountMinSketchSet; @@ -51,3 +65,4 @@ pub use stats::SelectivityEstimator; pub use stats::StatContext; pub use stats::TopNSet; pub(crate) use stats::cap_stat_info_by_rows; +pub type PVisitAction = VisitAction; diff --git a/src/query/sql/src/planner/optimizer/ir/mutation_plan.rs b/src/query/sql/src/planner/optimizer/ir/mutation_plan.rs new file mode 100644 index 00000000000..0ddb7ae581e --- /dev/null +++ b/src/query/sql/src/planner/optimizer/ir/mutation_plan.rs @@ -0,0 +1,84 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; + +use super::PExpr; +use crate::optimizer::ir::SExpr; +use crate::plans::Mutation; + +/// Mutation binding and input-plan selection have distinct tree representations. +/// Execution accepts only the planned tree; logical preparation never sees PExpr. +#[derive(Clone, Debug)] +pub enum MutationPlan { + Logical(SExpr), + Planned(PExpr), +} + +impl From for MutationPlan { + fn from(expr: SExpr) -> Self { + Self::Logical(expr) + } +} + +impl MutationPlan { + pub fn logical(&self) -> Result<&SExpr> { + match self { + Self::Logical(expr) => Ok(expr), + Self::Planned(_) => Err(ErrorCode::Internal("Expected a bound logical mutation")), + } + } + + pub fn into_logical(self) -> Result { + match self { + Self::Logical(expr) => Ok(expr), + Self::Planned(_) => Err(ErrorCode::Internal("Expected a bound logical mutation")), + } + } + + pub fn planned(&self) -> Result<&PExpr> { + match self { + Self::Planned(expr) => Ok(expr), + Self::Logical(_) => Err(ErrorCode::Internal( + "Mutation must be physically planned before execution", + )), + } + } + + pub fn into_planned(self) -> Result { + match self { + Self::Planned(expr) => Ok(expr), + Self::Logical(_) => Err(ErrorCode::Internal( + "Mutation must be physically planned before execution", + )), + } + } + + pub fn mutation(&self) -> Result<&Mutation> { + let plan = match self { + Self::Logical(expr) => expr.plan(), + Self::Planned(expr) => expr.plan(), + }; + plan.as_mutation() + .ok_or_else(|| ErrorCode::Internal("Expected a mutation root")) + } + + pub fn input_udfs(&self) -> Result> { + match self { + Self::Logical(expr) => expr.child(0)?.get_udfs(), + Self::Planned(expr) => expr.child(0)?.get_udfs(), + } + } +} diff --git a/src/query/sql/src/planner/optimizer/ir/property/builder.rs b/src/query/sql/src/planner/optimizer/ir/property/builder.rs index 473adc46aa6..a477902888e 100644 --- a/src/query/sql/src/planner/optimizer/ir/property/builder.rs +++ b/src/query/sql/src/planner/optimizer/ir/property/builder.rs @@ -19,6 +19,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use crate::IndexType; +use crate::optimizer::ir::PExpr; use crate::optimizer::ir::StatContext; use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::expr::MExpr; @@ -32,6 +33,9 @@ use crate::plans::Operator; /// A helper to access children of `SExpr` and `MExpr` in /// a unified view. pub enum RelExpr<'a> { + PExpr { + expr: &'a PExpr, + }, SExpr { expr: &'a SExpr, }, @@ -47,6 +51,10 @@ pub enum RelExpr<'a> { } impl<'a> RelExpr<'a> { + pub fn with_p_expr(expr: &'a PExpr) -> Self { + Self::PExpr { expr } + } + pub fn with_s_expr(s_expr: &'a SExpr) -> Self { Self::SExpr { expr: s_expr } } @@ -70,6 +78,7 @@ impl<'a> RelExpr<'a> { #[recursive::recursive] pub fn derive_relational_prop(&self) -> Result> { match self { + RelExpr::PExpr { expr } => expr.derive_relational_prop(), RelExpr::SExpr { expr } => expr.derive_relational_prop(), RelExpr::MExpr { expr, .. } => expr.plan.derive_relational_prop(self), RelExpr::OptContext { expr, .. } => expr.plan.derive_relational_prop(self), @@ -78,6 +87,11 @@ impl<'a> RelExpr<'a> { pub fn derive_relational_prop_child(&self, index: usize) -> Result> { match self { + RelExpr::PExpr { expr } => { + let child = expr.child(index)?; + let rel_expr = RelExpr::with_p_expr(child); + rel_expr.derive_relational_prop() + } RelExpr::SExpr { expr } => { let child = expr.child(index)?; let rel_expr = RelExpr::with_s_expr(child); @@ -98,12 +112,8 @@ impl<'a> RelExpr<'a> { #[recursive::recursive] pub fn derive_cardinality(&self, stat_ctx: &StatContext) -> Result> { match self { - RelExpr::SExpr { expr } => { - let stat_info = expr - .stat_info - .get_or_try_init(|| expr.plan.derive_stats(self, stat_ctx))?; - Ok(stat_info.clone()) - } + RelExpr::PExpr { expr } => expr.derive_cardinality(stat_ctx), + RelExpr::SExpr { expr } => expr.derive_cardinality(stat_ctx), RelExpr::MExpr { expr, .. } => expr.plan.derive_stats(self, stat_ctx), RelExpr::OptContext { expr, .. } => expr.plan.derive_stats(self, stat_ctx), } @@ -115,6 +125,11 @@ impl<'a> RelExpr<'a> { stat_ctx: &StatContext, ) -> Result> { match self { + RelExpr::PExpr { expr } => { + let child = expr.child(index)?; + let rel_expr = RelExpr::with_p_expr(child); + rel_expr.derive_cardinality(stat_ctx) + } RelExpr::SExpr { expr } => { let child = expr.child(index)?; let rel_expr = RelExpr::with_s_expr(child); @@ -130,6 +145,7 @@ impl<'a> RelExpr<'a> { #[recursive::recursive] pub fn derive_physical_prop(&self) -> Result { let plan = match self { + RelExpr::PExpr { expr } => expr.plan(), RelExpr::SExpr { expr } => expr.plan(), RelExpr::MExpr { expr, .. } => &expr.plan, RelExpr::OptContext { expr, .. } => &expr.plan, @@ -141,6 +157,11 @@ impl<'a> RelExpr<'a> { pub fn derive_physical_prop_child(&self, index: usize) -> Result { match self { + RelExpr::PExpr { expr } => { + let child = expr.child(index)?; + let rel_expr = RelExpr::with_p_expr(child); + rel_expr.derive_physical_prop() + } RelExpr::SExpr { expr } => { let child = expr.child(index)?; let rel_expr = RelExpr::with_s_expr(child); @@ -166,6 +187,7 @@ impl<'a> RelExpr<'a> { input: &RequiredProperty, ) -> Result { let plan = match self { + RelExpr::PExpr { expr } => expr.plan(), RelExpr::SExpr { expr } => expr.plan(), RelExpr::MExpr { expr, .. } => &expr.plan, RelExpr::OptContext { expr, .. } => &expr.plan, @@ -181,6 +203,7 @@ impl<'a> RelExpr<'a> { input: &RequiredProperty, ) -> Result>> { let plan = match self { + RelExpr::PExpr { expr } => expr.plan(), RelExpr::SExpr { expr } => expr.plan(), RelExpr::MExpr { expr, .. } => &expr.plan, RelExpr::OptContext { expr, .. } => &expr.plan, diff --git a/src/query/sql/src/planner/optimizer/ir/property/enforcer.rs b/src/query/sql/src/planner/optimizer/ir/property/enforcer.rs index 093b466a986..1cc67a6be4a 100644 --- a/src/query/sql/src/planner/optimizer/ir/property/enforcer.rs +++ b/src/query/sql/src/planner/optimizer/ir/property/enforcer.rs @@ -21,7 +21,7 @@ use databend_common_expression::type_check::common_super_type; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::wrap_cast; -use crate::optimizer::ir::expr::SExpr; +use crate::optimizer::ir::PExpr; use crate::optimizer::ir::property::Distribution; use crate::optimizer::ir::property::PhysicalProperty; use crate::optimizer::ir::property::RelExpr; @@ -94,17 +94,17 @@ impl PropertyEnforcer { Self { ctx } } - /// Require and enforce physical property from a physical `SExpr` + /// Require and enforce physical property from a physical `PExpr` #[recursive::recursive] - pub fn require_property(&self, required: &RequiredProperty, s_expr: &SExpr) -> Result { - // First, we will require the child SExpr with input `RequiredProperty`. + pub fn require_property(&self, required: &RequiredProperty, s_expr: &PExpr) -> Result { + // First, we will require the child PExpr with input `RequiredProperty`. let children = s_expr .children() .map(|child| Ok(Arc::new(self.require_property(required, child)?))) .collect::>>()?; - let s_expr = SExpr::create(Arc::new(s_expr.plan().clone()), children, None, None, None); - let rel_expr = RelExpr::with_s_expr(&s_expr); + let s_expr = PExpr::create(Arc::new(s_expr.plan().clone()), children, None, None, None); + let rel_expr = RelExpr::with_p_expr(&s_expr); // Prepare containers for child properties let mut children = Vec::with_capacity(s_expr.arity()); @@ -182,11 +182,11 @@ impl PropertyEnforcer { children.push(Arc::new(enforced_child)); } - Ok(SExpr::create(Arc::new(plan), children, None, None, None)) + Ok(PExpr::create(Arc::new(plan), children, None, None, None)) } - /// Try to enforce physical property from a physical `SExpr` - pub fn enforce_property(&self, s_expr: &SExpr, required: &RequiredProperty) -> Result { + /// Try to enforce physical property from a physical `PExpr` + pub fn enforce_property(&self, s_expr: &PExpr, required: &RequiredProperty) -> Result { // Enforce distribution if needed let s_expr = if !required.distribution.satisfied_by(&Distribution::Any) { self.enforce_distribution(&required.distribution, s_expr)? @@ -201,9 +201,9 @@ impl PropertyEnforcer { pub fn enforce_distribution( &self, distribution: &Distribution, - s_expr: &SExpr, - ) -> Result { - let physical_prop = RelExpr::with_s_expr(s_expr).derive_physical_prop()?; + s_expr: &PExpr, + ) -> Result { + let physical_prop = RelExpr::with_p_expr(s_expr).derive_physical_prop()?; // Check if enforcement is needed if distribution.satisfied_by(&physical_prop.distribution) { @@ -215,7 +215,7 @@ impl PropertyEnforcer { // Apply the enforcer let exchange_op = enforcer.enforce()?; - let result = SExpr::create_unary(Arc::new(exchange_op), Arc::new(s_expr.clone())); + let result = PExpr::create_unary(Arc::new(exchange_op), Arc::new(s_expr.clone())); Ok(result) } diff --git a/src/query/sql/src/planner/optimizer/ir/query_plan.rs b/src/query/sql/src/planner/optimizer/ir/query_plan.rs new file mode 100644 index 00000000000..ef6c2dabe0b --- /dev/null +++ b/src/query/sql/src/planner/optimizer/ir/query_plan.rs @@ -0,0 +1,108 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; + +use super::PExpr; +use crate::optimizer::ir::SExpr; + +/// Query output from physical planning, including the enabled finalization passes. +/// Its private physical expression evolves independently of logical SExpr. +#[derive(Clone, Debug)] +pub struct PlannedQuery { + expr: PExpr, +} + +impl PlannedQuery { + pub(in crate::planner::optimizer) fn new(expr: PExpr) -> Self { + Self { expr } + } + + /// Selected physical expression, independent of the logical expression type. + pub fn expr(&self) -> &PExpr { + &self.expr + } + + pub(crate) fn into_expr(self) -> PExpr { + self.expr + } + + pub(crate) fn remove_root_merge(&self) -> Self { + if matches!( + self.expr.plan(), + crate::plans::RelOperator::Exchange(crate::plans::Exchange::Merge) + ) { + Self::new(self.expr.unary_child().clone()) + } else { + self.clone() + } + } +} + +/// Statement container for the two query lifecycles. Logical optimizers never accept +/// this enum: they consume SExpr; execution consumes only PlannedQuery. +#[derive(Clone, Debug)] +pub enum QueryPlan { + Logical(SExpr), + Planned(PlannedQuery), +} + +impl From for QueryPlan { + fn from(expr: SExpr) -> Self { + Self::Logical(expr) + } +} + +impl QueryPlan { + pub fn into_logical(self) -> Result { + match self { + Self::Logical(expr) => Ok(expr), + Self::Planned(_) => Err(ErrorCode::Internal( + "Expected a bound logical query, not a planned query", + )), + } + } + + pub fn planned(&self) -> Result<&PlannedQuery> { + match self { + Self::Planned(plan) => Ok(plan), + Self::Logical(_) => Err(ErrorCode::Internal( + "Query must be physically planned before execution", + )), + } + } + + /// Inspect a logical query; planned queries must be accessed through `planned`. + pub fn logical(&self) -> Result<&SExpr> { + match self { + Self::Logical(expr) => Ok(expr), + Self::Planned(_) => Err(ErrorCode::Internal("Expected logical query")), + } + } + + pub fn get_udfs(&self) -> Result> { + match self { + Self::Logical(expr) => expr.get_udfs(), + Self::Planned(plan) => plan.expr().get_udfs(), + } + } + + pub(crate) fn remove_root_merge(&self) -> Self { + match self { + Self::Planned(plan) => Self::Planned(plan.remove_root_merge()), + Self::Logical(_) => self.clone(), + } + } +} diff --git a/src/query/sql/src/planner/optimizer/mod.rs b/src/query/sql/src/planner/optimizer/mod.rs index cb7966450a1..b70e6604866 100644 --- a/src/query/sql/src/planner/optimizer/mod.rs +++ b/src/query/sql/src/planner/optimizer/mod.rs @@ -22,6 +22,7 @@ mod optimizer; mod optimizer_api; mod optimizer_context; pub mod optimizers; +mod physical_planner; pub mod pipeline; mod statistics; @@ -29,5 +30,6 @@ pub use optimizer::optimize; pub use optimizer::optimize_query; pub use optimizer_api::Optimizer; pub use optimizer_context::OptimizerContext; +pub use physical_planner::PhysicalPlanner; pub use statistics::collect_statistics::CollectStatisticsOptimizer; pub use statistics::collect_statistics::StatisticsTraceCollector; diff --git a/src/query/sql/src/planner/optimizer/mutation.rs b/src/query/sql/src/planner/optimizer/mutation.rs index 8751ba52b40..66a37b3de3e 100644 --- a/src/query/sql/src/planner/optimizer/mutation.rs +++ b/src/query/sql/src/planner/optimizer/mutation.rs @@ -18,16 +18,17 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use super::optimizer::query_logical_pipeline; -use super::optimizer::query_planning_pipeline; use crate::binder::MutationStrategy; use crate::binder::MutationType; -use crate::binder::target_probe; use crate::optimizer::OptimizerContext; +use crate::optimizer::PhysicalPlanner; +use crate::optimizer::ir::MutationPlan; +use crate::optimizer::ir::PExpr; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::distributed::BroadcastToShuffleOptimizer; use crate::optimizer::optimizers::recursive::RecursiveRuleOptimizer; use crate::optimizer::optimizers::rule::RuleID; -use crate::optimizer::pipeline::OptimizerPipeline; +use crate::optimizer::pipeline::configure_distributed_optimization; use crate::plans::Join; use crate::plans::JoinType; use crate::plans::MatchedEvaluator; @@ -81,16 +82,17 @@ impl PreparedMutation { } } -/// Select and finalize the input using the legacy mutation distribution policy. A local +/// Select and finalize the input using the mutation distribution policy. A local /// retry reuses the prepared logical input, rather than rerunning preparation on raw SQL. -async fn plan_input(opt_ctx: Arc, input: SExpr, local: bool) -> Result { - let pipeline = OptimizerPipeline::new(opt_ctx.clone(), input).await?; - if local { - // Pipeline configuration must not re-enable distribution for warehouse tables. - opt_ctx.set_enable_distributed_optimization(false); - } - let mut pipeline = query_planning_pipeline(opt_ctx, pipeline)?; - pipeline.execute().await +async fn plan_input(opt_ctx: Arc, input: SExpr, local: bool) -> Result { + configure_distributed_optimization(&opt_ctx, &input).await?; + let mut planner = PhysicalPlanner::new(opt_ctx); + let planned = if local { + planner.plan_local(input).await? + } else { + planner.plan(input).await? + }; + Ok(planned.into_expr()) } pub(super) async fn optimize_mutation( @@ -102,29 +104,29 @@ pub(super) async fn optimize_mutation( input, schema, } = PreparedMutation::prepare(opt_ctx.clone(), &s_expr).await?; - let mut input_s_expr = plan_input(opt_ctx.clone(), input.clone(), false).await?; + let mut planned_input = plan_input(opt_ctx.clone(), input.clone(), false).await?; - // Preserve the legacy mutation consumer policy until requirements are part of + // Apply the mutation consumer policy until requirements are part of // physical search: discard the query-root Exchange and retry locally if necessary. - if matches!(input_s_expr.plan(), RelOperator::Exchange(_)) { - input_s_expr = input_s_expr.child(0)?.clone(); + if matches!(planned_input.plan(), RelOperator::Exchange(_)) { + planned_input = planned_input.child(0)?.clone(); } - if input_s_expr.has_merge_exchange() { - input_s_expr = plan_input(opt_ctx.clone(), input, true).await?; + if planned_input.has_merge_exchange() { + planned_input = plan_input(opt_ctx.clone(), input, true).await?; } mutation.distributed = opt_ctx.get_enable_distributed_optimization(); - let inner_rel_op = input_s_expr.plan.rel_op(); - input_s_expr = match mutation.mutation_type { + let inner_rel_op = planned_input.plan.rel_op(); + planned_input = match mutation.mutation_type { MutationType::Merge => { if mutation.distributed && inner_rel_op == RelOp::Join { - let join = Join::try_from(input_s_expr.plan().clone())?; + let join = Join::try_from(planned_input.plan().clone())?; let broadcast_to_shuffle = BroadcastToShuffleOptimizer::create(); - let is_broadcast = broadcast_to_shuffle.matcher.matches(&input_s_expr) - && broadcast_to_shuffle.is_broadcast(&input_s_expr)?; + let is_broadcast = broadcast_to_shuffle.matcher.matches(&planned_input) + && broadcast_to_shuffle.is_broadcast(&planned_input)?; // If the mutation strategy is matched only, the join type is inner join, if it is a broadcast // join and the target table on the probe side, we can avoid row id shuffle after the join. - let target_probe = target_probe(&input_s_expr, mutation.target_table_index)?; + let target_probe = target_probe(&planned_input, mutation.target_table_index)?; if is_broadcast && target_probe && mutation.strategy == MutationStrategy::MatchedOnly @@ -135,23 +137,23 @@ pub(super) async fn optimize_mutation( // Change broadcast join to shuffle join if the join type is left or left-anti join, because // broadcast join can not deduplicate row ids. if is_broadcast && matches!(join.join_type, JoinType::Left | JoinType::LeftAnti) { - broadcast_to_shuffle.optimize(&input_s_expr)? + broadcast_to_shuffle.optimize(&planned_input)? } else { - input_s_expr + planned_input } } else { - input_s_expr + planned_input } } - MutationType::Update | MutationType::Delete => input_s_expr, + MutationType::Update | MutationType::Delete => planned_input, }; Ok(Plan::DataMutation { schema, - s_expr: Box::new(SExpr::create_unary( + s_expr: Box::new(MutationPlan::Planned(PExpr::create_unary( Arc::new(RelOperator::Mutation(mutation)), - Arc::new(input_s_expr), - )), + Arc::new(planned_input), + ))), metadata: opt_ctx.get_metadata(), }) } @@ -211,3 +213,21 @@ fn prepare_direct_source(s_expr: &SExpr, mutation: &mut Mutation) -> Result Ok(None), } } + +fn target_probe(s_expr: &PExpr, target_table_index: usize) -> Result { + if !matches!(s_expr.plan(), RelOperator::Join(_)) { + return Ok(false); + } + + fn contains_target_table(s_expr: &PExpr, target_table_index: usize) -> bool { + if let RelOperator::Scan(scan) = s_expr.plan() { + scan.table_index == target_table_index + } else { + s_expr + .children() + .any(|child| contains_target_table(child, target_table_index)) + } + } + + Ok(contains_target_table(s_expr.child(0)?, target_table_index)) +} diff --git a/src/query/sql/src/planner/optimizer/optimizer.rs b/src/query/sql/src/planner/optimizer/optimizer.rs index 4f61243d14f..fd13ad6ef61 100644 --- a/src/query/sql/src/planner/optimizer/optimizer.rs +++ b/src/query/sql/src/planner/optimizer/optimizer.rs @@ -23,17 +23,17 @@ use log::info; use crate::InsertInputSource; use crate::optimizer::OptimizerContext; +use crate::optimizer::PhysicalPlanner; use crate::optimizer::ir::Memo; +use crate::optimizer::ir::PlannedQuery; +use crate::optimizer::ir::QueryPlan; use crate::optimizer::ir::SExpr; use crate::optimizer::mutation::optimize_mutation; use crate::optimizer::optimizers::CTEFilterPushdownOptimizer; -use crate::optimizer::optimizers::CascadesOptimizer; use crate::optimizer::optimizers::CommonSubexpressionOptimizer; use crate::optimizer::optimizers::DPhpyOptimizer; use crate::optimizer::optimizers::EliminateSelfJoinOptimizer; -use crate::optimizer::optimizers::operator::CleanupUnusedCTEOptimizer; use crate::optimizer::optimizers::operator::DeduplicateJoinConditionOptimizer; -use crate::optimizer::optimizers::operator::FinalizeSpatialJoinOptimizer; use crate::optimizer::optimizers::operator::PullUpFilterOptimizer; use crate::optimizer::optimizers::operator::RuleNormalizeAggregateOptimizer; use crate::optimizer::optimizers::operator::RuleStatsAggregateOptimizer; @@ -70,8 +70,13 @@ pub async fn optimize(opt_ctx: Arc, plan: Plan) -> Result, plan: Plan) -> Result, plan: Plan) -> Result { if let deref!( Plan::Query { ref s_expr, .. }) = plan { - let memo = get_optimized_memo(opt_ctx.clone(), *s_expr.clone()).await?; + let memo = + get_optimized_memo(opt_ctx.clone(), s_expr.logical()?.clone()).await?; Ok(Plan::Explain { config, kind: ExplainKind::Memo(memo.display()?), @@ -173,7 +180,9 @@ pub async fn optimize(opt_ctx: Arc, plan: Plan) -> Result optimize_mutation(opt_ctx, *s_expr).await, + Plan::DataMutation { s_expr, .. } => { + optimize_mutation(opt_ctx, (*s_expr).into_logical()?).await + } // distributed insert will be optimized in `physical_plan_builder` Plan::Insert(mut plan) => { @@ -207,10 +216,10 @@ pub async fn optimize(opt_ctx: Arc, plan: Plan) -> Result, plan: Plan) -> Result, s_expr: SExpr) -> Result { +pub async fn optimize_query(opt_ctx: Arc, s_expr: SExpr) -> Result { optimize_query_inner(opt_ctx, s_expr, None).await } @@ -278,7 +287,7 @@ async fn optimize_query_with_output_columns( opt_ctx: Arc, s_expr: SExpr, output_columns: std::collections::HashSet, -) -> Result { +) -> Result { optimize_query_inner(opt_ctx, s_expr, Some(output_columns)).await } @@ -286,10 +295,13 @@ async fn optimize_query_inner( opt_ctx: Arc, s_expr: SExpr, output_columns: Option>, -) -> Result { - let pipeline = query_logical_pipeline(opt_ctx.clone(), s_expr, output_columns).await?; - let mut pipeline = query_planning_pipeline(opt_ctx, pipeline)?; - pipeline.execute().await +) -> Result { + let mut pipeline = query_logical_pipeline(opt_ctx.clone(), s_expr, output_columns).await?; + let input = pipeline.execute().await?; + PhysicalPlanner::new(opt_ctx) + .with_trace_collector(pipeline.get_trace_collector(), pipeline.num_optimizers()) + .plan(input) + .await } /// Build the common logical passes without selecting distributions or execution plans. @@ -353,26 +365,6 @@ pub(super) async fn query_logical_pipeline( Ok(pipeline) } -/// Append the existing planning and cleanup passes. Ordinary queries keep a single -/// pipeline; mutation inputs enter here only after their logical preparation. -pub(super) fn query_planning_pipeline( - opt_ctx: Arc, - pipeline: OptimizerPipeline, -) -> Result { - Ok(pipeline - // Cascades optimizer may fail due to timeout, fallback to heuristic optimizer in this case. - .add(CascadesOptimizer::new(opt_ctx.clone())?) - // Eliminate unnecessary scalar calculations to clean up the final plan - .add(RecursiveRuleOptimizer::new( - opt_ctx.clone(), - [RuleID::EliminateEvalScalar].as_slice(), - )) - // Clean up unused CTEs - .add(CleanupUnusedCTEOptimizer) - // Finalize derived join annotations after all logical rewrites. - .add(FinalizeSpatialJoinOptimizer::new(opt_ctx.clone()))) -} - fn rewrite_insert_multi_table_whens( opt_ctx: Arc, plan: &mut crate::plans::InsertMultiTable, @@ -381,7 +373,7 @@ fn rewrite_insert_multi_table_whens( return Ok(()); }; - let mut source_expr = s_expr.as_ref().clone(); + let mut source_expr = s_expr.logical()?.clone(); let mut rewritten_any = false; for (idx, when) in plan.whens.iter_mut().enumerate() { @@ -420,7 +412,7 @@ fn rewrite_insert_multi_table_whens( } if rewritten_any { - *s_expr = Box::new(source_expr); + *s_expr = Box::new(QueryPlan::Logical(source_expr)); } Ok(()) @@ -446,10 +438,11 @@ async fn get_optimized_memo(opt_ctx: Arc, s_expr: SExpr) -> Re RuleID::SplitAggregate, ])) // Cost based optimization - .add(DPhpyOptimizer::new(opt_ctx.clone())) - .add(CascadesOptimizer::new(opt_ctx.clone())?); - - let _s_expr = pipeline.execute().await?; + .add(DPhpyOptimizer::new(opt_ctx.clone())); - Ok(pipeline.memo()) + let input = pipeline.execute().await?; + PhysicalPlanner::new(opt_ctx) + .with_trace_collector(pipeline.get_trace_collector(), pipeline.num_optimizers()) + .search_memo(input) + .await } diff --git a/src/query/sql/src/planner/optimizer/optimizer_api.rs b/src/query/sql/src/planner/optimizer/optimizer_api.rs index a683199f198..5db4cbc9b30 100644 --- a/src/query/sql/src/planner/optimizer/optimizer_api.rs +++ b/src/query/sql/src/planner/optimizer/optimizer_api.rs @@ -16,24 +16,18 @@ use std::sync::Arc; use databend_common_exception::Result; -use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; use crate::optimizer::pipeline::OptimizerTraceCollector; -/// Trait defining the interface for query optimizers. +/// Interface for same-type rewrite passes. Logical passes use SExpr by default; +/// physical passes use PExpr. Logical-to-physical search has a separate entry point. #[async_trait::async_trait] -pub trait Optimizer: Send + Sync { +pub trait Optimizer: Send + Sync { /// Returns a unique identifier for this optimizer. fn name(&self) -> String; /// Consume the given expression and return the optimized version. - async fn optimize(&mut self, expr: SExpr) -> Result; - - /// Get the memo if this optimizer maintains one. - /// Default implementation returns None for optimizers that don't use a memo. - fn memo(&self) -> Option<&Memo> { - None - } + async fn optimize(&mut self, expr: Expr) -> Result; /// Set the trace collector for this optimizer. /// Default implementation does nothing. diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs index fa26abfa191..b3f1eb83188 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs @@ -20,11 +20,11 @@ use log::debug; use log::info; use crate::IndexType; -use crate::optimizer::Optimizer; use crate::optimizer::OptimizerContext; use crate::optimizer::cost::CostModel; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::Memo; +use crate::optimizer::ir::PExpr; use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::cascades::cost::DefaultCostModel; @@ -50,6 +50,8 @@ pub struct CascadesOptimizer { } impl CascadesOptimizer { + pub const NAME: &'static str = "CascadesOptimizer"; + pub fn new(opt_ctx: Arc) -> Result { let table_ctx = opt_ctx.get_table_ctx(); let settings = table_ctx.get_settings(); @@ -75,6 +77,10 @@ impl CascadesOptimizer { }) } + pub fn memo(&self) -> &Memo { + &self.memo + } + pub(crate) fn enforce_distribution(&self) -> bool { self.opt_ctx.get_enable_distributed_optimization() } @@ -86,7 +92,7 @@ impl CascadesOptimizer { } #[recursive::recursive] - pub fn optimize_sync(&mut self, s_expr: SExpr) -> Result { + pub fn optimize_sync(&mut self, s_expr: SExpr) -> Result { let opt_ctx = self.opt_ctx.clone(); let distributed = opt_ctx.get_enable_distributed_optimization(); @@ -128,7 +134,7 @@ impl CascadesOptimizer { Ok(optimized_expr) } - fn optimize_without_staging(&mut self, s_expr: &SExpr) -> Result { + fn optimize_without_staging(&mut self, s_expr: &SExpr) -> Result { match self.optimize_internal(s_expr.clone()) { Ok(expr) => Ok(expr), Err(e) => { @@ -139,15 +145,15 @@ impl CascadesOptimizer { if self.opt_ctx.get_enable_distributed_optimization() { let distributed_optimizer = DistributedOptimizer::new(self.opt_ctx.clone()); - distributed_optimizer.optimize(s_expr) + distributed_optimizer.optimize(&PExpr::from(s_expr.clone())) } else { - Ok(s_expr.clone()) + Ok(PExpr::from(s_expr.clone())) } } } } - fn optimize_internal(&mut self, s_expr: SExpr) -> Result { + fn optimize_internal(&mut self, s_expr: SExpr) -> Result { // Update rule set based on current flags // This ensures we use the most up-to-date flag values, regardless of when the optimizer was created let table_ctx = self.opt_ctx.get_table_ctx(); @@ -243,7 +249,7 @@ impl CascadesOptimizer { &self, group_index: IndexType, required_property: &RequiredProperty, - ) -> Result { + ) -> Result { let group = self.memo.group(group_index)?; let cost_context = group.best_prop(required_property).ok_or_else(|| { ErrorCode::Internal(format!("Cannot find best cost of group: {group_index}",)) @@ -262,23 +268,8 @@ impl CascadesOptimizer { .map(|(index, required_prop)| Ok(Arc::new(self.find_best_plan(*index, required_prop)?))) .collect::>>()?; - let result = SExpr::create(m_expr.plan.clone(), children, None, None, None); + let result = PExpr::create(m_expr.plan.clone(), children, None, None, None); Ok(result) } } - -#[async_trait::async_trait] -impl Optimizer for CascadesOptimizer { - fn name(&self) -> String { - "CascadesOptimizer".to_string() - } - - async fn optimize(&mut self, s_expr: SExpr) -> Result { - self.optimize_sync(s_expr) - } - - fn memo(&self) -> Option<&Memo> { - Some(&self.memo) - } -} diff --git a/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed.rs b/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed.rs index 657d2168320..41ccb353688 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed.rs @@ -19,10 +19,10 @@ use databend_common_exception::Result; use crate::optimizer::OptimizerContext; use crate::optimizer::ir::Distribution; +use crate::optimizer::ir::PExpr; use crate::optimizer::ir::PropertyEnforcer; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RequiredProperty; -use crate::optimizer::ir::SExpr; use crate::plans::Exchange; /// DistributedOptimizer optimizes a query plan for distributed execution. @@ -45,7 +45,7 @@ impl DistributedOptimizer { /// Enforce distribution properties without lowering Exchange-sensitive /// operators into their partial and final stages. #[recursive::recursive] - pub fn optimize(&self, s_expr: &SExpr) -> Result { + pub fn optimize(&self, s_expr: &PExpr) -> Result { // Step 1: Set the initial distribution requirement (Any) let required = RequiredProperty { distribution: Distribution::Any, @@ -56,7 +56,7 @@ impl DistributedOptimizer { let mut result = enforcer.require_property(&required, s_expr)?; // Step 3: Check if the result satisfies the required distribution property - let rel_expr = RelExpr::with_s_expr(&result); + let rel_expr = RelExpr::with_p_expr(&result); let physical_prop = rel_expr.derive_physical_prop()?; let root_required = RequiredProperty { distribution: Distribution::Serial, @@ -65,7 +65,7 @@ impl DistributedOptimizer { // Step 4: If not satisfied, manually enforce serial distribution if !root_required.satisfied_by(&physical_prop) { // Add an Exchange::Merge operator - result = SExpr::create_unary(Arc::new(Exchange::Merge.into()), Arc::new(result)); + result = PExpr::create_unary(Arc::new(Exchange::Merge.into()), Arc::new(result)); } Ok(result) diff --git a/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed_merge.rs b/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed_merge.rs index 72b14e2b8d2..e2cf4d27564 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed_merge.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed_merge.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use databend_common_exception::Result; use crate::optimizer::ir::Matcher; -use crate::optimizer::ir::SExpr; +use crate::optimizer::ir::PExpr; use crate::plans::Exchange; use crate::plans::Join; use crate::plans::RelOp; @@ -33,7 +33,7 @@ impl BroadcastToShuffleOptimizer { } } - pub fn is_broadcast(&self, s_expr: &SExpr) -> Result { + pub fn is_broadcast(&self, s_expr: &PExpr) -> Result { let right = s_expr.child(1)?; Ok(matches!( right.plan(), @@ -41,7 +41,7 @@ impl BroadcastToShuffleOptimizer { )) } - pub fn optimize(&self, s_expr: &SExpr) -> Result { + pub fn optimize(&self, s_expr: &PExpr) -> Result { let left_exchange_input = s_expr.child(0)?; let right_exchange = s_expr.child(1)?; @@ -58,13 +58,13 @@ impl BroadcastToShuffleOptimizer { .unzip(); let new_join_children = vec![ - Arc::new(SExpr::create_unary( + Arc::new(PExpr::create_unary( Arc::new(RelOperator::Exchange(Exchange::NodeToNodeHash( left_conditions, ))), Arc::new(left_exchange_input.clone()), )), - Arc::new(SExpr::create_unary( + Arc::new(PExpr::create_unary( Arc::new(RelOperator::Exchange(Exchange::NodeToNodeHash( right_conditions, ))), diff --git a/src/query/sql/src/planner/optimizer/optimizers/distributed/materialized_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/distributed/materialized_cte.rs index 1f447eb19fa..ae0d25254b1 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/distributed/materialized_cte.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/distributed/materialized_cte.rs @@ -20,10 +20,10 @@ use databend_common_expression::Scalar; use databend_common_expression::types::NumberScalar; use crate::optimizer::ir::Distribution; +use crate::optimizer::ir::ExprVisitor; +use crate::optimizer::ir::PExpr; +use crate::optimizer::ir::PVisitAction as VisitAction; use crate::optimizer::ir::RelExpr; -use crate::optimizer::ir::SExpr; -use crate::optimizer::ir::SExprVisitor; -use crate::optimizer::ir::VisitAction; use crate::plans::ConstantExpr; use crate::plans::Exchange; use crate::plans::RelOperator; @@ -31,7 +31,7 @@ use crate::plans::ScalarExpr; /// Outcome of aligning materialized CTE placement with the distributed plan. pub enum MaterializedCTEDistribution { - Distributed(SExpr), + Distributed(PExpr), RequiresLocal, } @@ -46,7 +46,7 @@ impl MaterializedCTEDistributionOptimizer { /// Resolve materialized CTE placement after distribution properties are /// settled, but before Exchange-sensitive operators are split into stages. - pub fn optimize(&self, s_expr: &SExpr) -> Result { + pub fn optimize(&self, s_expr: &PExpr) -> Result { let result = s_expr .accept(&mut SerialProducerRedistributor)? .unwrap_or_else(|| s_expr.clone()); @@ -71,13 +71,13 @@ impl MaterializedCTEDistributionOptimizer { struct SerialProducerRedistributor; impl SerialProducerRedistributor { - fn match_merge_backed_producer(expr: &SExpr) -> Result> { + fn match_merge_backed_producer(expr: &PExpr) -> Result> { let RelOperator::Sequence(_) = expr.plan() else { return Ok(None); }; let producer = expr.left_child(); - let physical_prop = RelExpr::with_s_expr(producer).derive_physical_prop()?; + let physical_prop = RelExpr::with_p_expr(producer).derive_physical_prop()?; if physical_prop.distribution != Distribution::Serial { return Ok(None); } @@ -90,7 +90,7 @@ impl SerialProducerRedistributor { let mut expr = producer.unary_child(); loop { - let physical_prop = RelExpr::with_s_expr(expr).derive_physical_prop()?; + let physical_prop = RelExpr::with_p_expr(expr).derive_physical_prop()?; if physical_prop.distribution != Distribution::Serial { return Ok(None); } @@ -105,12 +105,12 @@ impl SerialProducerRedistributor { } } -impl SExprVisitor for SerialProducerRedistributor { - fn visit(&mut self, _expr: &SExpr) -> Result { +impl ExprVisitor for SerialProducerRedistributor { + fn visit(&mut self, _expr: &PExpr) -> Result { Ok(VisitAction::Continue) } - fn post_visit(&mut self, expr: &SExpr) -> Result { + fn post_visit(&mut self, expr: &PExpr) -> Result { let Some(producer) = Self::match_merge_backed_producer(expr)? else { return Ok(VisitAction::Continue); }; @@ -132,15 +132,15 @@ struct SerialSequenceFinder { found: bool, } -impl SExprVisitor for SerialSequenceFinder { - fn visit(&mut self, expr: &SExpr) -> Result { +impl ExprVisitor for SerialSequenceFinder { + fn visit(&mut self, expr: &PExpr) -> Result { if self.found { return Ok(VisitAction::SkipChildren); } if matches!(expr.plan(), RelOperator::Sequence(_)) { let left = expr.left_child(); - let physical_prop = RelExpr::with_s_expr(left).derive_physical_prop()?; + let physical_prop = RelExpr::with_p_expr(left).derive_physical_prop()?; if physical_prop.distribution == Distribution::Serial { self.found = true; return Ok(VisitAction::SkipChildren); diff --git a/src/query/sql/src/planner/optimizer/optimizers/distributed/sort_and_limit.rs b/src/query/sql/src/planner/optimizer/optimizers/distributed/sort_and_limit.rs index 98feb748b26..4e49d07917a 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/distributed/sort_and_limit.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/distributed/sort_and_limit.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use databend_common_exception::Result; use crate::optimizer::ir::Matcher; -use crate::optimizer::ir::SExpr; +use crate::optimizer::ir::PExpr; use crate::plans::Exchange; use crate::plans::Limit; use crate::plans::RelOp; @@ -117,7 +117,7 @@ impl SortAndLimitPushDownOptimizer { } #[recursive::recursive] - pub fn optimize(&self, s_expr: &SExpr) -> Result { + pub fn optimize(&self, s_expr: &PExpr) -> Result { let mut replaced_children = Vec::with_capacity(s_expr.arity()); for child in s_expr.children.iter() { let new_child = self.optimize(child)?; @@ -129,7 +129,7 @@ impl SortAndLimitPushDownOptimizer { self.apply_limit(&apply_topn_res) } - fn apply_sort(&self, s_expr: &SExpr) -> Result { + fn apply_sort(&self, s_expr: &PExpr) -> Result { if !self.sort_matcher.matches(s_expr) { return Ok(s_expr.clone()); } @@ -148,9 +148,9 @@ impl SortAndLimitPushDownOptimizer { let sort = s_expr.plan.as_sort().unwrap(); - let new_exchange = SExpr::create_unary( + let new_exchange = PExpr::create_unary( Arc::new(Exchange::MergeSort.into()), - SExpr::create_unary( + PExpr::create_unary( Arc::new( Sort { after_exchange: Some(false), @@ -161,7 +161,7 @@ impl SortAndLimitPushDownOptimizer { exchange_sexpr.unary_child_arc(), ), ); - Ok(SExpr::create_unary( + Ok(PExpr::create_unary( Arc::new( Sort { after_exchange: Some(true), @@ -173,7 +173,7 @@ impl SortAndLimitPushDownOptimizer { )) } - fn apply_topn(&self, s_expr: &SExpr) -> Result { + fn apply_topn(&self, s_expr: &PExpr) -> Result { if !self.topn_matcher.matches(s_expr) { return Ok(s_expr.clone()); } @@ -191,9 +191,9 @@ impl SortAndLimitPushDownOptimizer { let top_n = s_expr.plan.as_top_n().unwrap(); - let new_exchange = SExpr::create_unary( + let new_exchange = PExpr::create_unary( Arc::new(Exchange::Merge.into()), - SExpr::create_unary( + PExpr::create_unary( Arc::new( TopN { after_exchange: Some(false), @@ -204,7 +204,7 @@ impl SortAndLimitPushDownOptimizer { exchange_sexpr.unary_child_arc(), ), ); - Ok(SExpr::create_unary( + Ok(PExpr::create_unary( Arc::new( TopN { after_exchange: Some(true), @@ -216,7 +216,7 @@ impl SortAndLimitPushDownOptimizer { )) } - fn apply_limit(&self, s_expr: &SExpr) -> Result { + fn apply_limit(&self, s_expr: &PExpr) -> Result { if !self.limit_matcher.matches(s_expr) { return Ok(s_expr.clone()); } @@ -240,7 +240,7 @@ impl SortAndLimitPushDownOptimizer { debug_assert!(exchange_sexpr.children.len() == 1); let child = exchange_sexpr.child(0)?.clone(); - let new_child = SExpr::create_unary(Arc::new(limit.into()), Arc::new(child)); + let new_child = PExpr::create_unary(Arc::new(limit.into()), Arc::new(child)); let new_exchange = exchange_sexpr.replace_children(vec![Arc::new(new_child)]); Ok(s_expr.replace_children(vec![Arc::new(new_exchange)])) } diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs index ae7e42e2c8e..9d0fb5cb010 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use databend_common_exception::Result; use crate::optimizer::Optimizer; -use crate::optimizer::ir::SExpr; +use crate::optimizer::ir::PExpr; use crate::plans::RelOperator; /// Optimizer that removes unused CTEs from the query plan. @@ -28,7 +28,7 @@ pub struct CleanupUnusedCTEOptimizer; impl CleanupUnusedCTEOptimizer { /// Collect all CTE names that are referenced by MaterializeCTERef nodes and count their references - fn collect_referenced_ctes(s_expr: &SExpr) -> Result> { + fn collect_referenced_ctes(s_expr: &PExpr) -> Result> { let mut referenced_ctes = HashMap::new(); Self::collect_referenced_ctes_recursive(s_expr, &mut referenced_ctes)?; Ok(referenced_ctes) @@ -37,7 +37,7 @@ impl CleanupUnusedCTEOptimizer { /// Recursively traverse the expression tree to find MaterializeCTERef nodes and count references #[recursive::recursive] fn collect_referenced_ctes_recursive( - s_expr: &SExpr, + s_expr: &PExpr, referenced_ctes: &mut HashMap, ) -> Result<()> { // Check if current node is a MaterializeCTERef @@ -58,9 +58,9 @@ impl CleanupUnusedCTEOptimizer { /// Remove unused CTEs from the expression tree and update ref_count #[recursive::recursive] fn remove_unused_ctes( - mut s_expr: SExpr, + mut s_expr: PExpr, referenced_ctes: &HashMap, - ) -> Result { + ) -> Result { if let RelOperator::Sequence(_) = s_expr.plan() && matches!(s_expr.left_child().plan(), RelOperator::MaterializedCTE(_)) { @@ -85,9 +85,9 @@ impl CleanupUnusedCTEOptimizer { Arc::unwrap_or_clone(left_child.children.pop().unwrap()), referenced_ctes, )?; - let left_child_expr = SExpr::create_unary(cte, left_input); + let left_child_expr = PExpr::create_unary(cte, left_input); let right_child_expr = Self::remove_unused_ctes(right_child, referenced_ctes)?; - return Ok(SExpr::create_binary( + return Ok(PExpr::create_binary( s_expr.plan, left_child_expr, right_child_expr, @@ -108,12 +108,12 @@ impl CleanupUnusedCTEOptimizer { } #[async_trait::async_trait] -impl Optimizer for CleanupUnusedCTEOptimizer { +impl Optimizer for CleanupUnusedCTEOptimizer { fn name(&self) -> String { "CleanupUnusedCTEOptimizer".to_string() } - async fn optimize(&mut self, s_expr: SExpr) -> Result { + async fn optimize(&mut self, s_expr: PExpr) -> Result { // Collect all referenced CTEs and their ref_count let referenced_ctes = Self::collect_referenced_ctes(&s_expr)?; @@ -135,22 +135,22 @@ mod tests { use crate::plans::Sequence; use crate::plans::UnionAll; - fn dummy_scan() -> SExpr { - SExpr::create_leaf(DummyTableScan::new()) + fn dummy_scan() -> PExpr { + PExpr::create_leaf(DummyTableScan::new()) } - fn cte_ref(cte_name: &str) -> SExpr { - SExpr::create_leaf(RelOperator::MaterializedCTERef(MaterializedCTERef { + fn cte_ref(cte_name: &str) -> PExpr { + PExpr::create_leaf(RelOperator::MaterializedCTERef(MaterializedCTERef { cte_name: cte_name.to_string(), output_columns: vec![], - def: dummy_scan(), + def: crate::optimizer::ir::SExpr::create_leaf(DummyTableScan::new()), column_mapping: HashMap::new(), stat_info: None, })) } - fn union_all(left: SExpr, right: SExpr) -> SExpr { - SExpr::create_binary( + fn union_all(left: PExpr, right: PExpr) -> PExpr { + PExpr::create_binary( UnionAll { left_outputs: vec![], right_outputs: vec![], @@ -163,7 +163,7 @@ mod tests { ) } - fn materialized_cte_ref_count(s_expr: &SExpr, cte_name: &str) -> Option { + fn materialized_cte_ref_count(s_expr: &PExpr, cte_name: &str) -> Option { if let RelOperator::MaterializedCTE(cte) = s_expr.plan() && cte.cte_name == cte_name { @@ -177,20 +177,20 @@ mod tests { #[test] fn test_cleanup_updates_nested_materialized_cte_ref_count() { - let inner_producer = SExpr::create_unary( + let inner_producer = PExpr::create_unary( MaterializedCTE::new("inner".to_string(), None), Arc::new(dummy_scan()), ); - let outer_definition = SExpr::create_binary( + let outer_definition = PExpr::create_binary( Sequence, Arc::new(inner_producer), Arc::new(union_all(cte_ref("inner"), cte_ref("inner"))), ); - let outer_producer = SExpr::create_unary( + let outer_producer = PExpr::create_unary( MaterializedCTE::new("outer".to_string(), None), Arc::new(outer_definition), ); - let root = SExpr::create_binary( + let root = PExpr::create_binary( Sequence, Arc::new(outer_producer), Arc::new(cte_ref("outer")), diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/eliminate_eval_scalar.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/eliminate_eval_scalar.rs new file mode 100644 index 00000000000..6c5baa43e36 --- /dev/null +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/eliminate_eval_scalar.rs @@ -0,0 +1,162 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; +use std::time::Instant; + +use databend_common_exception::Result; + +use crate::ColumnSet; +use crate::ScalarExpr; +use crate::optimizer::Optimizer; +use crate::optimizer::OptimizerContext; +use crate::optimizer::ir::Matcher; +use crate::optimizer::ir::PExpr; +use crate::optimizer::ir::RelExpr; +use crate::optimizer::optimizers::rule::RuleID; +use crate::optimizer::pipeline::OptimizerTraceCollector; +use crate::plans::EvalScalar; +use crate::plans::Operator; +use crate::plans::RelOp; + +/// Post-search cleanup for EvalScalar nodes. Logical recursive rewrites and their +/// materialized-view handling are deliberately not part of this physical pass. +pub struct EliminateEvalScalarOptimizer { + ctx: Arc, + trace_collector: Option>, +} + +impl EliminateEvalScalarOptimizer { + pub fn new(ctx: Arc) -> Self { + Self { + ctx, + trace_collector: None, + } + } + + /// Preserve children-first traversal and repeat after a successful elimination. + #[recursive::recursive] + pub fn optimize_sync(&self, mut current: PExpr) -> Result { + loop { + let mut children = Vec::with_capacity(current.children.len()); + for child in std::mem::take(&mut current.children) { + children.push(Arc::new(self.optimize_sync(Arc::unwrap_or_clone(child))?)); + } + current = current.replace_children(children); + match self.eliminate(¤t)? { + Some(expr) => current = expr, + None => return Ok(current), + } + } + } + + fn eliminate(&self, expr: &PExpr) -> Result> { + let id = RuleID::EliminateEvalScalar; + if self.ctx.is_optimizer_disabled(&id.to_string()) { + return Ok(None); + } + let start = Instant::now(); + let before = expr; + let mut expr = expr.clone(); + let matcher = Matcher::MatchOp { + op_type: RelOp::EvalScalar, + children: vec![Matcher::Leaf], + }; + let result = if matcher.matches(&expr) && !expr.applied_rule(&id) { + expr.set_applied_rule(&id); + Self::eliminate_eval_scalar(&expr)? + } else { + None + }; + if self.ctx.get_enable_trace() + && let Some(collector) = &self.trace_collector + { + collector.trace_rule( + id.to_string(), + self.name(), + start.elapsed(), + before, + result.as_ref().unwrap_or(&expr), + &self.ctx.get_metadata().read(), + self.ctx.get_stat_context(), + )?; + } + Ok(result) + } + + fn eliminate_eval_scalar(expr: &PExpr) -> Result> { + // Eliminate empty EvalScalar + let eval_scalar: EvalScalar = expr.plan().clone().try_into()?; + if eval_scalar.items.is_empty() { + return Ok(Some(expr.child(0)?.clone())); + } + + let child = expr.child(0)?; + let child_output_cols = child + .plan() + .derive_relational_prop(&RelExpr::with_p_expr(child))? + .output_columns + .clone(); + let eval_scalar_output_cols: ColumnSet = + eval_scalar.items.iter().map(|x| x.index).collect(); + + if eval_scalar_output_cols.is_subset(&child_output_cols) { + // check if there's f(#x) as #x, if so we can't eliminate the eval scalar + for item in eval_scalar.items { + match item.scalar { + ScalarExpr::ConstantExpr(_) | ScalarExpr::TypedConstantExpr(_, _) => { + // A constant with an existing output index shadows the child column. + // It cannot be eliminated as an identity projection. + return Ok(None); + } + ScalarExpr::FunctionCall(func) => { + if func.arguments.len() == 1 { + if let ScalarExpr::BoundColumnRef(bound_column_ref) = &func.arguments[0] + && bound_column_ref.column.index == item.index + { + return Ok(None); + } + } + } + ScalarExpr::CastExpr(cast) => { + if let ScalarExpr::BoundColumnRef(bound_column_ref) = cast.argument.as_ref() + && bound_column_ref.column.index == item.index + { + return Ok(None); + } + } + _ => {} + } + } + return Ok(Some(expr.child(0)?.clone())); + } + Ok(None) + } +} + +#[async_trait::async_trait] +impl Optimizer for EliminateEvalScalarOptimizer { + fn name(&self) -> String { + // Preserve the existing setting/trace identifier despite separating the pass. + "RecursiveRuleOptimizer[EliminateEvalScalar]".to_string() + } + + async fn optimize(&mut self, expr: PExpr) -> Result { + self.optimize_sync(expr) + } + + fn set_trace_collector(&mut self, collector: Arc) { + self.trace_collector = Some(collector); + } +} diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/join/finalize_spatial_join.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/join/finalize_spatial_join.rs index b6012b25f0c..29abf64a51d 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/join/finalize_spatial_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/join/finalize_spatial_join.rs @@ -18,8 +18,8 @@ use databend_common_exception::Result; use crate::optimizer::Optimizer; use crate::optimizer::OptimizerContext; +use crate::optimizer::ir::PExpr; use crate::optimizer::ir::RelExpr; -use crate::optimizer::ir::SExpr; use crate::plans::RelOperator; use crate::plans::spatial_join_gate; @@ -33,7 +33,7 @@ impl FinalizeSpatialJoinOptimizer { Self { ctx } } - pub fn optimize_sync(&mut self, s_expr: SExpr) -> Result { + pub fn optimize_sync(&mut self, s_expr: PExpr) -> Result { if !self .ctx .get_table_ctx() @@ -47,7 +47,7 @@ impl FinalizeSpatialJoinOptimizer { } #[recursive::recursive] - fn finalize_spatial_join(mut s_expr: SExpr) -> Result { + fn finalize_spatial_join(mut s_expr: PExpr) -> Result { let mut children = Vec::with_capacity(s_expr.children.len()); for child in std::mem::take(&mut s_expr.children) { children.push(Arc::new(Self::finalize_spatial_join( @@ -58,8 +58,8 @@ impl FinalizeSpatialJoinOptimizer { let mut result = s_expr.replace_children(children); if let RelOperator::Join(join) = result.plan() { - let left_prop = RelExpr::with_s_expr(result.left_child()).derive_relational_prop()?; - let right_prop = RelExpr::with_s_expr(result.right_child()).derive_relational_prop()?; + let left_prop = RelExpr::with_p_expr(result.left_child()).derive_relational_prop()?; + let right_prop = RelExpr::with_p_expr(result.right_child()).derive_relational_prop()?; let spatial_join = spatial_join_gate(join, &left_prop.output_columns, &right_prop.output_columns) .map(Box::new); @@ -77,12 +77,12 @@ impl FinalizeSpatialJoinOptimizer { } #[async_trait::async_trait] -impl Optimizer for FinalizeSpatialJoinOptimizer { +impl Optimizer for FinalizeSpatialJoinOptimizer { fn name(&self) -> String { "FinalizeSpatialJoinOptimizer".to_string() } - async fn optimize(&mut self, s_expr: SExpr) -> Result { + async fn optimize(&mut self, s_expr: PExpr) -> Result { self.optimize_sync(s_expr) } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs index 8ed6717ffe7..746acf01bdd 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs @@ -32,3 +32,6 @@ pub use filter::NormalizeDisjunctiveFilterOptimizer; pub use filter::PullUpFilterOptimizer; pub use join::FinalizeSpatialJoinOptimizer; pub use join::SingleToInnerOptimizer; + +mod eliminate_eval_scalar; +pub use eliminate_eval_scalar::EliminateEvalScalarOptimizer; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs index 31e84226693..943eb79895c 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use crate::optimizer::OptimizerContext; @@ -21,7 +22,6 @@ use crate::optimizer::optimizers::rule::RuleCommuteJoin; use crate::optimizer::optimizers::rule::RuleCommuteJoinBaseTable; use crate::optimizer::optimizers::rule::RuleDeduplicateSort; use crate::optimizer::optimizers::rule::RuleEagerAggregation; -use crate::optimizer::optimizers::rule::RuleEliminateEvalScalar; use crate::optimizer::optimizers::rule::RuleEliminateFilter; use crate::optimizer::optimizers::rule::RuleEliminateSelfJoin; use crate::optimizer::optimizers::rule::RuleEliminateSort; @@ -71,7 +71,9 @@ impl RuleFactory { let metadata = ctx.get_metadata(); match id { RuleID::EliminateUnion => Ok(Box::new(RuleEliminateUnion::new(metadata))), - RuleID::EliminateEvalScalar => Ok(Box::new(RuleEliminateEvalScalar::new())), + RuleID::EliminateEvalScalar => Err(ErrorCode::Internal( + "EliminateEvalScalar requires a physical expression", + )), RuleID::FilterNulls => Ok(Box::new(RuleFilterNulls::new( ctx.get_enable_distributed_optimization(), ctx.get_stat_context().clone(), diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/mod.rs index 1a2f1ac41a6..2f6ff4f519e 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/mod.rs @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod rule_eliminate_eval_scalar; mod rule_merge_eval_scalar; mod rule_normalize_scalar; -pub use rule_eliminate_eval_scalar::RuleEliminateEvalScalar; pub use rule_merge_eval_scalar::RuleMergeEvalScalar; pub use rule_normalize_scalar::RuleNormalizeScalarFilter; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs deleted file mode 100644 index 2429be24ee9..00000000000 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_exception::Result; - -use crate::ColumnSet; -use crate::ScalarExpr; -use crate::optimizer::ir::Matcher; -use crate::optimizer::ir::RelExpr; -use crate::optimizer::ir::SExpr; -use crate::optimizer::optimizers::rule::Rule; -use crate::optimizer::optimizers::rule::RuleID; -use crate::optimizer::optimizers::rule::TransformResult; -use crate::plans::EvalScalar; -use crate::plans::Operator; -use crate::plans::RelOp; - -pub struct RuleEliminateEvalScalar { - id: RuleID, - matchers: Vec, -} - -impl Default for RuleEliminateEvalScalar { - fn default() -> Self { - Self::new() - } -} - -impl RuleEliminateEvalScalar { - pub fn new() -> Self { - Self { - id: RuleID::EliminateEvalScalar, - // EvalScalar - // \ - // * - matchers: vec![Matcher::MatchOp { - op_type: RelOp::EvalScalar, - children: vec![Matcher::Leaf], - }], - } - } -} - -impl Rule for RuleEliminateEvalScalar { - fn id(&self) -> RuleID { - self.id - } - - fn apply(&self, s_expr: &SExpr, state: &mut TransformResult) -> Result<()> { - // Eliminate empty EvalScalar - let eval_scalar: EvalScalar = s_expr.plan().clone().try_into()?; - if eval_scalar.items.is_empty() { - state.add_result(s_expr.child(0)?.clone()); - return Ok(()); - } - - let child = s_expr.child(0)?; - let child_output_cols = child - .plan() - .derive_relational_prop(&RelExpr::with_s_expr(child))? - .output_columns - .clone(); - let eval_scalar_output_cols: ColumnSet = - eval_scalar.items.iter().map(|x| x.index).collect(); - - if eval_scalar_output_cols.is_subset(&child_output_cols) { - // check if there's f(#x) as #x, if so we can't eliminate the eval scalar - for item in eval_scalar.items { - match item.scalar { - ScalarExpr::ConstantExpr(_) | ScalarExpr::TypedConstantExpr(_, _) => { - // A constant with an existing output index shadows the child column. - // It cannot be eliminated as an identity projection. - return Ok(()); - } - ScalarExpr::FunctionCall(func) => { - if func.arguments.len() == 1 { - if let ScalarExpr::BoundColumnRef(bound_column_ref) = &func.arguments[0] - { - if bound_column_ref.column.index == item.index { - return Ok(()); - } - } - } - } - ScalarExpr::CastExpr(cast) => { - if let ScalarExpr::BoundColumnRef(bound_column_ref) = cast.argument.as_ref() - { - if bound_column_ref.column.index == item.index { - return Ok(()); - } - } - } - _ => {} - } - } - state.add_result(s_expr.child(0)?.clone()); - return Ok(()); - } - Ok(()) - } - - fn matchers(&self) -> &[Matcher] { - &self.matchers - } -} diff --git a/src/query/sql/src/planner/optimizer/physical_planner.rs b/src/query/sql/src/planner/optimizer/physical_planner.rs new file mode 100644 index 00000000000..dbdb45e8149 --- /dev/null +++ b/src/query/sql/src/planner/optimizer/physical_planner.rs @@ -0,0 +1,207 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; +use std::time::Instant; + +use databend_common_exception::Result; + +use super::ir::PExpr; +use super::ir::PlannedQuery; +use super::optimizers::operator::EliminateEvalScalarOptimizer; +use crate::optimizer::Optimizer; +use crate::optimizer::OptimizerContext; +use crate::optimizer::ir::Memo; +use crate::optimizer::ir::SExpr; +use crate::optimizer::optimizers::CascadesOptimizer; +use crate::optimizer::optimizers::operator::CleanupUnusedCTEOptimizer; +use crate::optimizer::optimizers::operator::FinalizeSpatialJoinOptimizer; +use crate::optimizer::pipeline::OptimizerTraceCollector; + +/// Selects a physical expression from a logical input, then runs physical cleanup +/// and finalization separately from the logical optimizer pipeline. +pub struct PhysicalPlanner { + opt_ctx: Arc, + trace: Arc, + memo: Option, + trace_offset: usize, +} + +impl PhysicalPlanner { + pub fn new(opt_ctx: Arc) -> Self { + Self { + opt_ctx, + trace: Arc::new(OptimizerTraceCollector::new()), + memo: None, + trace_offset: 0, + } + } + + pub fn with_trace_collector( + mut self, + trace: Arc, + offset: usize, + ) -> Self { + self.trace = trace; + self.trace_offset = offset; + self + } + + pub fn memo(&self) -> Memo { + self.memo + .clone() + .unwrap_or_else(|| Memo::new(self.opt_ctx.get_stat_context().clone())) + } + + pub async fn plan(&mut self, input: SExpr) -> Result { + self.plan_inner(input, false, true).await + } + + /// Force local planning after the caller configures input distribution, without + /// rerunning logical preparation. + pub(crate) async fn plan_local(&mut self, input: SExpr) -> Result { + self.plan_inner(input, true, true).await + } + + pub(crate) async fn search_memo(&mut self, input: SExpr) -> Result { + self.plan_inner(input, false, false).await?; + Ok(self.memo()) + } + + async fn plan_inner( + &mut self, + input: SExpr, + local: bool, + finalize: bool, + ) -> Result { + self.memo = None; + if local { + self.opt_ctx.set_enable_distributed_optimization(false); + } + let mut expr = self.run_search(input, if finalize { 4 } else { 1 })?; + if finalize { + expr = self + .run_pass( + EliminateEvalScalarOptimizer::new(self.opt_ctx.clone()), + expr, + 1, + 4, + ) + .await?; + expr = self.run_pass(CleanupUnusedCTEOptimizer, expr, 2, 4).await?; + expr = self + .run_pass( + FinalizeSpatialJoinOptimizer::new(self.opt_ctx.clone()), + expr, + 3, + 4, + ) + .await?; + } + if self.opt_ctx.get_enable_trace() { + self.trace.log_report(); + log::info!( + "Final planned query:\n{}", + expr.pretty_format( + &self.opt_ctx.get_metadata().read(), + self.opt_ctx.get_stat_context() + )? + ); + } + Ok(PlannedQuery::new(expr)) + } + + /// Search is a logical-to-physical transition, not a same-type rewrite pass. + fn run_search(&mut self, input: SExpr, total: usize) -> Result { + #[cfg(debug_assertions)] + { + input.validate_types(&self.opt_ctx.get_metadata())?; + input.validate_column_scope(&self.opt_ctx.get_metadata())?; + } + let mut search = CascadesOptimizer::new(self.opt_ctx.clone())?; + if self.opt_ctx.is_optimizer_disabled(CascadesOptimizer::NAME) { + return Ok(input.into()); + } + // Only tracing needs a physical copy of the input for the existing diff API. + // Search itself consumes the original logical expression directly. + let before = self + .opt_ctx + .get_enable_trace() + .then(|| PExpr::from(input.clone())); + let start = Instant::now(); + let output = search.optimize_sync(input)?; + self.validate(&output).map_err(|e| { + e.add_message_back(" (after physical planning pass `CascadesOptimizer`)") + })?; + self.memo = Some(search.memo().clone()); + if let Some(before) = before { + self.trace.trace_optimizer( + CascadesOptimizer::NAME.to_string(), + self.trace_offset, + total + self.trace_offset, + start.elapsed(), + &before, + &output, + &self.opt_ctx.get_metadata().read(), + self.opt_ctx.get_stat_context(), + )?; + } + Ok(output) + } + + /// Run a physical rewrite pass with skip-list handling, validation and tracing. + async fn run_pass>( + &mut self, + mut pass: T, + input: PExpr, + index: usize, + total: usize, + ) -> Result { + let name = pass.name(); + if self.opt_ctx.is_optimizer_disabled(&name) { + return Ok(input); + } + let tracing = self.opt_ctx.get_enable_trace(); + if tracing { + pass.set_trace_collector(self.trace.clone()); + } + let before = tracing.then(|| input.clone()); + let start = Instant::now(); + let output = pass.optimize(input).await?; + self.validate(&output) + .map_err(|e| e.add_message_back(format!(" (after physical planning pass `{name}`)")))?; + if let Some(before) = before { + self.trace.trace_optimizer( + name, + index + self.trace_offset, + total + self.trace_offset, + start.elapsed(), + &before, + &output, + &self.opt_ctx.get_metadata().read(), + self.opt_ctx.get_stat_context(), + )?; + } + Ok(output) + } + + fn validate(&self, _expr: &PExpr) -> Result<()> { + #[cfg(debug_assertions)] + { + _expr.validate_types(&self.opt_ctx.get_metadata())?; + _expr.validate_column_scope(&self.opt_ctx.get_metadata())?; + } + Ok(()) + } +} diff --git a/src/query/sql/src/planner/optimizer/pipeline/mod.rs b/src/query/sql/src/planner/optimizer/pipeline/mod.rs index 53623a6aa06..456905bbf8b 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/mod.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/mod.rs @@ -18,5 +18,6 @@ mod pipeline; mod trace; pub use pipeline::OptimizerPipeline; +pub(crate) use pipeline::configure_distributed_optimization; pub use trace::OptimizerTrace; pub use trace::OptimizerTraceCollector; diff --git a/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs b/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs index f2a7199cd00..68e1d1fddbd 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs @@ -22,19 +22,15 @@ use super::common::contains_local_table_scan; use super::common::contains_warehouse_table_scan; use crate::optimizer::Optimizer; use crate::optimizer::OptimizerContext; -use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; use crate::optimizer::pipeline::OptimizerTraceCollector; -/// A pipeline of optimizers that are executed in sequence. +/// Sequential logical rewrite passes over SExpr. Physical planning runs separately. pub struct OptimizerPipeline { /// The optimizer context opt_ctx: Arc, /// The sequence of optimizers to be applied optimizers: Vec>, - /// The memo captured during optimization (if any) - memo: Option, - /// The trace collector for generating reports trace_collector: Arc, @@ -44,18 +40,13 @@ pub struct OptimizerPipeline { impl OptimizerPipeline { /// Create a new optimizer pipeline pub async fn new(opt_ctx: Arc, s_expr: SExpr) -> Result { - let pipeline = Self { + configure_distributed_optimization(&opt_ctx, &s_expr).await?; + Ok(Self { opt_ctx, optimizers: Vec::new(), - memo: None, trace_collector: Arc::new(OptimizerTraceCollector::new()), s_expr, - }; - - pipeline - .configure_distributed_optimization(&pipeline.s_expr) - .await?; - Ok(pipeline) + }) } /// Add an optimizer to the pipeline @@ -78,26 +69,7 @@ impl OptimizerPipeline { if condition { self.add(optimizer) } else { self } } - /// Configure distributed optimization based on table types - async fn configure_distributed_optimization(&self, s_expr: &SExpr) -> Result<()> { - let metadata = self.opt_ctx.get_metadata(); - - if contains_local_table_scan(s_expr, &metadata) { - self.opt_ctx.set_enable_distributed_optimization(false); - info!("Disable distributed optimization due to local table scan."); - } else if contains_warehouse_table_scan(s_expr, &metadata) { - let warehouse = self.opt_ctx.get_table_ctx().get_warehouse_cluster().await?; - - if !warehouse.is_empty() { - self.opt_ctx.set_enable_distributed_optimization(true); - info!("Enable distributed optimization due to warehouse table scan."); - } - } - - Ok(()) - } - - /// Execute the pipeline on the given expression + /// Execute the logical passes on the stored input expression pub async fn execute(&mut self) -> Result { // Then apply all optimizers in sequence let mut current_expr = self.s_expr.clone(); @@ -139,10 +111,6 @@ impl OptimizerPipeline { // Calculate duration let duration = start_time.elapsed(); - if let Some(memo) = optimizer.memo() { - self.memo = Some(memo.clone()); - } - // Only trace if tracing is enabled if let Some(before_expr) = before_expr { let metadata_ref = self.opt_ctx.get_metadata(); @@ -174,18 +142,9 @@ impl OptimizerPipeline { Ok(current_expr) } - /// Get the memo captured during optimization - /// - /// If no memo was captured during optimization, an empty memo is created and returned. - /// This ensures the method always returns a valid Memo. - pub fn memo(&self) -> Memo { - match &self.memo { - Some(memo) => memo.clone(), - None => { - // Create and return an empty memo - Memo::new(self.opt_ctx.get_stat_context().clone()) - } - } + /// Number of enabled passes, used to continue trace ordering in physical planning. + pub fn num_optimizers(&self) -> usize { + self.optimizers.len() } /// Get the trace collector @@ -193,3 +152,22 @@ impl OptimizerPipeline { self.trace_collector.clone() } } + +/// Shared input configuration, independent of logical pipeline execution. +pub(crate) async fn configure_distributed_optimization( + opt_ctx: &Arc, + s_expr: &SExpr, +) -> Result<()> { + let metadata = opt_ctx.get_metadata(); + if contains_local_table_scan(s_expr, &metadata) { + opt_ctx.set_enable_distributed_optimization(false); + info!("Disable distributed optimization due to local table scan."); + } else if contains_warehouse_table_scan(s_expr, &metadata) { + let warehouse = opt_ctx.get_table_ctx().get_warehouse_cluster().await?; + if !warehouse.is_empty() { + opt_ctx.set_enable_distributed_optimization(true); + info!("Enable distributed optimization due to warehouse table scan."); + } + } + Ok(()) +} diff --git a/src/query/sql/src/planner/optimizer/pipeline/trace/expr_diff.rs b/src/query/sql/src/planner/optimizer/pipeline/trace/expr_diff.rs index 6e2b9818e66..53e818831b3 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/trace/expr_diff.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/trace/expr_diff.rs @@ -17,12 +17,13 @@ use similar::ChangeTag; use similar::TextDiff; use crate::Metadata; -use crate::optimizer::ir::SExpr; +use crate::optimizer::ir::Expr; +use crate::optimizer::ir::RelExprKind; use crate::optimizer::ir::StatContext; use crate::planner::format::FormatOptions; use crate::planner::format::MetadataIdHumanizer; -impl SExpr { +impl Expr { /// Compares this SExpr with another SExpr and returns a diff of their string representations. /// /// # Arguments @@ -33,7 +34,7 @@ impl SExpr { /// A string containing the diff between the two SExpr instances. pub fn diff( &self, - other: &SExpr, + other: &Self, metadata: &Metadata, stat_context: &StatContext, ) -> Result { diff --git a/src/query/sql/src/planner/optimizer/pipeline/trace/trace.rs b/src/query/sql/src/planner/optimizer/pipeline/trace/trace.rs index e67e89e225e..1d1a60c99f9 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/trace/trace.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/trace/trace.rs @@ -21,7 +21,6 @@ use log::info; use parking_lot::Mutex; use crate::Metadata; -use crate::optimizer::ir::SExpr; use crate::optimizer::ir::StatContext; /// Represents a trace entry for a rule execution @@ -129,14 +128,14 @@ impl OptimizerTraceCollector { /// * `before` - Expression state before optimization /// * `after` - Expression state after optimization /// * `metadata` - Metadata for expression comparison - pub fn trace_optimizer( + pub fn trace_optimizer( &self, name: String, index: usize, total: usize, time: Duration, - before: &SExpr, - after: &SExpr, + before: &crate::optimizer::ir::Expr, + after: &crate::optimizer::ir::Expr, metadata: &Metadata, stat_context: &StatContext, ) -> Result<()> { @@ -164,13 +163,13 @@ impl OptimizerTraceCollector { /// * `before` - Expression state before rule application /// * `after` - Expression state after rule application /// * `metadata` - Metadata for expression comparison - pub fn trace_rule( + pub fn trace_rule( &self, rule_name: String, optimizer_name: String, time: Duration, - before: &SExpr, - after: &SExpr, + before: &crate::optimizer::ir::Expr, + after: &crate::optimizer::ir::Expr, metadata: &Metadata, stat_context: &StatContext, ) -> Result<()> { diff --git a/src/query/sql/src/planner/planner.rs b/src/query/sql/src/planner/planner.rs index ec8fc10d0b2..736d1d52e11 100644 --- a/src/query/sql/src/planner/planner.rs +++ b/src/query/sql/src/planner/planner.rs @@ -346,7 +346,7 @@ impl Planner { plan.capture_bound_query_lineage(); } - // Step 4: Optimize the SExpr with optimizers, and generate optimized physical SExpr + // Step 4: Optimize logical inputs, then select and finalize query implementations. // Single-statement EXECUTE IMMEDIATE can apply inner settings during binding. let settings = self.ctx.get_settings(); let func_ctx = self.ctx.get_function_context()?; diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index f517069afcf..0a33dd69a2b 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -33,6 +33,7 @@ use super::ShowCreateDictionaryPlan; use crate::BindContext; use crate::MetadataRef; use crate::binder::ExplainConfig; +use crate::optimizer::ir::QueryPlan; use crate::optimizer::ir::SExpr; use crate::plans::AddTableColumnPlan; use crate::plans::AddTableConstraintPlan; @@ -223,7 +224,7 @@ use crate::plans::worker_schema; pub enum Plan { // `SELECT` statement Query { - s_expr: Box, + s_expr: Box, metadata: MetadataRef, bind_context: Box, rewrite_kind: Option, @@ -365,7 +366,7 @@ pub enum Plan { InsertMultiTable(Box), Replace(Box), DataMutation { - s_expr: Box, + s_expr: Box, schema: DataSchemaRef, metadata: MetadataRef, }, @@ -708,8 +709,10 @@ impl Plan { ignore_result, } = self { - if let RelOperator::Exchange(Exchange::Merge) = s_expr.plan.as_ref() { - let s_expr = Box::new(s_expr.child(0).unwrap().clone()); + if s_expr.planned().is_ok_and(|plan| { + matches!(plan.expr().plan(), RelOperator::Exchange(Exchange::Merge)) + }) { + let s_expr = Box::new(s_expr.remove_root_merge()); return Plan::Query { s_expr, metadata: metadata.clone(), @@ -745,7 +748,7 @@ impl Plan { }; Plan::Query { - s_expr: Box::new(s_expr), + s_expr: Box::new(s_expr.into()), metadata: metadata.clone(), bind_context: bind_context.clone(), rewrite_kind: rewrite_kind.clone(), diff --git a/src/query/sql/test-support/src/optimizer/mod.rs b/src/query/sql/test-support/src/optimizer/mod.rs index e7a08dd2bcc..9accbaa1c51 100644 --- a/src/query/sql/test-support/src/optimizer/mod.rs +++ b/src/query/sql/test-support/src/optimizer/mod.rs @@ -670,8 +670,13 @@ fn apply_stats( column_stats, histogram_stats, }; - if let Some(new_expr) = s_expr.accept(&mut applier)? { - **s_expr = new_expr; + let databend_common_sql::optimizer::ir::QueryPlan::Logical(expr) = s_expr.as_mut() else { + return Err(ErrorCode::Internal( + "Test statistics must be applied before physical planning", + )); + }; + if let Some(new_expr) = expr.accept(&mut applier)? { + *expr = new_expr; } } Ok(()) diff --git a/src/query/sql/tests/it/optimizer/collect_statistics.rs b/src/query/sql/tests/it/optimizer/collect_statistics.rs index c7d234fc5f7..853f9119c91 100644 --- a/src/query/sql/tests/it/optimizer/collect_statistics.rs +++ b/src/query/sql/tests/it/optimizer/collect_statistics.rs @@ -103,7 +103,9 @@ fn set_scan_change_type(plan: Plan, change_type: Option) -> Plan { formatted_ast, ignore_result, } => Plan::Query { - s_expr: Box::new(set_s_expr_scan_change_type(&s_expr, change_type)), + s_expr: Box::new( + set_s_expr_scan_change_type(s_expr.logical().unwrap(), change_type).into(), + ), metadata, bind_context, rewrite_kind, @@ -132,12 +134,14 @@ fn set_s_expr_scan_change_type(s_expr: &SExpr, change_type: Option) fn find_scan(plan: &Plan) -> &Scan { match plan { - Plan::Query { s_expr, .. } => find_scan_in_s_expr(s_expr).unwrap(), + Plan::Query { s_expr, .. } => { + find_scan_in_s_expr(s_expr.planned().unwrap().expr()).unwrap() + } _ => unreachable!("test optimizer should return Plan::Query"), } } -fn find_scan_in_s_expr(s_expr: &SExpr) -> Option<&Scan> { +fn find_scan_in_s_expr(s_expr: &databend_common_sql::optimizer::ir::PExpr) -> Option<&Scan> { match s_expr.plan() { RelOperator::Scan(scan) => Some(scan), _ => s_expr.children().find_map(find_scan_in_s_expr), diff --git a/src/query/sql/tests/it/optimizer/column_scope.rs b/src/query/sql/tests/it/optimizer/column_scope.rs index 3407f798208..5389d5ae753 100644 --- a/src/query/sql/tests/it/optimizer/column_scope.rs +++ b/src/query/sql/tests/it/optimizer/column_scope.rs @@ -17,8 +17,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::optimizer::ir::SExpr; -use databend_common_sql::optimizer::ir::SExprVisitor; -use databend_common_sql::optimizer::ir::VisitAction; use databend_common_sql::plans::Plan; use databend_common_sql::plans::RelOperator; use databend_common_sql::plans::ScalarExpr; @@ -80,7 +78,7 @@ async fn test_optimizer_rejects_plan_referencing_missing_column() -> Result<()> unreachable!("expected a query plan"); }; let broken = Plan::Query { - s_expr: Box::new(drop_scan_column(&s_expr, "b", &metadata)), + s_expr: Box::new(drop_scan_column(s_expr.logical()?, "b", &metadata).into()), metadata, bind_context, rewrite_kind, @@ -161,8 +159,13 @@ struct DeadProjectionCollector { expression_scans: usize, } -impl SExprVisitor for DeadProjectionCollector { - fn visit(&mut self, s_expr: &SExpr) -> Result { +impl databend_common_sql::optimizer::ir::ExprVisitor + for DeadProjectionCollector +{ + fn visit( + &mut self, + s_expr: &databend_common_sql::optimizer::ir::PExpr, + ) -> Result { match s_expr.plan() { RelOperator::ExpressionScan(_) => self.expression_scans += 1, RelOperator::EvalScalar(eval_scalar) => { @@ -183,7 +186,7 @@ impl SExprVisitor for DeadProjectionCollector { } _ => {} } - Ok(VisitAction::Continue) + Ok(databend_common_sql::optimizer::ir::PVisitAction::Continue) } } @@ -239,7 +242,7 @@ async fn test_lateral_values_drops_unresolvable_projection_items() -> Result<()> }; let mut collector = DeadProjectionCollector::default(); - s_expr.accept(&mut collector)?; + s_expr.planned()?.expr().accept(&mut collector)?; // Proves the case reached the lateral `VALUES` branch of the binder. assert!( collector.expression_scans > 0, diff --git a/src/query/sql/tests/it/optimizer/eager_aggregation.rs b/src/query/sql/tests/it/optimizer/eager_aggregation.rs index 7b6fde661da..52591535e12 100644 --- a/src/query/sql/tests/it/optimizer/eager_aggregation.rs +++ b/src/query/sql/tests/it/optimizer/eager_aggregation.rs @@ -50,7 +50,7 @@ async fn write_rule_results(file: &mut impl std::io::Write, case: &SqlTestCase) let settings = ctx.get_settings(); let opt_ctx = OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?) .with_settings(&settings)?; - let before_expr = optimize_before(opt_ctx.clone(), s_expr).await?; + let before_expr = optimize_before(opt_ctx.clone(), s_expr.logical()?).await?; let before_plan = plan.replace_query_s_expr(before_expr.clone()); write_case_header(file, case)?; @@ -352,8 +352,8 @@ GROUP BY ss_store_sk", }; let opt_ctx = OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?); - let split = - RecursiveRuleOptimizer::new(opt_ctx, &[RuleID::SplitAggregate]).optimize_sync(*s_expr)?; + let split = RecursiveRuleOptimizer::new(opt_ctx, &[RuleID::SplitAggregate]) + .optimize_sync((*s_expr).into_logical()?)?; let rewritten = RuleEagerAggregation::new(metadata.clone()).optimize_sync(&split)?; assert_no_initial_aggregate(&rewritten)?; rewritten.validate_types(&metadata)?; @@ -393,7 +393,7 @@ GROUP BY ss_store_sk" let opt_ctx = OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?); let split = RecursiveRuleOptimizer::new(opt_ctx, &[RuleID::SplitAggregate]) - .optimize_sync(*s_expr)?; + .optimize_sync((*s_expr).into_logical()?)?; let mut results = TransformResult::new(); RuleEagerAggregation::new(metadata.clone()).apply(&split, &mut results)?; assert_eq!(!results.results().is_empty(), eligible, "{aggregate}"); @@ -432,7 +432,7 @@ GROUP BY o_orderkey ORDER BY s DESC LIMIT 5", }; let opt_ctx = OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?); - let before = optimize_before(opt_ctx, &s_expr).await?; + let before = optimize_before(opt_ctx, s_expr.logical()?).await?; let mut extractor = Extractor { rule: RuleEagerAggregation::new(metadata.clone()), results: TransformResult::new(), @@ -472,7 +472,7 @@ GROUP BY ss_store_sk" let opt_ctx = OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?); let split = RecursiveRuleOptimizer::new(opt_ctx, &[RuleID::SplitAggregate]) - .optimize_sync(*s_expr)?; + .optimize_sync((*s_expr).into_logical()?)?; let mut results = TransformResult::new(); RuleEagerAggregation::new(metadata.clone()).apply(&split, &mut results)?; assert!(results.results().is_empty(), "{aggregate}"); diff --git a/src/query/sql/tests/it/optimizer/eliminate_eval_scalar.rs b/src/query/sql/tests/it/optimizer/eliminate_eval_scalar.rs new file mode 100644 index 00000000000..b93ec5562d1 --- /dev/null +++ b/src/query/sql/tests/it/optimizer/eliminate_eval_scalar.rs @@ -0,0 +1,134 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Write; + +use databend_common_catalog::table_context::TableContextSettings; +use databend_common_exception::Result; +use databend_common_sql::optimizer::OptimizerContext; +use databend_common_sql::optimizer::ir::ExprVisitor; +use databend_common_sql::optimizer::ir::PExpr; +use databend_common_sql::optimizer::ir::PVisitAction as VisitAction; +use databend_common_sql::optimizer::ir::StatContext; +use databend_common_sql::optimizer::optimize; +use databend_common_sql::optimizer::optimizers::operator::EliminateEvalScalarOptimizer; +use databend_common_sql::plans::Plan; + +use crate::framework::golden::SqlTestCase; +use crate::framework::golden::open_golden_file; +use crate::framework::golden::setup_context; +use crate::framework::golden::write_case_header; + +fn eval_count(expr: &PExpr) -> Result { + struct Count(usize); + impl ExprVisitor for Count { + fn visit(&mut self, expr: &PExpr) -> Result { + self.0 += usize::from(expr.plan().as_eval_scalar().is_some()); + Ok(VisitAction::Continue) + } + } + let mut count = Count(0); + expr.accept(&mut count)?; + Ok(count.0) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_physical_eliminate_eval_scalar() -> Result<()> { + let mut file = open_golden_file("optimizer", "eliminate_eval_scalar.txt")?; + for (name, sql, expected_evals) in [ + ("identity_projection", "SELECT k FROM t", 0), + ("computed_projection", "SELECT k + 1 FROM t", 1), + ("constant_projection", "SELECT 42 FROM t", 1), + ("cast_projection", "SELECT CAST(k AS String) FROM t", 1), + ( + "nested_identity", + "SELECT x FROM (SELECT k + 1 AS x FROM t) q", + 1, + ), + ] { + let skips: &[&str] = if expected_evals == 0 { + &[ + "", + "EliminateEvalScalar", + "RecursiveRuleOptimizer[EliminateEvalScalar]", + ] + } else { + &[""] + }; + for &skipped in skips { + let case = SqlTestCase { + name, + description: "Physical EvalScalar cleanup removes identity projections but keeps computations, and honors both skip levels.", + setup_sqls: &["CREATE TABLE t(k Int64 NOT NULL)"], + sql, + }; + let ctx = setup_context(&case).await?; + ctx.get_settings() + .set_optimizer_skip_list(skipped.to_string())?; + ctx.get_settings() + .set_setting("enable_optimizer_trace".to_string(), "1".to_string())?; + let raw = ctx.bind_sql(sql).await?; + let Plan::Query { metadata, .. } = &raw else { + unreachable!() + }; + let context = + OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?) + .with_settings(&ctx.get_settings())?; + let planned = optimize(context.clone(), raw.clone()).await?; + let Plan::Query { s_expr, .. } = &planned else { + unreachable!() + }; + let expr = s_expr.planned()?.expr(); + expr.validate_types(metadata)?; + expr.validate_column_scope(metadata)?; + assert_eq!(raw.schema(), planned.schema()); + if skipped.is_empty() { + assert_eq!(eval_count(expr)?, expected_evals, "{name}"); + } else { + // Rule-level skip remains in the cleanup entry; whole-pass skip is + // enforced by PhysicalPlanner. Directly run the same physical cleanup + // with an enabled context to prove the skipped node is reachable. + ctx.get_settings().set_optimizer_skip_list(String::new())?; + let enabled = OptimizerContext::new( + ctx.clone(), + metadata.clone(), + ctx.get_function_context()?, + ) + .with_settings(&ctx.get_settings())?; + let cleaned = + EliminateEvalScalarOptimizer::new(enabled).optimize_sync(expr.clone())?; + assert_eq!(eval_count(&cleaned)?, expected_evals, "{name}"); + assert!( + eval_count(expr)? > 0, + "skip did not retain identity projection" + ); + } + if skipped.is_empty() { + write_case_header(&mut file, &case)?; + writeln!( + file, + "raw_plan:\n{}", + raw.format_indent(Default::default(), &StatContext::default())? + )?; + } + writeln!(file, "skip: {skipped}")?; + writeln!( + file, + "optimized_plan:\n{}", + planned.format_indent(Default::default(), &StatContext::default())? + )?; + } + } + Ok(()) +} diff --git a/src/query/sql/tests/it/optimizer/eliminate_eval_scalar.txt b/src/query/sql/tests/it/optimizer/eliminate_eval_scalar.txt new file mode 100644 index 00000000000..f1dd0b8341e --- /dev/null +++ b/src/query/sql/tests/it/optimizer/eliminate_eval_scalar.txt @@ -0,0 +1,130 @@ +=== identity_projection === +description: Physical EvalScalar cleanup removes identity projections but keeps computations, and honors both skip levels. +sql: SELECT k FROM t +raw_plan: +EvalScalar +├── scalars: [t.k (#0) AS (#0)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +skip: +optimized_plan: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +skip: EliminateEvalScalar +optimized_plan: +EvalScalar +├── scalars: [t.k (#0) AS (#0)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +skip: RecursiveRuleOptimizer[EliminateEvalScalar] +optimized_plan: +EvalScalar +├── scalars: [t.k (#0) AS (#0)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +=== computed_projection === +description: Physical EvalScalar cleanup removes identity projections but keeps computations, and honors both skip levels. +sql: SELECT k + 1 FROM t +raw_plan: +EvalScalar +├── scalars: [plus(t.k (#0), 1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +skip: +optimized_plan: +EvalScalar +├── scalars: [plus(t.k (#0), 1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +=== constant_projection === +description: Physical EvalScalar cleanup removes identity projections but keeps computations, and honors both skip levels. +sql: SELECT 42 FROM t +raw_plan: +EvalScalar +├── scalars: [42 AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +skip: +optimized_plan: +EvalScalar +├── scalars: [42 AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +=== cast_projection === +description: Physical EvalScalar cleanup removes identity projections but keeps computations, and honors both skip levels. +sql: SELECT CAST(k AS String) FROM t +raw_plan: +EvalScalar +├── scalars: [CAST(t.k (#0) AS String) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +skip: +optimized_plan: +EvalScalar +├── scalars: [CAST(t.k (#0) AS String) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +=== nested_identity === +description: Physical EvalScalar cleanup removes identity projections but keeps computations, and honors both skip levels. +sql: SELECT x FROM (SELECT k + 1 AS x FROM t) q +raw_plan: +EvalScalar +├── scalars: [x (#1) AS (#1)] +└── EvalScalar + ├── scalars: [plus(t.k (#0), 1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +skip: +optimized_plan: +EvalScalar +├── scalars: [plus(t.k (#0), 1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + diff --git a/src/query/sql/tests/it/optimizer/hierarchical_grouping_sets.rs b/src/query/sql/tests/it/optimizer/hierarchical_grouping_sets.rs index cdf9c539825..af4cb578373 100644 --- a/src/query/sql/tests/it/optimizer/hierarchical_grouping_sets.rs +++ b/src/query/sql/tests/it/optimizer/hierarchical_grouping_sets.rs @@ -15,11 +15,11 @@ use databend_common_catalog::table_context::TableContextSettings; use databend_common_exception::Result; use databend_common_sql::optimizer::ir::Distribution; +use databend_common_sql::optimizer::ir::ExprVisitor; +use databend_common_sql::optimizer::ir::PExpr; +use databend_common_sql::optimizer::ir::PVisitAction as VisitAction; use databend_common_sql::optimizer::ir::RelExpr; -use databend_common_sql::optimizer::ir::SExpr; -use databend_common_sql::optimizer::ir::SExprVisitor; use databend_common_sql::optimizer::ir::StatContext; -use databend_common_sql::optimizer::ir::VisitAction; use databend_common_sql::plans::Operator; use databend_common_sql::plans::Plan; use databend_common_sql::plans::RelOp; @@ -76,10 +76,10 @@ fn assert_no_serial_sequence_producer(plan: &Plan) -> Result<()> { struct SequenceDistributionChecker; - impl SExprVisitor for SequenceDistributionChecker { - fn visit(&mut self, expr: &SExpr) -> Result { + impl ExprVisitor for SequenceDistributionChecker { + fn visit(&mut self, expr: &PExpr) -> Result { if expr.plan().rel_op() == RelOp::Sequence { - let left_prop = RelExpr::with_s_expr(expr.left_child()).derive_physical_prop()?; + let left_prop = RelExpr::with_p_expr(expr.left_child()).derive_physical_prop()?; assert_ne!( left_prop.distribution, Distribution::Serial, @@ -90,7 +90,10 @@ fn assert_no_serial_sequence_producer(plan: &Plan) -> Result<()> { } } - s_expr.accept(&mut SequenceDistributionChecker)?; + s_expr + .planned()? + .expr() + .accept(&mut SequenceDistributionChecker)?; Ok(()) } diff --git a/src/query/sql/tests/it/optimizer/insert_multi_table.rs b/src/query/sql/tests/it/optimizer/insert_multi_table.rs index 09a1237b8e7..f52655e174e 100644 --- a/src/query/sql/tests/it/optimizer/insert_multi_table.rs +++ b/src/query/sql/tests/it/optimizer/insert_multi_table.rs @@ -17,10 +17,10 @@ use std::io::Write; use databend_common_catalog::table_context::TableContextSettings; use databend_common_exception::Result; use databend_common_sql::optimizer::OptimizerContext; -use databend_common_sql::optimizer::ir::SExpr; -use databend_common_sql::optimizer::ir::SExprVisitor; +use databend_common_sql::optimizer::ir::ExprVisitor; +use databend_common_sql::optimizer::ir::PExpr; +use databend_common_sql::optimizer::ir::PVisitAction as VisitAction; use databend_common_sql::optimizer::ir::StatContext; -use databend_common_sql::optimizer::ir::VisitAction; use databend_common_sql::optimizer::optimize; use databend_common_sql::plans::AggregateMode; use databend_common_sql::plans::Plan; @@ -120,7 +120,7 @@ async fn test_when_subqueries_are_planned_with_source() -> Result<()> { unreachable!() }; assert_eq!(source_columns, bind_context.result_columns()); - let source = s_expr.as_ref(); + let source = s_expr.planned()?.expr(); source.validate_types(&insert.meta_data)?; source.validate_column_scope(&insert.meta_data)?; let property = source.derive_relational_prop()?; @@ -137,8 +137,8 @@ async fn test_when_subqueries_are_planned_with_source() -> Result<()> { struct Check { joins: usize, } - impl SExprVisitor for Check { - fn visit(&mut self, expr: &SExpr) -> Result { + impl ExprVisitor for Check { + fn visit(&mut self, expr: &PExpr) -> Result { self.joins += usize::from(expr.plan().as_join().is_some()); if let Some(aggregate) = expr.plan().as_aggregate() { assert_ne!( @@ -153,15 +153,17 @@ async fn test_when_subqueries_are_planned_with_source() -> Result<()> { let mut check = Check { joins: 0 }; source.accept(&mut check)?; assert_eq!(check.joins > 0, has_join, "{name}"); - write_case_header(&mut file, &case)?; + if !cbo && !distributed { + write_case_header(&mut file, &case)?; + writeln!( + file, + "raw_source:\n{}", + bound + .input_source + .format_indent(Default::default(), &StatContext::default())? + )?; + } writeln!(file, "cbo: {cbo}, distributed: {distributed}")?; - writeln!( - file, - "raw_source:\n{}", - bound - .input_source - .format_indent(Default::default(), &StatContext::default())? - )?; writeln!( file, "planned_source:\n{}", diff --git a/src/query/sql/tests/it/optimizer/insert_multi_table.txt b/src/query/sql/tests/it/optimizer/insert_multi_table.txt index fb1d0179baf..a2c7a7eb126 100644 --- a/src/query/sql/tests/it/optimizer/insert_multi_table.txt +++ b/src/query/sql/tests/it/optimizer/insert_multi_table.txt @@ -1,7 +1,6 @@ === no_subquery === description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. sql: INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t -cbo: false, distributed: false raw_source: EvalScalar ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] @@ -11,6 +10,7 @@ EvalScalar ├── order by: [] └── limit: NONE +cbo: false, distributed: false planned_source: Scan ├── table: default.t (#0) @@ -19,19 +19,7 @@ Scan └── limit: NONE when_0: t.k (#0) > CAST(0 AS Int64 NULL) -=== no_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: false, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Scan ├── table: default.t (#0) @@ -40,19 +28,7 @@ Scan └── limit: NONE when_0: t.k (#0) > CAST(0 AS Int64 NULL) -=== no_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: false -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Scan ├── table: default.t (#0) @@ -61,19 +37,7 @@ Scan └── limit: NONE when_0: t.k (#0) > CAST(0 AS Int64 NULL) -=== no_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k > 0 THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Scan ├── table: default.t (#0) @@ -85,7 +49,6 @@ when_0: t.k (#0) > CAST(0 AS Int64 NULL) === constant_subquery === description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. sql: INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t -cbo: false, distributed: false raw_source: EvalScalar ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] @@ -95,6 +58,7 @@ EvalScalar ├── order by: [] └── limit: NONE +cbo: false, distributed: false planned_source: Scan ├── table: default.t (#0) @@ -103,19 +67,7 @@ Scan └── limit: NONE when_0: t.k (#0) = CAST(1 AS Int64 NULL) -=== constant_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: false, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Scan ├── table: default.t (#0) @@ -124,19 +76,7 @@ Scan └── limit: NONE when_0: t.k (#0) = CAST(1 AS Int64 NULL) -=== constant_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: false -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Scan ├── table: default.t (#0) @@ -145,19 +85,7 @@ Scan └── limit: NONE when_0: t.k (#0) = CAST(1 AS Int64 NULL) -=== constant_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k = (SELECT 1) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Scan ├── table: default.t (#0) @@ -169,7 +97,6 @@ when_0: t.k (#0) = CAST(1 AS Int64 NULL) === in_subquery === description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t -cbo: false, distributed: false raw_source: EvalScalar ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] @@ -179,6 +106,7 @@ EvalScalar ├── order by: [] └── limit: NONE +cbo: false, distributed: false planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -196,19 +124,7 @@ Join(RightMark) └── limit: NONE when_0: 4 (#4) -=== in_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: false, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -226,19 +142,7 @@ Join(RightMark) └── limit: NONE when_0: 4 (#4) -=== in_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: false -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -256,19 +160,7 @@ Join(RightMark) └── limit: NONE when_0: 4 (#4) -=== in_subquery === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -289,7 +181,6 @@ when_0: 4 (#4) === correlated_exists === description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. sql: INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s -cbo: false, distributed: false raw_source: EvalScalar ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] @@ -299,6 +190,7 @@ EvalScalar ├── order by: [] └── limit: NONE +cbo: false, distributed: false planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -320,19 +212,7 @@ Join(RightMark) └── limit: NONE when_0: is_true(5 (#5)) -=== correlated_exists === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s cbo: false, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -354,19 +234,7 @@ Join(RightMark) └── limit: NONE when_0: is_true(5 (#5)) -=== correlated_exists === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s cbo: true, distributed: false -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -388,19 +256,7 @@ Join(RightMark) └── limit: NONE when_0: is_true(5 (#5)) -=== correlated_exists === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN EXISTS (SELECT 1 FROM lookup WHERE lookup.k = s.k) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t s cbo: true, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -425,7 +281,6 @@ when_0: is_true(5 (#5)) === scalar_aggregate === description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. sql: INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t -cbo: false, distributed: false raw_source: EvalScalar ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] @@ -435,6 +290,7 @@ EvalScalar ├── order by: [] └── limit: NONE +cbo: false, distributed: false planned_source: Join(RightSingle) ├── build keys: [] @@ -458,19 +314,7 @@ Join(RightSingle) └── limit: NONE when_0: t.k (#0) > scalar_subquery_3 (#3) -=== scalar_aggregate === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: false, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightSingle) ├── build keys: [] @@ -494,19 +338,7 @@ Join(RightSingle) └── limit: NONE when_0: t.k (#0) > scalar_subquery_3 (#3) -=== scalar_aggregate === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: false -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightSingle) ├── build keys: [] @@ -530,19 +362,7 @@ Join(RightSingle) └── limit: NONE when_0: t.k (#0) > scalar_subquery_3 (#3) -=== scalar_aggregate === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k > (SELECT max(k) FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightSingle) ├── build keys: [] @@ -569,7 +389,6 @@ when_0: t.k (#0) > scalar_subquery_3 (#3) === multiple_first === description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. sql: INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t -cbo: false, distributed: false raw_source: EvalScalar ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] @@ -579,6 +398,7 @@ EvalScalar ├── order by: [] └── limit: NONE +cbo: false, distributed: false planned_source: Join(RightSingle) ├── build keys: [] @@ -612,19 +432,7 @@ Join(RightSingle) when_0: 6 (#6) when_1: t.k (#0) > scalar_subquery_4 (#4) -=== multiple_first === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t cbo: false, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightSingle) ├── build keys: [] @@ -658,19 +466,7 @@ Join(RightSingle) when_0: 6 (#6) when_1: t.k (#0) > scalar_subquery_4 (#4) -=== multiple_first === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t cbo: true, distributed: false -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightSingle) ├── build keys: [] @@ -704,19 +500,7 @@ Join(RightSingle) when_0: 6 (#6) when_1: t.k (#0) > scalar_subquery_4 (#4) -=== multiple_first === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT FIRST WHEN k IN (SELECT k FROM lookup) THEN INTO dst WHEN k > (SELECT min(k) FROM lookup) THEN INTO dst2 ELSE INTO dst SELECT k, v FROM t cbo: true, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightSingle) ├── build keys: [] @@ -753,7 +537,6 @@ when_1: t.k (#0) > scalar_subquery_4 (#4) === nullable_not_in === description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. sql: INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t -cbo: false, distributed: false raw_source: EvalScalar ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] @@ -763,6 +546,7 @@ EvalScalar ├── order by: [] └── limit: NONE +cbo: false, distributed: false planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -780,19 +564,7 @@ Join(RightMark) └── limit: NONE when_0: NOT 4 (#4) -=== nullable_not_in === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: false, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -810,19 +582,7 @@ Join(RightMark) └── limit: NONE when_0: NOT 4 (#4) -=== nullable_not_in === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: false -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -840,19 +600,7 @@ Join(RightMark) └── limit: NONE when_0: NOT 4 (#4) -=== nullable_not_in === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k NOT IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t cbo: true, distributed: true -raw_source: -EvalScalar -├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] -└── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -873,7 +621,6 @@ when_0: NOT 4 (#4) === source_limit === description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2 -cbo: false, distributed: false raw_source: Limit ├── limit: [2] @@ -889,6 +636,7 @@ Limit ├── order by: [] └── limit: NONE +cbo: false, distributed: false planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -910,25 +658,7 @@ Join(RightMark) └── limit: 2 when_0: 4 (#4) -=== source_limit === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2 cbo: false, distributed: true -raw_source: -Limit -├── limit: [2] -├── offset: [0] -└── Sort - ├── sort keys: [t.k (#0) ASC NULLS LAST] - ├── limit: [NONE] - └── EvalScalar - ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] - └── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -950,25 +680,7 @@ Join(RightMark) └── limit: 2 when_0: 4 (#4) -=== source_limit === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2 cbo: true, distributed: false -raw_source: -Limit -├── limit: [2] -├── offset: [0] -└── Sort - ├── sort keys: [t.k (#0) ASC NULLS LAST] - ├── limit: [NONE] - └── EvalScalar - ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] - └── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] @@ -990,25 +702,7 @@ Join(RightMark) └── limit: 2 when_0: 4 (#4) -=== source_limit === -description: WHEN decorrelation introduces logical nodes before source planning; branch symbols and source column order survive optimization. -sql: INSERT ALL WHEN k IN (SELECT k FROM lookup) THEN INTO dst ELSE INTO dst2 SELECT k, v FROM t ORDER BY k LIMIT 2 cbo: true, distributed: true -raw_source: -Limit -├── limit: [2] -├── offset: [0] -└── Sort - ├── sort keys: [t.k (#0) ASC NULLS LAST] - ├── limit: [NONE] - └── EvalScalar - ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] - └── Scan - ├── table: default.t (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - planned_source: Join(RightMark) ├── build keys: [lookup.k (#2)] diff --git a/src/query/sql/tests/it/optimizer/join_cardinality/common.rs b/src/query/sql/tests/it/optimizer/join_cardinality/common.rs index c4e7a351712..d76af50b43d 100644 --- a/src/query/sql/tests/it/optimizer/join_cardinality/common.rs +++ b/src/query/sql/tests/it/optimizer/join_cardinality/common.rs @@ -20,8 +20,8 @@ use databend_common_sql::ColumnEntry; use databend_common_sql::Metadata; use databend_common_sql::Symbol; use databend_common_sql::optimizer::ir::ColumnStat; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::RelExpr; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::optimizer::ir::StatInfo; use databend_common_sql::plans::JoinType; @@ -146,7 +146,7 @@ pub(super) fn write_join_stat_info( pub(super) fn collect_join_cardinalities( file: &mut impl Write, metadata: &Metadata, - expr: &SExpr, + expr: &PExpr, expected_join_type: JoinType, case_name: &str, ) -> Result { @@ -156,7 +156,7 @@ pub(super) fn collect_join_cardinalities( join.join_type, expected_join_type, "unexpected join type for {case_name}" ); - let stat_info = RelExpr::with_s_expr(expr).derive_cardinality(&StatContext::default())?; + let stat_info = RelExpr::with_p_expr(expr).derive_cardinality(&StatContext::default())?; writeln!( file, "join : {:<11} cardinality={:.3}", diff --git a/src/query/sql/tests/it/optimizer/join_cardinality/histogram.rs b/src/query/sql/tests/it/optimizer/join_cardinality/histogram.rs index c5be4877d10..f10d5873dea 100644 --- a/src/query/sql/tests/it/optimizer/join_cardinality/histogram.rs +++ b/src/query/sql/tests/it/optimizer/join_cardinality/histogram.rs @@ -123,8 +123,13 @@ async fn write_sql_join_input( writeln!(file, "query : {}", query.name)?; writeln!(file, "sql : {}", query.sql)?; - let joins = - collect_join_cardinalities(file, &metadata, &s_expr, expected_join_type, case.name)?; + let joins = collect_join_cardinalities( + file, + &metadata, + s_expr.planned()?.expr(), + expected_join_type, + case.name, + )?; assert_eq!(joins, 1); Ok(()) } diff --git a/src/query/sql/tests/it/optimizer/join_cardinality/histogram/decimal.rs b/src/query/sql/tests/it/optimizer/join_cardinality/histogram/decimal.rs index d97050d9017..604a7a2df54 100644 --- a/src/query/sql/tests/it/optimizer/join_cardinality/histogram/decimal.rs +++ b/src/query/sql/tests/it/optimizer/join_cardinality/histogram/decimal.rs @@ -99,7 +99,13 @@ async fn write_decimal_case( let metadata = metadata.read(); writeln!(file, "sql : {sql}")?; - let joins = collect_join_cardinalities(file, &metadata, &s_expr, JoinType::Inner, name)?; + let joins = collect_join_cardinalities( + file, + &metadata, + s_expr.planned()?.expr(), + JoinType::Inner, + name, + )?; assert_eq!(joins, 1); writeln!(file)?; Ok(()) diff --git a/src/query/sql/tests/it/optimizer/join_cardinality/histogram/single.rs b/src/query/sql/tests/it/optimizer/join_cardinality/histogram/single.rs index 3486473d667..3cc1389374b 100644 --- a/src/query/sql/tests/it/optimizer/join_cardinality/histogram/single.rs +++ b/src/query/sql/tests/it/optimizer/join_cardinality/histogram/single.rs @@ -14,11 +14,19 @@ use std::io::Write; +use databend_common_catalog::table_context::TableContextSettings; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_sql::optimizer::CollectStatisticsOptimizer; +use databend_common_sql::optimizer::Optimizer; +use databend_common_sql::optimizer::OptimizerContext; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::RelExpr; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::StatContext; +use databend_common_sql::optimizer::optimizers::operator::SubqueryDecorrelatorOptimizer; +use databend_common_sql::optimizer::optimizers::recursive::RecursiveRuleOptimizer; +use databend_common_sql::optimizer::optimizers::rule::DEFAULT_REWRITE_RULES; use databend_common_sql::optimizer::optimizers::rule::Rule; use databend_common_sql::optimizer::optimizers::rule::RuleCommuteJoin; use databend_common_sql::optimizer::optimizers::rule::TransformResult; @@ -58,7 +66,7 @@ fn large_overlap_stats() -> TableStats { } } -fn find_join(expr: &SExpr, join_type: JoinType) -> Option<&SExpr> { +fn find_join(expr: &PExpr, join_type: JoinType) -> Option<&PExpr> { if matches!(expr.plan(), RelOperator::Join(join) if join.join_type == join_type) { return Some(expr); } @@ -66,6 +74,14 @@ fn find_join(expr: &SExpr, join_type: JoinType) -> Option<&SExpr> { .find_map(|child| find_join(child, join_type)) } +fn find_logical_join(expr: &SExpr, join_type: JoinType) -> Option<&SExpr> { + if matches!(expr.plan(), RelOperator::Join(join) if join.join_type == join_type) { + return Some(expr); + } + expr.children() + .find_map(|child| find_logical_join(child, join_type)) +} + async fn write_optimizer_commuted_right_single( file: &mut impl Write, case: &JoinTestCase, @@ -88,20 +104,37 @@ async fn write_optimizer_commuted_right_single( ) .await?; + let raw = ctx.bind_sql(case.input.sql).await?; let Plan::Query { s_expr, metadata, .. - } = ctx - .optimize_plan(ctx.bind_sql(case.input.sql).await?) - .await? + } = ctx.optimize_plan(raw.clone()).await? else { return Err(ErrorCode::Internal("SELECT should bind to a query plan")); }; let mut state = TransformResult::new(); - let (right_single, optimizer) = match find_join(&s_expr, JoinType::RightSingle) { + let physical_commuted; + let (right_single, optimizer) = match find_join(s_expr.planned()?.expr(), JoinType::RightSingle) + { Some(right_single) => (right_single, "full optimizer"), None => { - let left_single = find_join(&s_expr, JoinType::LeftSingle) - .ok_or_else(|| ErrorCode::Internal("optimizer did not derive SINGLE from SQL"))?; + // CommuteJoin is a logical rule. Derive its input from bound SQL before + // physical selection rather than converting an execution tree backwards. + let Plan::Query { s_expr, .. } = raw else { + unreachable!() + }; + let context = + OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?); + let logical = SubqueryDecorrelatorOptimizer::new(context.clone(), None) + .optimize_sync((*s_expr).into_logical()?)?; + let logical = CollectStatisticsOptimizer::new(context.clone()) + .optimize(logical) + .await?; + let logical = RecursiveRuleOptimizer::new(context, &DEFAULT_REWRITE_RULES) + .optimize_sync(logical)?; + let left_single = + find_logical_join(&logical, JoinType::LeftSingle).ok_or_else(|| { + ErrorCode::Internal("logical optimizer did not derive SINGLE from SQL") + })?; RuleCommuteJoin::new(StatContext::default()).apply(left_single, &mut state)?; let left_cardinality = RelExpr::with_s_expr(left_single.child(0)?) .derive_cardinality(&StatContext::default())? @@ -118,7 +151,8 @@ async fn write_optimizer_commuted_right_single( .ok_or_else(|| ErrorCode::Internal(format!( "join commute rule did not derive RIGHT SINGLE: left={left_cardinality}, right={right_cardinality}" )))?; - (right_single, "CommuteJoin") + physical_commuted = PExpr::from(right_single.clone()); + (&physical_commuted, "CommuteJoin") } }; diff --git a/src/query/sql/tests/it/optimizer/join_cardinality/statistics.rs b/src/query/sql/tests/it/optimizer/join_cardinality/statistics.rs index 5aacbc0d447..095d4b90201 100644 --- a/src/query/sql/tests/it/optimizer/join_cardinality/statistics.rs +++ b/src/query/sql/tests/it/optimizer/join_cardinality/statistics.rs @@ -138,8 +138,13 @@ async fn write_sql_join_statistics_case( return Err(ErrorCode::Internal("SELECT should bind to a query plan")); }; let metadata = metadata.read(); - let joins = - collect_join_cardinalities(file, &metadata, &s_expr, case.expected_join_type, case.name)?; + let joins = collect_join_cardinalities( + file, + &metadata, + s_expr.planned()?.expr(), + case.expected_join_type, + case.name, + )?; assert_eq!(joins, 1); writeln!(file)?; Ok(()) diff --git a/src/query/sql/tests/it/optimizer/mod.rs b/src/query/sql/tests/it/optimizer/mod.rs index bb67214e5f4..e7c487d44fe 100644 --- a/src/query/sql/tests/it/optimizer/mod.rs +++ b/src/query/sql/tests/it/optimizer/mod.rs @@ -25,6 +25,7 @@ mod column_scope; mod decorrelate_correlated_aliases; mod distributed_join; mod eager_aggregation; +mod eliminate_eval_scalar; mod hierarchical_grouping_sets; mod insert_multi_table; mod join_cardinality; @@ -32,6 +33,7 @@ mod materialized_cte_distribution; mod mutation; mod normalize_scalar; mod outer_join_to_anti; +mod physical_planner; mod planning_context; mod push_down_filter_project_set; mod push_down_rank_limit_aggregate; diff --git a/src/query/sql/tests/it/optimizer/mutation.rs b/src/query/sql/tests/it/optimizer/mutation.rs index f9c53cb492a..adbbc5813f0 100644 --- a/src/query/sql/tests/it/optimizer/mutation.rs +++ b/src/query/sql/tests/it/optimizer/mutation.rs @@ -18,6 +18,7 @@ use databend_common_catalog::table_context::TableContextSettings; use databend_common_exception::Result; use databend_common_sql::binder::MutationStrategy; use databend_common_sql::optimizer::OptimizerContext; +use databend_common_sql::optimizer::ir::MutationPlan; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::optimizer::optimize; use databend_common_sql::plans::Plan; @@ -153,8 +154,8 @@ async fn test_mutation_preparation() -> Result<()> { }, ), ]; - for distributed in [false, true] { - for (name, sql, expected) in cases { + for (name, sql, expected) in cases { + for distributed in [false, true] { let case = SqlTestCase { name, description: "Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy.", @@ -172,11 +173,25 @@ async fn test_mutation_preparation() -> Result<()> { } let raw = ctx.bind_sql(sql).await?; let Plan::DataMutation { - metadata, schema, .. + s_expr: bound_expr, + metadata, + schema, } = &raw else { unreachable!() }; + assert!(matches!(bound_expr.as_ref(), MutationPlan::Logical(_))); + assert!( + bound_expr.planned().is_err(), + "bound mutation must not reach execution" + ); + raw.capture_bound_query_lineage(); + let bound_lineage = raw.query_lineage()?; + let bound_udfs = bound_expr + .input_udfs()? + .into_iter() + .cloned() + .collect::>(); let opt_ctx = OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?) .with_settings(&ctx.get_settings())?; @@ -190,6 +205,17 @@ async fn test_mutation_preparation() -> Result<()> { else { unreachable!() }; + assert!(matches!(s_expr.as_ref(), MutationPlan::Planned(_))); + assert!( + s_expr.logical().is_err(), + "planned mutation must not reenter logical preparation" + ); + assert_eq!(bound_lineage, optimized.query_lineage()?); + assert_eq!( + bound_udfs, + s_expr.input_udfs()?.into_iter().cloned().collect() + ); + let s_expr = s_expr.planned()?; let mutation = s_expr.plan().as_mutation().unwrap(); assert_eq!( schema, optimized_schema, @@ -235,13 +261,15 @@ async fn test_mutation_preparation() -> Result<()> { } s_expr.child(0)?.validate_types(metadata)?; s_expr.child(0)?.validate_column_scope(metadata)?; - write_case_header(&mut file, &case)?; + if !distributed { + write_case_header(&mut file, &case)?; + writeln!( + file, + "raw_plan:\n{}", + raw.format_indent(Default::default(), &StatContext::default())? + )?; + } writeln!(file, "requested_distributed: {distributed}")?; - writeln!( - file, - "raw_plan:\n{}", - raw.format_indent(Default::default(), &StatContext::default())? - )?; writeln!( file, "optimized_plan:\n{}", diff --git a/src/query/sql/tests/it/optimizer/mutation.txt b/src/query/sql/tests/it/optimizer/mutation.txt index 1ca9a55852e..4f3aae929db 100644 --- a/src/query/sql/tests/it/optimizer/mutation.txt +++ b/src/query/sql/tests/it/optimizer/mutation.txt @@ -1,7 +1,6 @@ === direct_update === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: UPDATE target SET v = v + 1 WHERE k > 10 -requested_distributed: false raw_plan: MergeInto: target_table: default.default.target @@ -13,6 +12,7 @@ target_table: default.default.target ├── filters: [gt(target.k (#0), 10)] └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) +requested_distributed: false optimized_plan: MergeInto: target_table: default.default.target @@ -24,10 +24,21 @@ target_table: default.default.target mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=Some(2), direct_filter_count=1, matched_update_count=1 +requested_distributed: true +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [FunctionCall(FunctionCall { span: None, func_name: "gt", params: [], arguments: [BoundColumnRef(BoundColumnRef { span: Some(34..35), column: ColumnBinding { database_name: Some("default"), table_name: Some("target"), column_position: Some(1), table_index: Some(0), column_name: "k", column_name_lower: None, index: 0, data_type: Number(Int64), visibility: Visible, virtual_expr: None, is_srf: false } }), ConstantExpr(ConstantExpr { span: None, value: Number(10_i64) })] })], predicate_column_index: Some(2), read_partition_columns: {0} }) + +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=Some(2), direct_filter_count=1, matched_update_count=1 + === direct_update_false === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: UPDATE target SET v = v + 1 WHERE false -requested_distributed: false raw_plan: MergeInto: target_table: default.default.target @@ -39,6 +50,7 @@ target_table: default.default.target ├── filters: [false] └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) +requested_distributed: false optimized_plan: MergeInto: target_table: default.default.target @@ -50,10 +62,21 @@ target_table: default.default.target mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 +requested_distributed: true +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: true +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── EmptyResultScan + +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + === direct_delete_filter === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: DELETE FROM target WHERE k > 10 -requested_distributed: false raw_plan: MergeInto: target_table: default.default.target @@ -65,6 +88,7 @@ target_table: default.default.target ├── filters: [gt(target.k (#0), 10)] └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) +requested_distributed: false optimized_plan: MergeInto: target_table: default.default.target @@ -76,34 +100,21 @@ target_table: default.default.target mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=1, matched_update_count=0 -=== direct_delete_all === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: DELETE FROM target -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) - +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target -├── distributed: false +├── distributed: true ├── target_build_optimization: false ├── can_try_update_column_only: false ├── matched delete: [condition: None] -└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [FunctionCall(FunctionCall { span: None, func_name: "gt", params: [], arguments: [BoundColumnRef(BoundColumnRef { span: Some(25..26), column: ColumnBinding { database_name: Some("default"), table_name: Some("target"), column_position: Some(1), table_index: Some(0), column_name: "k", column_name_lower: None, index: 0, data_type: Number(Int64), visibility: Visible, virtual_expr: None, is_srf: false } }), ConstantExpr(ConstantExpr { span: None, value: Number(10_i64) })] })], predicate_column_index: None, read_partition_columns: {0} }) -mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=1, matched_update_count=0 -=== direct_delete_true === +=== direct_delete_all === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: DELETE FROM target WHERE true -requested_distributed: false +sql: DELETE FROM target raw_plan: MergeInto: target_table: default.default.target @@ -111,10 +122,9 @@ target_table: default.default.target ├── target_build_optimization: false ├── can_try_update_column_only: false ├── matched delete: [condition: None] -└── Filter - ├── filters: [true] - └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) +requested_distributed: false optimized_plan: MergeInto: target_table: default.default.target @@ -126,90 +136,21 @@ target_table: default.default.target mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 -=== direct_delete_false === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: DELETE FROM target WHERE false -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -└── Filter - ├── filters: [false] - └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) - +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target -├── distributed: false +├── distributed: true ├── target_build_optimization: false ├── can_try_update_column_only: false ├── matched delete: [condition: None] -└── EmptyResultScan - -mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 - -=== subquery_update === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: UPDATE target SET v = v + 1 WHERE k IN (SELECT k FROM source) -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] -└── Filter - ├── filters: [marker (#5)] - └── Join(RightMark) - ├── build keys: [source.k (#2)] - ├── probe keys: [target.k (#0)] - ├── other filters: [] - ├── EvalScalar - │ ├── scalars: [source.k (#2) AS (#2)] - │ └── Scan - │ ├── table: default.source (#1) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] -└── Join(LeftSemi) - ├── build keys: [source.k (#2)] - ├── probe keys: [target.k (#0)] - ├── other filters: [] - ├── Scan - │ ├── table: default.source (#1) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE +└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) -mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 +mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 -=== subquery_delete === +=== direct_delete_true === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: DELETE FROM target WHERE k IN (SELECT k FROM source) -requested_distributed: false +sql: DELETE FROM target WHERE true raw_plan: MergeInto: target_table: default.default.target @@ -218,538 +159,22 @@ target_table: default.default.target ├── can_try_update_column_only: false ├── matched delete: [condition: None] └── Filter - ├── filters: [marker (#5)] - └── Join(RightMark) - ├── build keys: [source.k (#2)] - ├── probe keys: [target.k (#0)] - ├── other filters: [] - ├── EvalScalar - │ ├── scalars: [source.k (#2) AS (#2)] - │ └── Scan - │ ├── table: default.source (#1) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -└── Join(LeftSemi) - ├── build keys: [source.k (#2)] - ├── probe keys: [target.k (#0)] - ├── other filters: [] - ├── Scan - │ ├── table: default.source (#1) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + ├── filters: [true] + └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) -=== matched_merge === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = source.v (#1)] -└── Join(Inner) - ├── build keys: [source.k (#0)] - ├── probe keys: [target.k (#2)] - ├── other filters: [] - ├── Scan - │ ├── table: default.source (#0) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#1) - ├── filters: [] - ├── order by: [] - └── limit: NONE - optimized_plan: MergeInto: target_table: default.default.target ├── distributed: false ├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = source.v (#1)] -└── Join(Inner) - ├── build keys: [source.k (#0)] - ├── probe keys: [target.k (#2)] - ├── other filters: [] - ├── Scan - │ ├── table: default.source (#0) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#1) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 - -=== insert_only_merge === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: MERGE INTO target t USING source s ON t.k = s.k WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] -└── Join(RightAnti) - ├── build keys: [source.k (#0)] - ├── probe keys: [target.k (#2)] - ├── other filters: [] - ├── Scan - │ ├── table: default.source (#0) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#1) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] -└── Join(LeftAnti) - ├── build keys: [target.k (#2)] - ├── probe keys: [source.k (#0)] - ├── other filters: [] - ├── Scan - │ ├── table: default.target (#1) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.source (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -mutation_state: strategy=NotMatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 - -=== mixed_merge === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = source.v (#1)] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] -└── Join(Right) - ├── build keys: [source.k (#0)] - ├── probe keys: [target.k (#2)] - ├── other filters: [] - ├── Scan - │ ├── table: default.source (#0) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#1) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = source.v (#1)] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] -└── Join(Left) - ├── build keys: [target.k (#2)] - ├── probe keys: [source.k (#0)] - ├── other filters: [] - ├── Scan - │ ├── table: default.target (#1) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.source (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 - -=== aggregate_source_local_retry === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = max(v) (#3)] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] -└── Join(Right) - ├── build keys: [max(k) (#2)] - ├── probe keys: [target.k (#4)] - ├── other filters: [] - ├── EvalScalar - │ ├── scalars: [max(k) (#2) AS (#2), max(v) (#3) AS (#3)] - │ └── Aggregate(Initial) - │ ├── group items: [] - │ ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] - │ └── EvalScalar - │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] - │ └── Scan - │ ├── table: default.source (#0) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#1) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = max(v) (#3)] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] -└── Join(Left) - ├── build keys: [target.k (#4)] - ├── probe keys: [max(k) (#2)] - ├── other filters: [] - ├── Scan - │ ├── table: default.target (#1) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Aggregate(Final) - ├── group items: [] - ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] - └── Aggregate(Partial) - ├── group items: [] - ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] - └── Scan - ├── table: default.source (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 - -=== empty_target_local_retry === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = max(v) (#3)] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] -└── Join(Right) - ├── build keys: [max(k) (#2)] - ├── probe keys: [target.k (#4)] - ├── other filters: [] - ├── EvalScalar - │ ├── scalars: [max(k) (#2) AS (#2), max(v) (#3) AS (#3)] - │ └── Aggregate(Initial) - │ ├── group items: [] - │ ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] - │ └── EvalScalar - │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] - │ └── Scan - │ ├── table: default.source (#0) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Filter - ├── filters: [gt(target.k (#4), 10), lt(target.k (#4), 0)] - └── Scan - ├── table: default.target (#1) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] -└── Join(Left) - ├── build keys: [target.k (#4)] - ├── probe keys: [max(k) (#2)] - ├── other filters: [] - ├── EmptyResultScan - └── Aggregate(Final) - ├── group items: [] - ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] - └── Aggregate(Partial) - ├── group items: [] - ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] - └── Scan - ├── table: default.source (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 - -=== empty_target_merge === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: MERGE INTO target t USING source s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = source.v (#1)] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] -└── Join(Right) - ├── build keys: [source.k (#0)] - ├── probe keys: [target.k (#2)] - ├── other filters: [] - ├── Scan - │ ├── table: default.source (#0) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Filter - ├── filters: [gt(target.k (#2), 10), lt(target.k (#2), 0)] - └── Scan - ├── table: default.target (#1) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] -└── Join(Left) - ├── build keys: [target.k (#2)] - ├── probe keys: [source.k (#0)] - ├── other filters: [] - ├── EmptyResultScan - └── Scan - ├── table: default.source (#0) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 - -=== empty_source_merge === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: MERGE INTO target t USING (SELECT * FROM source WHERE false) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: false -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = source.v (#1)] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] -└── Join(Right) - ├── build keys: [source.k (#0)] - ├── probe keys: [target.k (#2)] - ├── other filters: [] - ├── EvalScalar - │ ├── scalars: [source.k (#0) AS (#0), source.v (#1) AS (#1)] - │ └── Filter - │ ├── filters: [false] - │ └── Scan - │ ├── table: default.source (#0) - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── Scan - ├── table: default.target (#1) - ├── filters: [] - ├── order by: [] - └── limit: NONE - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: true -├── matched update: [condition: None,update set v = source.v (#1)] -├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] -└── EmptyResultScan - -mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 - -=== direct_update === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: UPDATE target SET v = v + 1 WHERE k > 10 -requested_distributed: true -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] -└── Filter - ├── filters: [gt(target.k (#0), 10)] - └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: true -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] -└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [FunctionCall(FunctionCall { span: None, func_name: "gt", params: [], arguments: [BoundColumnRef(BoundColumnRef { span: Some(34..35), column: ColumnBinding { database_name: Some("default"), table_name: Some("target"), column_position: Some(1), table_index: Some(0), column_name: "k", column_name_lower: None, index: 0, data_type: Number(Int64), visibility: Visible, virtual_expr: None, is_srf: false } }), ConstantExpr(ConstantExpr { span: None, value: Number(10_i64) })] })], predicate_column_index: Some(2), read_partition_columns: {0} }) - -mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=Some(2), direct_filter_count=1, matched_update_count=1 - -=== direct_update_false === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: UPDATE target SET v = v + 1 WHERE false -requested_distributed: true -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] -└── Filter - ├── filters: [false] - └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Update, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: true -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] -└── EmptyResultScan - -mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 - -=== direct_delete_filter === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: DELETE FROM target WHERE k > 10 -requested_distributed: true -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -└── Filter - ├── filters: [gt(target.k (#0), 10)] - └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) - -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: true -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [FunctionCall(FunctionCall { span: None, func_name: "gt", params: [], arguments: [BoundColumnRef(BoundColumnRef { span: Some(25..26), column: ColumnBinding { database_name: Some("default"), table_name: Some("target"), column_position: Some(1), table_index: Some(0), column_name: "k", column_name_lower: None, index: 0, data_type: Number(Int64), visibility: Visible, virtual_expr: None, is_srf: false } }), ConstantExpr(ConstantExpr { span: None, value: Number(10_i64) })] })], predicate_column_index: None, read_partition_columns: {0} }) - -mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=1, matched_update_count=0 - -=== direct_delete_all === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: DELETE FROM target -requested_distributed: true -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false ├── can_try_update_column_only: false ├── matched delete: [condition: None] └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) -optimized_plan: -MergeInto: -target_table: default.default.target -├── distributed: true -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -└── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) - -mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 - -=== direct_delete_true === -description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. -sql: DELETE FROM target WHERE true -requested_distributed: true -raw_plan: -MergeInto: -target_table: default.default.target -├── distributed: false -├── target_build_optimization: false -├── can_try_update_column_only: false -├── matched delete: [condition: None] -└── Filter - ├── filters: [true] - └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) - +mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=false, truncate=true, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -764,7 +189,6 @@ mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effec === direct_delete_false === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: DELETE FROM target WHERE false -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -776,6 +200,19 @@ target_table: default.default.target ├── filters: [false] └── MutationSource(MutationSource { schema: TableSchema { fields: [TableField { name: "k", default_expr: None, data_type: Number(Int64), column_id: 0, computed_expr: None, auto_increment_expr: None }, TableField { name: "v", default_expr: None, data_type: Number(Int64), column_id: 1, computed_expr: None, auto_increment_expr: None }], metadata: {}, next_column_id: 2 }, columns: {0, 1}, table_index: 0, mutation_type: Delete, secure_predicates: [], user_predicates: [], predicate_column_index: None, read_partition_columns: {} }) +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── EmptyResultScan + +mutation_state: strategy=Direct, distributed=false, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -790,7 +227,6 @@ mutation_state: strategy=Direct, distributed=true, row_id_shuffle=true, no_effec === subquery_update === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: UPDATE target SET v = v + 1 WHERE k IN (SELECT k FROM source) -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -817,6 +253,32 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched update: [condition: None,update set v = plus(target.v (#1), 1)] +└── Join(LeftSemi) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -847,7 +309,6 @@ mutation_state: strategy=MatchedOnly, distributed=true, row_id_shuffle=true, no_ === subquery_delete === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: DELETE FROM target WHERE k IN (SELECT k FROM source) -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -874,6 +335,32 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +└── Join(LeftSemi) + ├── build keys: [source.k (#2)] + ├── probe keys: [target.k (#0)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -904,7 +391,6 @@ mutation_state: strategy=MatchedOnly, distributed=true, row_id_shuffle=true, no_ === matched_merge === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -927,6 +413,32 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +└── Join(Inner) + ├── build keys: [source.k (#0)] + ├── probe keys: [target.k (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.source (#0) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.target (#1) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -957,7 +469,6 @@ mutation_state: strategy=MatchedOnly, distributed=true, row_id_shuffle=false, no === insert_only_merge === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: MERGE INTO target t USING source s ON t.k = s.k WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -980,6 +491,32 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(LeftAnti) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=NotMatchedOnly, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -1013,7 +550,6 @@ mutation_state: strategy=NotMatchedOnly, distributed=true, row_id_shuffle=true, === mixed_merge === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: MERGE INTO target t USING source s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -1037,6 +573,33 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -1071,7 +634,6 @@ mutation_state: strategy=MixedMatched, distributed=true, row_id_shuffle=true, no === aggregate_source_local_retry === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -1102,6 +664,39 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = max(v) (#3)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#4)] + ├── probe keys: [max(k) (#2)] + ├── other filters: [] + ├── Scan + │ ├── table: default.target (#1) + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -1136,7 +731,6 @@ mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, n === empty_target_local_retry === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: MERGE INTO target t USING (SELECT max(k) AS k, max(v) AS v FROM source) s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -1169,6 +763,35 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(max(k) (#2) AS Int64),CAST(max(v) (#3) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#4)] + ├── probe keys: [max(k) (#2)] + ├── other filters: [] + ├── EmptyResultScan + └── Aggregate(Final) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Aggregate(Partial) + ├── group items: [] + ├── aggregate functions: [max(source.k (#0)) AS (#2), max(source.v (#1)) AS (#3)] + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -1199,7 +822,6 @@ mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, n === empty_target_merge === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: MERGE INTO target t USING source s ON t.k = s.k AND t.k > 10 AND t.k < 0 WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -1225,6 +847,29 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: false +├── matched delete: [condition: None] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── Join(Left) + ├── build keys: [target.k (#2)] + ├── probe keys: [source.k (#0)] + ├── other filters: [] + ├── EmptyResultScan + └── Scan + ├── table: default.source (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=false, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=0 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target @@ -1255,7 +900,6 @@ mutation_state: strategy=MixedMatched, distributed=true, row_id_shuffle=true, no === empty_source_merge === description: Mutation logical preparation precedes plan selection; distribution finalization keeps its legacy policy. sql: MERGE INTO target t USING (SELECT * FROM source WHERE false) s ON t.k = s.k WHEN MATCHED THEN UPDATE SET v = s.v WHEN NOT MATCHED THEN INSERT (k, v) VALUES (s.k, s.v) -requested_distributed: true raw_plan: MergeInto: target_table: default.default.target @@ -1283,6 +927,20 @@ target_table: default.default.target ├── order by: [] └── limit: NONE +requested_distributed: false +optimized_plan: +MergeInto: +target_table: default.default.target +├── distributed: false +├── target_build_optimization: false +├── can_try_update_column_only: true +├── matched update: [condition: None,update set v = source.v (#1)] +├── unmatched insert: [condition: None,insert into (k,v) values(CAST(source.k (#0) AS Int64),CAST(source.v (#1) AS Int64))] +└── EmptyResultScan + +mutation_state: strategy=MixedMatched, distributed=false, row_id_shuffle=true, no_effect=true, truncate=false, predicate_column=None, direct_filter_count=0, matched_update_count=1 + +requested_distributed: true optimized_plan: MergeInto: target_table: default.default.target diff --git a/src/query/sql/tests/it/optimizer/physical_planner.rs b/src/query/sql/tests/it/optimizer/physical_planner.rs new file mode 100644 index 00000000000..767d0d46b28 --- /dev/null +++ b/src/query/sql/tests/it/optimizer/physical_planner.rs @@ -0,0 +1,341 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Write; + +use databend_common_catalog::table_context::TableContextSettings; +use databend_common_exception::Result; +use databend_common_sql::optimizer::OptimizerContext; +use databend_common_sql::optimizer::ir::QueryPlan; +use databend_common_sql::optimizer::ir::StatContext; +use databend_common_sql::optimizer::optimize; +use databend_common_sql::plans::Plan; + +use crate::framework::LiteTableContext; +use crate::framework::golden::open_golden_file; + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_query_planning_boundary() -> Result<()> { + let mut file = open_golden_file("optimizer", "physical_planner.txt")?; + for distributed in [false, true] { + for cbo in [false, true] { + let ctx = LiteTableContext::create().await?; + ctx.set_cluster_node_num(if distributed { 2 } else { 1 }); + ctx.set_table_warehouse_distribution(distributed); + ctx.register_setup_sql("CREATE TABLE t(k Int64 NOT NULL, v Int64 NOT NULL)") + .await?; + ctx.get_settings() + .set_setting("enable_cbo".to_string(), u8::from(cbo).to_string())?; + ctx.get_settings() + .set_setting("enable_optimizer_trace".to_string(), "1".to_string())?; + let raw = ctx + .bind_sql("SELECT k, sum(v) FROM t GROUP BY k ORDER BY k LIMIT 5") + .await?; + let Plan::Query { + s_expr, metadata, .. + } = &raw + else { + unreachable!() + }; + assert!(matches!(s_expr.as_ref(), QueryPlan::Logical(_))); + assert!( + s_expr.planned().is_err(), + "bound input must not reach execution" + ); + let context = + OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?) + .with_settings(&ctx.get_settings())?; + context.set_enable_distributed_optimization(distributed); + let planned = optimize(context, raw.clone()).await?; + let Plan::Query { s_expr, .. } = &planned else { + unreachable!() + }; + assert!(matches!(s_expr.as_ref(), QueryPlan::Planned(_))); + assert!( + (*s_expr.clone()).into_logical().is_err(), + "selected implementations are not logical inputs" + ); + let implementation = s_expr.planned()?; + if distributed { + assert!(implementation.expr().has_merge_exchange()); + } + implementation.expr().validate_types(metadata)?; + implementation.expr().validate_column_scope(metadata)?; + assert_eq!(raw.schema(), planned.schema()); + writeln!( + file, + "=== query_boundary: distributed={distributed}, cbo={cbo} ===" + )?; + writeln!( + file, + "sql: SELECT k, sum(v) FROM t GROUP BY k ORDER BY k LIMIT 5" + )?; + writeln!( + file, + "raw_plan:\n{}", + raw.format_indent(Default::default(), &StatContext::default())? + )?; + writeln!( + file, + "planned_query:\n{}", + planned.format_indent(Default::default(), &StatContext::default())? + )?; + let without_merge = planned.remove_exchange_for_select(); + let Plan::Query { s_expr, .. } = without_merge else { + unreachable!() + }; + assert!( + s_expr.planned().is_ok(), + "consumer adapter must preserve planned state" + ); + } + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_cascades_logical_input_physical_output() -> Result<()> { + use databend_common_sql::optimizer::ir::Distribution; + use databend_common_sql::optimizer::ir::PExpr; + use databend_common_sql::optimizer::ir::RequiredProperty; + use databend_common_sql::optimizer::optimizers::CascadesOptimizer; + + for cbo in [false, true] { + let ctx = LiteTableContext::create().await?; + ctx.register_setup_sql("CREATE TABLE t(k Int64 NOT NULL)") + .await?; + ctx.get_settings() + .set_setting("enable_cbo".to_string(), u8::from(cbo).to_string())?; + let Plan::Query { + s_expr, metadata, .. + } = ctx.bind_sql("SELECT k FROM t").await? + else { + unreachable!() + }; + let context = + OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?) + .with_settings(&ctx.get_settings())?; + let mut search = CascadesOptimizer::new(context)?; + let output: PExpr = search.optimize_sync((*s_expr).into_logical()?)?; + output.validate_types(&metadata)?; + output.validate_column_scope(&metadata)?; + let root = search + .memo() + .root() + .expect("search initializes the memo even on fallback"); + assert_eq!( + root.best_prop(&RequiredProperty { + distribution: Distribution::Any + }) + .is_some(), + cbo + ); + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_physical_planner_explain_and_skip_compatibility() -> Result<()> { + for skipped in ["", "CascadesOptimizer"] { + for sql in [ + "SELECT sum(v) FROM t", + "EXPLAIN MEMO SELECT sum(v) FROM t", + "EXPLAIN DECORRELATED SELECT k FROM t", + ] { + let ctx = LiteTableContext::create().await?; + ctx.register_setup_sql("CREATE TABLE t(k Int64 NOT NULL, v Int64 NOT NULL)") + .await?; + ctx.get_settings() + .set_optimizer_skip_list(skipped.to_string())?; + let raw = ctx.bind_sql(sql).await?; + let metadata = match &raw { + Plan::Query { metadata, .. } => metadata.clone(), + Plan::Explain { plan, .. } => match plan.as_ref() { + Plan::Query { metadata, .. } => metadata.clone(), + _ => unreachable!(), + }, + _ => unreachable!(), + }; + let context = OptimizerContext::new(ctx.clone(), metadata, ctx.get_function_context()?) + .with_settings(&ctx.get_settings())?; + let planned = optimize(context, raw).await?; + match planned { + Plan::Query { s_expr, .. } => { + s_expr.planned()?; + } + Plan::Explain { plan, .. } => { + let Plan::Query { s_expr, .. } = plan.as_ref() else { + unreachable!() + }; + // MEMO displays the search result, DECORRELATED stops before selection. + assert!(matches!(s_expr.as_ref(), QueryPlan::Logical(_))); + } + _ => unreachable!(), + } + } + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_nested_queries_keep_planned_state() -> Result<()> { + let mut file = open_golden_file("optimizer", "physical_planner_nested.txt")?; + for sql in [ + "INSERT INTO dst SELECT k, v FROM t", + "REPLACE INTO dst ON(k) SELECT k, v FROM t", + "CREATE TABLE copied ENGINE=NULL AS SELECT k, v FROM t", + "INSERT ALL WHEN k IN (SELECT k FROM t) THEN INTO dst VALUES(k, v) ELSE INTO dst2 VALUES(k, v) SELECT k, v FROM t", + ] { + let ctx = LiteTableContext::create().await?; + for setup in [ + "CREATE TABLE t(k Int64 NOT NULL, v Int64 NOT NULL)", + "CREATE TABLE dst(k Int64 NOT NULL, v Int64 NOT NULL)", + "CREATE TABLE dst2(k Int64 NOT NULL, v Int64 NOT NULL)", + ] { + ctx.register_setup_sql(setup).await?; + } + let raw = ctx.bind_sql(sql).await?; + fn source(plan: &Plan) -> &Plan { + match plan { + Plan::Insert(plan) => match &plan.source { + databend_common_sql::InsertInputSource::SelectPlan(query) => query, + _ => unreachable!(), + }, + Plan::Replace(plan) => match &plan.source { + databend_common_sql::InsertInputSource::SelectPlan(query) => query, + _ => unreachable!(), + }, + Plan::CreateTable(plan) => plan.as_select.as_deref().unwrap(), + Plan::InsertMultiTable(plan) => &plan.input_source, + _ => unreachable!(), + } + } + let Plan::Query { metadata, .. } = source(&raw) else { + unreachable!() + }; + let context = + OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?) + .with_settings(&ctx.get_settings())?; + let planned = optimize(context, raw.clone()).await?; + let Plan::Query { s_expr, .. } = source(&planned) else { + unreachable!() + }; + s_expr.planned()?.expr().validate_types(metadata)?; + s_expr.planned()?.expr().validate_column_scope(metadata)?; + writeln!(file, "sql: {sql}")?; + writeln!( + file, + "raw_source:\n{}", + source(&raw).format_indent(Default::default(), &StatContext::default())? + )?; + writeln!( + file, + "planned_source:\n{}", + source(&planned).format_indent(Default::default(), &StatContext::default())? + )?; + } + Ok(()) +} + +/// SQL produces the trees under test; compare both representations before allowing +/// their implementations to diverge. Test forward conversion and invalidation. +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_physical_expression_fork_equivalence() -> Result<()> { + use std::hash::Hash; + use std::hash::Hasher; + + use databend_common_sql::optimizer::ir::ExprVisitor; + use databend_common_sql::optimizer::ir::PExpr; + use databend_common_sql::optimizer::ir::PVisitAction as VisitAction; + use databend_common_sql::optimizer::ir::RelExpr; + + for sql in [ + "SELECT k, sum(v) FROM t GROUP BY k ORDER BY k LIMIT 5", + "WITH c AS (SELECT k, v FROM t) SELECT a.k FROM c a JOIN c b ON a.k = b.k", + "SELECT a.k FROM t a LEFT JOIN t b ON a.k = b.k WHERE b.v > 0", + ] { + let ctx = LiteTableContext::create().await?; + ctx.register_setup_sql("CREATE TABLE t(k Int64 NOT NULL, v Int64 NOT NULL)") + .await?; + let raw = ctx.bind_sql(sql).await?; + let Plan::Query { + s_expr, metadata, .. + } = &raw + else { + unreachable!() + }; + let logical = s_expr.logical()?.clone(); + let property = logical.derive_relational_prop()?; + let statistics = + RelExpr::with_s_expr(&logical).derive_cardinality(&StatContext::default())?; + let physical = PExpr::from(logical.clone()); + assert!(std::sync::Arc::ptr_eq( + &property, + &physical.derive_relational_prop()? + )); + assert!(std::sync::Arc::ptr_eq( + &statistics, + &RelExpr::with_p_expr(&physical).derive_cardinality(&StatContext::default())? + )); + physical.validate_types(metadata)?; + physical.validate_column_scope(metadata)?; + assert_eq!( + logical.pretty_format(&metadata.read(), &StatContext::default())?, + physical.pretty_format(&metadata.read(), &StatContext::default())? + ); + fn hash(expr: &impl Hash) -> u64 { + let mut h = std::collections::hash_map::DefaultHasher::new(); + expr.hash(&mut h); + h.finish() + } + assert_eq!(hash(&logical), hash(&physical)); + let replaced = physical.replace_plan(physical.plan.clone()); + // Some operators return their child's Arc directly. Compare replacement + // behavior rather than assuming a newly allocated property value. + assert_eq!( + format!( + "{:?}", + logical + .replace_plan(logical.plan.clone()) + .derive_relational_prop()? + ), + format!("{:?}", replaced.derive_relational_prop()?) + ); + assert_eq!(physical, replaced); + struct Count(usize); + impl ExprVisitor for Count { + fn visit(&mut self, _: &PExpr) -> Result { + self.0 += 1; + Ok(VisitAction::Continue) + } + } + let mut count = Count(0); + assert!(physical.accept(&mut count)?.is_none()); + assert!(count.0 > 1); + let planned = ctx.optimize_plan(raw).await?; + let Plan::Query { + s_expr, metadata, .. + } = &planned + else { + unreachable!() + }; + let selected = s_expr.planned()?.expr(); + selected.validate_types(metadata)?; + selected.validate_column_scope(metadata)?; + // A selected implementation is inspected directly, never reconstructed as + // a logical tree merely to exercise the old property interface. + RelExpr::with_p_expr(selected).derive_physical_prop()?; + } + Ok(()) +} diff --git a/src/query/sql/tests/it/optimizer/physical_planner.txt b/src/query/sql/tests/it/optimizer/physical_planner.txt new file mode 100644 index 00000000000..d4f02693728 --- /dev/null +++ b/src/query/sql/tests/it/optimizer/physical_planner.txt @@ -0,0 +1,170 @@ +=== query_boundary: distributed=false, cbo=false === +sql: SELECT k, sum(v) FROM t GROUP BY k ORDER BY k LIMIT 5 +raw_plan: +Limit +├── limit: [5] +├── offset: [0] +└── Sort + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [NONE] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), sum(v) (#2) AS (#2)] + └── Aggregate(Initial) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_query: +TopN +├── sort keys: [t.k (#0) ASC NULLS LAST] +├── limit: [5] +├── offset: [0] +└── Aggregate(Final) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── Aggregate(Partial) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [t.k (#0) ASC] + └── limit: NONE + +=== query_boundary: distributed=false, cbo=true === +sql: SELECT k, sum(v) FROM t GROUP BY k ORDER BY k LIMIT 5 +raw_plan: +Limit +├── limit: [5] +├── offset: [0] +└── Sort + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [NONE] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), sum(v) (#2) AS (#2)] + └── Aggregate(Initial) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_query: +TopN +├── sort keys: [t.k (#0) ASC NULLS LAST] +├── limit: [5] +├── offset: [0] +└── Aggregate(Final) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── Aggregate(Partial) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [t.k (#0) ASC] + └── limit: NONE + +=== query_boundary: distributed=true, cbo=false === +sql: SELECT k, sum(v) FROM t GROUP BY k ORDER BY k LIMIT 5 +raw_plan: +Limit +├── limit: [5] +├── offset: [0] +└── Sort + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [NONE] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), sum(v) (#2) AS (#2)] + └── Aggregate(Initial) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_query: +TopN +├── sort keys: [t.k (#0) ASC NULLS LAST] +├── limit: [5] +├── offset: [0] +└── Exchange(Merge) + └── TopN + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [5] + ├── offset: [0] + └── Aggregate(Final) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── Aggregate(Partial) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── Exchange(Hash) + ├── Exchange(Hash): keys: [t.k (#0)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [t.k (#0) ASC] + └── limit: NONE + +=== query_boundary: distributed=true, cbo=true === +sql: SELECT k, sum(v) FROM t GROUP BY k ORDER BY k LIMIT 5 +raw_plan: +Limit +├── limit: [5] +├── offset: [0] +└── Sort + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [NONE] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), sum(v) (#2) AS (#2)] + └── Aggregate(Initial) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── EvalScalar + ├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_query: +TopN +├── sort keys: [t.k (#0) ASC NULLS LAST] +├── limit: [5] +├── offset: [0] +└── Exchange(Merge) + └── TopN + ├── sort keys: [t.k (#0) ASC NULLS LAST] + ├── limit: [5] + ├── offset: [0] + └── Aggregate(Final) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── Aggregate(Partial) + ├── group items: [t.k (#0) AS (#0)] + ├── aggregate functions: [sum(t.v (#1)) AS (#2)] + └── Exchange(Hash) + ├── Exchange(Hash): keys: [t.k (#0)] + └── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [t.k (#0) ASC] + └── limit: NONE + diff --git a/src/query/sql/tests/it/optimizer/physical_planner_nested.txt b/src/query/sql/tests/it/optimizer/physical_planner_nested.txt new file mode 100644 index 00000000000..a3d85252ddf --- /dev/null +++ b/src/query/sql/tests/it/optimizer/physical_planner_nested.txt @@ -0,0 +1,77 @@ +sql: INSERT INTO dst SELECT k, v FROM t +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +sql: REPLACE INTO dst ON(k) SELECT k, v FROM t +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +sql: CREATE TABLE copied ENGINE=NULL AS SELECT k, v FROM t +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Scan +├── table: default.t (#0) +├── filters: [] +├── order by: [] +└── limit: NONE + +sql: INSERT ALL WHEN k IN (SELECT k FROM t) THEN INTO dst VALUES(k, v) ELSE INTO dst2 VALUES(k, v) SELECT k, v FROM t +raw_source: +EvalScalar +├── scalars: [t.k (#0) AS (#0), t.v (#1) AS (#1)] +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + +planned_source: +Join(RightMark) +├── build keys: [t.k (#2)] +├── probe keys: [t.k (#0)] +├── other filters: [] +├── Scan +│ ├── table: default.t (#1) +│ ├── filters: [] +│ ├── order by: [] +│ └── limit: NONE +└── Scan + ├── table: default.t (#0) + ├── filters: [] + ├── order by: [] + └── limit: NONE + diff --git a/src/query/sql/tests/it/optimizer/stat_derivation.rs b/src/query/sql/tests/it/optimizer/stat_derivation.rs index a6de4d25a7a..a516246156b 100644 --- a/src/query/sql/tests/it/optimizer/stat_derivation.rs +++ b/src/query/sql/tests/it/optimizer/stat_derivation.rs @@ -27,8 +27,8 @@ use databend_common_sql::optimizer::CollectStatisticsOptimizer; use databend_common_sql::optimizer::Optimizer; use databend_common_sql::optimizer::OptimizerContext; use databend_common_sql::optimizer::ir::ColumnStat; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::RelExpr; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::optimizer::ir::StatInfo; use databend_common_sql::optimizer::optimizers::recursive::RecursiveRuleOptimizer; @@ -84,7 +84,7 @@ fn column_statistics() -> HashMap { ]) } -fn find_operator(expr: &SExpr, operator: RelOp) -> Option<&SExpr> { +fn find_operator(expr: &PExpr, operator: RelOp) -> Option<&PExpr> { if expr.plan().rel_op() == operator { return Some(expr); } @@ -166,10 +166,10 @@ async fn write_case(file: &mut impl Write, case: &StatsCase) -> Result<()> { let opt_ctx = OptimizerContext::new(ctx.clone(), metadata.clone(), ctx.get_function_context()?); let mut collector = CollectStatisticsOptimizer::new(opt_ctx.clone()); - let s_expr = collector.optimize(*s_expr).await?; + let s_expr = collector.optimize((*s_expr).into_logical()?).await?; let s_expr = RecursiveRuleOptimizer::new(opt_ctx, &[RuleID::PushDownLimitSort]) .optimize_sync(s_expr)?; - (s_expr, metadata) + (PExpr::from(s_expr), metadata) } else { let Plan::Query { s_expr, metadata, .. @@ -177,12 +177,12 @@ async fn write_case(file: &mut impl Write, case: &StatsCase) -> Result<()> { else { return Err(ErrorCode::Internal("expected optimized query plan")); }; - (*s_expr, metadata) + (s_expr.planned()?.expr().clone(), metadata) }; let target = find_operator(&s_expr, case.operator.clone()).ok_or_else(|| { ErrorCode::Internal(format!("cannot find {:?} in optimized plan", case.operator)) })?; - let stats = RelExpr::with_s_expr(target).derive_cardinality(&StatContext::default())?; + let stats = RelExpr::with_p_expr(target).derive_cardinality(&StatContext::default())?; write_case_title(file, case.name, case.description)?; writeln!(file, "sql: {}", case.sql)?; diff --git a/src/query/sql/tests/it/optimizer/union_all.rs b/src/query/sql/tests/it/optimizer/union_all.rs index ca49cc3136e..028e2fb057b 100644 --- a/src/query/sql/tests/it/optimizer/union_all.rs +++ b/src/query/sql/tests/it/optimizer/union_all.rs @@ -20,13 +20,12 @@ use databend_common_catalog::BasicColumnStatistics; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::ColumnEntry; -use databend_common_sql::FormatOptions; use databend_common_sql::Metadata; use databend_common_sql::MetadataRef; use databend_common_sql::Symbol; use databend_common_sql::optimizer::ir::ColumnStat; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::RelExpr; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::optimizer::ir::StatInfo; use databend_common_sql::plans::Operator; @@ -57,7 +56,7 @@ struct UnionCase { } struct StatTarget { - // Child directions starting from the optimized Plan::Query SExpr root. + // Child directions starting from the optimized Plan::Query PExpr root. path: Vec, operator: RelOp, } @@ -78,7 +77,7 @@ impl Child { } fn collect_operator_paths( - expr: &SExpr, + expr: &PExpr, operator: &RelOp, path: &mut Vec, paths: &mut Vec>, @@ -91,21 +90,21 @@ fn collect_operator_paths( path.push(match index { 0 => Child::Left, 1 => Child::Right, - _ => unreachable!("SExpr nodes have at most two children"), + _ => unreachable!("PExpr nodes have at most two children"), }); collect_operator_paths(child, operator, path, paths); path.pop(); } } -fn possible_operator_paths(root: &SExpr, operator: &RelOp) -> Vec> { +fn possible_operator_paths(root: &PExpr, operator: &RelOp) -> Vec> { let mut paths = Vec::new(); collect_operator_paths(root, operator, &mut Vec::new(), &mut paths); paths } fn invalid_target_path( - root: &SExpr, + root: &PExpr, target: &StatTarget, reason: impl std::fmt::Display, ) -> ErrorCode { @@ -193,22 +192,14 @@ fn write_stat_info(file: &mut impl Write, metadata: &Metadata, stat_info: &StatI Ok(()) } -fn format_node(metadata: &MetadataRef, expr: &SExpr) -> Result { - Plan::Query { - s_expr: Box::new(expr.clone()), - metadata: metadata.clone(), - bind_context: Default::default(), - rewrite_kind: None, - formatted_ast: None, - ignore_result: false, - } - .format_indent(FormatOptions::default(), &StatContext::default()) +fn format_node(metadata: &MetadataRef, expr: &PExpr) -> Result { + expr.pretty_format(&metadata.read(), &StatContext::default()) } fn write_derived_stats( file: &mut impl Write, metadata: &MetadataRef, - root: &SExpr, + root: &PExpr, target: &StatTarget, ) -> Result<()> { let mut expr = root; @@ -240,7 +231,7 @@ fn write_derived_stats( writeln!(file, "path: {:?}", target.path)?; writeln!(file, "node:")?; writeln!(file, "{}", format_node(metadata, expr)?)?; - let stat_info = RelExpr::with_s_expr(expr).derive_cardinality(&StatContext::default())?; + let stat_info = RelExpr::with_p_expr(expr).derive_cardinality(&StatContext::default())?; write_stat_info(file, &metadata.read(), &stat_info)?; Ok(()) } @@ -262,7 +253,7 @@ async fn write_case(file: &mut impl Write, case: UnionCase) -> Result<()> { write_case_title(file, case.name, case.description)?; writeln!(file, "sql: {}", case.sql)?; for target in &case.targets { - write_derived_stats(file, &metadata, &s_expr, target)?; + write_derived_stats(file, &metadata, s_expr.planned()?.expr(), target)?; } writeln!(file)?; Ok(()) diff --git a/src/query/sql/tests/it/planner.rs b/src/query/sql/tests/it/planner.rs index 1d3e6141270..2e276a4006c 100644 --- a/src/query/sql/tests/it/planner.rs +++ b/src/query/sql/tests/it/planner.rs @@ -26,7 +26,7 @@ use databend_common_meta_app::schema::CatalogOption; use databend_common_sql::FormatOptions; use databend_common_sql::MetadataRef; use databend_common_sql::Planner; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::plans::Operator; use databend_common_sql::plans::Plan; @@ -280,10 +280,10 @@ fn format_statistics_trace_summary(plan: &Plan) -> Result { )); }; - Ok(statistics_trace_summary_tree(s_expr, metadata)?.format_pretty()?) + Ok(statistics_trace_summary_tree(s_expr.planned()?.expr(), metadata)?.format_pretty()?) } -fn statistics_trace_summary_tree(s_expr: &SExpr, metadata: &MetadataRef) -> Result { +fn statistics_trace_summary_tree(s_expr: &PExpr, metadata: &MetadataRef) -> Result { match s_expr.plan() { RelOperator::MaterializedCTE(cte) => { let children = s_expr diff --git a/src/query/sql/tests/it/semantic/binder.rs b/src/query/sql/tests/it/semantic/binder.rs index d1fe0e5ed08..e0402fa27e1 100644 --- a/src/query/sql/tests/it/semantic/binder.rs +++ b/src/query/sql/tests/it/semantic/binder.rs @@ -47,7 +47,7 @@ async fn test_window_input_reuse_preserves_nondeterministic_expressions() -> Res else { panic!("expected query plan"); }; - let RelOperator::EvalScalar(projection) = s_expr.plan() else { + let RelOperator::EvalScalar(projection) = s_expr.logical()?.plan() else { panic!("expected SELECT projection"); }; for (name, reused) in [("n", true), ("r", false)] { @@ -275,7 +275,7 @@ async fn test_binder_mutation_internal_column_strategy() -> Result<()> { let Plan::DataMutation { s_expr, .. } = plan else { panic!("expected mutation plan for {sql}"); }; - let RelOperator::Mutation(mutation) = s_expr.plan() else { + let RelOperator::Mutation(mutation) = s_expr.logical()?.plan() else { panic!("expected mutation operator for {sql}"); }; assert_eq!(mutation.strategy, expected_strategy, "sql: {sql}"); @@ -496,7 +496,7 @@ async fn test_many_window_expressions_bind_as_flat_window_group() -> Result<()> }; let mut stats = WindowPlanStats::default(); - collect_window_plan_stats(&s_expr, &mut stats); + collect_window_plan_stats(s_expr.logical()?, &mut stats); assert_eq!(stats.window_group_nodes, 1); assert_eq!(stats.window_nodes, 0); @@ -526,7 +526,7 @@ async fn test_mixed_partition_windows_bind_as_partitioned_window_groups() -> Res }; let mut stats = WindowPlanStats::default(); - collect_window_plan_stats(&s_expr, &mut stats); + collect_window_plan_stats(s_expr.logical()?, &mut stats); assert_eq!(stats.window_group_nodes, 2); assert_eq!(stats.window_nodes, 0); diff --git a/src/tests/planner_replay/src/main.rs b/src/tests/planner_replay/src/main.rs index 4e4954e7c46..2b92f6771a9 100644 --- a/src/tests/planner_replay/src/main.rs +++ b/src/tests/planner_replay/src/main.rs @@ -20,7 +20,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::FormatOptions; use databend_common_sql::MetadataRef; -use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::optimizer::ir::PExpr; use databend_common_sql::optimizer::ir::StatContext; use databend_common_sql::plans::Operator; use databend_common_sql::plans::Plan; @@ -145,10 +145,10 @@ fn format_join_explain(plan: Plan) -> Result { )); }; - Ok(join_format_tree(&s_expr, &metadata)?.format_pretty()?) + Ok(join_format_tree(s_expr.planned()?.expr(), &metadata)?.format_pretty()?) } -fn join_format_tree(s_expr: &SExpr, metadata: &MetadataRef) -> Result { +fn join_format_tree(s_expr: &PExpr, metadata: &MetadataRef) -> Result { match s_expr.plan() { RelOperator::Join(join) => { let build_child = From a0c79dcf0413c8584e6db596b2734c4662654707 Mon Sep 17 00:00:00 2001 From: coldWater Date: Thu, 1 Oct 2026 13:17:31 +0800 Subject: [PATCH 4/4] refactor(query): require bound snapshots for planned lineage Read lineage for planned queries and mutations only from binding-time snapshots. Return no lineage when a planned statement has no snapshot, rather than attempting to reconstruct SQL dependencies from an execution tree. Keep direct extraction for bound statements and cover snapshot/no-snapshot behavior for INSERT, CTAS, multi-table INSERT, UPDATE and DELETE. --- src/query/sql/src/planner/lineage.rs | 31 +++++++++---- src/query/sql/tests/it/planner/lineage.rs | 53 +++++++++++++++++++++++ 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/src/query/sql/src/planner/lineage.rs b/src/query/sql/src/planner/lineage.rs index 668fae39d6c..82480cafc50 100644 --- a/src/query/sql/src/planner/lineage.rs +++ b/src/query/sql/src/planner/lineage.rs @@ -179,14 +179,23 @@ enum SourceExpr { impl Plan { /// Query lineage for this plan. /// - /// Prefers the snapshot captured by [`Plan::capture_bound_query_lineage`] on the bound plan. - /// Plans that never went through the planner (or carry no query part) are extracted directly. + /// Planned queries and mutations read only the snapshot captured by + /// [`Plan::capture_bound_query_lineage`]. Without that snapshot, lineage is unavailable: + /// execution rewrites cannot reconstruct what the user originally wrote. + /// Bound plans and statements without a query tree can still be inspected directly. pub fn query_lineage(&self) -> Result> { if let Some(metadata) = self.lineage_metadata() { if let Some(captured) = metadata.read().bound_query_lineage() { return Ok(captured.lineage.clone()); } } + match self.lineage_query_part() { + Some(Plan::Query { s_expr, .. }) if s_expr.planned().is_ok() => return Ok(None), + Some(Plan::DataMutation { s_expr, .. }) if s_expr.planned().is_ok() => { + return Ok(None); + } + _ => {} + } self.extract_query_lineage() } @@ -215,9 +224,9 @@ impl Plan { RelationExtractor::new(self).extract_query_lineage() } - /// Metadata of the query part whose shape the optimizer may change. - fn lineage_metadata(&self) -> Option<&MetadataRef> { - let query = match self { + /// Query part whose shape the optimizer may change. + fn lineage_query_part(&self) -> Option<&Plan> { + match self { Plan::CreateTable(plan) => plan.as_select.as_deref(), Plan::CreateView(plan) => plan.query_plan.as_deref(), Plan::CreateMaterializedView(plan) => Some(plan.query_plan.as_ref()), @@ -232,11 +241,15 @@ impl Plan { Plan::CopyIntoTable(plan) => plan.query.as_deref(), Plan::CopyIntoLocation(plan) => Some(plan.from.as_ref()), Plan::InsertMultiTable(plan) => Some(&plan.input_source), - Plan::DataMutation { metadata, .. } => return Some(metadata), + Plan::DataMutation { .. } => Some(self), _ => None, - }?; - match query { - Plan::Query { metadata, .. } => Some(metadata), + } + } + + /// Metadata containing the binding-time lineage snapshot. + fn lineage_metadata(&self) -> Option<&MetadataRef> { + match self.lineage_query_part()? { + Plan::Query { metadata, .. } | Plan::DataMutation { metadata, .. } => Some(metadata), _ => None, } } diff --git a/src/query/sql/tests/it/planner/lineage.rs b/src/query/sql/tests/it/planner/lineage.rs index 31f69a54b71..b7e970b0139 100644 --- a/src/query/sql/tests/it/planner/lineage.rs +++ b/src/query/sql/tests/it/planner/lineage.rs @@ -37,6 +37,59 @@ use databend_common_sql_test_support::init_testing_globals_with_config; use crate::framework::LiteTableContext; +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_planned_lineage_requires_bound_snapshot() -> Result<()> { + use databend_common_sql::optimizer::OptimizerContext; + use databend_common_sql::optimizer::optimize; + + for sql in [ + "INSERT INTO dst SELECT a FROM src", + "CREATE TABLE copied ENGINE=NULL AS SELECT a FROM src", + "INSERT ALL WHEN a > 0 THEN INTO dst VALUES(a) SELECT a FROM src", + "UPDATE src SET a = b WHERE a > 0", + "DELETE FROM src WHERE a > 0", + ] { + for capture in [false, true] { + let ctx = lineage_test_context().await?; + ctx.register_setup_sql("CREATE TABLE src(a INT, b INT)") + .await?; + ctx.register_setup_sql("CREATE TABLE dst(x INT)").await?; + let raw = ctx.bind_sql(sql).await?; + let expected = raw.query_lineage()?.expect("bound SQL has lineage"); + if capture { + raw.capture_bound_query_lineage(); + } + let metadata = match &raw { + Plan::Insert(plan) => match &plan.source { + databend_common_sql::InsertInputSource::SelectPlan(query) => { + match query.as_ref() { + Plan::Query { metadata, .. } => metadata.clone(), + _ => unreachable!(), + } + } + _ => unreachable!(), + }, + Plan::CreateTable(plan) => match plan.as_select.as_deref().unwrap() { + Plan::Query { metadata, .. } => metadata.clone(), + _ => unreachable!(), + }, + Plan::InsertMultiTable(plan) => plan.meta_data.clone(), + Plan::DataMutation { metadata, .. } => metadata.clone(), + _ => unreachable!(), + }; + let context = OptimizerContext::new(ctx.clone(), metadata, ctx.get_function_context()?) + .with_settings(&ctx.get_settings())?; + let planned = optimize(context, raw).await?; + assert_eq!( + planned.query_lineage()?, + capture.then_some(expected), + "{sql}" + ); + } + } + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_query_lineage_insert_select_from_sql() -> Result<()> { let ctx = lineage_test_context().await?;