Skip to content

Commit dca458f

Browse files
committed
test: cover CrossJoinExec row metrics across batched inputs
Add Rust tests that assert build_input_rows, input_rows, and output_rows stay stable when CrossJoinExec inputs are split into multiple batches, including empty build and probe sides. Related to apache#25333.
1 parent 6ffc45d commit dca458f

1 file changed

Lines changed: 169 additions & 3 deletions

File tree

‎datafusion/physical-plan/src/joins/cross_join.rs‎

Lines changed: 169 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -706,11 +706,15 @@ impl CrossJoinStream {
706706
mod tests {
707707
use super::*;
708708
use crate::common;
709-
use crate::test::{assert_join_metrics, build_table_scan_i32};
709+
use crate::test::{
710+
TestMemoryExec, assert_join_metrics, build_table_i32, build_table_scan_i32,
711+
};
710712

711713
use datafusion_common::{assert_contains, test_util::batches_to_sort_string};
714+
use datafusion_execution::config::SessionConfig;
712715
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
713-
use insta::assert_snapshot;
716+
use insta::{allow_duplicates, assert_snapshot};
717+
use rstest::rstest;
714718

715719
async fn join_collect(
716720
left: Arc<dyn ExecutionPlan>,
@@ -727,6 +731,61 @@ mod tests {
727731
Ok((columns_header, batches, metrics))
728732
}
729733

734+
fn prepare_task_ctx(batch_size: usize) -> Arc<TaskContext> {
735+
let session_config = SessionConfig::default().with_batch_size(batch_size);
736+
Arc::new(TaskContext::default().with_session_config(session_config))
737+
}
738+
739+
/// Split a 3-column i32 table into `RecordBatch`es of at most `batch_size` rows.
740+
///
741+
/// `CrossJoinExec` does not re-chunk `TestMemoryExec` input by session
742+
/// `batch_size`, so this is how tests produce multi-batch build/probe sides.
743+
fn build_table_scan_i32_chunked(
744+
a: (&str, &Vec<i32>),
745+
b: (&str, &Vec<i32>),
746+
c: (&str, &Vec<i32>),
747+
batch_size: usize,
748+
) -> Arc<dyn ExecutionPlan> {
749+
let full = build_table_i32(a, b, c);
750+
let n = full.num_rows();
751+
let batches = if n == 0 {
752+
vec![full.clone()]
753+
} else {
754+
(0..n)
755+
.step_by(batch_size)
756+
.map(|i| full.slice(i, batch_size.min(n - i)))
757+
.collect()
758+
};
759+
TestMemoryExec::try_new_exec(&[batches], full.schema(), None).unwrap()
760+
}
761+
762+
fn metric_count(metrics: &MetricsSet, name: &str) -> usize {
763+
metrics
764+
.sum_by_name(name)
765+
.unwrap_or_else(|| panic!("did not find {name} metric"))
766+
.as_usize()
767+
}
768+
769+
/// Row counts that must stay the same no matter how the inputs are batched.
770+
fn assert_cross_join_row_metrics(
771+
metrics: &MetricsSet,
772+
build_input_rows: usize,
773+
input_rows: usize,
774+
output_rows: usize,
775+
) {
776+
assert_join_metrics!(metrics, output_rows);
777+
assert_eq!(
778+
metric_count(metrics, "build_input_rows"),
779+
build_input_rows,
780+
"build_input_rows"
781+
);
782+
assert_eq!(
783+
metric_count(metrics, "input_rows"),
784+
input_rows,
785+
"input_rows"
786+
);
787+
}
788+
730789
#[tokio::test]
731790
async fn test_stats_cartesian_product() {
732791
let left_row_count = 11;
@@ -965,7 +1024,114 @@ mod tests {
9651024
+----+----+----+----+----+----+
9661025
");
9671026

968-
assert_join_metrics!(metrics, 6);
1027+
assert_cross_join_row_metrics(&metrics, 3, 2, 6);
1028+
1029+
Ok(())
1030+
}
1031+
1032+
/// `build_input_rows`, `input_rows`, and `output_rows` must count each row
1033+
/// once even when the build and probe sides arrive as several batches.
1034+
///
1035+
/// These values are independent of batching: `output_rows` is the Cartesian
1036+
/// product, `build_input_rows` is the left row count, and `input_rows` is
1037+
/// the right row count (when the build side is non-empty).
1038+
#[rstest]
1039+
#[tokio::test]
1040+
async fn join_metrics_count_rows_once_across_batches(
1041+
#[values(8192, 10, 5, 2, 1)] batch_size: usize,
1042+
) -> Result<()> {
1043+
let task_ctx = prepare_task_ctx(batch_size);
1044+
1045+
let left = build_table_scan_i32_chunked(
1046+
("a1", &vec![1, 2, 3]),
1047+
("b1", &vec![4, 5, 6]),
1048+
("c1", &vec![7, 8, 9]),
1049+
batch_size,
1050+
);
1051+
let right = build_table_scan_i32_chunked(
1052+
("a2", &vec![10, 11]),
1053+
("b2", &vec![12, 13]),
1054+
("c2", &vec![14, 15]),
1055+
batch_size,
1056+
);
1057+
1058+
let (columns, batches, metrics) = join_collect(left, right, task_ctx).await?;
1059+
1060+
assert_eq!(columns, vec!["a1", "b1", "c1", "a2", "b2", "c2"]);
1061+
1062+
allow_duplicates! {
1063+
assert_snapshot!(batches_to_sort_string(&batches), @r"
1064+
+----+----+----+----+----+----+
1065+
| a1 | b1 | c1 | a2 | b2 | c2 |
1066+
+----+----+----+----+----+----+
1067+
| 1 | 4 | 7 | 10 | 12 | 14 |
1068+
| 1 | 4 | 7 | 11 | 13 | 15 |
1069+
| 2 | 5 | 8 | 10 | 12 | 14 |
1070+
| 2 | 5 | 8 | 11 | 13 | 15 |
1071+
| 3 | 6 | 9 | 10 | 12 | 14 |
1072+
| 3 | 6 | 9 | 11 | 13 | 15 |
1073+
+----+----+----+----+----+----+
1074+
");
1075+
}
1076+
1077+
// 3 left rows × 2 right rows, regardless of how those rows are batched.
1078+
assert_cross_join_row_metrics(&metrics, 3, 2, 6);
1079+
1080+
Ok(())
1081+
}
1082+
1083+
/// An empty build side terminates before the probe stream is read, so
1084+
/// `input_rows` stays 0 even when the right side has rows.
1085+
#[tokio::test]
1086+
async fn join_metrics_empty_build_skips_probe() -> Result<()> {
1087+
let task_ctx = prepare_task_ctx(1);
1088+
1089+
let left = build_table_scan_i32_chunked(
1090+
("a1", &vec![]),
1091+
("b1", &vec![]),
1092+
("c1", &vec![]),
1093+
1,
1094+
);
1095+
let right = build_table_scan_i32_chunked(
1096+
("a2", &vec![10, 11]),
1097+
("b2", &vec![12, 13]),
1098+
("c2", &vec![14, 15]),
1099+
1,
1100+
);
1101+
1102+
let (_, batches, metrics) = join_collect(left, right, task_ctx).await?;
1103+
1104+
let num_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
1105+
assert_eq!(num_rows, 0);
1106+
assert_cross_join_row_metrics(&metrics, 0, 0, 0);
1107+
1108+
Ok(())
1109+
}
1110+
1111+
/// An empty probe side still loads the build side, so `build_input_rows`
1112+
/// is the left row count and `output_rows` is 0.
1113+
#[tokio::test]
1114+
async fn join_metrics_empty_probe() -> Result<()> {
1115+
let task_ctx = prepare_task_ctx(1);
1116+
1117+
let left = build_table_scan_i32_chunked(
1118+
("a1", &vec![1, 2, 3]),
1119+
("b1", &vec![4, 5, 6]),
1120+
("c1", &vec![7, 8, 9]),
1121+
1,
1122+
);
1123+
let right = build_table_scan_i32_chunked(
1124+
("a2", &vec![]),
1125+
("b2", &vec![]),
1126+
("c2", &vec![]),
1127+
1,
1128+
);
1129+
1130+
let (_, batches, metrics) = join_collect(left, right, task_ctx).await?;
1131+
1132+
let num_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
1133+
assert_eq!(num_rows, 0);
1134+
assert_cross_join_row_metrics(&metrics, 3, 0, 0);
9691135

9701136
Ok(())
9711137
}

0 commit comments

Comments
 (0)