Skip to content

Commit 562f87d

Browse files
authored
refactor(proto): migrate HashJoinExec serde (apache#23853)
## Which issue does this PR close? - Closes apache#23507. ## Rationale for this change Part of epic apache#23494. Moves `HashJoinExec` protobuf serialization from central dispatch. ## What changes are included in this PR? Add protobuf serialization and deserialization to the `HashJoinExec` physical plan implementation and deprecate the corresponding central proto methods. The wire format remains unchanged. ## Are these changes tested? Yes, existing HashJoin round-trip tests cover this plan. ## Are there any user-facing changes? Existing methods are deprecated with no immediate API change.
1 parent 0d8482c commit 562f87d

2 files changed

Lines changed: 217 additions & 223 deletions

File tree

  • datafusion
    • physical-plan/src/joins/hash_join
    • proto/src/physical_plan

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1761,6 +1761,197 @@ impl ExecutionPlan for HashJoinExec {
17611761
.ok()
17621762
.map(|exec| Arc::new(exec) as _)
17631763
}
1764+
#[cfg(feature = "proto")]
1765+
fn try_to_proto(
1766+
&self,
1767+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
1768+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
1769+
use datafusion_proto_models::protobuf;
1770+
1771+
let left = ctx.encode_child(self.left())?;
1772+
let right = ctx.encode_child(self.right())?;
1773+
1774+
let on = self
1775+
.on()
1776+
.iter()
1777+
.map(|(l, r)| -> Result<protobuf::JoinOn> {
1778+
Ok(protobuf::JoinOn {
1779+
left: Some(ctx.encode_expr(l)?),
1780+
right: Some(ctx.encode_expr(r)?),
1781+
})
1782+
})
1783+
.collect::<Result<Vec<_>>>()?;
1784+
1785+
let join_type = crate::joins::proto::join_type_to_proto(*self.join_type());
1786+
let null_equality =
1787+
crate::joins::proto::null_equality_to_proto(self.null_equality());
1788+
// `PartitionMode` is specific to `HashJoinExec`, so its conversion stays
1789+
// inline (by-name on purpose: the enums are numbered differently).
1790+
let partition_mode = match self.partition_mode() {
1791+
PartitionMode::CollectLeft => protobuf::PartitionMode::CollectLeft,
1792+
PartitionMode::Partitioned => protobuf::PartitionMode::Partitioned,
1793+
PartitionMode::Auto => protobuf::PartitionMode::Auto,
1794+
};
1795+
1796+
let filter = self
1797+
.filter()
1798+
.map(|f| crate::joins::proto::join_filter_to_proto(f, ctx))
1799+
.transpose()?;
1800+
1801+
let dynamic_filter = self
1802+
.dynamic_filter_expr()
1803+
.map(|df| {
1804+
let df_expr: Arc<dyn PhysicalExpr> =
1805+
Arc::clone(df) as Arc<dyn PhysicalExpr>;
1806+
ctx.encode_expr(&df_expr)
1807+
})
1808+
.transpose()?;
1809+
1810+
Ok(Some(protobuf::PhysicalPlanNode {
1811+
physical_plan_type: Some(
1812+
protobuf::physical_plan_node::PhysicalPlanType::HashJoin(Box::new(
1813+
protobuf::HashJoinExecNode {
1814+
left: Some(Box::new(left)),
1815+
right: Some(Box::new(right)),
1816+
on,
1817+
join_type: join_type.into(),
1818+
partition_mode: partition_mode.into(),
1819+
null_equality: null_equality.into(),
1820+
filter,
1821+
// Proto3 `repeated` cannot distinguish `None` from
1822+
// `Some(vec![])`. `Some(vec![])` (reachable via
1823+
// `try_embed_projection` for e.g. `SELECT count(1) … JOIN …`)
1824+
// changes the output schema, so it is encoded with the
1825+
// single-element sentinel `[u32::MAX]` (never a valid column
1826+
// index); every other state is sent as-is. See
1827+
// `try_from_proto` for the matching decoder.
1828+
projection: match self.projection.as_ref() {
1829+
None => Vec::new(),
1830+
Some(v) if v.is_empty() => vec![u32::MAX],
1831+
Some(v) => v.iter().map(|x| *x as u32).collect(),
1832+
},
1833+
null_aware: self.null_aware,
1834+
dynamic_filter,
1835+
},
1836+
)),
1837+
),
1838+
}))
1839+
}
1840+
}
1841+
1842+
#[cfg(feature = "proto")]
1843+
impl HashJoinExec {
1844+
/// Reconstruct a [`HashJoinExec`] from its protobuf representation.
1845+
pub fn try_from_proto(
1846+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
1847+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
1848+
) -> Result<Arc<dyn ExecutionPlan>> {
1849+
use datafusion_common::internal_datafusion_err;
1850+
use datafusion_proto_models::protobuf;
1851+
use std::any::Any;
1852+
1853+
let hashjoin = crate::expect_plan_variant!(
1854+
node,
1855+
protobuf::physical_plan_node::PhysicalPlanType::HashJoin,
1856+
"HashJoinExec",
1857+
);
1858+
1859+
let left =
1860+
ctx.decode_required_child(hashjoin.left.as_deref(), "HashJoinExec", "left")?;
1861+
let right = ctx.decode_required_child(
1862+
hashjoin.right.as_deref(),
1863+
"HashJoinExec",
1864+
"right",
1865+
)?;
1866+
let left_schema = left.schema();
1867+
let right_schema = right.schema();
1868+
1869+
let on: Vec<(PhysicalExprRef, PhysicalExprRef)> = hashjoin
1870+
.on
1871+
.iter()
1872+
.map(|col| {
1873+
let l = ctx.decode_required_expr(
1874+
col.left.as_ref(),
1875+
left_schema.as_ref(),
1876+
"HashJoinExec",
1877+
"on.left",
1878+
)?;
1879+
let r = ctx.decode_required_expr(
1880+
col.right.as_ref(),
1881+
right_schema.as_ref(),
1882+
"HashJoinExec",
1883+
"on.right",
1884+
)?;
1885+
Ok((l, r))
1886+
})
1887+
.collect::<Result<_>>()?;
1888+
1889+
let join_type = crate::joins::proto::join_type_from_proto(
1890+
hashjoin.join_type,
1891+
"HashJoinExec",
1892+
)?;
1893+
let null_equality = crate::joins::proto::null_equality_from_proto(
1894+
hashjoin.null_equality,
1895+
"HashJoinExec",
1896+
)?;
1897+
// `PartitionMode` is specific to `HashJoinExec`, so its conversion stays
1898+
// inline (by-name on purpose: the enums are numbered differently).
1899+
let partition_mode = match protobuf::PartitionMode::try_from(
1900+
hashjoin.partition_mode,
1901+
)
1902+
.map_err(|_| {
1903+
internal_datafusion_err!(
1904+
"HashJoinExec: unknown PartitionMode {}",
1905+
hashjoin.partition_mode
1906+
)
1907+
})? {
1908+
protobuf::PartitionMode::CollectLeft => PartitionMode::CollectLeft,
1909+
protobuf::PartitionMode::Partitioned => PartitionMode::Partitioned,
1910+
protobuf::PartitionMode::Auto => PartitionMode::Auto,
1911+
};
1912+
1913+
let filter = hashjoin
1914+
.filter
1915+
.as_ref()
1916+
.map(|f| crate::joins::proto::join_filter_from_proto(f, ctx, "HashJoinExec"))
1917+
.transpose()?;
1918+
1919+
// Preserve the empty-projection sentinel written by `try_to_proto`.
1920+
let projection = match hashjoin.projection.as_slice() {
1921+
[] => None,
1922+
[u32::MAX] => Some(Vec::new()),
1923+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
1924+
};
1925+
1926+
let mut hash_join = HashJoinExec::try_new(
1927+
left,
1928+
right,
1929+
on,
1930+
filter,
1931+
&join_type,
1932+
projection,
1933+
partition_mode,
1934+
null_equality,
1935+
hashjoin.null_aware,
1936+
)?;
1937+
1938+
if let Some(dynamic_filter_proto) = &hashjoin.dynamic_filter {
1939+
// The dynamic filter is a `DynamicFilterPhysicalExpr` over the probe
1940+
// (right) side; decode against the right schema then downcast.
1941+
let dynamic_filter_expr =
1942+
ctx.decode_expr(dynamic_filter_proto, right_schema.as_ref())?;
1943+
let df = (dynamic_filter_expr as Arc<dyn Any + Send + Sync>)
1944+
.downcast::<DynamicFilterPhysicalExpr>()
1945+
.map_err(|_| {
1946+
internal_datafusion_err!(
1947+
"HashJoinExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"
1948+
)
1949+
})?;
1950+
hash_join = hash_join.with_dynamic_filter_expr(df)?;
1951+
}
1952+
1953+
Ok(Arc::new(hash_join))
1954+
}
17641955
}
17651956

17661957
/// Determines which sides of a join are "preserved" for filter pushdown.

0 commit comments

Comments
 (0)