Skip to content

Commit 991fd23

Browse files
authored
ci: Ensure only the main datafusion crate is used in the examples (#25777)
## Which issue does this PR close? - Closes #25578. ## Rationale for this change See the issue and #14305. ## What changes are included in this PR? - New CI job that checks if `datafusion-examples` uses subcrates directly which are already exported by the main `datafusion` crate. - Updated the examples to only use the main `datafusion` crate. ## What is the testing strategy for this PR? This PR. ## Are there any user-facing changes? Examples now do not use subcrates directly without needing to.
1 parent 51c2e04 commit 991fd23

16 files changed

Lines changed: 118 additions & 61 deletions

File tree

‎.asf.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ github:
8787
- "check Cargo.toml formatting"
8888
- "check configs.md and ***_functions.md is up-to-date"
8989
- "check example README is up-to-date"
90+
- "Check datafusion crates used in datafusion-examples"
9091
- "Verify MSRV (Min Supported Rust Version)"
9192
# needs to be updated as part of the release process
9293
# .asf.yaml doesn't support wildcard branch protection rules, only exact branch names

‎.github/workflows/rust.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -897,6 +897,16 @@ jobs:
897897
run: |
898898
bash ci/scripts/check_examples_docs.sh
899899
900+
# Ensures that the examples only use the main `datafusion` crate and not its subcrates, which are
901+
# already exported by `datafusion` (there are a few exceptions, like `datafusion-proto`).
902+
# See: https://github.com/apache/datafusion/pull/14305
903+
examples-check-datafusion-crates:
904+
name: Check datafusion crates used in datafusion-examples
905+
runs-on: ubuntu-slim
906+
steps:
907+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
908+
- run: python3 ci/scripts/check_examples_datafusion_crates.py
909+
900910
# Verify MSRV for the crates which are directly used by other projects:
901911
# - datafusion
902912
# - datafusion-substrait

‎Cargo.lock‎

Lines changed: 0 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env python3
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
# Checks if datafusion-examples uses `datafusion-...` subcrates unnecessarily, as they are already
20+
# exported by the `datafusion` crate.
21+
22+
import tomllib
23+
import sys
24+
25+
allowed_datafusion_crates = {
26+
"datafusion",
27+
"datafusion-proto",
28+
"datafusion-substrait",
29+
}
30+
31+
crates = set()
32+
with open("datafusion-examples/Cargo.toml", "rb") as f:
33+
cargo = tomllib.load(f)
34+
for section in ("dependencies", "dev-dependencies", "build-dependencies"):
35+
crates.update(
36+
value.get("package", name) if isinstance(value, dict) else name
37+
for name, value in cargo.get(section, {}).items()
38+
)
39+
40+
crates = [
41+
crate for crate in crates if "datafusion" in crate and crate not in allowed_datafusion_crates
42+
]
43+
44+
if len(crates) > 0:
45+
print(
46+
"datafusion-examples should only use the main datafusion crate and not its subcrates.",
47+
file=sys.stderr,
48+
)
49+
print(
50+
f"Please remove the following crates and use `datafusion::...` instead: {crates}.",
51+
file=sys.stderr,
52+
)
53+
54+
sys.exit(1)

‎datafusion-examples/Cargo.toml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,23 +38,19 @@ workspace = true
3838
[dependencies]
3939
arrow = { workspace = true }
4040
arrow-schema = { workspace = true }
41+
# note: only use the main datafusion crate for examples
4142
datafusion = { workspace = true, default-features = true, features = ["parquet_encryption"] }
42-
datafusion-common = { workspace = true }
4343
nom = "8.0.0"
4444
tempfile = { workspace = true }
4545
tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] }
4646

4747
[dev-dependencies]
4848
arrow-flight = { workspace = true }
4949
async-trait = { workspace = true }
50+
base64 = "0.23.0"
5051
bytes = { workspace = true }
5152
dashmap = { workspace = true }
52-
# note only use main datafusion crate for examples
53-
base64 = "0.23.0"
54-
datafusion-expr = { workspace = true }
55-
datafusion-physical-expr-adapter = { workspace = true }
5653
datafusion-proto = { workspace = true, features = ["parquet"] }
57-
datafusion-sql = { workspace = true }
5854
env_logger = { workspace = true }
5955
futures = { workspace = true }
6056
insta = { workspace = true }

‎datafusion-examples/examples/custom_data_source/adapter_serialization.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,11 @@ use datafusion::execution::context::SessionContext;
5050
use datafusion::execution::object_store::ObjectStoreUrl;
5151
use datafusion::parquet::arrow::ArrowWriter;
5252
use datafusion::physical_expr::PhysicalExpr;
53-
use datafusion::physical_plan::ExecutionPlan;
54-
use datafusion::prelude::SessionConfig;
55-
use datafusion_physical_expr_adapter::{
53+
use datafusion::physical_expr_adapter::{
5654
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
5755
};
56+
use datafusion::physical_plan::ExecutionPlan;
57+
use datafusion::prelude::SessionConfig;
5858
use datafusion_proto::bytes::{
5959
physical_plan_from_bytes_with_proto_converter,
6060
physical_plan_to_bytes_with_proto_converter,

‎datafusion-examples/examples/custom_data_source/custom_file_casts.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ use datafusion::execution::object_store::ObjectStoreUrl;
3434
use datafusion::parquet::arrow::ArrowWriter;
3535
use datafusion::physical_expr::PhysicalExpr;
3636
use datafusion::physical_expr::expressions::CastExpr;
37-
use datafusion::prelude::SessionConfig;
38-
use datafusion_physical_expr_adapter::{
37+
use datafusion::physical_expr_adapter::{
3938
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
4039
};
40+
use datafusion::prelude::SessionConfig;
4141
use object_store::memory::InMemory;
4242
use object_store::path::Path;
4343
use object_store::{ObjectStore, ObjectStoreExt, PutPayload};

‎datafusion-examples/examples/custom_data_source/default_column_values.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,12 @@ use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableType};
3838
use datafusion::parquet::arrow::ArrowWriter;
3939
use datafusion::parquet::file::properties::WriterProperties;
4040
use datafusion::physical_expr::PhysicalExpr;
41-
use datafusion::physical_plan::ExecutionPlan;
42-
use datafusion::prelude::{SessionConfig, lit};
43-
use datafusion_physical_expr_adapter::{
41+
use datafusion::physical_expr_adapter::{
4442
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
4543
replace_columns_with_literals,
4644
};
45+
use datafusion::physical_plan::ExecutionPlan;
46+
use datafusion::prelude::{SessionConfig, lit};
4747
use futures::StreamExt;
4848
use object_store::memory::InMemory;
4949
use object_store::path::Path;

‎datafusion-examples/examples/data_io/json_shredding.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,11 @@ use datafusion::parquet::arrow::ArrowWriter;
3939
use datafusion::parquet::file::properties::WriterProperties;
4040
use datafusion::physical_expr::PhysicalExpr;
4141
use datafusion::physical_expr::{ScalarFunctionExpr, expressions};
42-
use datafusion::prelude::SessionConfig;
43-
use datafusion::scalar::ScalarValue;
44-
use datafusion_physical_expr_adapter::{
42+
use datafusion::physical_expr_adapter::{
4543
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
4644
};
45+
use datafusion::prelude::SessionConfig;
46+
use datafusion::scalar::ScalarValue;
4747
use object_store::memory::InMemory;
4848
use object_store::path::Path;
4949
use object_store::{ObjectStoreExt, PutPayload};

‎datafusion-examples/examples/dataframe/cache_factory.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use async_trait::async_trait;
2626
use datafusion::catalog::Session;
2727
use datafusion::catalog::memory::MemorySourceConfig;
2828
use datafusion::common::DFSchemaRef;
29+
use datafusion::common::HashMap;
2930
use datafusion::error::Result;
3031
use datafusion::execution::context::QueryPlanner;
3132
use datafusion::execution::session_state::CacheFactory;
@@ -39,7 +40,6 @@ use datafusion::physical_planner::{
3940
DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner,
4041
};
4142
use datafusion::prelude::*;
42-
use datafusion_common::HashMap;
4343
use datafusion_examples::utils::{datasets::ExampleDataset, write_csv_to_parquet};
4444

4545
/// This example demonstrates how to leverage [CacheFactory] to implement custom caching strategies for dataframes in DataFusion.

0 commit comments

Comments
 (0)