Skip to content

Commit c73321f

Browse files
committed
test: trim overlapping prefetch tests and merge the test object stores
Drop the parquet_exec scan test that repeats the CometExecSuite test, run the Scala test with the config on and off only, and fold GatedStore into the recording store used by the range read tests.
1 parent 84a0950 commit c73321f

3 files changed

Lines changed: 99 additions & 217 deletions

File tree

‎native/core/src/parquet/eager_page_index_reader_factory.rs‎

Lines changed: 87 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -1298,21 +1298,82 @@ mod tests {
12981298
},
12991299
};
13001300

1301+
/// An `InMemory` store that counts the gets and range calls of each path. Reads of the path
1302+
/// set by `with_gate` wait until `release` is called, and each held read keeps a receiver of
1303+
/// `gate`.
13011304
#[derive(Debug)]
1302-
struct RecordingRangeStore {
1305+
struct RecordingStore {
13031306
inner: InMemory,
1304-
range_calls: AtomicUsize,
1305-
get_calls: AtomicUsize,
1307+
gets: Mutex<HashMap<Path, usize>>,
1308+
range_calls: Mutex<HashMap<Path, usize>>,
1309+
gated: Option<Path>,
1310+
gate: tokio::sync::watch::Sender<bool>,
1311+
}
1312+
1313+
impl RecordingStore {
1314+
fn new() -> Self {
1315+
Self {
1316+
inner: InMemory::new(),
1317+
gets: Mutex::new(HashMap::new()),
1318+
range_calls: Mutex::new(HashMap::new()),
1319+
gated: None,
1320+
gate: tokio::sync::watch::channel(false).0,
1321+
}
1322+
}
1323+
1324+
fn with_gate(mut self, gated: &str) -> Self {
1325+
self.gated = Some(Path::from(gated));
1326+
self
1327+
}
1328+
1329+
fn release(&self) {
1330+
self.gate.send_replace(true);
1331+
}
1332+
1333+
/// The gets and range calls of `location`.
1334+
fn reads(&self, location: &str) -> usize {
1335+
let location = Path::from(location);
1336+
[&self.gets, &self.range_calls]
1337+
.iter()
1338+
.map(|calls| calls.lock().unwrap().get(&location).copied().unwrap_or(0))
1339+
.sum()
1340+
}
1341+
1342+
fn get_calls(&self) -> usize {
1343+
self.gets.lock().unwrap().values().sum()
1344+
}
1345+
1346+
fn range_calls(&self) -> usize {
1347+
self.range_calls.lock().unwrap().values().sum()
1348+
}
1349+
1350+
/// Waits until a read of the gated path is held at the gate.
1351+
async fn wait_for_held_read(&self) {
1352+
tokio::time::timeout(std::time::Duration::from_secs(10), async {
1353+
while self.gate.receiver_count() == 0 {
1354+
tokio::task::yield_now().await;
1355+
}
1356+
})
1357+
.await
1358+
.expect("the gated path was never read");
1359+
}
1360+
1361+
async fn hold_if_gated(&self, location: &Path) {
1362+
if self.gated.as_ref() == Some(location) {
1363+
let mut gate = self.gate.subscribe();
1364+
gate.wait_for(|released| *released).await.unwrap();
1365+
}
1366+
}
13061367
}
13071368

1308-
impl Display for RecordingRangeStore {
1369+
impl Display for RecordingStore {
13091370
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1310-
write!(f, "recording-range-store")
1371+
write!(f, "recording-store")
13111372
}
13121373
}
13131374

13141375
#[async_trait]
1315-
impl ObjectStore for RecordingRangeStore {
1376+
impl ObjectStore for RecordingStore {
13161377
async fn put_opts(
13171378
&self,
13181379
p: &Path,
@@ -1331,7 +1392,8 @@ mod tests {
13311392
}
13321393

13331394
async fn get_opts(&self, p: &Path, o: GetOptions) -> ObjectStoreResult<GetResult> {
1334-
self.get_calls.fetch_add(1, Ordering::Relaxed);
1395+
*self.gets.lock().unwrap().entry(p.clone()).or_default() += 1;
1396+
self.hold_if_gated(p).await;
13351397
self.inner.get_opts(p, o).await
13361398
}
13371399

@@ -1340,7 +1402,13 @@ mod tests {
13401402
p: &Path,
13411403
ranges: &[Range<u64>],
13421404
) -> ObjectStoreResult<Vec<Bytes>> {
1343-
self.range_calls.fetch_add(1, Ordering::Relaxed);
1405+
*self
1406+
.range_calls
1407+
.lock()
1408+
.unwrap()
1409+
.entry(p.clone())
1410+
.or_default() += 1;
1411+
self.hold_if_gated(p).await;
13441412
self.inner.get_ranges(p, ranges).await
13451413
}
13461414

@@ -1394,11 +1462,7 @@ mod tests {
13941462
}
13951463

13961464
async fn assert_range_read_contract(source: ScanIoSource) {
1397-
let store = Arc::new(RecordingRangeStore {
1398-
inner: InMemory::new(),
1399-
range_calls: AtomicUsize::new(0),
1400-
get_calls: AtomicUsize::new(0),
1401-
});
1465+
let store = Arc::new(RecordingStore::new());
14021466
let location = Path::from("ranges.parquet");
14031467
store
14041468
.put(&location, Bytes::from_static(b"0123456789").into())
@@ -1426,11 +1490,8 @@ mod tests {
14261490
vec![Bytes::from_static(b"01"), Bytes::from_static(b"45")]
14271491
);
14281492
let remote = source == ScanIoSource::ObjectStore;
1429-
assert_eq!(
1430-
store.range_calls.load(Ordering::Relaxed),
1431-
usize::from(!remote)
1432-
);
1433-
assert_eq!(store.get_calls.load(Ordering::Relaxed), usize::from(remote));
1493+
assert_eq!(store.range_calls(), usize::from(!remote));
1494+
assert_eq!(store.get_calls(), usize::from(remote));
14341495
assert_eq!(
14351496
metrics
14361497
.clone_inner()
@@ -1701,118 +1762,6 @@ mod tests {
17011762
));
17021763
}
17031764

1704-
/// An `InMemory` store whose reads of one path wait until `open` is called, counting the
1705-
/// reads of each path. Each held read keeps a receiver of `gate`.
1706-
#[derive(Debug)]
1707-
struct GatedStore {
1708-
inner: InMemory,
1709-
gated: Path,
1710-
gate: tokio::sync::watch::Sender<bool>,
1711-
reads: Mutex<HashMap<Path, usize>>,
1712-
}
1713-
1714-
impl GatedStore {
1715-
fn new(gated: &str) -> Self {
1716-
Self {
1717-
inner: InMemory::new(),
1718-
gated: Path::from(gated),
1719-
gate: tokio::sync::watch::channel(false).0,
1720-
reads: Mutex::new(HashMap::new()),
1721-
}
1722-
}
1723-
1724-
fn open(&self) {
1725-
self.gate.send_replace(true);
1726-
}
1727-
1728-
fn reads(&self, location: &str) -> usize {
1729-
let reads = self.reads.lock().unwrap();
1730-
reads.get(&Path::from(location)).copied().unwrap_or(0)
1731-
}
1732-
1733-
/// Waits until a read of the gated path is held at the gate.
1734-
async fn wait_for_held_read(&self) {
1735-
tokio::time::timeout(std::time::Duration::from_secs(10), async {
1736-
while self.gate.receiver_count() == 0 {
1737-
tokio::task::yield_now().await;
1738-
}
1739-
})
1740-
.await
1741-
.expect("the gated path was never read");
1742-
}
1743-
}
1744-
1745-
impl Display for GatedStore {
1746-
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1747-
write!(f, "gated-store")
1748-
}
1749-
}
1750-
1751-
#[async_trait]
1752-
impl ObjectStore for GatedStore {
1753-
async fn put_opts(
1754-
&self,
1755-
p: &Path,
1756-
v: PutPayload,
1757-
o: PutOptions,
1758-
) -> ObjectStoreResult<PutResult> {
1759-
self.inner.put_opts(p, v, o).await
1760-
}
1761-
1762-
async fn put_multipart_opts(
1763-
&self,
1764-
p: &Path,
1765-
o: PutMultipartOptions,
1766-
) -> ObjectStoreResult<Box<dyn MultipartUpload>> {
1767-
self.inner.put_multipart_opts(p, o).await
1768-
}
1769-
1770-
async fn get_opts(&self, p: &Path, o: GetOptions) -> ObjectStoreResult<GetResult> {
1771-
*self.reads.lock().unwrap().entry(p.clone()).or_default() += 1;
1772-
if p == &self.gated {
1773-
let mut gate = self.gate.subscribe();
1774-
gate.wait_for(|open| *open).await.unwrap();
1775-
}
1776-
self.inner.get_opts(p, o).await
1777-
}
1778-
1779-
fn delete_stream(
1780-
&self,
1781-
paths: futures::stream::BoxStream<'static, ObjectStoreResult<Path>>,
1782-
) -> futures::stream::BoxStream<'static, ObjectStoreResult<Path>> {
1783-
self.inner.delete_stream(paths)
1784-
}
1785-
1786-
fn list(
1787-
&self,
1788-
p: Option<&Path>,
1789-
) -> futures::stream::BoxStream<'static, ObjectStoreResult<ObjectMeta>> {
1790-
self.inner.list(p)
1791-
}
1792-
1793-
async fn list_with_delimiter(&self, p: Option<&Path>) -> ObjectStoreResult<ListResult> {
1794-
self.inner.list_with_delimiter(p).await
1795-
}
1796-
1797-
async fn copy_opts(
1798-
&self,
1799-
from: &Path,
1800-
to: &Path,
1801-
options: CopyOptions,
1802-
) -> ObjectStoreResult<()> {
1803-
self.inner.copy_opts(from, to, options).await
1804-
}
1805-
1806-
async fn rename_opts(
1807-
&self,
1808-
from: &Path,
1809-
to: &Path,
1810-
options: RenameOptions,
1811-
) -> ObjectStoreResult<()> {
1812-
self.inner.rename_opts(from, to, options).await
1813-
}
1814-
}
1815-
18161765
async fn put_int_files(store: &InMemory, locations: &[&str]) -> Vec<PartitionedFile> {
18171766
let mut files = vec![];
18181767
for location in locations {
@@ -1914,7 +1863,7 @@ mod tests {
19141863
/// read instead of starting its own.
19151864
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19161865
async fn an_open_waits_for_the_load_of_its_metadata_in_flight() {
1917-
let store = Arc::new(GatedStore::new("b.parquet"));
1866+
let store = Arc::new(RecordingStore::new().with_gate("b.parquet"));
19181867
let files = put_int_files(&store.inner, &["a.parquet", "b.parquet"]).await;
19191868
let metrics = ExecutionPlanMetricsSet::new();
19201869
let factory = prefetching_factory(Arc::clone(&store) as _, &files, &metrics);
@@ -1924,7 +1873,7 @@ mod tests {
19241873
let mut metadata = Box::pin(open(&factory, &files[1], &metrics));
19251874
assert!((&mut metadata).now_or_never().is_none());
19261875
assert_eq!(store.reads("b.parquet"), 1, "the open read b itself");
1927-
store.open();
1876+
store.release();
19281877
assert_eq!(metadata.await.unwrap().file_metadata().num_rows(), 2);
19291878

19301879
assert_eq!(metric(&metrics, "scan_io_metadata_cache_misses"), 2);
@@ -1937,7 +1886,7 @@ mod tests {
19371886
/// metadata, so they share its next file too, and opening them doesn't start a second load.
19381887
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19391888
async fn the_next_file_skips_further_ranges_of_the_same_path() {
1940-
let store = Arc::new(GatedStore::new("unused.parquet"));
1889+
let store = Arc::new(RecordingStore::new());
19411890
let a = put_int_file(&store.inner, "a.parquet", None).await;
19421891
let b = put_int_file(&store.inner, "b.parquet", None).await;
19431892
let files = [
@@ -1952,19 +1901,9 @@ mod tests {
19521901
open(&factory, file, &metrics).await.unwrap();
19531902
}
19541903

1955-
// The ranges of a share one load. The second range finds it already in the cache.
1956-
let next = |file: &PartitionedFile| {
1957-
let prefetch = factory.next_file_prefetch.as_ref().unwrap();
1958-
prefetch
1959-
.next_files
1960-
.get(&(file.object_meta.location.clone(), file.range.clone()))
1961-
.map(|(next, _)| next.location.to_string())
1962-
};
1963-
assert_eq!(next(&files[0]).as_deref(), Some("b.parquet"));
1964-
assert_eq!(next(&files[1]).as_deref(), Some("b.parquet"));
1965-
assert_eq!(next(&files[2]), None);
1966-
// b is read once, by the load that the first range of a started.
1967-
let single_load = Arc::new(GatedStore::new("unused.parquet"));
1904+
// b is read once, by the load that the first range of a started. A second range of a
1905+
// with no next file would drop that load, and b's open would read it again.
1906+
let single_load = Arc::new(RecordingStore::new());
19681907
let alone = put_int_file(&single_load.inner, "b.parquet", None).await;
19691908
let alone_metrics = ExecutionPlanMetricsSet::new();
19701909
let alone_factory = prefetching_factory(
@@ -2018,7 +1957,7 @@ mod tests {
20181957
/// reports the error.
20191958
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
20201959
async fn a_failed_prefetch_leaves_the_open_to_report_its_own_error() {
2021-
let store = Arc::new(GatedStore::new("unused.parquet"));
1960+
let store = Arc::new(RecordingStore::new());
20221961
let a = put_int_file(&store.inner, "a.parquet", None).await;
20231962
let content = Bytes::from_static(b"not a parquet file");
20241963
let bad = PartitionedFile::new("bad.parquet".to_string(), content.len() as u64);
@@ -2049,7 +1988,7 @@ mod tests {
20491988
/// Dropping the scan cancels a metadata load that is still in flight.
20501989
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
20511990
async fn dropping_the_scan_aborts_a_metadata_load_in_flight() {
2052-
let store = Arc::new(GatedStore::new("b.parquet"));
1991+
let store = Arc::new(RecordingStore::new().with_gate("b.parquet"));
20531992
let files = put_int_files(&store.inner, &["a.parquet", "b.parquet"]).await;
20541993
let metrics = ExecutionPlanMetricsSet::new();
20551994
let factory = prefetching_factory(Arc::clone(&store) as _, &files, &metrics);

0 commit comments

Comments
 (0)