Skip to content

Commit 791f429

Browse files
authored
fix(mito2): validate bulk insert batches against the region schema (#9698)
* fix(mito2): validate bulk insert batches against the region schema Bulk inserts were only checked for the time index column, unlike row writes, which WriteRequest::check_schema checks against the region schema. A batch with columns of another type, nulls in the time index or in non-nullable columns, duplicate or unknown columns reached the region worker: a time index that is not a timestamp or holds only nulls made the worker fail, and other mismatches failed after the WAL append or were stored as written, depending on the SST format. Move the column checks of check_schema into a core shared by row writes and bulk inserts, and check bulk insert batches at the same points as rows: when the engine builds the worker request, and again in the worker if the region schema changed since. check_schema keeps its behavior. Bulk insert columns must have the column's Arrow type, except that tags may use any string encoding, as the memtables already convert them. Sparse batches must carry one encoded primary key column of binary without nulls. A batch that misses columns gets them filled by the worker even if it claims to match the current schema. The metric engine checks logical bulk inserts against the logical region before rewriting them into the physical region, as the rewrite copies columns without checking them and would encode a duplicated tag into the primary key twice. Signed-off-by: jeremyhi <[email protected]> * fix(mito2): report an error when a write is interrupted by a panic WriteNotify answers its waiter when it is dropped and reported success unless an error had been set, also when it was dropped while unwinding from a panic in the middle of the write. Report an error in that case. Signed-off-by: jeremyhi <[email protected]> * fix(mito2): tolerate a poisoned series lock when dropping a series map SeriesMap::drop reads every series to update the memtable metrics and unwrapped the lock, so dropping the map while unwinding from a panic that poisoned a series lock panicked again. Read through the poisoned lock instead. Signed-off-by: jeremyhi <[email protected]> --------- Signed-off-by: jeremyhi <[email protected]>
1 parent 4684b95 commit 791f429

8 files changed

Lines changed: 751 additions & 101 deletions

File tree

‎src/metric-engine/src/engine/bulk_insert.rs‎

Lines changed: 94 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@ use std::collections::HashSet;
1616

1717
use api::v1::{ArrowIpc, SemanticType};
1818
use bytes::Bytes;
19+
use common_error::ext::BoxedError;
1920
use common_grpc::flight::{FlightEncoder, FlightMessage};
2021
use datatypes::arrow::record_batch::RecordBatch;
21-
use snafu::{OptionExt, ensure};
22+
use mito2::request::check_bulk_insert_batch;
23+
use snafu::{OptionExt, ResultExt, ensure};
2224
use store_api::codec::PrimaryKeyEncoding;
2325
use store_api::metadata::RegionMetadataRef;
2426
use store_api::region_request::{AffectedRows, RegionBulkInsertsRequest, RegionRequest};
@@ -111,6 +113,11 @@ impl MetricEngineInner {
111113
&batch,
112114
&logical_metadata,
113115
)?;
116+
// The rewrite below copies columns without checking them, and would encode a
117+
// duplicated tag into the primary key twice.
118+
check_bulk_insert_batch(&logical_metadata, &batch)
119+
.map_err(BoxedError::new)
120+
.context(error::MitoWriteOperationSnafu)?;
114121
let modified_batch = modify_batch_sparse(
115122
batch.clone(),
116123
region_id.table_id(),
@@ -307,6 +314,92 @@ mod tests {
307314
})
308315
}
309316

317+
#[tokio::test]
318+
async fn test_bulk_insert_rejects_duplicate_tag() {
319+
use common_error::status_code::StatusCode;
320+
321+
let env = TestEnv::new().await;
322+
env.init_metric_region().await;
323+
let logical_region_id = env.default_logical_region_id();
324+
325+
let batch = build_logical_batch(0, 2);
326+
let mut fields = batch.schema().fields().to_vec();
327+
let mut columns = batch.columns().to_vec();
328+
fields.push(fields[2].clone());
329+
columns.push(columns[2].clone());
330+
let batch = RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap();
331+
let err = env
332+
.metric()
333+
.handle_request(
334+
logical_region_id,
335+
build_bulk_request(logical_region_id, batch, false),
336+
)
337+
.await
338+
.unwrap_err();
339+
assert_eq!(err.status_code(), StatusCode::InvalidArguments, "{err:?}");
340+
}
341+
342+
#[tokio::test]
343+
async fn test_bulk_insert_physical_region_rejects_malformed_primary_key() {
344+
use common_error::status_code::StatusCode;
345+
use datatypes::arrow::array::{ArrayRef, BinaryArray, Int64Array};
346+
use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME;
347+
348+
let env = TestEnv::new().await;
349+
env.init_metric_region().await;
350+
let physical_region_id = env.default_physical_region_id();
351+
let logical_region_id = env.default_logical_region_id();
352+
let tag_columns = vec![TagColumnInfo {
353+
name: "job".to_string(),
354+
index: 2,
355+
column_id: 2,
356+
}];
357+
let batch = modify_batch_sparse(
358+
build_logical_batch(0, 2),
359+
logical_region_id.table_id(),
360+
&tag_columns,
361+
&[0, 1],
362+
)
363+
.unwrap();
364+
let primary_key_index = batch.schema().index_of(PRIMARY_KEY_COLUMN_NAME).unwrap();
365+
let with_primary_key = |primary_key: Option<ArrayRef>, extra: Option<ArrayRef>| {
366+
let mut fields = batch.schema().fields().to_vec();
367+
let mut columns = batch.columns().to_vec();
368+
fields.remove(primary_key_index);
369+
columns.remove(primary_key_index);
370+
for array in [primary_key, extra].into_iter().flatten() {
371+
fields.push(Arc::new(Field::new(
372+
PRIMARY_KEY_COLUMN_NAME,
373+
array.data_type().clone(),
374+
true,
375+
)));
376+
columns.push(array);
377+
}
378+
RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap()
379+
};
380+
let encoded = batch.column(primary_key_index).clone();
381+
let rejected = [
382+
with_primary_key(None, None),
383+
with_primary_key(Some(Arc::new(Int64Array::from(vec![1, 2]))), None),
384+
with_primary_key(
385+
Some(Arc::new(BinaryArray::from_opt_vec(vec![Some(b"a"), None]))),
386+
None,
387+
),
388+
with_primary_key(Some(encoded.clone()), Some(encoded)),
389+
];
390+
for batch in rejected {
391+
let err = env
392+
.metric()
393+
.handle_request(
394+
physical_region_id,
395+
build_bulk_request(physical_region_id, batch, false),
396+
)
397+
.await
398+
.unwrap_err();
399+
assert_eq!(err.status_code(), StatusCode::InvalidArguments, "{err:?}");
400+
}
401+
}
402+
310403
async fn init_dense_metric_region(env: &TestEnv) -> RegionId {
311404
let physical_region_id = env.default_physical_region_id();
312405
env.create_physical_region(

‎src/mito2/src/engine.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ mod batch_catchup_test;
2525
#[cfg(test)]
2626
mod batch_open_test;
2727
#[cfg(test)]
28+
mod bulk_insert_validation_test;
29+
#[cfg(test)]
2830
mod bump_committed_sequence_test;
2931
#[cfg(test)]
3032
mod catchup_test;

0 commit comments

Comments
 (0)