Skip to content

Commit 93c3b11

Browse files
authored
fix(mito2): re-check a filled write as sent after a schema change (#9701)
* fix(mito2): re-check a filled write as sent after a schema change The engine fills the columns a write leaves out with their defaults when it accepts the write, appending them to the rows without adding them to `name_to_index` and `has_null`. If the region schema changes before the worker handles the write, the worker checks the request again and `check_schema` looks up a filled column in `name_to_index`, which panics and stops the region worker. If the filled column was dropped, the write fails with an unknown column that the client never sent. Remove the filled columns before checking, so the request is checked as sent and filled again for the current schema. Nothing is filled yet when the engine checks the request, so that check does no extra work. Signed-off-by: jeremyhi <[email protected]> * test(mito2): rename put_request helper to build_put_request Signed-off-by: jeremyhi <[email protected]> --------- Signed-off-by: jeremyhi <[email protected]>
1 parent da88ff8 commit 93c3b11

2 files changed

Lines changed: 169 additions & 3 deletions

File tree

‎src/mito2/src/engine/alter_test.rs‎

Lines changed: 160 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,16 +34,18 @@ use datafusion_expr::{col, lit};
3434
use datatypes::arrow::array::{ArrayRef, Float64Array, StringArray, TimestampMillisecondArray};
3535
use datatypes::arrow::datatypes::{DataType, Field, Schema, TimeUnit};
3636
use datatypes::prelude::ConcreteDataType;
37-
use datatypes::schema::{ColumnSchema, FulltextAnalyzer, FulltextBackend, FulltextOptions};
37+
use datatypes::schema::{
38+
ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextBackend, FulltextOptions,
39+
};
3840
use store_api::codec::PrimaryKeyEncoding;
3941
use store_api::logstore::provider::Provider;
4042
use store_api::metadata::{ColumnMetadata, RegionMetadata};
4143
use store_api::metric_engine_consts::{PRIMARY_KEY_ENCODING, TABLE_COLUMN_METADATA_EXTENSION_KEY};
4244
use store_api::region_engine::{RegionEngine, RegionManifestInfo, RegionRole};
4345
use store_api::region_request::{
4446
AddColumn, AddColumnLocation, AlterKind, ModifyColumnType, PathType, RegionAlterRequest,
45-
RegionBulkInsertsRequest, RegionCompactRequest, RegionOpenRequest, RegionPutRequest,
46-
RegionRequest, SetIndexOption, SetRegionOption, UnsetRegionOption,
47+
RegionBulkInsertsRequest, RegionCompactRequest, RegionCreateRequest, RegionOpenRequest,
48+
RegionPutRequest, RegionRequest, SetIndexOption, SetRegionOption, UnsetRegionOption,
4749
};
4850
use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME;
4951
use store_api::storage::{ColumnId, RegionId, ScanRequest};
@@ -52,6 +54,7 @@ use crate::config::MitoConfig;
5254
use crate::engine::MitoEngine;
5355
use crate::engine::listener::{AlterFlushListener, NotifyRegionChangeResultListener};
5456
use crate::error;
57+
use crate::request::WorkerRequest;
5558
use crate::sst::FormatType;
5659
use crate::test_util::batch_util::sort_batches_and_print;
5760
use crate::test_util::sst_util::{new_sparse_primary_key, sst_region_metadata_with_encoding};
@@ -3714,3 +3717,157 @@ async fn test_alter_time_index_widen_strict_window_compaction() {
37143717
// Each instant exactly once, rescaled to microseconds.
37153718
assert_eq!(vec![0, 1_000_000, 2_000_000, 3_000_000, 4_000_000], values);
37163719
}
3720+
3721+
/// Returns a create request with a field `field_1` that defaults to 42, and the
3722+
/// schema of rows that leave `field_1` out, so the engine fills it on puts.
3723+
fn create_request_with_default_field() -> (RegionCreateRequest, Vec<api::v1::ColumnSchema>) {
3724+
let mut request = CreateRequestBuilder::new().build();
3725+
let column_schemas = rows_schema(&request);
3726+
request.column_metadatas.push(ColumnMetadata {
3727+
column_schema: ColumnSchema::new("field_1", ConcreteDataType::float64_datatype(), true)
3728+
.with_default_constraint(Some(ColumnDefaultConstraint::Value(
3729+
datatypes::value::Value::Float64(42.0.into()),
3730+
)))
3731+
.unwrap(),
3732+
semantic_type: SemanticType::Field,
3733+
column_id: 3,
3734+
});
3735+
(request, column_schemas)
3736+
}
3737+
3738+
fn add_nullable_field2() -> RegionAlterRequest {
3739+
RegionAlterRequest {
3740+
kind: AlterKind::AddColumns {
3741+
columns: vec![AddColumn {
3742+
column_metadata: ColumnMetadata {
3743+
column_schema: ColumnSchema::new(
3744+
"field_2",
3745+
ConcreteDataType::float64_datatype(),
3746+
true,
3747+
),
3748+
semantic_type: SemanticType::Field,
3749+
column_id: 4,
3750+
},
3751+
location: None,
3752+
}],
3753+
},
3754+
}
3755+
}
3756+
3757+
fn build_put_request(
3758+
schema: Vec<api::v1::ColumnSchema>,
3759+
start: usize,
3760+
end: usize,
3761+
) -> RegionRequest {
3762+
RegionRequest::Put(RegionPutRequest {
3763+
rows: Rows {
3764+
schema,
3765+
rows: build_rows(start, end),
3766+
},
3767+
hint: None,
3768+
skip_wal: false,
3769+
partition_expr_version: None,
3770+
})
3771+
}
3772+
3773+
/// Sends a put that the engine boundary checked and filled with `metadata`,
3774+
/// the region metadata before an alter, and returns the worker's answer.
3775+
async fn put_filled_with_stale_metadata(
3776+
engine: &MitoEngine,
3777+
region_id: RegionId,
3778+
metadata: store_api::metadata::RegionMetadataRef,
3779+
schema: Vec<api::v1::ColumnSchema>,
3780+
) -> error::Result<usize> {
3781+
let (request, receiver) = WorkerRequest::try_from_region_request(
3782+
region_id,
3783+
build_put_request(schema, 0, 2),
3784+
Some(metadata),
3785+
)
3786+
.unwrap();
3787+
engine
3788+
.inner
3789+
.workers
3790+
.submit_to_worker(region_id, request)
3791+
.await
3792+
.unwrap();
3793+
receiver
3794+
.await
3795+
.expect("the worker dropped the request without an answer")
3796+
}
3797+
3798+
/// A put filled with a default value at the engine boundary is checked again by
3799+
/// the worker after a schema change; the filled column must not break the check.
3800+
#[tokio::test]
3801+
async fn test_put_filled_at_boundary_rechecked_after_add_column() {
3802+
common_telemetry::init_default_ut_logging();
3803+
let mut env = TestEnv::new().await;
3804+
let engine = env.create_engine(MitoConfig::default()).await;
3805+
let region_id = RegionId::new(1, 1);
3806+
let (request, column_schemas) = create_request_with_default_field();
3807+
engine
3808+
.handle_request(region_id, RegionRequest::Create(request))
3809+
.await
3810+
.unwrap();
3811+
let metadata = engine.get_region(region_id).unwrap().metadata();
3812+
engine
3813+
.handle_request(region_id, RegionRequest::Alter(add_nullable_field2()))
3814+
.await
3815+
.unwrap();
3816+
3817+
let result =
3818+
put_filled_with_stale_metadata(&engine, region_id, metadata, column_schemas.clone()).await;
3819+
assert_eq!(2, result.unwrap());
3820+
3821+
// The worker still serves the region.
3822+
let response = engine
3823+
.handle_request(region_id, build_put_request(column_schemas, 2, 3))
3824+
.await
3825+
.unwrap();
3826+
assert_eq!(1, response.affected_rows);
3827+
let scanner = engine
3828+
.scanner(region_id, ScanRequest::default())
3829+
.await
3830+
.unwrap();
3831+
let batches = RecordBatches::try_collect(scanner.scan().await.unwrap())
3832+
.await
3833+
.unwrap();
3834+
let expected = "\
3835+
+-------+---------+---------------------+---------+---------+
3836+
| tag_0 | field_0 | ts | field_1 | field_2 |
3837+
+-------+---------+---------------------+---------+---------+
3838+
| 0 | 0.0 | 1970-01-01T00:00:00 | 42.0 | |
3839+
| 1 | 1.0 | 1970-01-01T00:00:01 | 42.0 | |
3840+
| 2 | 2.0 | 1970-01-01T00:00:02 | 42.0 | |
3841+
+-------+---------+---------------------+---------+---------+";
3842+
assert_eq!(expected, batches.pretty_print().unwrap());
3843+
}
3844+
3845+
/// A put that left out a column which is dropped before the worker handles the
3846+
/// put is still valid.
3847+
#[tokio::test]
3848+
async fn test_put_filled_at_boundary_rechecked_after_drop_column() {
3849+
common_telemetry::init_default_ut_logging();
3850+
let mut env = TestEnv::new().await;
3851+
let engine = env.create_engine(MitoConfig::default()).await;
3852+
let region_id = RegionId::new(1, 1);
3853+
let (request, column_schemas) = create_request_with_default_field();
3854+
engine
3855+
.handle_request(region_id, RegionRequest::Create(request))
3856+
.await
3857+
.unwrap();
3858+
let metadata = engine.get_region(region_id).unwrap().metadata();
3859+
engine
3860+
.handle_request(
3861+
region_id,
3862+
RegionRequest::Alter(RegionAlterRequest {
3863+
kind: AlterKind::DropColumns {
3864+
names: vec!["field_1".to_string()],
3865+
},
3866+
}),
3867+
)
3868+
.await
3869+
.unwrap();
3870+
3871+
let result = put_filled_with_stale_metadata(&engine, region_id, metadata, column_schemas).await;
3872+
assert_eq!(2, result.unwrap());
3873+
}

‎src/mito2/src/request.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,15 @@ impl WriteRequest {
241241

242242
/// Checks the schema and fill missing columns.
243243
pub(crate) fn maybe_fill_missing_columns(&mut self, metadata: &RegionMetadata) -> Result<()> {
244+
// Removes the columns filled for an older schema, so the request is checked
245+
// as sent and filled again for this schema.
246+
let num_columns = self.name_to_index.len();
247+
if self.rows.schema.len() > num_columns {
248+
self.rows.schema.truncate(num_columns);
249+
for row in &mut self.rows.rows {
250+
row.values.truncate(num_columns);
251+
}
252+
}
244253
if let Err(e) = self.check_schema(metadata) {
245254
if e.is_fill_default() {
246255
// TODO(yingwen): Add metrics for this case.

0 commit comments

Comments
 (0)