@@ -34,16 +34,18 @@ use datafusion_expr::{col, lit};
3434use datatypes:: arrow:: array:: { ArrayRef , Float64Array , StringArray , TimestampMillisecondArray } ;
3535use datatypes:: arrow:: datatypes:: { DataType , Field , Schema , TimeUnit } ;
3636use datatypes:: prelude:: ConcreteDataType ;
37- use datatypes:: schema:: { ColumnSchema , FulltextAnalyzer , FulltextBackend , FulltextOptions } ;
37+ use datatypes:: schema:: {
38+ ColumnDefaultConstraint , ColumnSchema , FulltextAnalyzer , FulltextBackend , FulltextOptions ,
39+ } ;
3840use store_api:: codec:: PrimaryKeyEncoding ;
3941use store_api:: logstore:: provider:: Provider ;
4042use store_api:: metadata:: { ColumnMetadata , RegionMetadata } ;
4143use store_api:: metric_engine_consts:: { PRIMARY_KEY_ENCODING , TABLE_COLUMN_METADATA_EXTENSION_KEY } ;
4244use store_api:: region_engine:: { RegionEngine , RegionManifestInfo , RegionRole } ;
4345use store_api:: region_request:: {
4446 AddColumn , AddColumnLocation , AlterKind , ModifyColumnType , PathType , RegionAlterRequest ,
45- RegionBulkInsertsRequest , RegionCompactRequest , RegionOpenRequest , RegionPutRequest ,
46- RegionRequest , SetIndexOption , SetRegionOption , UnsetRegionOption ,
47+ RegionBulkInsertsRequest , RegionCompactRequest , RegionCreateRequest , RegionOpenRequest ,
48+ RegionPutRequest , RegionRequest , SetIndexOption , SetRegionOption , UnsetRegionOption ,
4749} ;
4850use store_api:: storage:: consts:: PRIMARY_KEY_COLUMN_NAME ;
4951use store_api:: storage:: { ColumnId , RegionId , ScanRequest } ;
@@ -52,6 +54,7 @@ use crate::config::MitoConfig;
5254use crate :: engine:: MitoEngine ;
5355use crate :: engine:: listener:: { AlterFlushListener , NotifyRegionChangeResultListener } ;
5456use crate :: error;
57+ use crate :: request:: WorkerRequest ;
5558use crate :: sst:: FormatType ;
5659use crate :: test_util:: batch_util:: sort_batches_and_print;
5760use crate :: test_util:: sst_util:: { new_sparse_primary_key, sst_region_metadata_with_encoding} ;
@@ -3714,3 +3717,157 @@ async fn test_alter_time_index_widen_strict_window_compaction() {
37143717 // Each instant exactly once, rescaled to microseconds.
37153718 assert_eq ! ( vec![ 0 , 1_000_000 , 2_000_000 , 3_000_000 , 4_000_000 ] , values) ;
37163719}
3720+
3721+ /// Returns a create request with a field `field_1` that defaults to 42, and the
3722+ /// schema of rows that leave `field_1` out, so the engine fills it on puts.
3723+ fn create_request_with_default_field ( ) -> ( RegionCreateRequest , Vec < api:: v1:: ColumnSchema > ) {
3724+ let mut request = CreateRequestBuilder :: new ( ) . build ( ) ;
3725+ let column_schemas = rows_schema ( & request) ;
3726+ request. column_metadatas . push ( ColumnMetadata {
3727+ column_schema : ColumnSchema :: new ( "field_1" , ConcreteDataType :: float64_datatype ( ) , true )
3728+ . with_default_constraint ( Some ( ColumnDefaultConstraint :: Value (
3729+ datatypes:: value:: Value :: Float64 ( 42.0 . into ( ) ) ,
3730+ ) ) )
3731+ . unwrap ( ) ,
3732+ semantic_type : SemanticType :: Field ,
3733+ column_id : 3 ,
3734+ } ) ;
3735+ ( request, column_schemas)
3736+ }
3737+
3738+ fn add_nullable_field2 ( ) -> RegionAlterRequest {
3739+ RegionAlterRequest {
3740+ kind : AlterKind :: AddColumns {
3741+ columns : vec ! [ AddColumn {
3742+ column_metadata: ColumnMetadata {
3743+ column_schema: ColumnSchema :: new(
3744+ "field_2" ,
3745+ ConcreteDataType :: float64_datatype( ) ,
3746+ true ,
3747+ ) ,
3748+ semantic_type: SemanticType :: Field ,
3749+ column_id: 4 ,
3750+ } ,
3751+ location: None ,
3752+ } ] ,
3753+ } ,
3754+ }
3755+ }
3756+
3757+ fn build_put_request (
3758+ schema : Vec < api:: v1:: ColumnSchema > ,
3759+ start : usize ,
3760+ end : usize ,
3761+ ) -> RegionRequest {
3762+ RegionRequest :: Put ( RegionPutRequest {
3763+ rows : Rows {
3764+ schema,
3765+ rows : build_rows ( start, end) ,
3766+ } ,
3767+ hint : None ,
3768+ skip_wal : false ,
3769+ partition_expr_version : None ,
3770+ } )
3771+ }
3772+
3773+ /// Sends a put that the engine boundary checked and filled with `metadata`,
3774+ /// the region metadata before an alter, and returns the worker's answer.
3775+ async fn put_filled_with_stale_metadata (
3776+ engine : & MitoEngine ,
3777+ region_id : RegionId ,
3778+ metadata : store_api:: metadata:: RegionMetadataRef ,
3779+ schema : Vec < api:: v1:: ColumnSchema > ,
3780+ ) -> error:: Result < usize > {
3781+ let ( request, receiver) = WorkerRequest :: try_from_region_request (
3782+ region_id,
3783+ build_put_request ( schema, 0 , 2 ) ,
3784+ Some ( metadata) ,
3785+ )
3786+ . unwrap ( ) ;
3787+ engine
3788+ . inner
3789+ . workers
3790+ . submit_to_worker ( region_id, request)
3791+ . await
3792+ . unwrap ( ) ;
3793+ receiver
3794+ . await
3795+ . expect ( "the worker dropped the request without an answer" )
3796+ }
3797+
3798+ /// A put filled with a default value at the engine boundary is checked again by
3799+ /// the worker after a schema change; the filled column must not break the check.
3800+ #[ tokio:: test]
3801+ async fn test_put_filled_at_boundary_rechecked_after_add_column ( ) {
3802+ common_telemetry:: init_default_ut_logging ( ) ;
3803+ let mut env = TestEnv :: new ( ) . await ;
3804+ let engine = env. create_engine ( MitoConfig :: default ( ) ) . await ;
3805+ let region_id = RegionId :: new ( 1 , 1 ) ;
3806+ let ( request, column_schemas) = create_request_with_default_field ( ) ;
3807+ engine
3808+ . handle_request ( region_id, RegionRequest :: Create ( request) )
3809+ . await
3810+ . unwrap ( ) ;
3811+ let metadata = engine. get_region ( region_id) . unwrap ( ) . metadata ( ) ;
3812+ engine
3813+ . handle_request ( region_id, RegionRequest :: Alter ( add_nullable_field2 ( ) ) )
3814+ . await
3815+ . unwrap ( ) ;
3816+
3817+ let result =
3818+ put_filled_with_stale_metadata ( & engine, region_id, metadata, column_schemas. clone ( ) ) . await ;
3819+ assert_eq ! ( 2 , result. unwrap( ) ) ;
3820+
3821+ // The worker still serves the region.
3822+ let response = engine
3823+ . handle_request ( region_id, build_put_request ( column_schemas, 2 , 3 ) )
3824+ . await
3825+ . unwrap ( ) ;
3826+ assert_eq ! ( 1 , response. affected_rows) ;
3827+ let scanner = engine
3828+ . scanner ( region_id, ScanRequest :: default ( ) )
3829+ . await
3830+ . unwrap ( ) ;
3831+ let batches = RecordBatches :: try_collect ( scanner. scan ( ) . await . unwrap ( ) )
3832+ . await
3833+ . unwrap ( ) ;
3834+ let expected = "\
3835+ +-------+---------+---------------------+---------+---------+
3836+ | tag_0 | field_0 | ts | field_1 | field_2 |
3837+ +-------+---------+---------------------+---------+---------+
3838+ | 0 | 0.0 | 1970-01-01T00:00:00 | 42.0 | |
3839+ | 1 | 1.0 | 1970-01-01T00:00:01 | 42.0 | |
3840+ | 2 | 2.0 | 1970-01-01T00:00:02 | 42.0 | |
3841+ +-------+---------+---------------------+---------+---------+" ;
3842+ assert_eq ! ( expected, batches. pretty_print( ) . unwrap( ) ) ;
3843+ }
3844+
3845+ /// A put that left out a column which is dropped before the worker handles the
3846+ /// put is still valid.
3847+ #[ tokio:: test]
3848+ async fn test_put_filled_at_boundary_rechecked_after_drop_column ( ) {
3849+ common_telemetry:: init_default_ut_logging ( ) ;
3850+ let mut env = TestEnv :: new ( ) . await ;
3851+ let engine = env. create_engine ( MitoConfig :: default ( ) ) . await ;
3852+ let region_id = RegionId :: new ( 1 , 1 ) ;
3853+ let ( request, column_schemas) = create_request_with_default_field ( ) ;
3854+ engine
3855+ . handle_request ( region_id, RegionRequest :: Create ( request) )
3856+ . await
3857+ . unwrap ( ) ;
3858+ let metadata = engine. get_region ( region_id) . unwrap ( ) . metadata ( ) ;
3859+ engine
3860+ . handle_request (
3861+ region_id,
3862+ RegionRequest :: Alter ( RegionAlterRequest {
3863+ kind : AlterKind :: DropColumns {
3864+ names : vec ! [ "field_1" . to_string( ) ] ,
3865+ } ,
3866+ } ) ,
3867+ )
3868+ . await
3869+ . unwrap ( ) ;
3870+
3871+ let result = put_filled_with_stale_metadata ( & engine, region_id, metadata, column_schemas) . await ;
3872+ assert_eq ! ( 2 , result. unwrap( ) ) ;
3873+ }
0 commit comments