@@ -1628,6 +1628,7 @@ mod tests {
16281628 use crate :: cfg:: BATCH_BUILDER_MAX_OUTSTANDING_PARTS ;
16291629 use crate :: internal:: paths:: { BlobKey , PartialBlobKey } ;
16301630 use crate :: tests:: { all_ok, new_test_client} ;
1631+ use mz_persist_types:: codec_impls:: StringSchema ;
16311632
16321633 #[ mz_ore:: test( tokio:: test) ]
16331634 #[ cfg_attr( miri, ignore) ] // unsupported operation: returning ready events from epoll_wait is not yet implemented
@@ -1759,6 +1760,108 @@ mod tests {
17591760 }
17601761 }
17611762
1763+ fn string_part ( updates : & [ ( ( & str , & str ) , u64 , i64 ) ] ) -> Part {
1764+ let mut builder = PartBuilder :: new ( & StringSchema , & StringSchema ) ;
1765+ for ( ( k, v) , t, d) in updates {
1766+ builder. push ( & k. to_string ( ) , & v. to_string ( ) , * t, * d) ;
1767+ }
1768+ builder. finish ( )
1769+ }
1770+
1771+ #[ mz_ore:: test( tokio:: test) ]
1772+ #[ cfg_attr( miri, ignore) ] // unsupported operation: returning ready events from epoll_wait is not yet implemented
1773+ async fn batch_builder_add_part_before_lower ( ) {
1774+ let client = new_test_client ( & ConfigUpdates :: default ( ) ) . await ;
1775+ let ( write, _) = client
1776+ . expect_open :: < String , String , u64 , i64 > ( ShardId :: new ( ) )
1777+ . await ;
1778+ let mut builder = write. builder ( Antichain :: from_elem ( 2 ) ) ;
1779+ let part = string_part ( & [ ( ( "1" , "one" ) , 2 , 1 ) , ( ( "2" , "two" ) , 1 , 1 ) ] ) ;
1780+ match builder. add_part ( part) . await {
1781+ Err ( InvalidUsage :: UpdateNotBeyondLower { ts, .. } ) => assert_eq ! ( ts, 1 ) ,
1782+ other => panic ! ( "expected UpdateNotBeyondLower, got {other:?}" ) ,
1783+ }
1784+ }
1785+
1786+ #[ mz_ore:: test( tokio:: test) ]
1787+ #[ cfg_attr( miri, ignore) ] // unsupported operation: returning ready events from epoll_wait is not yet implemented
1788+ async fn batch_builder_add_part_flushing ( ) {
1789+ let small = || string_part ( & [ ( ( "1" , "one" ) , 1 , 1 ) ] ) ;
1790+ let big = string_part ( & [
1791+ ( ( "2" , "two" ) , 1 , 1 ) ,
1792+ ( ( "3" , "three" ) , 1 , 1 ) ,
1793+ ( ( "4" , "four" ) , 1 , 1 ) ,
1794+ ( ( "5" , "five" ) , 1 , 1 ) ,
1795+ ] ) ;
1796+ let small_bytes = small ( ) . goodbytes ( ) ;
1797+ assert ! (
1798+ big. goodbytes( ) > 2 * small_bytes,
1799+ "big part must exceed the target"
1800+ ) ;
1801+
1802+ let cache = PersistClientCache :: new_no_metrics ( ) ;
1803+ // Two small parts fit the target exactly. Disable inline writes so every flushed part is
1804+ // a separate blob that `part_count` sees.
1805+ cache. cfg . set_config ( & BLOB_TARGET_SIZE , 2 * small_bytes) ;
1806+ cache. cfg . set_config ( & INLINE_WRITES_SINGLE_MAX_BYTES , 0 ) ;
1807+ cache. cfg . set_config ( & INLINE_WRITES_TOTAL_MAX_BYTES , 0 ) ;
1808+ let client = cache
1809+ . open ( PersistLocation :: new_in_mem ( ) )
1810+ . await
1811+ . expect ( "client construction failed" ) ;
1812+ let ( mut write, mut read) = client
1813+ . expect_open :: < String , String , u64 , i64 > ( ShardId :: new ( ) )
1814+ . await ;
1815+
1816+ let mut builder = write. builder ( Antichain :: from_elem ( 0 ) ) ;
1817+ // Parts accumulate while they fit the target.
1818+ let added = builder. add_part ( small ( ) ) . await . expect ( "valid usage" ) ;
1819+ assert ! ( matches!( added, Added :: Record ) , "first part accumulates" ) ;
1820+ let added = builder. add_part ( small ( ) ) . await . expect ( "valid usage" ) ;
1821+ assert ! (
1822+ matches!( added, Added :: Record ) ,
1823+ "second part fits the target"
1824+ ) ;
1825+ // The third would exceed the target, so the first two flush together.
1826+ let added = builder. add_part ( small ( ) ) . await . expect ( "valid usage" ) ;
1827+ assert ! ( matches!( added, Added :: RecordAndParts ) , "third part flushes" ) ;
1828+ // An oversized part flushes the accumulated part first, and then on its own.
1829+ let added = builder. add_part ( big) . await . expect ( "valid usage" ) ;
1830+ assert ! (
1831+ matches!( added, Added :: RecordAndParts ) ,
1832+ "oversized part flushes"
1833+ ) ;
1834+ let added = builder. add_part ( small ( ) ) . await . expect ( "valid usage" ) ;
1835+ assert ! (
1836+ matches!( added, Added :: Record ) ,
1837+ "accounting resets after a flush"
1838+ ) ;
1839+
1840+ let batch = builder
1841+ . finish ( Antichain :: from_elem ( 2 ) )
1842+ . await
1843+ . expect ( "valid usage" ) ;
1844+ // [small, small], [small], [big], [small]
1845+ assert_eq ! ( batch. batch. part_count( ) , 4 ) ;
1846+ write
1847+ . append_batch ( batch, Antichain :: from_elem ( 0 ) , Antichain :: from_elem ( 2 ) )
1848+ . await
1849+ . expect ( "valid usage" )
1850+ . expect ( "unexpected upper" ) ;
1851+ let mut expected = vec ! [
1852+ ( ( "1" . to_owned( ) , "one" . to_owned( ) ) , 1 , 4 ) ,
1853+ ( ( "2" . to_owned( ) , "two" . to_owned( ) ) , 1 , 1 ) ,
1854+ ( ( "3" . to_owned( ) , "three" . to_owned( ) ) , 1 , 1 ) ,
1855+ ( ( "4" . to_owned( ) , "four" . to_owned( ) ) , 1 , 1 ) ,
1856+ ( ( "5" . to_owned( ) , "five" . to_owned( ) ) , 1 , 1 ) ,
1857+ ] ;
1858+ expected. sort ( ) ;
1859+ assert_eq ! (
1860+ read. expect_snapshot_and_fetch( 1 ) . await ,
1861+ all_ok( & expected, 1 )
1862+ ) ;
1863+ }
1864+
17621865 #[ mz_ore:: test( tokio:: test) ]
17631866 #[ cfg_attr( miri, ignore) ] // unsupported operation: returning ready events from epoll_wait is not yet implemented
17641867 async fn batch_delete ( ) {
0 commit comments