|
| 1 | +// Copyright 2025 Apache Doris Community |
| 2 | +// Licensed under the Apache License, Version 2.0 |
| 3 | + |
| 4 | +//! Unit Test for PBlock Parser |
| 5 | +//! |
| 6 | +//! CLAUDE.MD Principle #2: Use BE source as specification |
| 7 | +//! This test creates synthetic PBlock data following the exact format |
| 8 | +//! documented in PBLOCK_FORMAT_SPECIFICATION.md and verifies parsing. |
| 9 | +//! |
| 10 | +//! Format reverse-engineered from: |
| 11 | +//! - /be/src/vec/core/block.cpp (Block::serialize/deserialize) |
| 12 | +//! - /be/src/vec/data_types/*.cpp (type-specific serialization) |
| 13 | +//! |
| 14 | +//! Run: cargo run --example test_pblock_parser_unit |
| 15 | +
|
| 16 | +use fe_backend_client::pblock_parser_v2::pblock_to_rows; |
| 17 | +use fe_backend_client::generated::doris::{PBlock, PColumnMeta, p_generic_type::TypeId}; |
| 18 | + |
| 19 | +fn main() -> Result<(), Box<dyn std::error::Error>> { |
| 20 | + println!("═══════════════════════════════════════════════════════"); |
| 21 | + println!(" PBlock Parser Unit Test"); |
| 22 | + println!(" Format: PBLOCK_FORMAT_SPECIFICATION.md"); |
| 23 | + println!("═══════════════════════════════════════════════════════\n"); |
| 24 | + |
| 25 | + // Test 1: Simple BIGINT column |
| 26 | + println!("Test 1: Single BIGINT column (3 rows)"); |
| 27 | + println!("─────────────────────────────────────────"); |
| 28 | + test_bigint_column()?; |
| 29 | + |
| 30 | + // Test 2: Two columns (BIGINT + BIGINT) |
| 31 | + println!("\nTest 2: Two BIGINT columns (2 rows)"); |
| 32 | + println!("─────────────────────────────────────────"); |
| 33 | + test_two_bigint_columns()?; |
| 34 | + |
| 35 | + // Test 3: BIGINT + STRING columns |
| 36 | + println!("\nTest 3: BIGINT + STRING columns (2 rows)"); |
| 37 | + println!("─────────────────────────────────────────"); |
| 38 | + test_bigint_string_columns()?; |
| 39 | + |
| 40 | + println!("\n═══════════════════════════════════════════════════════"); |
| 41 | + println!("✅ All PBlock Parser Tests PASSED!"); |
| 42 | + println!(" - Number type decoding: ✅"); |
| 43 | + println!(" - String type decoding: ✅"); |
| 44 | + println!(" - Column-to-row transposition: ✅"); |
| 45 | + println!(" - Multi-column support: ✅"); |
| 46 | + println!("═══════════════════════════════════════════════════════"); |
| 47 | + |
| 48 | + Ok(()) |
| 49 | +} |
| 50 | + |
| 51 | +/// Test Case 1: Single BIGINT column with 3 values |
| 52 | +/// |
| 53 | +/// Format (from data_type_number_base.cpp): |
| 54 | +/// [Header: 17 bytes] [Data: 24 bytes] |
| 55 | +/// |
| 56 | +/// Bytes: |
| 57 | +/// 0 | 0x00 | is_const = false |
| 58 | +/// 1-8 | 0x03 (little-endian) | row_num = 3 |
| 59 | +/// 9-16 | 0x03 (little-endian) | real_need_copy_num = 3 |
| 60 | +/// 17-24 | 0x01 (i64 LE) | value[0] = 1 |
| 61 | +/// 25-32 | 0x02 (i64 LE) | value[1] = 2 |
| 62 | +/// 33-40 | 0x03 (i64 LE) | value[2] = 3 |
| 63 | +fn test_bigint_column() -> Result<(), Box<dyn std::error::Error>> { |
| 64 | + // Construct column_values bytes following modern format (be_exec_version >= 4) |
| 65 | + let mut column_values = Vec::new(); |
| 66 | + |
| 67 | + // Column header |
| 68 | + column_values.push(0x00); // is_const = false |
| 69 | + column_values.extend_from_slice(&3u64.to_le_bytes()); // row_num = 3 |
| 70 | + column_values.extend_from_slice(&3u64.to_le_bytes()); // real_need_copy_num = 3 |
| 71 | + |
| 72 | + // Data: 3 BIGINT values |
| 73 | + column_values.extend_from_slice(&1i64.to_le_bytes()); // value 1 |
| 74 | + column_values.extend_from_slice(&2i64.to_le_bytes()); // value 2 |
| 75 | + column_values.extend_from_slice(&3i64.to_le_bytes()); // value 3 |
| 76 | + |
| 77 | + // Create PBlock |
| 78 | + let pblock = PBlock { |
| 79 | + be_exec_version: Some(5), // Modern format |
| 80 | + compressed: Some(false), |
| 81 | + compression_type: None, |
| 82 | + uncompressed_size: None, |
| 83 | + column_metas: vec![ |
| 84 | + PColumnMeta { |
| 85 | + name: Some("col1".to_string()), |
| 86 | + r#type: Some(TypeId::Int64 as i32), |
| 87 | + is_nullable: Some(false), |
| 88 | + ..Default::default() |
| 89 | + } |
| 90 | + ], |
| 91 | + column_values: Some(column_values), |
| 92 | + }; |
| 93 | + |
| 94 | + // Parse |
| 95 | + let rows = pblock_to_rows(&pblock)?; |
| 96 | + |
| 97 | + // Verify |
| 98 | + assert_eq!(rows.len(), 3, "Expected 3 rows"); |
| 99 | + println!("✓ Parsed {} rows", rows.len()); |
| 100 | + |
| 101 | + for (i, row) in rows.iter().enumerate() { |
| 102 | + println!(" Row {}: {:?}", i + 1, row); |
| 103 | + assert_eq!(row.values.len(), 1, "Expected 1 column"); |
| 104 | + } |
| 105 | + |
| 106 | + // Check values |
| 107 | + use fe_qe::result::Value; |
| 108 | + match &rows[0].values[0] { |
| 109 | + Value::BigInt(v) => assert_eq!(*v, 1, "Row 1 should be 1"), |
| 110 | + _ => panic!("Expected BigInt"), |
| 111 | + } |
| 112 | + match &rows[1].values[0] { |
| 113 | + Value::BigInt(v) => assert_eq!(*v, 2, "Row 2 should be 2"), |
| 114 | + _ => panic!("Expected BigInt"), |
| 115 | + } |
| 116 | + match &rows[2].values[0] { |
| 117 | + Value::BigInt(v) => assert_eq!(*v, 3, "Row 3 should be 3"), |
| 118 | + _ => panic!("Expected BigInt"), |
| 119 | + } |
| 120 | + |
| 121 | + println!("✓ Values correct: [1, 2, 3]"); |
| 122 | + |
| 123 | + Ok(()) |
| 124 | +} |
| 125 | + |
| 126 | +/// Test Case 2: Two BIGINT columns |
| 127 | +fn test_two_bigint_columns() -> Result<(), Box<dyn std::error::Error>> { |
| 128 | + let mut column_values = Vec::new(); |
| 129 | + |
| 130 | + // Column 1: [10, 20] |
| 131 | + column_values.push(0x00); // is_const = false |
| 132 | + column_values.extend_from_slice(&2u64.to_le_bytes()); // row_num = 2 |
| 133 | + column_values.extend_from_slice(&2u64.to_le_bytes()); // real_need_copy_num = 2 |
| 134 | + column_values.extend_from_slice(&10i64.to_le_bytes()); |
| 135 | + column_values.extend_from_slice(&20i64.to_le_bytes()); |
| 136 | + |
| 137 | + // Column 2: [100, 200] |
| 138 | + column_values.push(0x00); // is_const = false |
| 139 | + column_values.extend_from_slice(&2u64.to_le_bytes()); // row_num = 2 |
| 140 | + column_values.extend_from_slice(&2u64.to_le_bytes()); // real_need_copy_num = 2 |
| 141 | + column_values.extend_from_slice(&100i64.to_le_bytes()); |
| 142 | + column_values.extend_from_slice(&200i64.to_le_bytes()); |
| 143 | + |
| 144 | + let pblock = PBlock { |
| 145 | + be_exec_version: Some(5), |
| 146 | + compressed: Some(false), |
| 147 | + compression_type: None, |
| 148 | + uncompressed_size: None, |
| 149 | + column_metas: vec![ |
| 150 | + PColumnMeta { |
| 151 | + name: Some("col1".to_string()), |
| 152 | + r#type: Some(TypeId::Int64 as i32), |
| 153 | + is_nullable: Some(false), |
| 154 | + ..Default::default() |
| 155 | + }, |
| 156 | + PColumnMeta { |
| 157 | + name: Some("col2".to_string()), |
| 158 | + r#type: Some(TypeId::Int64 as i32), |
| 159 | + is_nullable: Some(false), |
| 160 | + ..Default::default() |
| 161 | + } |
| 162 | + ], |
| 163 | + column_values: Some(column_values), |
| 164 | + }; |
| 165 | + |
| 166 | + let rows = pblock_to_rows(&pblock)?; |
| 167 | + |
| 168 | + assert_eq!(rows.len(), 2); |
| 169 | + println!("✓ Parsed {} rows", rows.len()); |
| 170 | + |
| 171 | + for (i, row) in rows.iter().enumerate() { |
| 172 | + println!(" Row {}: {:?}", i + 1, row); |
| 173 | + assert_eq!(row.values.len(), 2, "Expected 2 columns"); |
| 174 | + } |
| 175 | + |
| 176 | + println!("✓ Transposition correct: columnar → rows"); |
| 177 | + |
| 178 | + Ok(()) |
| 179 | +} |
| 180 | + |
| 181 | +/// Test Case 3: BIGINT + STRING columns |
| 182 | +/// |
| 183 | +/// String format (from data_type_string.cpp): |
| 184 | +/// [Header: 17 bytes] [Offsets: N*8 bytes] [Total len: 8 bytes] [String data] |
| 185 | +fn test_bigint_string_columns() -> Result<(), Box<dyn std::error::Error>> { |
| 186 | + let mut column_values = Vec::new(); |
| 187 | + |
| 188 | + // Column 1: BIGINT [42, 99] |
| 189 | + column_values.push(0x00); |
| 190 | + column_values.extend_from_slice(&2u64.to_le_bytes()); // row_num = 2 |
| 191 | + column_values.extend_from_slice(&2u64.to_le_bytes()); // real_need_copy_num = 2 |
| 192 | + column_values.extend_from_slice(&42i64.to_le_bytes()); |
| 193 | + column_values.extend_from_slice(&99i64.to_le_bytes()); |
| 194 | + |
| 195 | + // Column 2: STRING ["abc", "de"] |
| 196 | + // Offsets are cumulative: "abc" ends at 3, "de" ends at 5 |
| 197 | + column_values.push(0x00); |
| 198 | + column_values.extend_from_slice(&2u64.to_le_bytes()); // row_num = 2 |
| 199 | + column_values.extend_from_slice(&2u64.to_le_bytes()); // real_need_copy_num = 2 |
| 200 | + |
| 201 | + // Offsets array |
| 202 | + column_values.extend_from_slice(&3u64.to_le_bytes()); // offset[0] = 3 (end of "abc") |
| 203 | + column_values.extend_from_slice(&5u64.to_le_bytes()); // offset[1] = 5 (end of "de") |
| 204 | + |
| 205 | + // Total value length |
| 206 | + column_values.extend_from_slice(&5u64.to_le_bytes()); // total 5 bytes |
| 207 | + |
| 208 | + // String data |
| 209 | + column_values.extend_from_slice(b"abc"); // 3 bytes |
| 210 | + column_values.extend_from_slice(b"de"); // 2 bytes |
| 211 | + |
| 212 | + let pblock = PBlock { |
| 213 | + be_exec_version: Some(5), |
| 214 | + compressed: Some(false), |
| 215 | + compression_type: None, |
| 216 | + uncompressed_size: None, |
| 217 | + column_metas: vec![ |
| 218 | + PColumnMeta { |
| 219 | + name: Some("id".to_string()), |
| 220 | + r#type: Some(TypeId::Int64 as i32), |
| 221 | + is_nullable: Some(false), |
| 222 | + ..Default::default() |
| 223 | + }, |
| 224 | + PColumnMeta { |
| 225 | + name: Some("name".to_string()), |
| 226 | + r#type: Some(TypeId::String as i32), |
| 227 | + is_nullable: Some(false), |
| 228 | + ..Default::default() |
| 229 | + } |
| 230 | + ], |
| 231 | + column_values: Some(column_values), |
| 232 | + }; |
| 233 | + |
| 234 | + let rows = pblock_to_rows(&pblock)?; |
| 235 | + |
| 236 | + assert_eq!(rows.len(), 2); |
| 237 | + println!("✓ Parsed {} rows", rows.len()); |
| 238 | + |
| 239 | + for (i, row) in rows.iter().enumerate() { |
| 240 | + println!(" Row {}: {:?}", i + 1, row); |
| 241 | + assert_eq!(row.values.len(), 2, "Expected 2 columns"); |
| 242 | + } |
| 243 | + |
| 244 | + // Verify values |
| 245 | + use fe_qe::result::Value; |
| 246 | + match (&rows[0].values[0], &rows[0].values[1]) { |
| 247 | + (Value::BigInt(id), Value::String(name)) => { |
| 248 | + assert_eq!(*id, 42); |
| 249 | + assert_eq!(name, "abc"); |
| 250 | + } |
| 251 | + _ => panic!("Row 1 type mismatch"), |
| 252 | + } |
| 253 | + |
| 254 | + match (&rows[1].values[0], &rows[1].values[1]) { |
| 255 | + (Value::BigInt(id), Value::String(name)) => { |
| 256 | + assert_eq!(*id, 99); |
| 257 | + assert_eq!(name, "de"); |
| 258 | + } |
| 259 | + _ => panic!("Row 2 type mismatch"), |
| 260 | + } |
| 261 | + |
| 262 | + println!("✓ Values correct: [(42, 'abc'), (99, 'de')]"); |
| 263 | + |
| 264 | + Ok(()) |
| 265 | +} |
0 commit comments