Skip to content

Commit a52a32f

Browse files
committed
migrate statpopgen to use FixedSizeList instead of List
Also fixes a subtle bug in the duckdb exporter, where duckdb `ARRAY` is quite different from their `LIST` type. Signed-off-by: Connor Tsui <[email protected]>
1 parent 2c8ff68 commit a52a32f

3 files changed

Lines changed: 199 additions & 77 deletions

File tree

‎bench-vortex/src/statpopgen/builder.rs‎

Lines changed: 156 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
use std::sync::Arc;
55

66
use arrow_array::builder::{
7-
ArrayBuilder, BooleanBuilder, Float32Builder, Int32Builder, ListBuilder, StringBuilder,
8-
UInt64Builder,
7+
ArrayBuilder, BooleanBuilder, FixedSizeListBuilder, Float32Builder, Int32Builder, ListBuilder,
8+
StringBuilder, UInt64Builder,
99
};
1010
use arrow_array::{ArrayRef, RecordBatch};
1111
use arrow_schema::{ArrowError, SchemaRef};
@@ -38,25 +38,29 @@ pub struct GnomADBuilder<'a> {
3838
/// - `chrX`
3939
/// - `chrMT`
4040
pub CHROM_builder: StringBuilder,
41+
4142
/// The 1-indexed position on the contig at which this variant was found.
4243
///
4344
/// Typically the first 100,000 and last 100,000 positions are not recorded in a VCF because
4445
/// they are difficult to sequence. In particular, they contain long repetitive runs that are
4546
/// difficult to capture with "short read sequencing".
4647
pub POS_builder: UInt64Builder,
48+
4749
/// A unique identifier for this variant.
4850
///
4951
/// While contig, position, reference allele, and alternate alele uniquely identify a variant,
5052
/// the "reference" contig may change. A variant ID is, by definition, invariant to the
5153
/// reference "build".
5254
pub ID_builder: StringBuilder,
55+
5356
/// The reference allele of this variant.
5457
///
5558
/// # Examples
5659
///
5760
/// - `A`
5861
/// - `ATG`
5962
pub REF_builder: StringBuilder,
63+
6064
/// The list of alternate alleles of this variant.
6165
///
6266
/// An empty list of alternate alleles is unusual in an analysis-ready, jointly-called VCF.
@@ -71,8 +75,10 @@ pub struct GnomADBuilder<'a> {
7175
/// - `["A"]`
7276
/// - `["AA", "G", "ATG"]`
7377
pub ALT_builder: ListBuilder<StringBuilder>,
78+
7479
/// The quality score of this variant.
7580
pub QUAL_builder: Float32Builder,
81+
7682
/// A list of "filter" values of this variant.
7783
///
7884
/// The header of the VCF lists possible FILTER values other than the string "PASS".
@@ -117,49 +123,60 @@ pub struct GnomADBuilder<'a> {
117123
/// `1|1`. We do not support these in the GT field.
118124
///
119125
/// Every list is the same length; however, individual positions may be missing.
120-
pub GT_builder: ListBuilder<UInt64Builder>,
126+
pub GT_builder: FixedSizeListBuilder<UInt64Builder>,
127+
121128
/// The genotype quality.
122129
///
123130
/// A small non-negative integer indicating our confidence in this genotype. It is usually the
124-
/// difference between the lowest and second lowest PL. Larger values indicate higher confidence.
125-
pub GQ_builder: ListBuilder<Int32Builder>,
131+
/// difference between the lowest and second lowest PL. Larger values indicate higher
132+
/// confidence.
133+
pub GQ_builder: FixedSizeListBuilder<Int32Builder>,
134+
126135
/// The genotype depth.
127136
///
128137
/// Varies by sequencing technology and service provider, but typically the number of reads
129138
/// which influenced this genotype call.
130-
pub DP_builder: ListBuilder<Int32Builder>,
139+
pub DP_builder: FixedSizeListBuilder<Int32Builder>,
140+
131141
/// The allele depth.
132142
///
133143
/// For each alternate allele, how many reads contained this allele.
134144
///
135145
/// The outer list is always equal to the number of samples and does not vary. The inner list is
136146
/// equal to the number of alternate alleles at this variant.
137-
pub AD_builder: ListBuilder<ListBuilder<Int32Builder>>,
147+
pub AD_builder: FixedSizeListBuilder<ListBuilder<Int32Builder>>,
148+
138149
/// The minimum depth.
139150
///
140151
/// From the VCF header: "Minimum DP observed within the GVCF block". I believe this is mostly
141152
/// relevant for homozygous reference calls.
142-
pub MIN_DP_builder: ListBuilder<Int32Builder>,
153+
pub MIN_DP_builder: FixedSizeListBuilder<Int32Builder>,
154+
143155
/// A phased genotype.
144156
///
145157
/// We encode them as `0|0`: 0, `0|1`: 1, `1|0`: 2, and `1|1`: 3. Almost all these values are
146158
/// null.
147-
pub PGT_builder: ListBuilder<Int32Builder>,
159+
pub PGT_builder: FixedSizeListBuilder<Int32Builder>,
160+
148161
/// The phase ID.
149162
///
150163
/// An identifier used to reconstruct the two distinguished copies of the contig.
151-
pub PID_builder: ListBuilder<StringBuilder>,
164+
pub PID_builder: FixedSizeListBuilder<StringBuilder>,
165+
152166
/// The phred-scaled (log) likelihood of each possible genotype call.
153167
///
154168
/// Smaller is better. Has length equal to the number of possible unphased genotypes. For
155169
/// biallelic variants, that number is three: 0/0, 0/1, 1/1.
156-
pub PL_builder: ListBuilder<ListBuilder<Int32Builder>>,
157-
/// Per-sample component statistics which comprise the Fisher's Exact Test to detect strand bias.
170+
pub PL_builder: FixedSizeListBuilder<ListBuilder<Int32Builder>>,
171+
172+
/// Per-sample component statistics which comprise the Fisher's Exact Test to detect strand
173+
/// bias.
158174
///
159175
/// A single copy of a chromosome is a double-helix. Opposing positions on that double-helix are
160176
/// complementary. Sometimes when you're sequencing one of the two helices is preferred, for
161-
/// complicated chemical reasons. This is a test of how biased we were towards one or the other strand.
162-
pub SB_builder: ListBuilder<ListBuilder<Int32Builder>>,
177+
/// complicated chemical reasons. This is a test of how biased we were towards one or the other
178+
/// strand.
179+
pub SB_builder: FixedSizeListBuilder<FixedSizeListBuilder<Int32Builder>>,
163180
}
164181

165182
pub enum InfoArrayBuilder {
@@ -232,7 +249,7 @@ impl InfoArrayBuilder {
232249

233250
impl<'a> GnomADBuilder<'a> {
234251
#[allow(non_snake_case)]
235-
pub fn new(header: &'a Header, schema: SchemaRef) -> Self {
252+
pub fn new(header: &'a Header, schema: SchemaRef, num_samples: i32) -> Self {
236253
let info_builder: HashMap<&'a str, InfoArrayBuilder> = header
237254
.infos()
238255
.iter()
@@ -252,15 +269,24 @@ impl<'a> GnomADBuilder<'a> {
252269
ALT_builder: Default::default(),
253270
QUAL_builder: Default::default(),
254271
FILTER_builder: Default::default(),
255-
GT_builder: Default::default(),
256-
GQ_builder: Default::default(),
257-
DP_builder: Default::default(),
258-
AD_builder: Default::default(),
259-
MIN_DP_builder: Default::default(),
260-
PGT_builder: Default::default(),
261-
PID_builder: Default::default(),
262-
PL_builder: Default::default(),
263-
SB_builder: Default::default(),
272+
GT_builder: FixedSizeListBuilder::new(UInt64Builder::new(), num_samples),
273+
GQ_builder: FixedSizeListBuilder::new(Int32Builder::new(), num_samples),
274+
DP_builder: FixedSizeListBuilder::new(Int32Builder::new(), num_samples),
275+
AD_builder: FixedSizeListBuilder::new(
276+
ListBuilder::new(Int32Builder::new()),
277+
num_samples,
278+
),
279+
MIN_DP_builder: FixedSizeListBuilder::new(Int32Builder::new(), num_samples),
280+
PGT_builder: FixedSizeListBuilder::new(Int32Builder::new(), num_samples),
281+
PID_builder: FixedSizeListBuilder::new(StringBuilder::new(), num_samples),
282+
PL_builder: FixedSizeListBuilder::new(
283+
ListBuilder::new(Int32Builder::new()),
284+
num_samples,
285+
),
286+
SB_builder: FixedSizeListBuilder::new(
287+
FixedSizeListBuilder::new(Int32Builder::new(), 4),
288+
num_samples,
289+
),
264290
}
265291
}
266292

@@ -295,50 +321,118 @@ impl<'a> GnomADBuilder<'a> {
295321

296322
self.consume_info(header, record.info())?;
297323

324+
// Process each sample and ensure we append exactly `num_samples` values.
298325
for sample in record.samples().iter() {
326+
// Collect all fields for this sample.
327+
let mut sample_fields = HashMap::new();
299328
for result in sample.iter(header) {
300329
let (field, value) = result?;
301-
match field {
302-
"GT" => self
303-
.GT_builder
304-
.values()
305-
.append_option(parse_genotype(value)?),
306-
"GQ" => self
307-
.GQ_builder
308-
.values()
309-
.append_option(parse_int32_format(value)?),
310-
"DP" => self
311-
.DP_builder
312-
.values()
313-
.append_option(parse_int32_format(value)?),
314-
"AD" => self
315-
.AD_builder
316-
.values()
317-
.append_option(parse_list_int32_format(value)?),
318-
"MIN_DP" => self
319-
.MIN_DP_builder
320-
.values()
321-
.append_option(parse_int32_format(value)?),
322-
"PGT" => self
323-
.PGT_builder
324-
.values()
325-
.append_option(parse_pgt_format(value)?),
326-
"PID" => self
327-
.PID_builder
328-
.values()
329-
.append_option(parse_string_format(value)?),
330-
"PL" => self
331-
.PL_builder
332-
.values()
333-
.append_option(parse_list_int32_format(value)?),
334-
"SB" => self
335-
.SB_builder
336-
.values()
337-
.append_option(parse_list_int32_format(value)?),
338-
_ => vortex_bail!("unknown field {field}"),
330+
sample_fields.insert(field, value);
331+
}
332+
333+
// GT field.
334+
if let Some(value) = sample_fields.remove("GT") {
335+
self.GT_builder
336+
.values()
337+
.append_option(parse_genotype(value)?);
338+
} else {
339+
self.GT_builder.values().append_null();
340+
}
341+
342+
// GQ field.
343+
if let Some(value) = sample_fields.remove("GQ") {
344+
self.GQ_builder
345+
.values()
346+
.append_option(parse_int32_format(value)?);
347+
} else {
348+
self.GQ_builder.values().append_null();
349+
}
350+
351+
// DP field.
352+
if let Some(value) = sample_fields.remove("DP") {
353+
self.DP_builder
354+
.values()
355+
.append_option(parse_int32_format(value)?);
356+
} else {
357+
self.DP_builder.values().append_null();
358+
}
359+
360+
// AD field.
361+
if let Some(value) = sample_fields.remove("AD") {
362+
self.AD_builder
363+
.values()
364+
.append_option(parse_list_int32_format(value)?);
365+
} else {
366+
self.AD_builder.values().append_null();
367+
}
368+
369+
// MIN_DP field.
370+
if let Some(value) = sample_fields.remove("MIN_DP") {
371+
self.MIN_DP_builder
372+
.values()
373+
.append_option(parse_int32_format(value)?);
374+
} else {
375+
self.MIN_DP_builder.values().append_null();
376+
}
377+
378+
// PGT field.
379+
if let Some(value) = sample_fields.remove("PGT") {
380+
self.PGT_builder
381+
.values()
382+
.append_option(parse_pgt_format(value)?);
383+
} else {
384+
self.PGT_builder.values().append_null();
385+
}
386+
387+
// PID field.
388+
if let Some(value) = sample_fields.remove("PID") {
389+
self.PID_builder
390+
.values()
391+
.append_option(parse_string_format(value)?);
392+
} else {
393+
self.PID_builder.values().append_null();
394+
}
395+
396+
// PL field.
397+
if let Some(value) = sample_fields.remove("PL") {
398+
self.PL_builder
399+
.values()
400+
.append_option(parse_list_int32_format(value)?);
401+
} else {
402+
self.PL_builder.values().append_null();
403+
}
404+
405+
// SB field.
406+
if let Some(value) = sample_fields.remove("SB") {
407+
if let Some(sb_values) = parse_list_int32_format(value)? {
408+
// SB must have exactly 4 values when present.
409+
if sb_values.len() != 4 {
410+
vortex_bail!(
411+
"SB field must have exactly 4 values, got {}",
412+
sb_values.len()
413+
);
414+
}
415+
for v in sb_values {
416+
self.SB_builder.values().values().append_option(v);
417+
}
418+
self.SB_builder.values().append(true);
419+
} else {
420+
// SB field is null/missing for this sample.
421+
for _ in 0..4 {
422+
self.SB_builder.values().values().append_null();
423+
}
424+
self.SB_builder.values().append(false);
339425
}
426+
} else {
427+
// SB field is not present for this sample.
428+
for _ in 0..4 {
429+
self.SB_builder.values().values().append_null();
430+
}
431+
self.SB_builder.values().append(false);
340432
}
341433
}
434+
435+
// After processing all samples, append to the `FixedSizeListBuilder`'s outer null map.
342436
self.GT_builder.append(true);
343437
self.GQ_builder.append(true);
344438
self.DP_builder.append(true);

‎bench-vortex/src/statpopgen/download_vcf.rs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,13 @@ impl StatPopGenBenchmark {
6767

6868
// Read and print the first 100,000 records
6969
let header = vcf_reader.read_header().await?;
70+
71+
let num_samples = header
72+
.sample_names()
73+
.len()
74+
.try_into()
75+
.context("Too many samples for i32")?;
76+
7077
let progress = ProgressBar::new(self.n_rows);
7178
progress.set_style(
7279
ProgressStyle::with_template(
@@ -75,15 +82,15 @@ impl StatPopGenBenchmark {
7582
.expect("style is ok"),
7683
);
7784
let mut record = Record::default();
78-
let schema = schema_from_vcf_header(&header);
79-
let mut builder = GnomADBuilder::new(&header, schema.clone());
85+
let schema = schema_from_vcf_header(&header, num_samples);
86+
let mut builder = GnomADBuilder::new(&header, schema.clone(), num_samples);
8087
let file = File::create(parquet_output_path).await?;
8188
let mut writer = AsyncArrowWriter::try_new(file, schema.clone(), None)
8289
.context("Failed to create parquet writer")?;
8390
for i in progress.wrap_iter(0..self.n_rows) {
8491
if i % ROW_GROUP_SIZE_IN_VARIANTS == 0 {
8592
let rb = builder.finish()?;
86-
builder = GnomADBuilder::new(&header, schema.clone());
93+
builder = GnomADBuilder::new(&header, schema.clone(), num_samples);
8794
writer
8895
.write(&rb)
8996
.await
@@ -161,10 +168,10 @@ impl StatPopGenBenchmark {
161168
let dtype = DType::from_arrow(parquet.schema().as_ref());
162169
let mut vortex_stream = parquet
163170
.build()
164-
.map_err(|err| VortexError::generic(Box::new(err)))?
171+
.context("Failed to build parquet stream")?
165172
.map(|record_batch| {
166173
record_batch
167-
.map_err(|err| VortexError::generic(Box::new(err)))
174+
.map_err(|e| VortexError::generic(Box::new(e)))
168175
.map(|rb| ArrayRef::from_arrow(rb, false))
169176
})
170177
.boxed();

0 commit comments

Comments
 (0)