44use std:: sync:: Arc ;
55
66use arrow_array:: builder:: {
7- ArrayBuilder , BooleanBuilder , Float32Builder , Int32Builder , ListBuilder , StringBuilder ,
8- UInt64Builder ,
7+ ArrayBuilder , BooleanBuilder , FixedSizeListBuilder , Float32Builder , Int32Builder , ListBuilder ,
8+ StringBuilder , UInt64Builder ,
99} ;
1010use arrow_array:: { ArrayRef , RecordBatch } ;
1111use arrow_schema:: { ArrowError , SchemaRef } ;
@@ -38,25 +38,29 @@ pub struct GnomADBuilder<'a> {
3838 /// - `chrX`
3939 /// - `chrMT`
4040 pub CHROM_builder : StringBuilder ,
41+
4142 /// The 1-indexed position on the contig at which this variant was found.
4243 ///
4344 /// Typically the first 100,000 and last 100,000 positions are not recorded in a VCF because
4445 /// they are difficult to sequence. In particular, they contain long repetitive runs that are
4546 /// difficult to capture with "short read sequencing".
4647 pub POS_builder : UInt64Builder ,
48+
4749 /// A unique identifier for this variant.
4850 ///
4951 /// While contig, position, reference allele, and alternate alele uniquely identify a variant,
5052 /// the "reference" contig may change. A variant ID is, by definition, invariant to the
5153 /// reference "build".
5254 pub ID_builder : StringBuilder ,
55+
5356 /// The reference allele of this variant.
5457 ///
5558 /// # Examples
5659 ///
5760 /// - `A`
5861 /// - `ATG`
5962 pub REF_builder : StringBuilder ,
63+
6064 /// The list of alternate alleles of this variant.
6165 ///
6266 /// An empty list of alternate alleles is unusual in an analysis-ready, jointly-called VCF.
@@ -71,8 +75,10 @@ pub struct GnomADBuilder<'a> {
7175 /// - `["A"]`
7276 /// - `["AA", "G", "ATG"]`
7377 pub ALT_builder : ListBuilder < StringBuilder > ,
78+
7479 /// The quality score of this variant.
7580 pub QUAL_builder : Float32Builder ,
81+
7682 /// A list of "filter" values of this variant.
7783 ///
7884 /// The header of the VCF lists possible FILTER values other than the string "PASS".
@@ -117,49 +123,60 @@ pub struct GnomADBuilder<'a> {
117123 /// `1|1`. We do not support these in the GT field.
118124 ///
119125 /// Every list is the same length; however, individual positions may be missing.
120- pub GT_builder : ListBuilder < UInt64Builder > ,
126+ pub GT_builder : FixedSizeListBuilder < UInt64Builder > ,
127+
121128 /// The genotype quality.
122129 ///
123130 /// A small non-negative integer indicating our confidence in this genotype. It is usually the
124- /// difference between the lowest and second lowest PL. Larger values indicate higher confidence.
125- pub GQ_builder : ListBuilder < Int32Builder > ,
131+ /// difference between the lowest and second lowest PL. Larger values indicate higher
132+ /// confidence.
133+ pub GQ_builder : FixedSizeListBuilder < Int32Builder > ,
134+
126135 /// The genotype depth.
127136 ///
128137 /// Varies by sequencing technology and service provider, but typically the number of reads
129138 /// which influenced this genotype call.
130- pub DP_builder : ListBuilder < Int32Builder > ,
139+ pub DP_builder : FixedSizeListBuilder < Int32Builder > ,
140+
131141 /// The allele depth.
132142 ///
133143 /// For each alternate allele, how many reads contained this allele.
134144 ///
135145 /// The outer list is always equal to the number of samples and does not vary. The inner list is
136146 /// equal to the number of alternate alleles at this variant.
137- pub AD_builder : ListBuilder < ListBuilder < Int32Builder > > ,
147+ pub AD_builder : FixedSizeListBuilder < ListBuilder < Int32Builder > > ,
148+
138149 /// The minimum depth.
139150 ///
140151 /// From the VCF header: "Minimum DP observed within the GVCF block". I believe this is mostly
141152 /// relevant for homozygous reference calls.
142- pub MIN_DP_builder : ListBuilder < Int32Builder > ,
153+ pub MIN_DP_builder : FixedSizeListBuilder < Int32Builder > ,
154+
143155 /// A phased genotype.
144156 ///
145157 /// We encode them as `0|0`: 0, `0|1`: 1, `1|0`: 2, and `1|1`: 3. Almost all these values are
146158 /// null.
147- pub PGT_builder : ListBuilder < Int32Builder > ,
159+ pub PGT_builder : FixedSizeListBuilder < Int32Builder > ,
160+
148161 /// The phase ID.
149162 ///
150163 /// An identifier used to reconstruct the two distinguished copies of the contig.
151- pub PID_builder : ListBuilder < StringBuilder > ,
164+ pub PID_builder : FixedSizeListBuilder < StringBuilder > ,
165+
152166 /// The phred-scaled (log) likelihood of each possible genotype call.
153167 ///
154168 /// Smaller is better. Has length equal to the number of possible unphased genotypes. For
155169 /// biallelic variants, that number is three: 0/0, 0/1, 1/1.
156- pub PL_builder : ListBuilder < ListBuilder < Int32Builder > > ,
157- /// Per-sample component statistics which comprise the Fisher's Exact Test to detect strand bias.
170+ pub PL_builder : FixedSizeListBuilder < ListBuilder < Int32Builder > > ,
171+
172+ /// Per-sample component statistics which comprise the Fisher's Exact Test to detect strand
173+ /// bias.
158174 ///
159175 /// A single copy of a chromosome is a double-helix. Opposing positions on that double-helix are
160176 /// complementary. Sometimes when you're sequencing one of the two helices is preferred, for
161- /// complicated chemical reasons. This is a test of how biased we were towards one or the other strand.
162- pub SB_builder : ListBuilder < ListBuilder < Int32Builder > > ,
177+ /// complicated chemical reasons. This is a test of how biased we were towards one or the other
178+ /// strand.
179+ pub SB_builder : FixedSizeListBuilder < FixedSizeListBuilder < Int32Builder > > ,
163180}
164181
165182pub enum InfoArrayBuilder {
@@ -232,7 +249,7 @@ impl InfoArrayBuilder {
232249
233250impl < ' a > GnomADBuilder < ' a > {
234251 #[ allow( non_snake_case) ]
235- pub fn new ( header : & ' a Header , schema : SchemaRef ) -> Self {
252+ pub fn new ( header : & ' a Header , schema : SchemaRef , num_samples : i32 ) -> Self {
236253 let info_builder: HashMap < & ' a str , InfoArrayBuilder > = header
237254 . infos ( )
238255 . iter ( )
@@ -252,15 +269,24 @@ impl<'a> GnomADBuilder<'a> {
252269 ALT_builder : Default :: default ( ) ,
253270 QUAL_builder : Default :: default ( ) ,
254271 FILTER_builder : Default :: default ( ) ,
255- GT_builder : Default :: default ( ) ,
256- GQ_builder : Default :: default ( ) ,
257- DP_builder : Default :: default ( ) ,
258- AD_builder : Default :: default ( ) ,
259- MIN_DP_builder : Default :: default ( ) ,
260- PGT_builder : Default :: default ( ) ,
261- PID_builder : Default :: default ( ) ,
262- PL_builder : Default :: default ( ) ,
263- SB_builder : Default :: default ( ) ,
272+ GT_builder : FixedSizeListBuilder :: new ( UInt64Builder :: new ( ) , num_samples) ,
273+ GQ_builder : FixedSizeListBuilder :: new ( Int32Builder :: new ( ) , num_samples) ,
274+ DP_builder : FixedSizeListBuilder :: new ( Int32Builder :: new ( ) , num_samples) ,
275+ AD_builder : FixedSizeListBuilder :: new (
276+ ListBuilder :: new ( Int32Builder :: new ( ) ) ,
277+ num_samples,
278+ ) ,
279+ MIN_DP_builder : FixedSizeListBuilder :: new ( Int32Builder :: new ( ) , num_samples) ,
280+ PGT_builder : FixedSizeListBuilder :: new ( Int32Builder :: new ( ) , num_samples) ,
281+ PID_builder : FixedSizeListBuilder :: new ( StringBuilder :: new ( ) , num_samples) ,
282+ PL_builder : FixedSizeListBuilder :: new (
283+ ListBuilder :: new ( Int32Builder :: new ( ) ) ,
284+ num_samples,
285+ ) ,
286+ SB_builder : FixedSizeListBuilder :: new (
287+ FixedSizeListBuilder :: new ( Int32Builder :: new ( ) , 4 ) ,
288+ num_samples,
289+ ) ,
264290 }
265291 }
266292
@@ -295,50 +321,118 @@ impl<'a> GnomADBuilder<'a> {
295321
296322 self . consume_info ( header, record. info ( ) ) ?;
297323
324+ // Process each sample and ensure we append exactly `num_samples` values.
298325 for sample in record. samples ( ) . iter ( ) {
326+ // Collect all fields for this sample.
327+ let mut sample_fields = HashMap :: new ( ) ;
299328 for result in sample. iter ( header) {
300329 let ( field, value) = result?;
301- match field {
302- "GT" => self
303- . GT_builder
304- . values ( )
305- . append_option ( parse_genotype ( value) ?) ,
306- "GQ" => self
307- . GQ_builder
308- . values ( )
309- . append_option ( parse_int32_format ( value) ?) ,
310- "DP" => self
311- . DP_builder
312- . values ( )
313- . append_option ( parse_int32_format ( value) ?) ,
314- "AD" => self
315- . AD_builder
316- . values ( )
317- . append_option ( parse_list_int32_format ( value) ?) ,
318- "MIN_DP" => self
319- . MIN_DP_builder
320- . values ( )
321- . append_option ( parse_int32_format ( value) ?) ,
322- "PGT" => self
323- . PGT_builder
324- . values ( )
325- . append_option ( parse_pgt_format ( value) ?) ,
326- "PID" => self
327- . PID_builder
328- . values ( )
329- . append_option ( parse_string_format ( value) ?) ,
330- "PL" => self
331- . PL_builder
332- . values ( )
333- . append_option ( parse_list_int32_format ( value) ?) ,
334- "SB" => self
335- . SB_builder
336- . values ( )
337- . append_option ( parse_list_int32_format ( value) ?) ,
338- _ => vortex_bail ! ( "unknown field {field}" ) ,
330+ sample_fields. insert ( field, value) ;
331+ }
332+
333+ // GT field.
334+ if let Some ( value) = sample_fields. remove ( "GT" ) {
335+ self . GT_builder
336+ . values ( )
337+ . append_option ( parse_genotype ( value) ?) ;
338+ } else {
339+ self . GT_builder . values ( ) . append_null ( ) ;
340+ }
341+
342+ // GQ field.
343+ if let Some ( value) = sample_fields. remove ( "GQ" ) {
344+ self . GQ_builder
345+ . values ( )
346+ . append_option ( parse_int32_format ( value) ?) ;
347+ } else {
348+ self . GQ_builder . values ( ) . append_null ( ) ;
349+ }
350+
351+ // DP field.
352+ if let Some ( value) = sample_fields. remove ( "DP" ) {
353+ self . DP_builder
354+ . values ( )
355+ . append_option ( parse_int32_format ( value) ?) ;
356+ } else {
357+ self . DP_builder . values ( ) . append_null ( ) ;
358+ }
359+
360+ // AD field.
361+ if let Some ( value) = sample_fields. remove ( "AD" ) {
362+ self . AD_builder
363+ . values ( )
364+ . append_option ( parse_list_int32_format ( value) ?) ;
365+ } else {
366+ self . AD_builder . values ( ) . append_null ( ) ;
367+ }
368+
369+ // MIN_DP field.
370+ if let Some ( value) = sample_fields. remove ( "MIN_DP" ) {
371+ self . MIN_DP_builder
372+ . values ( )
373+ . append_option ( parse_int32_format ( value) ?) ;
374+ } else {
375+ self . MIN_DP_builder . values ( ) . append_null ( ) ;
376+ }
377+
378+ // PGT field.
379+ if let Some ( value) = sample_fields. remove ( "PGT" ) {
380+ self . PGT_builder
381+ . values ( )
382+ . append_option ( parse_pgt_format ( value) ?) ;
383+ } else {
384+ self . PGT_builder . values ( ) . append_null ( ) ;
385+ }
386+
387+ // PID field.
388+ if let Some ( value) = sample_fields. remove ( "PID" ) {
389+ self . PID_builder
390+ . values ( )
391+ . append_option ( parse_string_format ( value) ?) ;
392+ } else {
393+ self . PID_builder . values ( ) . append_null ( ) ;
394+ }
395+
396+ // PL field.
397+ if let Some ( value) = sample_fields. remove ( "PL" ) {
398+ self . PL_builder
399+ . values ( )
400+ . append_option ( parse_list_int32_format ( value) ?) ;
401+ } else {
402+ self . PL_builder . values ( ) . append_null ( ) ;
403+ }
404+
405+ // SB field.
406+ if let Some ( value) = sample_fields. remove ( "SB" ) {
407+ if let Some ( sb_values) = parse_list_int32_format ( value) ? {
408+ // SB must have exactly 4 values when present.
409+ if sb_values. len ( ) != 4 {
410+ vortex_bail ! (
411+ "SB field must have exactly 4 values, got {}" ,
412+ sb_values. len( )
413+ ) ;
414+ }
415+ for v in sb_values {
416+ self . SB_builder . values ( ) . values ( ) . append_option ( v) ;
417+ }
418+ self . SB_builder . values ( ) . append ( true ) ;
419+ } else {
420+ // SB field is null/missing for this sample.
421+ for _ in 0 ..4 {
422+ self . SB_builder . values ( ) . values ( ) . append_null ( ) ;
423+ }
424+ self . SB_builder . values ( ) . append ( false ) ;
339425 }
426+ } else {
427+ // SB field is not present for this sample.
428+ for _ in 0 ..4 {
429+ self . SB_builder . values ( ) . values ( ) . append_null ( ) ;
430+ }
431+ self . SB_builder . values ( ) . append ( false ) ;
340432 }
341433 }
434+
435+ // After processing all samples, append to the `FixedSizeListBuilder`'s outer null map.
342436 self . GT_builder . append ( true ) ;
343437 self . GQ_builder . append ( true ) ;
344438 self . DP_builder . append ( true ) ;
0 commit comments