126126# FACTORY FUNCTIONS
127127
128128
129- def _resize (masked , new_size ):
129+ def _growable_arrays (masked ):
130130 """
131- Masked arrays can not be resized inplace, and `np.resize` and
132- `ma.resize` are both incompatible with structured arrays.
133- Therefore, we do all this.
131+ Return zeroed data and mask arrays with the dtype and length of *masked*.
132+
133+ Masked arrays can not be resized in place, but these plain arrays own
134+ their memory, so ``ndarray.resize`` can grow and trim them without
135+ holding a second copy of the table. Wrap them with `_to_masked` once
136+ they are filled.
134137 """
135- new_array = ma .zeros ((new_size ,), dtype = masked .dtype )
136- length = min (len (masked ), new_size )
137- new_array [:length ] = masked [:length ]
138+ data = np .zeros (len (masked ), dtype = masked .dtype )
139+ mask = np .zeros (len (masked ), dtype = ma .make_mask_descr (masked .dtype ))
140+ return data , mask
141+
138142
139- return new_array
143+ def _to_masked (data , mask ):
144+ """Make a masked array from the arrays of `_growable_arrays` without copying them."""
145+ return ma .MaskedArray (data , mask = mask , copy = False , keep_mask = False )
140146
141147
142148def _lookup_by_attr_factory (attr , unique , iterator , element_name , doc ):
@@ -2988,9 +2994,9 @@ def _parse_tabledata(self, iterator, colnumbers, config):
29882994 # allocation is by factors of 1.5.
29892995 invalid = config .get ("invalid" , "exception" )
29902996
2991- # Need to have only one reference so that we can resize the
2992- # array
2993- array = self .array
2997+ # These must be the only references to the arrays, so that they
2998+ # can be resized in place
2999+ array , mask = _growable_arrays ( self .array )
29943000 del self .array
29953001
29963002 fields = self .all_fields
@@ -3077,9 +3083,10 @@ def _parse_tabledata(self, iterator, colnumbers, config):
30773083 while numrows + chunk_size > alloc_rows :
30783084 alloc_rows = self ._resize_strategy (alloc_rows )
30793085 if alloc_rows != len (array ):
3080- array = _resize (array , alloc_rows )
3086+ array .resize (alloc_rows , refcheck = False )
3087+ mask .resize (alloc_rows , refcheck = False )
30813088 array [numrows : numrows + chunk_size ] = array_chunk
3082- array . mask [numrows : numrows + chunk_size ] = mask_chunk
3089+ mask [numrows : numrows + chunk_size ] = mask_chunk
30833090 numrows += chunk_size
30843091 array_chunk = []
30853092 mask_chunk = []
@@ -3091,17 +3098,17 @@ def _parse_tabledata(self, iterator, colnumbers, config):
30913098 # put the last chunk values in there.
30923099 alloc_rows = numrows + len (array_chunk )
30933100
3094- array = _resize (array , alloc_rows )
3101+ array .resize (alloc_rows , refcheck = False )
3102+ mask .resize (alloc_rows , refcheck = False )
30953103 array [numrows :] = array_chunk
3096- if alloc_rows != 0 :
3097- array .mask [numrows :] = mask_chunk
3104+ mask [numrows :] = mask_chunk
30983105 numrows += len (array_chunk )
30993106
31003107 if self .nrows is not None and self .nrows >= 0 and self .nrows != numrows :
31013108 warn_or_raise (W18 , W18 , (self .nrows , numrows ), config , pos )
31023109 self ._nrows = numrows
31033110
3104- return array
3111+ return _to_masked ( array , mask )
31053112
31063113 def _get_binary_data_stream (self , iterator , config ):
31073114 have_local_stream = False
@@ -3171,9 +3178,9 @@ def careful_read(length):
31713178 def _parse_binary (self , mode , iterator , colnumbers , config , pos ):
31723179 careful_read = self ._get_binary_data_stream (iterator , config )
31733180
3174- # Need to have only one reference so that we can resize the
3175- # array
3176- array = self .array
3181+ # These must be the only references to the arrays, so that they
3182+ # can be resized in place
3183+ array , mask = _growable_arrays ( self .array )
31773184 del self .array
31783185
31793186 fields = self .all_fields
@@ -3185,7 +3192,8 @@ def _parse_binary(self, mode, iterator, colnumbers, config, pos):
31853192 # Resize result arrays if necessary
31863193 if numrows >= alloc_rows :
31873194 alloc_rows = self ._resize_strategy (alloc_rows )
3188- array = _resize (array , alloc_rows )
3195+ array .resize (alloc_rows , refcheck = False )
3196+ mask .resize (alloc_rows , refcheck = False )
31893197
31903198 row_data = []
31913199 row_mask_data = []
@@ -3231,10 +3239,12 @@ def _parse_binary(self, mode, iterator, colnumbers, config, pos):
32313239 row_mask [j ] = row_mask_data [i ]
32323240
32333241 array [numrows ] = tuple (row )
3234- array . mask [numrows ] = tuple (row_mask )
3242+ mask [numrows ] = tuple (row_mask )
32353243 numrows += 1
32363244
3237- return _resize (array , numrows )
3245+ array .resize (numrows , refcheck = False )
3246+ mask .resize (numrows , refcheck = False )
3247+ return _to_masked (array , mask )
32383248
32393249 def _parse_fits (self , iterator , extnum , config ):
32403250 for start , tag , data , pos in iterator :
0 commit comments