Repository navigation
Expand file tree
/
Copy pathaws_remote_fits.py
More file actions
292 lines (218 loc) · 9.22 KB
/
Copy pathaws_remote_fits.py
File metadata and controls
292 lines (218 loc) · 9.22 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
"""Adapted from code written by Tom Robitaille (@astrofrog) at
https://gist.github.com/astrofrog/404a5682311bae221fd5308c1498da94
Requires:
* boto3
* numpy
* astropy
Examples
--------
Import all the necessary modules:
>>> import boto3
>>> import numpy as np
>>> from astropy.io import fits
This imports the data class from a local module named
``aws_remote_fits.py``. If you have copied the code
for ``RemoteAWSFITSFile`` directly into the active Python
session, you can skip this:
>>> from aws_remote_fits import RemoteAWSFITSFile
Authenticate with AWS S3. Replace ``'xxx'`` with
your credentials:
>>> s3_client = boto3.client(
... 's3', aws_access_key_id='xxx', aws_secret_access_key='xxx')
Create an instance of ``RemoteAWSFITSFile`` that holds
the desired TESS cube file being hosted on S3:
>>> s3_file = RemoteAWSFITSFile(
... s3_client, 'stpubdata', 'tess/public/mast/tess-s0001-1-1-cube.fits')
On a successful read, you will see some attributes being
populated:
>>> print(s3_file.header)
TENSION= 'IMAGE ' / Image extension
BITPIX = -32 / array data type
NAXIS = 4 / number of array dimensions
NAXIS1 = 2
NAXIS2 = 1282
NAXIS3 = 2136
NAXIS4 = 2078
PCOUNT = 0 / number of parameters
GCOUNT = 1 / number of groups
END
...
>>> print(s3_file.shape)
(2078, 2136, 1282, 2)
This is the part where ranged GET request to S3 bucket happens.
This is almost the same as ``cube[:10, :10, 0, 0]``, except that
the result returned needs a reshape and a transpose afterwards:
>>> subcube = s3_file[slice(0, 10), slice(0, 10), slice(0, 1), slice(0, 1)]
>>> subcube = subcube.reshape(10, 10).T
Once you have the sub-cube, you can perform regular Numpy
operations on it:
>>> print(subcube.mean())
0.009653374
To write it to a file locally as a simple single-extension FITS:
>>> fits.writeto('subcube.fits', subcube, overwrite=True)
To make sure it round-trips:
>>> with fits.open('subcube.fits') as pf:
... data = pf[0].data.copy()
>>> np.testing.assert_array_equal(subcube, data)
"""
import numpy as np
from astropy.io import fits
class RemoteAWSFITSFile:
"""Class to handle partial download of FITS file
containing data cube stored in AWS S3 bucket.
Currently, this only handles the following file format, where
``ImageHDU`` has 4D array::
No. Name Ver Type Cards Dimensions Format
0 PRIMARY 1 PrimaryHDU 31 ()
1 1 ImageHDU 9 (..., ...) float32
Parameters
----------
s3_client : obj
An instance of authenticated ``boto3.client``.
s3_bucket : str
S3 bucket name. Example: ``'stpubdata'``
s3_key : str
S3 key to the file.
Example: ``'tess/public/mast/tess-s0001-1-1-cube.fits'``
"""
def __init__(self, s3_client, s3_bucket, s3_key):
self.s3_client = s3_client
self.s3_bucket = s3_bucket
self.s3_key = s3_key
# Load complete header which allows us to find start of data
header_block_size = 2879
# EXT=0 with primary header only, no data.
hdr_str = self.get_data_single_range(0, header_block_size)
self.primary_header = fits.Header.fromstring(hdr_str.decode('ascii'))
# EXT=1 with header and data, other extensions ignored.
hdr_str = self.get_data_single_range(
header_block_size + 1, 2 * header_block_size + 1)
self.header = fits.Header.fromstring(hdr_str.decode('ascii'))
self.data_start = 2 * header_block_size + 2
# Data type, see
# https://docs.astropy.org/en/stable/io/fits/usage/image.html
bitpix = self.header['BITPIX']
if bitpix == 8:
data_type = np.uint8
elif bitpix == 16:
data_type = np.int16
elif bitpix == 32:
data_type = np.int32
elif bitpix == -32:
data_type = np.float32
elif bitpix == -64:
data_type = np.float64
else:
raise ValueError('BITPIX={} not supported'.format(bitpix))
self.data_type = np.dtype(data_type).newbyteorder('>')
self.itemsize = np.dtype(data_type).itemsize
# When fitsinfo gives (2, 1282, 2136, 2078),
# shape gives (2078, 2136, 1282, 2) and is consistent with io.fits
self.shape = tuple([self.header['NAXIS{0}'.format(i + 1)]
for i in range(self.header['NAXIS'])][::-1])
if len(self.shape) != 4:
raise ValueError('Only 4D cube supported for now')
self.strides = (
self.shape[1] * self.shape[2] * self.shape[3] * self.itemsize,
self.shape[2] * self.shape[3] * self.itemsize,
self.shape[3] * self.itemsize,
self.itemsize)
def __getitem__(self, view):
"""``view`` is a tuple of slices."""
# Not sure what @astrofrog means here:
# TODO: normalize view to list of slices
# TODO: take into account stride
# TODO: what about BSCALE and BZERO?
# TODO: hard-coded to 4D currently
data_indiv = []
start, end = view[3].start, view[3].stop
i_stride_1 = start * self.strides[3]
i_stride_2 = end * self.strides[3]
for j in range(view[2].start, view[2].stop, view[2].step or 1):
j_stride = j * self.strides[2]
for k in range(view[1].start, view[1].stop, view[1].step or 1):
k_stride = k * self.strides[1]
for m in range(view[0].start, view[0].stop, view[0].step or 1):
m_stride = m * self.strides[0]
range_start = (self.data_start + i_stride_1 +
j_stride + k_stride + m_stride)
range_end = (self.data_start + i_stride_2 +
j_stride + k_stride + m_stride - 1)
content = self.get_data_single_range(
range_start, range_end)
data_indiv.append(content)
data = b''.join(data_indiv)
array = np.frombuffer(data, dtype=self.data_type)
new_shape = (view[0].stop - view[0].start,
(view[1].stop - view[1].start) // (view[1].step or 1),
(view[2].stop - view[2].start) // (view[2].step or 1),
(view[3].stop - view[3].start) // (view[3].step or 1))
return array.reshape(new_shape)
def get_data_single_range(self, start, end):
"""Perform a GET request for single range.
Parameters
----------
start, end : int
Starting and ending bytes to get.
Returns
-------
content : str
Bytes string from the request.
"""
resp = self.s3_client.get_object(
RequestPayer='requester', Bucket=self.s3_bucket, Key=self.s3_key,
Range='bytes={}-{}'.format(start, end))
return resp['Body'].read()
def subarray_by_bytes(fin, data_shape, data_type, view):
"""Offline version of the ``__get__`` method above for testing.
.. note:: Make sure algorithm here is in-sync with ``__get__``.
Parameters
----------
fin : obj
Binary file pointer to the FITS file, opened with
``open(filename, 'rb')``.
data_shape : tuple
Numpy array shape of the entire data array.
data_type : obj
Numpy data type for the array.
view : tuple
Tuple of slices.
Returns
-------
subarray : ndarray
Extracted subarray.
"""
header_block_size = 2879
data_start = 2 * header_block_size + 2
itemsize = np.dtype(data_type).itemsize
data_strides = (data_shape[1] * data_shape[2] * data_shape[3] * itemsize,
data_shape[2] * data_shape[3] * itemsize,
data_shape[3] * itemsize,
itemsize)
print('strides:', data_strides)
data_indiv = []
start, end = view[3].start, view[3].stop
i_stride_1 = start * data_strides[3]
i_stride_2 = end * data_strides[3]
for j in range(view[2].start, view[2].stop, view[2].step or 1):
j_stride = j * data_strides[2]
for k in range(view[1].start, view[1].stop, view[1].step or 1):
k_stride = k * data_strides[1]
for m in range(view[0].start, view[0].stop, view[0].step or 1):
m_stride = m * data_strides[0]
range_start = (data_start + i_stride_1 +
j_stride + k_stride + m_stride)
range_end = (data_start + i_stride_2 +
j_stride + k_stride + m_stride - 1)
print('Range={}-{}'.format(range_start, range_end))
fin.seek(range_start)
content = fin.read(range_end - range_start + 1)
data_indiv.append(content)
data = b''.join(data_indiv)
array = np.frombuffer(data, dtype=data_type)
print(array)
new_shape = (view[0].stop - view[0].start,
(view[1].stop - view[1].start) // (view[1].step or 1),
(view[2].stop - view[2].start) // (view[2].step or 1),
(view[3].stop - view[3].start) // (view[3].step or 1))
return array.reshape(new_shape)