Repository navigation
Expand file tree
/
Copy pathregisters.py
More file actions
288 lines (242 loc) · 9.8 KB
/
Copy pathregisters.py
File metadata and controls
288 lines (242 loc) · 9.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
"""
A module containing the mechanics of the specutils io registry.
"""
import inspect
import os
import pathlib
import sys
import warnings
from functools import wraps
from astropy.io import registry as io_registry
from ..spectra import Spectrum, SpectrumCollection, SpectrumList
__all__ = ['data_loader', 'custom_writer', 'get_loaders_by_extension', 'identify_spectrum_format']
def _astropy_has_priorities():
"""
Check if astropy has support for loader priorities
"""
sig = inspect.signature(io_registry.register_reader)
if sig.parameters.get("priority") is not None:
return True
return False
def data_loader(
label,
identifier=None,
dtype=Spectrum,
extensions=None,
priority=0,
force=False,
autogenerate_spectrumlist=True,
verbose=False,
lazy_loader=None,
):
"""
Wraps a function that can be added to an `~astropy.io.registry` for custom
file reading.
Parameters
----------
label : str
The label given to the function inside the registry.
identifier : func
The identified function used to verify that a file is to use a
particular file.
dtype : class
A class reference for which the data loader should be store.
extensions : list
A list of file extensions this loader supports loading from. In the
case that no identifier function is defined, but a list of file
extensions is, a simple identifier function will be created to check
for consistency with the extensions.
priority : int
Set the priority of the loader. Currently influences the sorting of the
returned loaders for a dtype.
force : bool, optional
Whether to override any existing function if already present.
Default is ``False``. Passed down to astropy registry.
autogenerate_spectrumlist : bool, optional
Whether to automatically register a SpectrumList reader for any
data_loader that reads Spectrum objects. Default is ``True``.
verbose : bool
Print extra info.
lazy_loader : Callable, optional
A loader function to create a lazy-loading SpectrumList.
"""
def identifier_wrapper(ident):
def wrapper(*args, **kwargs):
'''In case the identifier function raises an exception, log that and continue'''
try:
return ident(*args, **kwargs)
except Exception as e:
if verbose:
print(f"Tried to read this as {label} file, but could not: {repr(e)}")
return False
return wrapper
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
lazy_load = bool(kwargs.pop("lazy_load", False))
if lazy_load and dtype is Spectrum:
raise ValueError("Lazy loading is not supported for Spectrum objects.")
# check SpectrumList loaders for lazy option
if dtype is SpectrumList:
if lazy_load and lazy_loader is not None:
return lazy_loader(*args, **kwargs)
# If not lazy, use eager loader
kwargs.pop("cache_size", None)
return func(*args, **kwargs)
# Spectrum loaders
kwargs.pop("cache_size", None)
return func(*args, **kwargs)
if _astropy_has_priorities():
io_registry.register_reader(
label,
dtype,
wrapper,
priority=priority,
force=force,
)
else:
io_registry.register_reader(
label,
dtype,
wrapper,
force=force,
)
if identifier is None:
# If the identifier is not defined, but the extensions are, create
# a simple identifier based off file extension.
if extensions is not None:
if verbose:
print(f"'{label}' data loader provided for {dtype.__name__} without "
"explicit identifier. Creating identifier using "
"list of compatible extensions")
id_func = lambda *args, **kwargs: any([args[1].endswith(x)
for x in extensions])
# Otherwise, create a dummy identifier
else:
warnings.warn(f"'{label}' data loader provided for {dtype.__name__} without "
"explicit identifier or list of compatible "
"extensions")
id_func = lambda *args, **kwargs: True
else:
id_func = identifier_wrapper(identifier)
io_registry.register_identifier(
label, dtype, id_func, force=force,
)
# Include the file extensions as attributes on the function object
wrapper.extensions = extensions
if verbose:
print(f"Successfully loaded reader \"{label}\".")
# Optionally register a SpectrumList reader for any data_loader that
# reads Spectrum objects.
if dtype is Spectrum and autogenerate_spectrumlist:
def load_spectrum_list(*args, **kwargs):
return SpectrumList([ func(*args, **kwargs) ])
# Add these attributes to the SpectrumList reader as well
load_spectrum_list.extensions = extensions
load_spectrum_list.priority = priority
if _astropy_has_priorities():
io_registry.register_reader(
label, SpectrumList, load_spectrum_list, priority=priority,
force=force,
)
else:
io_registry.register_reader(
label, SpectrumList, load_spectrum_list, force=force,
)
io_registry.register_identifier(
label, SpectrumList, id_func, force=force,
)
if verbose:
print(f"Created SpectrumList reader for \"{label}\".")
return wrapper
return decorator
def custom_writer(label, dtype=Spectrum, priority=0, force=False):
def decorator(func):
if _astropy_has_priorities():
io_registry.register_writer(
label, dtype, func, priority=priority, force=force,
)
else:
io_registry.register_writer(label, dtype, func, force=force)
@wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
return decorator
def get_loaders_by_extension(extension):
"""
Retrieve a list of loader labels associated with a given extension.
Parameters
----------
extension : str
The extension for which associated loaders will be matched against.
Returns
-------
loaders : list
A list of loader names that are associated with the extension.
"""
def _registered_readers():
# With the implementation of priorities support in the astropy registry
# loaders, astropy version 4.2 and up return a tuple of ``func`` and
# ``priority``, while versions < 4.2 return just the ``func`` object.
# This function ignores priorities when calling extension loaders.
return [((fmt, cls), func[0])
if isinstance(func, tuple) else ((fmt, cls), func)
for (fmt, cls), func in io_registry._readers.items()]
return [fmt for (fmt, cls), func in _registered_readers()
if issubclass(cls, Spectrum) and
func.extensions is not None and
extension in func.extensions]
def _load_user_io():
# Get the path relative to the user's home directory
path = os.path.expanduser("~/.specutils")
# Import all python files from the directory if it exists
if os.path.exists(path):
for file in os.listdir(path):
if not file.endswith("py"):
continue
try:
import importlib.util as util
spec = util.spec_from_file_location(file[:-3],
os.path.join(path, file))
mod = util.module_from_spec(spec)
spec.loader.exec_module(mod)
except ImportError:
from importlib import import_module
sys.path.insert(0, path)
try:
import_module(file[:-3])
except ModuleNotFoundError: # noqa
pass
def identify_spectrum_format(filename, dtype=Spectrum):
""" Attempt to identify a spectrum file format
Given a filename, attempts to identify a valid file format
from the list of registered specutils loaders. Essentially a wrapper for
`~astropy.io.registry.identify_format` setting **origin** to ``read`` and
**data_class_required** to `~specutils.Spectrum`.
Parameters
----------
filename : str
A path to a file to be identified
dtype: object
class type of Spectrum, SpectrumList, or SpectrumCollection. Default is
Spectrum.
Returns
-------
valid_format : list, str
A list of valid file formats. If only one valid format found, returns
just that element.
"""
# check for valid string input
if not isinstance(filename, (str, pathlib.Path)) or not os.path.isfile(filename):
raise ValueError(f'{filename} is not a valid string path to a file')
# check for proper class type
assert dtype in \
[Spectrum, SpectrumList, SpectrumCollection], \
'dtype class must be either Spectrum, SpectrumList, or SpectrumCollection'
# identify the file format
valid_format = io_registry.identify_format(
'read', dtype, filename, None, {}, {})
if valid_format and len(valid_format) == 1:
return valid_format[0]
return valid_format