Repository navigation
Expand file tree
/
Copy pathutils.py
More file actions
739 lines (622 loc) · 20.9 KB
/
Copy pathutils.py
File metadata and controls
739 lines (622 loc) · 20.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
""" Module for general utilities which don't belong in another sub-package.
"""
from __future__ import print_function, absolute_import, division, unicode_literals
import json
import gzip, os
import warnings
import pdb
import numpy as np
from astropy import constants as const
from astropy import units as u
from astropy.units import Quantity, Unit
def between(a, vmin, vmax):
""" Return a boolean array True where vmin <= a < vmax.
Parameters
----------
a : array of shape N
Input array.
vmin, vmix: float
Minimum and maximum values to test between.
Returns
-------
c : array of shape N
Boolean array true where vmin < a < vmax.
Notes
-----
This is a convenience function equivalent to (vmin <= a) & (a < vmax).
Be careful of floating point issues when dealing with equalities.
"""
a = np.asarray(a)
c = a < vmax
c &= a >= vmin
return c
def compare_two_files(file1, file2, except_l2_has=None, verbose=False):
""" Compare 2 files, line by line
Parameters
----------
file1 : str
file2 : str
except_l2_has : str, optional
String in line of file2 that make the test to be skipped
Default is None
Returns
-------
test : bool
True if all lines are identical
"""
f1 = open(file1, 'r')
f2 = open(file2, 'r')
lines1 = f1.readlines()
lines2 = f2.readlines()
q=1
test = True
for l1,l2 in zip(lines1,lines2):
if except_l2_has is not None:
if except_l2_has in l2:
continue
sub_test = l1 == l2
test &= sub_test
if verbose & (~sub_test):
print("Line {} in file1 different than corresponding line {} in file2, details as follows:\n{}different than:\n{}\n".format(q,q,l1,l2))
q += 1
f1.close()
f2.close()
# Return
return test
def compare_two_dict(d1, d2):
""" Compare 2 dicts (using code from StackOverflow)
Parameters
----------
d1 : dict
Considered the new one
d2 : dict
Considered the new original
Returns
-------
added : set
Keys added to d1
removed : set
Keys removed from d2
modified : set
Entries that were modified
same : set
Entries that remained the same
"""
d1_keys = set(d1.keys())
d2_keys = set(d2.keys())
intersect_keys = d1_keys.intersection(d2_keys)
added = d1_keys - d2_keys
removed = d2_keys - d1_keys
modified = {o: (d1[o], d2[o]) for o in intersect_keys if d1[o] != d2[o]}
same = set(o for o in intersect_keys if d1[o] == d2[o])
# Return
return added, removed, modified, same
def compare_two_json(file1, file2):
""" Compare the values of 2 JSON files which may have
different ordering in the keys
Parameters
----------
file1 : str
file2 : str
Returns
-------
test : bool
"""
j1 = loadjson(file1)
j2 = loadjson(file2)
#
test = j1 == j2
return test
def convert_quantity_in_dict(idict):
""" Return a dict where Quantities (usually from a JSON file)
have been converted from unit/value
Parameters
----------
idict : dict
Input dict
Returns
-------
obj : dict or Quantity
"""
if 'unit' in idict.keys(): # Simple dict of Quantity (e.g. from jsonify)
obj = Quantity(idict['value'], unit=idict['unit'])
return obj
else: # Nested dict (possibly)
for key in idict.keys():
if isinstance(idict[key], dict):
idict[key] = convert_quantity_in_dict(idict[key])
return idict
def name_from_coord(coord, precision=(2,1)):
""" Generate a standard JXXXXXX.XX+XXXXXX.X name from a SkyCoord object
Parameters
----------
coord : SkyCoord
precision : tuple, optional
Number of decimal places to include in name
Returns
-------
name : str
In JXX format
"""
name = 'J{:s}{:s}'.format(coord.ra.to_string(unit=u.hour,sep='',pad=True,precision=precision[0]),
coord.dec.to_string(sep='',pad=True,alwayssign=True,precision=precision[1]))
# Return
return name
def radec_to_coord(radec, gal=False):
""" Converts one of many of Celestial Coordinates
`radec` formats to an astropy SkyCoord object. Assumes
J2000 equinox.
Parameters
----------
radec : str or tuple or SkyCoord or list
Examples:
'J124511+144523',
'124511+144523',
'J12:45:11+14:45:23',
('12:45:11','+14:45:23')
('12 45 11', +14 45 23)
('12:45:11','14:45:23') -- Assumes positive DEC
(123.123, 12.1224) -- Assumed deg
[(123.123, 12.1224), (125.123, 32.1224)]
gal : bool, optional
Input pair of floats are (l,b) in deg
Returns
-------
coord : SkyCoord
Converts to astropy.coordinate.SkyCoord (as needed)
Returns a SkyCoord array if input is a list
"""
from astropy.coordinates import SkyCoord
if gal:
frame = 'galactic'
else:
frame = 'icrs'
# RA/DEC
if isinstance(radec, (tuple)):
if isinstance(radec[0], str):
if radec[1][0] not in ['+', '-']: #
DEC = '+'+radec[1]
warnings.warn("Assuming your DEC is +")
else:
DEC = radec[1]
#
coord = SkyCoord(radec[0]+DEC, frame=frame,
unit=(u.hourangle, u.deg))
else:
if frame == 'galactic':
coord = SkyCoord(l=radec[0], b=radec[1], frame=frame, unit='deg')
else:
coord = SkyCoord(ra=radec[0], dec=radec[1], frame=frame, unit='deg')
elif isinstance(radec,SkyCoord):
coord = radec
elif isinstance(radec,str):
# Find first instance of a number (i.e. strip J, SDSS, etc.)
for ii in range(len(radec)):
if radec[ii].isdigit():
break
radec = radec[ii:]
#
if ':' in radec:
coord = SkyCoord(radec, frame='icrs', unit=(u.hourangle, u.deg))
else: # Add in :
if ('+' in radec) or ('-' in radec):
sign = max(radec.find('+'), radec.find('-'))
else:
raise ValueError("radec must include + or - for DEC")
newradec = (radec[0:2]+':'+radec[2:4]+':'+radec[4:sign+3] +':'+radec[sign+3:sign+5]+':'+radec[sign+5:])
coord = SkyCoord(newradec, frame='icrs', unit=(u.hourangle, u.deg))
elif isinstance(radec,list):
clist = []
for item in radec:
clist.append(radec_to_coord(item,gal=gal))
# Convert to SkyCoord array
ras = [ii.icrs.ra.value for ii in clist]
decs = [ii.icrs.dec.value for ii in clist]
return SkyCoord(ra=ras, dec=decs, unit='deg')
else:
raise IOError("Bad input type for radec")
# Return
return coord
def scipy_rebin(aa, *args):
""" Simple script to rebin an input array to a new shape.
Akin to IDL's routine Taken from scipy documentation:
http://wiki.scipy.org/Cookbook/Rebinning As in IDL, the new shape
must be a factor of the old one. The ugly 'evList trick' builds
and executes a python command.
"""
raise DeprecationWarning("This function is deprecated and likely broken")
"""
shape = aa.shape
lenShape = len(shape)
factor = np.asarray(shape)/np.asarray(args)
evList = ['aa.reshape('] + \
['args[%d],factor[%d],'%(i,i) for i in range(lenShape)] + \
[')'] + ['.mean(%d)'%(i+1) for i in range(lenShape)]
#print ''.join(evList)
return eval(''.join(evList))
"""
def jsonify(obj, debug=False):
""" Recursively process an object so it can be serialised in json
format.
WARNING - the input object may be modified if it's a dictionary or
list!
Parameters
----------
obj : any object
debug : bool, optional
Returns
-------
obj - the same obj is json_friendly format (arrays turned to
lists, np.int64 converted to int, np.float64 to float, and so on).
"""
if isinstance(obj, np.float64):
obj = float(obj)
elif isinstance(obj, np.float32):
obj = float(obj)
elif isinstance(obj, np.int32):
obj = int(obj)
elif isinstance(obj, np.int64):
obj = int(obj)
elif isinstance(obj, np.int16):
obj = int(obj)
elif isinstance(obj, np.bool_):
obj = bool(obj)
elif isinstance(obj, np.bytes_):
obj = str(obj)
elif isinstance(obj, Quantity):
if obj.size == 1:
obj = dict(value=obj.value, unit=obj.unit.to_string())
else:
obj = dict(value=obj.value.tolist(), unit=obj.unit.to_string())
elif isinstance(obj, np.ndarray): # Must come after Quantity
obj = obj.tolist()
elif isinstance(obj, dict):
for key, value in obj.items():
obj[key] = jsonify(value, debug=debug)
elif isinstance(obj, list):
for i,item in enumerate(obj):
obj[i] = jsonify(item, debug=debug)
elif isinstance(obj, tuple):
obj = list(obj)
for i,item in enumerate(obj):
obj[i] = jsonify(item, debug=debug)
obj = tuple(obj)
elif isinstance(obj, Unit):
obj = obj.name
elif obj is u.dimensionless_unscaled:
obj = 'dimensionless_unit'
if debug:
print(type(obj))
return obj
def savejson(filename, obj, overwrite=False, indent=None, easy_to_read=False,
**kwargs):
""" Save a python object to filename using the JSON encoder.
Parameters
----------
filename : str
obj : object
Frequently a dict
overwrite : bool, optional
indent : int, optional
Input to json.dump
easy_to_read : bool, optional
Another approach and obj must be a dict
kwargs : optional
Passed to json.dump
Returns
-------
"""
import io
if os.path.lexists(filename) and not overwrite:
raise IOError('%s exists' % filename)
if easy_to_read:
if not isinstance(obj, dict):
raise IOError("This approach requires obj to be a dict")
with io.open(filename, 'w', encoding='utf-8') as f:
f.write(json.dumps(obj, sort_keys=True, indent=4,
separators=(',', ': '), **kwargs))
else:
if filename.endswith('.gz'):
with gzip.open(filename, 'wt') as fh:
json.dump(obj, fh, indent=indent, **kwargs)
else:
with open(filename, 'wt') as fh:
json.dump(obj, fh, indent=indent, **kwargs)
def loadjson(filename):
""" Load a python object saved with savejson."""
if filename.endswith('.gz'):
with gzip.open(filename, "rb") as f:
obj = json.loads(f.read().decode("ascii"))
else:
with open(filename, 'rt') as fh:
obj = json.load(fh)
return obj
def rel_vel(wavelength, wv_obs):
""" Simple relative velocity method
Parameters
----------
wavelength : Quantity array
wv_obs : Quantity
Returns
-------
"""
if not isinstance(wavelength, Quantity):
raise ValueError('Input wavelength array needs to be a Quantity array')
if not isinstance(wv_obs, Quantity):
raise ValueError('Input wv_obs needs to be a Quantity')
return ((wavelength - wv_obs) * const.c / wv_obs).to('km/s')
def v_from_z(z1, z2):
""" Find the relativistic velocity between 2 redshifts.
Parameters
----------
z1 : float
One redshift.
z2 : float or array
Other redshift(s)
Returns
-------
v : Quantity (km/s)
Velocity
Notes
-----
"""
"""
R = (1+z1) / (1+z2)
v = const.c * (R**2 - 1)/(1+R**2)
return v.to('km/s')"""
raise DeprecationWarning("This function is deprecated, please use dv_from_z() instead.")
def z_from_v(z, v):
""" Find the redshift given z and v
Parameters
----------
z : float or array
Redshift
v : Quantity or float or array or array of Quantity
Velocities. If not Quantity it assumes km/s units.
Returns
-------
z : float or array
New redshifts
Notes
-----
"""
"""
# Check for unit
if not isinstance(v, u.quantity.Quantity):
# Assume km/s
v = v * u.Unit('km/s')
# b
bval = (v / const.c.to('km/s'))
# R
R = np.sqrt((1-bval)/(1+bval))
# Finally
znew = (1+z)/R - 1
return znew.value
"""
raise DeprecationWarning("This function is deprecated, instead please use either z_from_dv() or dz_from_dv()"
" depending on your needs.")
# Slightly different functions for passing from dv to dz, and viceversa (that NT prefers).
# May need to agree on one kind of conversion in the future
def give_dz(dv, zref, rel=True):
"""Same as dz_from_dv. This function will be deprecated."""
DeprecationWarning("This function will be deprecated. Please use instead dz_from_dv().")
raise DeprecationWarning('Gone :P')
return dz_from_dv(dv, zref, rel=rel)
def give_dv(z, zref, rel=True):
"""Same as dv_from_dz. This function will be deprecated."""
warnings.warn("This function will be deprecated. Please use instead dv_from_z().")
raise DeprecationWarning('Gone :P')
return dv_from_z(z, zref, rel=rel)
def dv_from_z(z, zref, rel=True):
"""Gives the rest-frame velocity difference dv
between z and zref. dv=0 at zref by definition.
Parameters
----------
z : float or np.ndarray or list or tuple
Redshifts to calculate dv on
zref : float or np.ndarray or list or tuple
Reference redshift where dv=0 by definition.
If the shape of zref is equal to the shape of z,
each dv is calculated at each zref, otherwise zref
is expected to be float.
rel : bool, optional
Whether to apply relativistic correction for
a locally flat space-time. Default is True.
Returns
-------
dv : Quantity or Quantity array
Rest-frame velocity difference between z and zref (dv=0 at zref by definition).
It has the same shape as z.
"""
# check format
if not isinstance(z, (float, np.ndarray, list, tuple)):
raise IOError('z must be float or np.ndarray or list or tuple.')
if not isinstance(zref, (float, np.ndarray, list, tuple)):
raise IOError('zref must be float or np.ndarray or list or tuple.')
if (not isinstance(zref, float)) and (np.shape(zref) != np.shape(z)):
raise IOError('If zref is not float, it must be of same shape as z.')
z = np.array(z)
zref = np.array(zref)
if rel:
dv = ((1 + z)**2 - (1 + zref)**2) / ((1 + z)**2 + (1 + zref)**2)
else:
dv = (z - zref) / (1. + zref)
return dv * const.c.to('km/s')
def dz_from_dv(dv, zref, rel=True):
"""Gives redshift difference for a given
velocity difference(s) with respect to zref.
Parameters
----------
dv : Quantity or Quantity array
Rest-frame velocity difference with respect to zref
zref : float or np.ndarray or list or tuple
Reference redshift where dv=0.
If shape of zref is equal than shape of dv,
each dz is calculated at each zref, otherwise zref
is expected to be float
rel : bool, optional
Whether to apply relativistic correction for
a locally flat space-time. Default is True.
Returns
-------
dz : np.array
Redshift difference for a given dv with respect to zref.
Same shape as dv.
Notes
-----
See also linetools.utils.z_from_dv()
"""
if not isinstance(dv, u.quantity.Quantity):
raise IOError('dv must be Quantity or Quantity array.')
if not isinstance(zref, (float, np.ndarray, list, tuple)):
raise IOError('zref must be float or np.ndarray or list or tuple.')
if (not isinstance(zref, float)) and (np.shape(zref) != np.shape(dv)):
raise IOError('If zref is not float, it must be of same shape as dv.')
zref = np.array(zref)
beta = dv / const.c
beta = beta.decompose()
# check dimensionless
if beta.unit != u.dimensionless_unscaled:
raise IOError('dv must have velocity units.')
beta = beta.value # beta is dimensionless
if rel:
aux = np.sqrt((1. + beta) / (1. - beta))
dz = (1. + zref) * (aux - 1.)
else:
dz = beta * (1. + zref)
return dz
def z_from_dv(dv, zref, rel=True):
"""Gives the redshift for a given
velocity difference(s) with respect to zref.
Parameters
----------
dv : Quantity or Quantity array
Rest-frame velocity difference with respect to zref
zref : float or np.ndarray or list or tuple
Reference redshift where dv=0.
If shape of zref is equal than shape of dv,
each z is calculated at each zref, otherwise zref
is expected to be float
rel : bool, optional
Whether to apply relativistic correction for
a locally flat space-time. Default is True.
Returns
z : np.array
Absolute redshift for a given dv with respect to zref.
Same shape as dv.
Notes
-----
See also linetools.utils.dz_from_dv()
"""
return dz_from_dv(dv, zref, rel=rel) + zref
def overlapping_chunks(chunk1, chunk2):
"""True if there is overlap between chunks
`chunk1` and `chunk2`. Otherwise False. Chunks are
assumed to represent continuous coverage, so the only
information that matters are the minimum and maximum
values of a given chunk. Chunks must be sorted though.
Parameters
----------
chunk1 : tuple, list, 1-d np.array, Quantity, Quantity array
A given chunk, assumed to represent a contiguous region
so only its minimum and maximum values matter. Still,
chunk must be sorted.
chunk2 : tuple, list, 1-d np.array, Quantity, Quantity array
Ditto.
Returns
-------
answer : bool
True if there is overlap, False otherwise.
"""
# Check units in case chunks are Quantity
if isinstance(chunk1, Quantity):
unit1 = chunk1.unit
chunk1 = np.array(chunk1.value)
if not isinstance(chunk2, Quantity):
raise ValueError('chunk2 must be Quantity because chunk1 is!')
try:
chunk2 = chunk2.to(unit1)
chunk2 = np.array(chunk2.value) # has the same units as chunk1
except u.core.UnitConversionError:
raise ValueError('If chunks are given as Quantity they must have convertible units!')
else:
chunk1 = np.array(chunk1) # not a quantity
# this may be redundant but cleaner code
if isinstance(chunk2, Quantity):
unit2 = chunk2.unit
chunk2 = np.array(chunk2.value)
if not isinstance(chunk1, Quantity):
raise ValueError('chunk1 must be Quantity because chunk2 is!')
else:
chunk2 = np.array(chunk2) # not a quantity
# here we have chunks as values (i.e no units)
# make sure the chunks are sorted
cond1 = np.sort(chunk1) != chunk1
cond2 = np.sort(chunk2) != chunk2
if (np.sum(cond1) > 0) or (np.sum(cond2) > 0):
raise ValueError('chunks must be sorted!')
# figure out the lowest of the chunks
# and sort them such that chunk1 is by definition
# the one with the lowest limit
if np.min(chunk1) <= np.min(chunk2):
pass
else: # invert them if necessary
aux = chunk1
chunk1 = chunk2
chunk2 = aux
# now the chunk1 is the one with the lowest value
# then, the only way they do not overlap is:
if np.min(chunk2) > np.max(chunk1):
return False
else: # overlap exists
return True
def is_local_minima(a):
"""For a given 1-d array a, it returns True for local minima
and False otherwise. A local minima corresponds to an element
in the array whose value is strictly lower than the values of
its two immediate neighbours. Both edges of array are always
False by definition.
Parameters
----------
a : list, 1-d np.array
An array of values
Returns
-------
answer : boolean
Whether a value in array `a` is a local minima
Notes
-----
See also is_local_maxima()
"""
a_l = np.roll(a, 1)
a_r = np.roll(a, -1)
is_lmin = (a < a_r) & (a < a_l)
# fill edges with False
is_lmin[0] = False
is_lmin[-1] = False
return is_lmin
def is_local_maxima(a):
"""For a given 1-d array a, it returns True for local maxima
and False otherwise. A local maxima corresponds to an element
in the array whose value is strictly larger than the values of
its two immediate neighbours. Both edges of array are
always False by definition.
Parameters
----------
a : list, 1-d np.array
An array of values
Returns
-------
answer : boolean
Whether a value in array `a` is a local maxima
Notes
-----
See also is_local_minima()
"""
a_l = np.roll(a, 1)
a_r = np.roll(a, -1)
is_lmax = (a > a_r) & (a > a_l)
# fill edges with False
is_lmax[0] = False
is_lmax[-1] = False
return is_lmax