Skip to content

Instantly share code, notes, and snippets.

@matteobachetti
Created January 15, 2021 17:51
Show Gist options
  • Select an option

  • Save matteobachetti/5d102ce518bd183bf48b0aec2b4fa793 to your computer and use it in GitHub Desktop.

Select an option

Save matteobachetti/5d102ce518bd183bf48b0aec2b4fa793 to your computer and use it in GitHub Desktop.
Roundtrip from QDP files
import sys
import copy
from itertools import groupby
from astropy.timeseries import LombScargle
import numpy as np
from astropy.table import Table
def line_type(line):
"""Interpret a QDP file line
Parameters
----------
line : str
a single line of the file
Returns
-------
type : str
Line type: "comment", "command", or "data"
Examples
--------
>>> line_type("READ SERR 3")
'command'
>>> line_type(" \\n !some gibberish")
'comment'
>>> line_type(" ")
'comment'
>>> line_type(" 21345.45")
'data,1'
>>> line_type(" 21345.45 NO")
'data,2'
>>> line_type(" 21345.45 ! a comment to disturb")
'data,1'
>>> line_type("NO NO NO NO NO")
'new'
>>> line_type(" some non-comment gibberish")
Traceback (most recent call last):
...
ValueError: Unrecognized QDP line...
"""
line = line.strip()
if line == "":
return "comment"
# Look for in-line comments (In lines containing other data or commands,
# not starting with those comments)
comment_in_line = line.find("!")
if comment_in_line > 1:
line = line[:comment_in_line]
probe = line.split()[0]
n = len(line.split())
if probe.startswith("!"):
return "comment"
# Full line of NOs -> new table
if probe.strip("NO ") == "":
return "new"
# A single NO here and there: missing data!
if probe == "NO":
return f"data,{n}"
if probe == "READ":
return "command"
try:
float(probe)
return f"data,{n}"
except:
pass
raise ValueError(f"Unrecognized QDP line: {line}")
def get_type_from_list_of_lines(list_of_lines):
"""Read through the list of QDP file lines and label each line by type
Parameters
----------
list_of_lines : List
List containing one file line in each entry
Returns
-------
contents : List
List containing the type for each line (see `line_type_and_data`)
ncol : int
The number of columns in the data lines. Must be the same throughout
the file
Examples
--------
>>> line0 = "! A comment"
>>> line1 = "543 12 456.0"
>>> list_of_lines = [line0, line1]
>>> types, ncol = get_type_from_list_of_lines(list_of_lines)
>>> types[0]
'comment'
>>> types[1]
'data,3'
>>> ncol
3
>>> list_of_lines.append("23")
>>> get_type_from_list_of_lines(list_of_lines)
Traceback (most recent call last):
...
ValueError: Inconsistent number of columns
"""
contents = []
for line in list_of_lines:
contents.append(line_type(line))
all_data_specs = [c for c in contents if c.startswith("data")]
# Verify that all data specs are the same (i.e. all data lines
# contain the same number of entries)
if len(list(set(all_data_specs))) != 1:
raise ValueError("Inconsistent number of columns")
ncol = int(all_data_specs[0].replace("data,", "").strip())
return contents, ncol
def analyze_qdp_file(qdp_file):
"""Read through the QDP file and label each line by type
Parameters
----------
qdp_file : str
File name
Returns
-------
contents : List
List containing the type for each line (see `line_type_and_data`)
ncol : int
The number of columns in the data lines. Must be the same throughout
the file
"""
with open(qdp_file) as fobj:
list_of_lines = list(fobj.readlines())
return get_type_from_list_of_lines(list_of_lines)
def interpret_err_lines(err_specs, ncols, input_colnames=None):
"""Give list of column names from the READ SERR and TERR commands
Parameters
----------
err_specs : dict, {'serr': [n0, n1, ...], 'terr': [n2, n3, ...]}
Error specifications for symmetric and two-sided errors
ncols : int
Number of data columns
Other parameters
----------------
input_colnames : List of strings
Name of data columns (defaults to ['col1', 'col2', ...]), _not_
including error columns.
Returns
-------
colnames : List
List containing the column names. Error columns will have the name
of the main column plus ``_err`` for symmetric errors, and ``_perr``
and ``_nerr`` for positive and negative errors respectively
Examples
--------
>>> err_specs = {'terr': [1], 'serr': [2]}
>>> ncols = 5
>>> col_in = ['MJD', 'Rate']
>>> cols = interpret_err_lines(err_specs, ncols, input_colnames=col_in)
>>> cols[0]
'MJD'
>>> cols[2]
'MJD_nerr'
>>> cols[4]
'Rate_err'
>>> interpret_err_lines(err_specs, 6, input_colnames=col_in)
Traceback (most recent call last):
...
ValueError: Inconsistent number of input colnames
"""
colnames = ["" for i in range(ncols)]
if err_specs is None:
serr_cols = terr_cols = []
else:
# I don't want to empty the original one when using `pop` below
err_specs = copy.deepcopy(err_specs)
serr_cols = err_specs.pop("serr", [])
terr_cols = err_specs.pop("terr", [])
if input_colnames is not None:
all_error_cols = len(serr_cols) + len(terr_cols) * 2
if all_error_cols + len(input_colnames) != ncols:
raise ValueError("Inconsistent number of input colnames")
shift = 0
for i in range(ncols):
col_num = i + 1 - shift
if colnames[i] != "":
continue
colname_root = f"col{col_num}"
if input_colnames is not None:
colname_root = input_colnames[col_num - 1]
colnames[i] = f"{colname_root}"
if col_num in serr_cols:
colnames[i + 1] = f"{colname_root}_err"
shift += 1
continue
if col_num in terr_cols:
colnames[i + 1] = f"{colname_root}_perr"
colnames[i + 2] = f"{colname_root}_nerr"
shift += 2
continue
assert not np.any([c == "" for c in colnames])
return colnames
def get_tables_from_qdp_file(qdp_file, input_colnames=None):
"""Give list of column names from the READ SERR and TERR commands
Parameters
----------
qdp_file : str
Input QDP file name
Other parameters
----------------
input_colnames : List of strings
Name of data columns (defaults to ['col1', 'col2', ...]), _not_
including error columns.
Returns
-------
tables : List of `Table` objects
List containing all the tables present inside the QDP file
"""
contents, ncol = analyze_qdp_file(qdp_file)
with open(qdp_file) as fobj:
lines = list(fobj.readlines())
file_line = 0
table_list = []
initial_comments = ""
comment_text = ""
colnames = None
err_specs = {}
for key, group in groupby(contents):
n_lines = len(list(group))
if key == "comment":
comment_text = ""
for line in lines[file_line : file_line + n_lines]:
comment_text += line.strip().lstrip("! ") + "\n"
if file_line == 0:
initial_comments = comment_text
elif key == "command":
if err_specs != {}:
warnings.warn(
"This file contains multiple command blocks. Please verify"
)
for line in lines[file_line : file_line + n_lines]:
command = line.strip().split()
err_specs[command[1].lower()] = [int(c) for c in command[2:]]
colnames = interpret_err_lines(
err_specs, ncol, input_colnames=input_colnames
)
elif key.startswith("data"):
data_rows = []
for line in lines[file_line : file_line + n_lines]:
values = []
for v in line.split():
if v == "NO":
values.append(np.nan)
else:
values.append(float(v))
data_rows.append(values)
new_table = Table(rows=data_rows, names=colnames)
new_table.meta["initial_comments"] = initial_comments
new_table.meta["comments"] = comment_text
table_list.append(new_table)
file_line += n_lines
return table_list
def understand_err_col(colnames):
"""Get which column names are error columns
Examples
--------
>>> colnames = ['a', 'a_err', 'b', 'b_perr', 'b_nerr']
>>> serr, terr = understand_err_col(colnames)
>>> np.allclose(serr, [1])
True
>>> np.allclose(terr, [2])
True
"""
shift = 0
serr = []
terr = []
for i, col in enumerate(colnames):
if col.endswith("_err"):
# The previous column, but they're numbered from 1!
# Plus, take shift into account
serr.append(i - shift)
shift += 1
elif col.endswith("_perr"):
terr.append(i - shift)
if not colnames[i + 1].endswith('_nerr'):
raise ValueError("Missing negative error")
shift += 2
return serr, terr
def write_table_to_qdp(table, filename, err_specs=None):
from astropy.io import ascii
with open(filename, 'w') as fobj:
if 'initial_comments' in table.meta and table.meta['initial_comments'].strip() != "":
for line in table.meta['initial_comments'].split("\n"):
line = line.strip()
if not line.startswith("!"):
line = "!" + line
print(line, file=fobj)
if err_specs is None:
serr_cols, terr_cols = understand_err_col(table.colnames)
else:
serr_cols = err_specs.pop("serr", [])
terr_cols = err_specs.pop("terr", [])
if serr_cols != []:
col_string = " ".join([str(val) for val in serr_cols])
print(f"READ SERR {col_string}", file=fobj)
if terr_cols != []:
col_string = " ".join([str(val) for val in terr_cols])
print(f"READ TERR {col_string}", file=fobj)
if 'comments' in table.meta and table.meta['comments'].strip() != "":
for line in table.meta['comments'].split("\n"):
line = line.strip()
if not line.startswith("!"):
line = "!" + line
print(line, file=fobj)
colnames = table.colnames
print("!" + " ".join(colnames), file=fobj)
for row in table:
print(" ".join([str(val) for val in row]), file=fobj)
def test_get_tables_from_qdp_file():
import tempfile
example_qdp = """
! Swift/XRT hardness ratio of trigger: XXXX, name: BUBU X-2
! Columns are as labelled
READ TERR 1
READ SERR 2
! WT -- hard data
!MJD Err (pos) Err(neg) Rate Error
53000.123456 2.37847222222222e-05 -2.37847222222222e-05 -0.212439 0.212439
55045.099887 1.14467592592593e-05 -1.14467592592593e-05 0.000000 0.000000
NO NO NO NO NO
! WT -- soft data
!MJD Err (pos) Err(neg) Rate Error
53000.123456 2.37847222222222e-05 -2.37847222222222e-05 0.726155 0.583890
55045.099887 1.14467592592593e-05 -1.14467592592593e-05 2.410935 1.393592
NO NO NO NO NO
! WT -- hardness ratio
!MJD Err (pos) Err(neg) Rate Error
53000.123456 2.37847222222222e-05 -2.37847222222222e-05 -0.292553 -0.374935
55045.099887 1.14467592592593e-05 -1.14467592592593e-05 0.000000 -nan
"""
fd, path = tempfile.mkstemp()
with open(path, "w") as fp:
print(example_qdp, file=fp)
tables = get_tables_from_qdp_file(fp.name, input_colnames=["MJD", "Rate"])
table0 = tables[0]
assert table0.meta["initial_comments"].lstrip().startswith("Swift")
assert table0.meta["comments"].lstrip().startswith("WT -- hard data")
table2 = tables[2]
assert table2.meta["initial_comments"].lstrip().startswith("Swift")
assert table2.meta["comments"].lstrip().startswith("WT -- hardness")
assert np.isclose(table2["MJD_nerr"][0], -2.37847222222222e-05)
def test_roundtrip():
import tempfile
example_qdp = """
! Swift/XRT hardness ratio of trigger: XXXX, name: BUBU X-2
! Columns are as labelled
READ TERR 1
READ SERR 2
! WT -- hard data
!MJD Err (pos) Err(neg) Rate Error
53000.123456 2.37847222222222e-05 -2.37847222222222e-05 NO 0.212439
55045.099887 1.14467592592593e-05 -1.14467592592593e-05 0.000000 0.000000
NO NO NO NO NO
! WT -- soft data
!MJD Err (pos) Err(neg) Rate Error
53000.123456 2.37847222222222e-05 -2.37847222222222e-05 0.726155 0.583890
55045.099887 1.14467592592593e-05 -1.14467592592593e-05 2.410935 1.393592
NO NO NO NO NO
! WT -- hardness ratio
!MJD Err (pos) Err(neg) Rate Error
53000.123456 2.37847222222222e-05 -2.37847222222222e-05 -0.292553 -0.374935
55045.099887 1.14467592592593e-05 -1.14467592592593e-05 0.000000 NO
"""
fd, path = tempfile.mkstemp()
fd2, path2 = tempfile.mkstemp()
with open(path, "w") as fp:
print(example_qdp, file=fp)
tables = get_tables_from_qdp_file(fp.name, input_colnames=["MJD", "Rate"])
write_table_to_qdp(tables[0], path2)
new_tables = get_tables_from_qdp_file(path2, input_colnames=["MJD", "Rate"])
assert np.allclose(new_tables[0]['MJD'], tables[0]['MJD'])
assert np.isnan(tables[0]['Rate'][0])
assert np.isnan(new_tables[0]['Rate'][0])
for meta_name in ['initial_comments', 'comments']:
assert meta_name in new_tables[0].meta
if __name__ == "__main__":
qdp_file = sys.argv[1]
tables = get_tables_from_qdp_file(qdp_file, input_colnames=["MJD", "Rate"])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment