Created
January 15, 2021 17:51
-
-
Save matteobachetti/5d102ce518bd183bf48b0aec2b4fa793 to your computer and use it in GitHub Desktop.
Roundtrip from QDP files
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import sys | |
| import copy | |
| from itertools import groupby | |
| from astropy.timeseries import LombScargle | |
| import numpy as np | |
| from astropy.table import Table | |
| def line_type(line): | |
| """Interpret a QDP file line | |
| Parameters | |
| ---------- | |
| line : str | |
| a single line of the file | |
| Returns | |
| ------- | |
| type : str | |
| Line type: "comment", "command", or "data" | |
| Examples | |
| -------- | |
| >>> line_type("READ SERR 3") | |
| 'command' | |
| >>> line_type(" \\n !some gibberish") | |
| 'comment' | |
| >>> line_type(" ") | |
| 'comment' | |
| >>> line_type(" 21345.45") | |
| 'data,1' | |
| >>> line_type(" 21345.45 NO") | |
| 'data,2' | |
| >>> line_type(" 21345.45 ! a comment to disturb") | |
| 'data,1' | |
| >>> line_type("NO NO NO NO NO") | |
| 'new' | |
| >>> line_type(" some non-comment gibberish") | |
| Traceback (most recent call last): | |
| ... | |
| ValueError: Unrecognized QDP line... | |
| """ | |
| line = line.strip() | |
| if line == "": | |
| return "comment" | |
| # Look for in-line comments (In lines containing other data or commands, | |
| # not starting with those comments) | |
| comment_in_line = line.find("!") | |
| if comment_in_line > 1: | |
| line = line[:comment_in_line] | |
| probe = line.split()[0] | |
| n = len(line.split()) | |
| if probe.startswith("!"): | |
| return "comment" | |
| # Full line of NOs -> new table | |
| if probe.strip("NO ") == "": | |
| return "new" | |
| # A single NO here and there: missing data! | |
| if probe == "NO": | |
| return f"data,{n}" | |
| if probe == "READ": | |
| return "command" | |
| try: | |
| float(probe) | |
| return f"data,{n}" | |
| except: | |
| pass | |
| raise ValueError(f"Unrecognized QDP line: {line}") | |
| def get_type_from_list_of_lines(list_of_lines): | |
| """Read through the list of QDP file lines and label each line by type | |
| Parameters | |
| ---------- | |
| list_of_lines : List | |
| List containing one file line in each entry | |
| Returns | |
| ------- | |
| contents : List | |
| List containing the type for each line (see `line_type_and_data`) | |
| ncol : int | |
| The number of columns in the data lines. Must be the same throughout | |
| the file | |
| Examples | |
| -------- | |
| >>> line0 = "! A comment" | |
| >>> line1 = "543 12 456.0" | |
| >>> list_of_lines = [line0, line1] | |
| >>> types, ncol = get_type_from_list_of_lines(list_of_lines) | |
| >>> types[0] | |
| 'comment' | |
| >>> types[1] | |
| 'data,3' | |
| >>> ncol | |
| 3 | |
| >>> list_of_lines.append("23") | |
| >>> get_type_from_list_of_lines(list_of_lines) | |
| Traceback (most recent call last): | |
| ... | |
| ValueError: Inconsistent number of columns | |
| """ | |
| contents = [] | |
| for line in list_of_lines: | |
| contents.append(line_type(line)) | |
| all_data_specs = [c for c in contents if c.startswith("data")] | |
| # Verify that all data specs are the same (i.e. all data lines | |
| # contain the same number of entries) | |
| if len(list(set(all_data_specs))) != 1: | |
| raise ValueError("Inconsistent number of columns") | |
| ncol = int(all_data_specs[0].replace("data,", "").strip()) | |
| return contents, ncol | |
| def analyze_qdp_file(qdp_file): | |
| """Read through the QDP file and label each line by type | |
| Parameters | |
| ---------- | |
| qdp_file : str | |
| File name | |
| Returns | |
| ------- | |
| contents : List | |
| List containing the type for each line (see `line_type_and_data`) | |
| ncol : int | |
| The number of columns in the data lines. Must be the same throughout | |
| the file | |
| """ | |
| with open(qdp_file) as fobj: | |
| list_of_lines = list(fobj.readlines()) | |
| return get_type_from_list_of_lines(list_of_lines) | |
| def interpret_err_lines(err_specs, ncols, input_colnames=None): | |
| """Give list of column names from the READ SERR and TERR commands | |
| Parameters | |
| ---------- | |
| err_specs : dict, {'serr': [n0, n1, ...], 'terr': [n2, n3, ...]} | |
| Error specifications for symmetric and two-sided errors | |
| ncols : int | |
| Number of data columns | |
| Other parameters | |
| ---------------- | |
| input_colnames : List of strings | |
| Name of data columns (defaults to ['col1', 'col2', ...]), _not_ | |
| including error columns. | |
| Returns | |
| ------- | |
| colnames : List | |
| List containing the column names. Error columns will have the name | |
| of the main column plus ``_err`` for symmetric errors, and ``_perr`` | |
| and ``_nerr`` for positive and negative errors respectively | |
| Examples | |
| -------- | |
| >>> err_specs = {'terr': [1], 'serr': [2]} | |
| >>> ncols = 5 | |
| >>> col_in = ['MJD', 'Rate'] | |
| >>> cols = interpret_err_lines(err_specs, ncols, input_colnames=col_in) | |
| >>> cols[0] | |
| 'MJD' | |
| >>> cols[2] | |
| 'MJD_nerr' | |
| >>> cols[4] | |
| 'Rate_err' | |
| >>> interpret_err_lines(err_specs, 6, input_colnames=col_in) | |
| Traceback (most recent call last): | |
| ... | |
| ValueError: Inconsistent number of input colnames | |
| """ | |
| colnames = ["" for i in range(ncols)] | |
| if err_specs is None: | |
| serr_cols = terr_cols = [] | |
| else: | |
| # I don't want to empty the original one when using `pop` below | |
| err_specs = copy.deepcopy(err_specs) | |
| serr_cols = err_specs.pop("serr", []) | |
| terr_cols = err_specs.pop("terr", []) | |
| if input_colnames is not None: | |
| all_error_cols = len(serr_cols) + len(terr_cols) * 2 | |
| if all_error_cols + len(input_colnames) != ncols: | |
| raise ValueError("Inconsistent number of input colnames") | |
| shift = 0 | |
| for i in range(ncols): | |
| col_num = i + 1 - shift | |
| if colnames[i] != "": | |
| continue | |
| colname_root = f"col{col_num}" | |
| if input_colnames is not None: | |
| colname_root = input_colnames[col_num - 1] | |
| colnames[i] = f"{colname_root}" | |
| if col_num in serr_cols: | |
| colnames[i + 1] = f"{colname_root}_err" | |
| shift += 1 | |
| continue | |
| if col_num in terr_cols: | |
| colnames[i + 1] = f"{colname_root}_perr" | |
| colnames[i + 2] = f"{colname_root}_nerr" | |
| shift += 2 | |
| continue | |
| assert not np.any([c == "" for c in colnames]) | |
| return colnames | |
| def get_tables_from_qdp_file(qdp_file, input_colnames=None): | |
| """Give list of column names from the READ SERR and TERR commands | |
| Parameters | |
| ---------- | |
| qdp_file : str | |
| Input QDP file name | |
| Other parameters | |
| ---------------- | |
| input_colnames : List of strings | |
| Name of data columns (defaults to ['col1', 'col2', ...]), _not_ | |
| including error columns. | |
| Returns | |
| ------- | |
| tables : List of `Table` objects | |
| List containing all the tables present inside the QDP file | |
| """ | |
| contents, ncol = analyze_qdp_file(qdp_file) | |
| with open(qdp_file) as fobj: | |
| lines = list(fobj.readlines()) | |
| file_line = 0 | |
| table_list = [] | |
| initial_comments = "" | |
| comment_text = "" | |
| colnames = None | |
| err_specs = {} | |
| for key, group in groupby(contents): | |
| n_lines = len(list(group)) | |
| if key == "comment": | |
| comment_text = "" | |
| for line in lines[file_line : file_line + n_lines]: | |
| comment_text += line.strip().lstrip("! ") + "\n" | |
| if file_line == 0: | |
| initial_comments = comment_text | |
| elif key == "command": | |
| if err_specs != {}: | |
| warnings.warn( | |
| "This file contains multiple command blocks. Please verify" | |
| ) | |
| for line in lines[file_line : file_line + n_lines]: | |
| command = line.strip().split() | |
| err_specs[command[1].lower()] = [int(c) for c in command[2:]] | |
| colnames = interpret_err_lines( | |
| err_specs, ncol, input_colnames=input_colnames | |
| ) | |
| elif key.startswith("data"): | |
| data_rows = [] | |
| for line in lines[file_line : file_line + n_lines]: | |
| values = [] | |
| for v in line.split(): | |
| if v == "NO": | |
| values.append(np.nan) | |
| else: | |
| values.append(float(v)) | |
| data_rows.append(values) | |
| new_table = Table(rows=data_rows, names=colnames) | |
| new_table.meta["initial_comments"] = initial_comments | |
| new_table.meta["comments"] = comment_text | |
| table_list.append(new_table) | |
| file_line += n_lines | |
| return table_list | |
| def understand_err_col(colnames): | |
| """Get which column names are error columns | |
| Examples | |
| -------- | |
| >>> colnames = ['a', 'a_err', 'b', 'b_perr', 'b_nerr'] | |
| >>> serr, terr = understand_err_col(colnames) | |
| >>> np.allclose(serr, [1]) | |
| True | |
| >>> np.allclose(terr, [2]) | |
| True | |
| """ | |
| shift = 0 | |
| serr = [] | |
| terr = [] | |
| for i, col in enumerate(colnames): | |
| if col.endswith("_err"): | |
| # The previous column, but they're numbered from 1! | |
| # Plus, take shift into account | |
| serr.append(i - shift) | |
| shift += 1 | |
| elif col.endswith("_perr"): | |
| terr.append(i - shift) | |
| if not colnames[i + 1].endswith('_nerr'): | |
| raise ValueError("Missing negative error") | |
| shift += 2 | |
| return serr, terr | |
| def write_table_to_qdp(table, filename, err_specs=None): | |
| from astropy.io import ascii | |
| with open(filename, 'w') as fobj: | |
| if 'initial_comments' in table.meta and table.meta['initial_comments'].strip() != "": | |
| for line in table.meta['initial_comments'].split("\n"): | |
| line = line.strip() | |
| if not line.startswith("!"): | |
| line = "!" + line | |
| print(line, file=fobj) | |
| if err_specs is None: | |
| serr_cols, terr_cols = understand_err_col(table.colnames) | |
| else: | |
| serr_cols = err_specs.pop("serr", []) | |
| terr_cols = err_specs.pop("terr", []) | |
| if serr_cols != []: | |
| col_string = " ".join([str(val) for val in serr_cols]) | |
| print(f"READ SERR {col_string}", file=fobj) | |
| if terr_cols != []: | |
| col_string = " ".join([str(val) for val in terr_cols]) | |
| print(f"READ TERR {col_string}", file=fobj) | |
| if 'comments' in table.meta and table.meta['comments'].strip() != "": | |
| for line in table.meta['comments'].split("\n"): | |
| line = line.strip() | |
| if not line.startswith("!"): | |
| line = "!" + line | |
| print(line, file=fobj) | |
| colnames = table.colnames | |
| print("!" + " ".join(colnames), file=fobj) | |
| for row in table: | |
| print(" ".join([str(val) for val in row]), file=fobj) | |
| def test_get_tables_from_qdp_file(): | |
| import tempfile | |
| example_qdp = """ | |
| ! Swift/XRT hardness ratio of trigger: XXXX, name: BUBU X-2 | |
| ! Columns are as labelled | |
| READ TERR 1 | |
| READ SERR 2 | |
| ! WT -- hard data | |
| !MJD Err (pos) Err(neg) Rate Error | |
| 53000.123456 2.37847222222222e-05 -2.37847222222222e-05 -0.212439 0.212439 | |
| 55045.099887 1.14467592592593e-05 -1.14467592592593e-05 0.000000 0.000000 | |
| NO NO NO NO NO | |
| ! WT -- soft data | |
| !MJD Err (pos) Err(neg) Rate Error | |
| 53000.123456 2.37847222222222e-05 -2.37847222222222e-05 0.726155 0.583890 | |
| 55045.099887 1.14467592592593e-05 -1.14467592592593e-05 2.410935 1.393592 | |
| NO NO NO NO NO | |
| ! WT -- hardness ratio | |
| !MJD Err (pos) Err(neg) Rate Error | |
| 53000.123456 2.37847222222222e-05 -2.37847222222222e-05 -0.292553 -0.374935 | |
| 55045.099887 1.14467592592593e-05 -1.14467592592593e-05 0.000000 -nan | |
| """ | |
| fd, path = tempfile.mkstemp() | |
| with open(path, "w") as fp: | |
| print(example_qdp, file=fp) | |
| tables = get_tables_from_qdp_file(fp.name, input_colnames=["MJD", "Rate"]) | |
| table0 = tables[0] | |
| assert table0.meta["initial_comments"].lstrip().startswith("Swift") | |
| assert table0.meta["comments"].lstrip().startswith("WT -- hard data") | |
| table2 = tables[2] | |
| assert table2.meta["initial_comments"].lstrip().startswith("Swift") | |
| assert table2.meta["comments"].lstrip().startswith("WT -- hardness") | |
| assert np.isclose(table2["MJD_nerr"][0], -2.37847222222222e-05) | |
| def test_roundtrip(): | |
| import tempfile | |
| example_qdp = """ | |
| ! Swift/XRT hardness ratio of trigger: XXXX, name: BUBU X-2 | |
| ! Columns are as labelled | |
| READ TERR 1 | |
| READ SERR 2 | |
| ! WT -- hard data | |
| !MJD Err (pos) Err(neg) Rate Error | |
| 53000.123456 2.37847222222222e-05 -2.37847222222222e-05 NO 0.212439 | |
| 55045.099887 1.14467592592593e-05 -1.14467592592593e-05 0.000000 0.000000 | |
| NO NO NO NO NO | |
| ! WT -- soft data | |
| !MJD Err (pos) Err(neg) Rate Error | |
| 53000.123456 2.37847222222222e-05 -2.37847222222222e-05 0.726155 0.583890 | |
| 55045.099887 1.14467592592593e-05 -1.14467592592593e-05 2.410935 1.393592 | |
| NO NO NO NO NO | |
| ! WT -- hardness ratio | |
| !MJD Err (pos) Err(neg) Rate Error | |
| 53000.123456 2.37847222222222e-05 -2.37847222222222e-05 -0.292553 -0.374935 | |
| 55045.099887 1.14467592592593e-05 -1.14467592592593e-05 0.000000 NO | |
| """ | |
| fd, path = tempfile.mkstemp() | |
| fd2, path2 = tempfile.mkstemp() | |
| with open(path, "w") as fp: | |
| print(example_qdp, file=fp) | |
| tables = get_tables_from_qdp_file(fp.name, input_colnames=["MJD", "Rate"]) | |
| write_table_to_qdp(tables[0], path2) | |
| new_tables = get_tables_from_qdp_file(path2, input_colnames=["MJD", "Rate"]) | |
| assert np.allclose(new_tables[0]['MJD'], tables[0]['MJD']) | |
| assert np.isnan(tables[0]['Rate'][0]) | |
| assert np.isnan(new_tables[0]['Rate'][0]) | |
| for meta_name in ['initial_comments', 'comments']: | |
| assert meta_name in new_tables[0].meta | |
| if __name__ == "__main__": | |
| qdp_file = sys.argv[1] | |
| tables = get_tables_from_qdp_file(qdp_file, input_colnames=["MJD", "Rate"]) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment