Skip to content

Commit 137067f

Browse files
SolarDrewCadair
andauthored
Provide documentation of what path interpolation parameters you can use in Globus download methods (#207)
* Typos * Add basic description of path keys in data download section * Add page on path interpolation keys with list of available ones * Move description and table to net file * Add changelog * Replace hard-coded table in docs with function to print the table * Add function to path-sanitize keys and use in FileManager.download * Apply suggestions from code review * Use astropy table and copy key format from sunpy * Factor out inventory keys as argument to format function * Test path formatting with defined input and output Co-authored-by: Stuart Mumford <[email protected]>
1 parent 8c061c7 commit 137067f

8 files changed

Lines changed: 164 additions & 8 deletions

File tree

‎changelog/207.doc.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add documentation for available path interpolation keys.

‎dkist/conftest.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,8 @@ def dataset(array, identity_gwcs):
203203
'asdfObjectKey': 'test_proposal/test_dataset/test_dataset.asdf',
204204
'browseMovieObjectKey': 'test_proposal/test_dataset/test_dataset.mp4',
205205
'qualityReportObjectKey': 'test_proposal/test_dataset/test_dataset.pdf',
206+
'wavelengthMin': 0,
207+
'wavelengthMax': 0,
206208
},
207209
'headers': Table()
208210
}

‎dkist/io/file_manager.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
from dkist.io.loaders import BaseFITSLoader
3434
from dkist.net import conf as net_conf
3535
from dkist.net.helpers import _orchestrate_transfer_task
36-
from dkist.utils.inventory import humanize_inventory
36+
from dkist.utils.inventory import humanize_inventory, path_format_inventory
3737

3838

3939
class BaseStripedExternalArray:
@@ -362,12 +362,12 @@ def download(self, path=None, destination_endpoint=None, progress=True, wait=Tru
362362
)
363363

364364
inv = self._ndcube.meta["inventory"]
365-
human_inv = humanize_inventory(inv)
365+
path_inv = path_format_inventory(humanize_inventory(inv))
366366

367367
base_path = Path(net_conf.dataset_path.format(**inv))
368368
destination_path = path or self.basepath or "/~/"
369369
destination_path = Path(destination_path).as_posix()
370-
destination_path = Path(destination_path.format(**human_inv))
370+
destination_path = Path(destination_path.format(**path_inv))
371371

372372
# TODO: If we are transferring the whole dataset then we should use the
373373
# directory not the list of all the files in it.

‎dkist/io/tests/test_file_manager.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ def test_download_path_interpolation(dataset, orchestrate_transfer_mock):
239239
folder / "test_dataset.pdf"]
240240
file_list = [base_path / fn for fn in file_list]
241241

242-
dataset.files.download(path="~/{Dataset ID}")
242+
dataset.files.download(path="~/{dataset_id}")
243243

244244
orchestrate_transfer_mock.assert_called_once_with(
245245
file_list,

‎dkist/utils/inventory.py‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
"""
22
Functions for working with dataset inventory.
33
"""
4+
import re
5+
import string
46
from typing import Dict
57
from collections import defaultdict
68

9+
from astropy.table import Table
10+
711
__all__ = ['dehumanize_inventory', 'humanize_inventory', 'INVENTORY_KEY_MAP']
812

913

@@ -43,8 +47,8 @@ def __missing__(self, key):
4347
"qualityAverageFriedParameter": "Average Fried Parameter",
4448
"qualityAveragePolarimetricAccuracy": "Average Polarimetric Accuracy",
4549
"recipeId": "Recipe ID",
46-
"recipeInstanceId": "Recipie Instance ID",
47-
"recipeRunId": "Recipie Run ID",
50+
"recipeInstanceId": "Recipe Instance ID",
51+
"recipeRunId": "Recipe Run ID",
4852
"startTime": "Start Time",
4953
"stokesParameters": "Stokes Parameters",
5054
"targetTypes": "Target Types",
@@ -60,6 +64,26 @@ def __missing__(self, key):
6064
})
6165

6266

67+
def _key_clean(key):
68+
key = re.sub('[%s]' % re.escape(string.punctuation), '_', key)
69+
key = key.replace(' ', '_')
70+
key = ''.join(char for char in key
71+
if char.isidentifier() or char.isnumeric())
72+
return key.lower()
73+
74+
75+
def path_format_keys(keymap):
76+
"""
77+
Return a list of all valid keys for path formatting.
78+
"""
79+
return tuple(map(_key_clean, keymap.values()))
80+
81+
82+
def _path_format_table(keymap=INVENTORY_KEY_MAP):
83+
t = Table({'Inventory Keyword': list(keymap.keys()), 'Path Key': path_format_keys(keymap)})
84+
return '\n'.join(t.pformat(max_lines=-1, html=True))
85+
86+
6387
def humanize_inventory(inventory: Dict[str, str]) -> Dict[str, str]:
6488
"""
6589
Convert an inventory dict to have human readable keys.
@@ -71,6 +95,17 @@ def humanize_inventory(inventory: Dict[str, str]) -> Dict[str, str]:
7195
return humanized_inventory
7296

7397

98+
def path_format_inventory(human_inv):
99+
"""
100+
Given a single humanized inventory record return a dict for formatting paths.
101+
"""
102+
# Putting this here because of circular imports
103+
from ..net.client import DKISTQueryResponseTable as Table
104+
105+
t = Table.from_results([human_inv], client=None)
106+
return t[0].response_block_map
107+
108+
74109
def dehumanize_inventory(humanized_inventory: Dict[str, str]) -> Dict[str, str]:
75110
"""
76111
Convert a human readable inventory dict back to the original keys.

‎dkist/utils/tests/test_inventory.py‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from dkist.utils.inventory import dehumanize_inventory, humanize_inventory
1+
from dkist.utils.inventory import _path_format_table, dehumanize_inventory, humanize_inventory
22

33

44
def test_humanize_loop():
@@ -20,3 +20,97 @@ def test_humanize_loop():
2020

2121

2222
assert old_inv == inv
23+
24+
25+
def test_path_format_table():
26+
output = """<thead><tr><th>Inventory Keyword</th><th>Path Key</th></tr></thead>
27+
<tr><td>asdfObjectKey</td><td>asdf_filename</td></tr>
28+
<tr><td>boundingBox</td><td>bounding_box</td></tr>
29+
<tr><td>browseMovieObjectKey</td><td>movie_filename</td></tr>
30+
<tr><td>browseMovieUrl</td><td>preview_url</td></tr>
31+
<tr><td>bucket</td><td>storage_bucket</td></tr>
32+
<tr><td>contributingExperimentIds</td><td>experiment_ids</td></tr>
33+
<tr><td>contributingProposalIds</td><td>proposal_ids</td></tr>
34+
<tr><td>createDate</td><td>creation_date</td></tr>
35+
<tr><td>datasetId</td><td>dataset_id</td></tr>
36+
<tr><td>datasetSize</td><td>dataset_size</td></tr>
37+
<tr><td>embargoEndDate</td><td>embargo_end_date</td></tr>
38+
<tr><td>endTime</td><td>end_time</td></tr>
39+
<tr><td>experimentDescription</td><td>experiment_description</td></tr>
40+
<tr><td>exposureTime</td><td>exposure_time</td></tr>
41+
<tr><td>filterWavelengths</td><td>filter_wavelengths</td></tr>
42+
<tr><td>frameCount</td><td>number_of_frames</td></tr>
43+
<tr><td>hasAllStokes</td><td>full_stokes</td></tr>
44+
<tr><td>instrumentName</td><td>instrument</td></tr>
45+
<tr><td>isDownloadable</td><td>downloadable</td></tr>
46+
<tr><td>isEmbargoed</td><td>embargoed</td></tr>
47+
<tr><td>observables</td><td>observables</td></tr>
48+
<tr><td>originalFrameCount</td><td>level_0_frame_count</td></tr>
49+
<tr><td>primaryExperimentId</td><td>primary_experiment_id</td></tr>
50+
<tr><td>primaryProposalId</td><td>primary_proposal_id</td></tr>
51+
<tr><td>qualityAverageFriedParameter</td><td>average_fried_parameter</td></tr>
52+
<tr><td>qualityAveragePolarimetricAccuracy</td><td>average_polarimetric_accuracy</td></tr>
53+
<tr><td>recipeId</td><td>recipe_id</td></tr>
54+
<tr><td>recipeInstanceId</td><td>recipe_instance_id</td></tr>
55+
<tr><td>recipeRunId</td><td>recipe_run_id</td></tr>
56+
<tr><td>startTime</td><td>start_time</td></tr>
57+
<tr><td>stokesParameters</td><td>stokes_parameters</td></tr>
58+
<tr><td>targetTypes</td><td>target_types</td></tr>
59+
<tr><td>updateDate</td><td>last_updated</td></tr>
60+
<tr><td>wavelengthMax</td><td>wavelength_max</td></tr>
61+
<tr><td>wavelengthMin</td><td>wavelength_min</td></tr>
62+
<tr><td>hasSpectralAxis</td><td>has_spectral_axis</td></tr>
63+
<tr><td>hasTemporalAxis</td><td>has_temporal_axis</td></tr>
64+
<tr><td>averageDatasetSpectralSampling</td><td>average_spectral_sampling</td></tr>
65+
<tr><td>averageDatasetSpatialSampling</td><td>average_spatial_sampling</td></tr>
66+
<tr><td>averageDatasetTemporalSampling</td><td>average_temporal_sampling</td></tr>
67+
<tr><td>qualityReportObjectKey</td><td>quality_report_filename</td></tr>
68+
</table>"""
69+
70+
test_keymap = {
71+
"asdfObjectKey": "asdf Filename",
72+
"boundingBox": "Bounding Box",
73+
"browseMovieObjectKey": "Movie Filename",
74+
"browseMovieUrl": "Preview URL",
75+
"bucket": "Storage Bucket",
76+
"contributingExperimentIds": "Experiment IDs",
77+
"contributingProposalIds": "Proposal IDs",
78+
"createDate": "Creation Date",
79+
"datasetId": "Dataset ID",
80+
"datasetSize": "Dataset Size",
81+
"embargoEndDate": "Embargo End Date",
82+
"endTime": "End Time",
83+
"experimentDescription": "Experiment Description",
84+
"exposureTime": "Exposure Time",
85+
"filterWavelengths": "Filter Wavelengths",
86+
"frameCount": "Number of Frames",
87+
"hasAllStokes": "Full Stokes",
88+
"instrumentName": "Instrument",
89+
"isDownloadable": "Downloadable",
90+
"isEmbargoed": "Embargoed",
91+
"observables": "Observables",
92+
"originalFrameCount": "Level 0 Frame count",
93+
"primaryExperimentId": "Primary Experiment ID",
94+
"primaryProposalId": "Primary Proposal ID",
95+
"qualityAverageFriedParameter": "Average Fried Parameter",
96+
"qualityAveragePolarimetricAccuracy": "Average Polarimetric Accuracy",
97+
"recipeId": "Recipe ID",
98+
"recipeInstanceId": "Recipe Instance ID",
99+
"recipeRunId": "Recipe Run ID",
100+
"startTime": "Start Time",
101+
"stokesParameters": "Stokes Parameters",
102+
"targetTypes": "Target Types",
103+
"updateDate": "Last Updated",
104+
"wavelengthMax": "Wavelength Max",
105+
"wavelengthMin": "Wavelength Min",
106+
"hasSpectralAxis": "Has Spectral Axis",
107+
"hasTemporalAxis": "Has Temporal Axis",
108+
"averageDatasetSpectralSampling": "Average Spectral Sampling",
109+
"averageDatasetSpatialSampling": "Average Spatial Sampling",
110+
"averageDatasetTemporalSampling": "Average Temporal Sampling",
111+
"qualityReportObjectKey": "Quality Report Filename",
112+
}
113+
table = _path_format_table(test_keymap)
114+
table = table[table.find('\n')+1:]
115+
116+
assert table == output

‎docs/conf.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@
7878
'sunpy.util.sphinx.doctest',
7979
'sunpy.util.sphinx.generate',
8080
]
81-
8281
# Add any paths that contain templates here, relative to this directory.
8382
# templates_path = ['_templates']
8483

‎docs/guide/net.rst‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,9 @@ Once we have loaded the dataset, if we wish to transfer all the FITS files a sin
124124
If this is the first time you have run this method, or your authentication has expired, a login page should open in your webbrowser to authenticate with the Globus system.
125125
By default this call will download all the FITS files to the current value of ``ds.files.basepath``, i.e. by default in the same directory as the loaded ASDF file.
126126
You can override this behaviour by using the ``path=`` keyword argument.
127+
The path argument can contain keys which will be replaced by the corresponding values in the dataset's metadata.
128+
For example, setting `path="~/data/dkist/{instrument}/"` will download all files and save them in separate folders named for the instrument.
129+
A full list of the available keys can be found in :ref:`interpolation-keys`.
127130

128131
The real power of using ``download()`` however, is that you don't have to transfer the FITS files for the frames you do not wish to study (yet).
129132
For instance, imagine the situation where you wish to first inspect the Stokes I profile to asses the viability of the data for your analysis, using this download method you can do this and your transfer will take about a quarter of the time.
@@ -183,3 +186,25 @@ Finally, if you know the dataset ID of a dataset you wish to download, you can j
183186
.. code-block:: python
184187
185188
transfer_complete_datasets("AAAAA")
189+
190+
.. _interpolation-keys:
191+
192+
Path interpolation keys
193+
-----------------------
194+
195+
When downloading DKIST data with ``ds.files.download()`` or ``Fido.fetch(), the ``path=`` keyword argument can be used to specify the target folder for the download.
196+
This path can include keys corresponding to metadata entries, and those values are then used to complete the download path.
197+
For example, to download a dataset into its own folder named with the dataset ID, with separate subfolders for each instrument in the dataset, you could set ``path="~/data/dkist/{dataset_id}/{instrument}/"``.
198+
This would take the values for the dataset ID and instrument name from either the ASDF file or the search results.
199+
200+
Here is a full list of the metadata keywords available for this purpose and their corresponding path interpolation keys:
201+
202+
.. generate:: html
203+
204+
from dkist.utils.inventory import _path_format_table
205+
print(_path_format_table())
206+
207+
The complete list of keys can be accessed using the function ``dkist.utils.inventory.path_format_keys()``::
208+
209+
>>> from dkist.utils.inventory import path_format_keys
210+
>>> path_format_keys()

0 commit comments

Comments
 (0)