Skip to content

Commit 266ffae

Browse files
authored
Merge pull request #1905 from takluyver/vds-set-virtual-sooner
Improvements to assembling virtual datasets
2 parents c731c3d + 1a415a5 commit 266ffae

6 files changed

Lines changed: 205 additions & 66 deletions

File tree

‎docs/high/group.rst‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,26 @@ Reference
427427
:param fillvalue:
428428
The value to use where there is no data.
429429

430+
.. method:: build_virtual_dataset()
431+
432+
Assemble a virtual dataset in this group.
433+
434+
This is used as a context manager::
435+
436+
with f.build_virtual_dataset('virt', (10, 1000), np.uint32) as layout:
437+
layout[0] = h5py.VirtualSource('foo.h5', 'data', (1000,))
438+
439+
Inside the context, you populate a :class:`VirtualLayout` object.
440+
The file is only modified when you leave the context, and if there's
441+
no error.
442+
443+
:param str name: Name of the dataset (absolute or relative)
444+
:param tuple shape: Shape of the dataset
445+
:param dtype: A numpy dtype for data read from the virtual dataset
446+
:param tuple maxshape: Maximum dimensions if the dataset can grow
447+
(optional). Use None for unlimited dimensions.
448+
:param fillvalue: The value used where no data is available.
449+
430450
.. attribute:: attrs
431451

432452
:ref:`attributes` for this group.

‎h5py/_hl/dataset.py‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -142,34 +142,6 @@ def make_new_dset(parent, shape=None, dtype=None, data=None, name=None,
142142
return dset_id
143143

144144

145-
def make_new_virtual_dset(parent, shape, sources, dtype, name=None,
146-
maxshape=None, fillvalue=None):
147-
""" Return a new low-level dataset identifier for a virtual dataset """
148-
149-
# create the creation property list
150-
dcpl = h5p.create(h5p.DATASET_CREATE)
151-
if fillvalue is not None:
152-
dcpl.set_fill_value(numpy.array([fillvalue]))
153-
154-
if maxshape is not None:
155-
maxshape = tuple(m if m is not None else h5s.UNLIMITED for m in maxshape)
156-
157-
virt_dspace = h5s.create_simple(shape, maxshape)
158-
159-
for vspace, fpath, dset, src_dspace in sources:
160-
dcpl.set_virtual(vspace, fpath, dset, src_dspace)
161-
162-
if isinstance(dtype, Datatype):
163-
# Named types are used as-is
164-
tid = dtype.id
165-
else:
166-
dtype = numpy.dtype(dtype)
167-
tid = h5t.py_create(dtype, logical=1)
168-
169-
return h5d.create(parent.id, name=name, tid=tid, space=virt_dspace,
170-
dcpl=dcpl)
171-
172-
173145
class AstypeWrapper(object):
174146
"""Wrapper to convert data on reading from a dataset.
175147
"""

‎h5py/_hl/group.py‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@
1111
Implements support for high-level access to HDF5 groups.
1212
"""
1313

14+
from contextlib import contextmanager
1415
import posixpath as pp
1516
import numpy
1617

1718

1819
from .compat import filename_decode, filename_encode
1920

20-
from .. import h5, h5g, h5i, h5o, h5r, h5t, h5l, h5p, h5s, h5d
21+
from .. import h5, h5g, h5i, h5o, h5r, h5t, h5l, h5p
2122
from . import base
2223
from .base import HLObject, MutableMappingHDF5, phil, with_phil
2324
from . import dataset
@@ -165,18 +166,6 @@ def create_virtual_dataset(self, name, layout, fillvalue=None):
165166
The value to use where there is no data.
166167
167168
"""
168-
from .vds import VDSmap
169-
# Encode filenames and dataset names appropriately.
170-
sources = []
171-
for vspace, file_name, dset_name, src_space in layout.sources:
172-
if file_name == self.file.filename:
173-
# use relative path if the source dataset is in the same
174-
# file, in order to keep the virtual dataset valid in case
175-
# the file is renamed.
176-
file_name = '.'
177-
sources.append(VDSmap(vspace, filename_encode(file_name),
178-
self._e(dset_name), src_space))
179-
180169
with phil:
181170
group = self
182171

@@ -186,14 +175,42 @@ def create_virtual_dataset(self, name, layout, fillvalue=None):
186175
parent_path, name = name.rsplit(b'/', 1)
187176
group = self.require_group(parent_path)
188177

189-
dsid = dataset.make_new_virtual_dset(group, layout.shape,
190-
sources=sources, dtype=layout.dtype, name=name,
191-
maxshape=layout.maxshape, fillvalue=fillvalue)
192-
178+
dsid = layout.make_dataset(
179+
group, name=name, fillvalue=fillvalue,
180+
)
193181
dset = dataset.Dataset(dsid)
194182

195183
return dset
196184

185+
@contextmanager
186+
def build_virtual_dataset(
187+
self, name, shape, dtype, maxshape=None, fillvalue=None
188+
):
189+
"""Assemble a virtual dataset in this group.
190+
191+
This is used as a context manager::
192+
193+
with f.build_virtual_dataset('virt', (10, 1000), np.uint32) as layout:
194+
layout[0] = h5py.VirtualSource('foo.h5', 'data', (1000,))
195+
196+
name
197+
(str) Name of the new dataset
198+
shape
199+
(tuple) Shape of the dataset
200+
dtype
201+
A numpy dtype for data read from the virtual dataset
202+
maxshape
203+
(tuple, optional) Maximum dimensions if the dataset can grow.
204+
Use None for unlimited dimensions.
205+
fillvalue
206+
The value used where no data is available.
207+
"""
208+
from .vds import VirtualLayout
209+
layout = VirtualLayout(shape, dtype, maxshape, self.file.filename)
210+
yield layout
211+
212+
self.create_virtual_dataset(name, layout, fillvalue)
213+
197214
def require_dataset(self, name, shape, dtype, exact=False, **kwds):
198215
""" Open a dataset, creating it if it doesn't exist.
199216

‎h5py/_hl/vds.py‎

Lines changed: 85 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,13 @@
1313

1414
from copy import deepcopy as copy
1515
from collections import namedtuple
16+
17+
import numpy as np
18+
19+
from .compat import filename_encode
20+
from .datatype import Datatype
1621
from .selections import SimpleSelection, select
17-
from .. import h5s, h5
22+
from .. import h5d, h5p, h5s, h5t, h5
1823
from .. import version
1924

2025

@@ -24,7 +29,6 @@ class VDSmap(namedtuple('VDSmap', ('vspace', 'file_name',
2429
'''
2530

2631

27-
2832
vds_support = False
2933
hdf5_version = version.hdf5_version_tuple[0:3]
3034

@@ -156,17 +160,90 @@ class VirtualLayout(object):
156160
maxshape
157161
The virtual dataset is resizable up to this shape. Use None for
158162
axes you want to be unlimited.
163+
filename
164+
The name of the destination file, if known in advance. Mappings from
165+
data in the same file will be stored with filename '.', allowing the
166+
file to be renamed later.
159167
"""
160-
def __init__(self, shape, dtype, maxshape=None):
168+
def __init__(self, shape, dtype, maxshape=None, filename=None):
161169
self.shape = (shape,) if isinstance(shape, int) else shape
162170
self.dtype = dtype
163171
self.maxshape = (maxshape,) if isinstance(maxshape, int) else maxshape
164-
self.sources = []
172+
self._filename = filename
173+
self._src_filenames = set()
174+
self.dcpl = h5p.create(h5p.DATASET_CREATE)
165175

166176
def __setitem__(self, key, source):
167177
sel = select(self.shape, key, dataset=None)
168178
_convert_space_for_key(sel.id, key)
169-
self.sources.append(VDSmap(sel.id,
170-
source.path,
171-
source.name,
172-
source.sel.id))
179+
src_filename = self._source_file_name(source.path, self._filename)
180+
181+
self.dcpl.set_virtual(
182+
sel.id, src_filename, source.name.encode('utf-8'), source.sel.id
183+
)
184+
if self._filename is None:
185+
self._src_filenames.add(src_filename)
186+
187+
@staticmethod
188+
def _source_file_name(src_filename, dst_filename) -> bytes:
189+
src_filename = filename_encode(src_filename)
190+
if dst_filename and (src_filename == filename_encode(dst_filename)):
191+
# use relative path if the source dataset is in the same
192+
# file, in order to keep the virtual dataset valid in case
193+
# the file is renamed.
194+
return b'.'
195+
return filename_encode(src_filename)
196+
197+
def _get_dcpl(self, dst_filename):
198+
"""Get the property list containing virtual dataset mappings
199+
200+
If the destination filename wasn't known when the VirtualLayout was
201+
created, it is handled here.
202+
"""
203+
dst_filename = filename_encode(dst_filename)
204+
if self._filename is not None:
205+
# filename was known in advance; check dst_filename matches
206+
if dst_filename != filename_encode(self._filename):
207+
raise Exception(f"{dst_filename!r} != {self._filename!r}")
208+
return self.dcpl
209+
210+
# destination file not known in advance
211+
if dst_filename in self._src_filenames:
212+
# At least 1 source file is the same as the destination file,
213+
# but we didn't know this when making the mapping. Copy the mappings
214+
# to a new property list, replacing the dest filename with '.'
215+
new_dcpl = h5p.create(h5p.DATASET_CREATE)
216+
for i in range(self.dcpl.get_virtual_count()):
217+
src_filename = self.dcpl.get_virtual_filename(i)
218+
new_dcpl.set_virtual(
219+
self.dcpl.get_virtual_vspace(i),
220+
self._source_file_name(src_filename, dst_filename),
221+
self.dcpl.get_virtual_dsetname(i).encode('utf-8'),
222+
self.dcpl.get_virtual_srcspace(i),
223+
)
224+
return new_dcpl
225+
else:
226+
return self.dcpl # Mappings are all from other files
227+
228+
def make_dataset(self, parent, name, fillvalue=None):
229+
""" Return a new low-level dataset identifier for a virtual dataset """
230+
dcpl = self._get_dcpl(parent.file.filename)
231+
232+
if fillvalue is not None:
233+
dcpl.set_fill_value(np.array([fillvalue]))
234+
235+
maxshape = self.maxshape
236+
if maxshape is not None:
237+
maxshape = tuple(m if m is not None else h5s.UNLIMITED for m in maxshape)
238+
239+
virt_dspace = h5s.create_simple(self.shape, maxshape)
240+
241+
if isinstance(self.dtype, Datatype):
242+
# Named types are used as-is
243+
tid = self.dtype.id
244+
else:
245+
dtype = np.dtype(self.dtype)
246+
tid = h5t.py_create(dtype, logical=1)
247+
248+
return h5d.create(parent.id, name=name, tid=tid, space=virt_dspace,
249+
dcpl=dcpl)

‎h5py/tests/test_vds/test_highlevel_vds.py‎

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,14 @@ def test_inspection(self):
284284
for n in range(1, 5)}
285285
assert {s.file_name for s in ds.virtual_sources()} == src_files
286286

287+
def test_mismatched_selections(self):
288+
layout = h5.VirtualLayout((4, 100), 'i4', maxshape=(4, None))
289+
290+
filename = osp.join(self.tmpdir, "1.h5")
291+
vsource = h5.VirtualSource(filename, 'data', shape=(100,))
292+
with self.assertRaisesRegex(ValueError, r'different number'):
293+
layout[0, :49] = vsource[0:100:2]
294+
287295
def tearDown(self):
288296
shutil.rmtree(self.tmpdir)
289297

@@ -360,19 +368,22 @@ def setUp(self):
360368
# dataset
361369
ds = f.create_dataset('data', (10,), 'f4')
362370
ds[:] = self.data2
363-
# virtual dataset
364-
layout = h5.VirtualLayout((2, 10), 'f4')
365-
vsource1 = h5.VirtualSource(self.f1, 'data', shape=(10,))
366-
vsource2 = h5.VirtualSource(self.f2, 'data', shape=(10,))
367-
layout[0] = vsource1
368-
layout[1] = vsource2
369-
f.create_virtual_dataset('virtual', layout)
371+
self.make_vds(f)
372+
373+
def make_vds(self, f):
374+
# virtual dataset
375+
layout = h5.VirtualLayout((2, 10), 'f4')
376+
vsource1 = h5.VirtualSource(self.f1, 'data', shape=(10,))
377+
vsource2 = h5.VirtualSource(self.f2, 'data', shape=(10,))
378+
layout[0] = vsource1
379+
layout[1] = vsource2
380+
f.create_virtual_dataset('virtual', layout)
370381

371382
def test_relative_vds(self):
372383
with h5.File(self.f2) as f:
373384
data = f['virtual'][:]
374-
assert (data[0] == self.data1).all()
375-
assert (data[1] == self.data2).all()
385+
np.testing.assert_array_equal(data[0], self.data1)
386+
np.testing.assert_array_equal(data[1], self.data2)
376387

377388
# move f2 -> f3
378389
f3 = osp.join(self.tmpdir, 'testfile3.h5')
@@ -381,8 +392,8 @@ def test_relative_vds(self):
381392
with h5.File(f3) as f:
382393
data = f['virtual'][:]
383394
assert data.dtype == 'f4'
384-
assert (data[0] == self.data1).all()
385-
assert (data[1] == self.data2).all()
395+
np.testing.assert_array_equal(data[0], self.data1)
396+
np.testing.assert_array_equal(data[1], self.data2)
386397

387398
# moving other file
388399
f4 = osp.join(self.tmpdir, 'testfile4.h5')
@@ -392,12 +403,20 @@ def test_relative_vds(self):
392403
data = f['virtual'][:]
393404
assert data.dtype == 'f4'
394405
# unavailable data is silently converted to default value
395-
assert (data[0] == 0).all()
396-
assert (data[1] == self.data2).all()
406+
np.testing.assert_array_equal(data[0], 0)
407+
np.testing.assert_array_equal(data[1], self.data2)
397408

398409
def tearDown(self):
399410
shutil.rmtree(self.tmpdir)
400411

412+
class RelativeLinkBuildVDSTestCase(RelativeLinkTestCase):
413+
# Test a link to the same file with the virtual dataset created by
414+
# File.build_virtual_dataset()
415+
def make_vds(self, f):
416+
with f.build_virtual_dataset('virtual', (2, 10), dtype='f4') as layout:
417+
layout[0] = h5.VirtualSource(self.f1, 'data', shape=(10,))
418+
layout[1] = h5.VirtualSource(self.f2, 'data', shape=(10,))
419+
401420
@ut.skipUnless(vds_support,
402421
'VDS requires HDF5 >= 1.9.233')
403422
class VDSUnlimitedTestCase(ut.TestCase):

‎news/vds-set-virtual-sooner.rst‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
New features
2+
------------
3+
4+
* A new :meth:`Group.build_virtual_dataset` method acting as a context manager
5+
to assemble virtual datasets.
6+
* If the source and target of a virtual dataset mapping have different numbers
7+
of points, an error should now be thrown when you make the mapping in the
8+
:class:`VirtualLayout`, rather than later when writing this into the file.
9+
This should make it easier to find the source of such errors.
10+
11+
Deprecations
12+
------------
13+
14+
* <news item>
15+
16+
Exposing HDF5 functions
17+
-----------------------
18+
19+
* <news item>
20+
21+
Bug fixes
22+
---------
23+
24+
* <news item>
25+
26+
Building h5py
27+
-------------
28+
29+
* <news item>
30+
31+
Development
32+
-----------
33+
34+
* <news item>

0 commit comments

Comments
 (0)