Skip to content

Commit 6328fb3

Browse files
scheibelpscheibelp
authored
Move external Python glue logic into core (#50605)
This avoids imports of internal API in the spack_repo package
1 parent 16463e6 commit 6328fb3

File tree

3 files changed

+97
-188
lines changed
  • lib/spack/spack/solver
  • var/spack
    • repos/spack_repo/builtin/build_systems
    • test_repos/spack_repo/builtin_mock/build_systems

3 files changed

+97
-188
lines changed

‎lib/spack/spack/solver/asp.py‎

Lines changed: 97 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import spack.compilers.flags
4848
import spack.config
4949
import spack.deptypes as dt
50+
import spack.detection
5051
import spack.environment as ev
5152
import spack.error
5253
import spack.package_base
@@ -3912,13 +3913,13 @@ def external_spec_selected(self, node, idx):
39123913
compiler_str = external_spec_deps[0]
39133914
self._specs[node].annotations.with_compiler(spack.spec.Spec(compiler_str))
39143915

3915-
# If this is an extension, update the dependencies to include the extendee
3916+
# Packages that are external - but normally depend on python -
3917+
# get an edge inserted to python as a post-concretization step
39163918
package = spack.repo.PATH.get_pkg_class(self._specs[node].fullname)(self._specs[node])
39173919
extendee_spec = package.extendee_spec
3918-
3919-
if extendee_spec:
3920-
extendee_node = SpecBuilder.make_node(pkg=extendee_spec.name)
3921-
package._update_external_dependencies(self._specs.get(extendee_node))
3920+
if extendee_spec and extendee_spec.name == "python":
3921+
candidate_python_to_attach = self._specs.get(SpecBuilder.make_node(pkg="python"))
3922+
_attach_python_to_external(package, extendee_spec=candidate_python_to_attach)
39223923

39233924
def depends_on(self, parent_node, dependency_node, type):
39243925
dependency_spec = self._specs[dependency_node]
@@ -4210,6 +4211,97 @@ def execute_explicit_splices(self):
42104211
return specs
42114212

42124213

4214+
def _attach_python_to_external(
4215+
dependent_package, extendee_spec: Optional[spack.spec.Spec] = None
4216+
) -> None:
4217+
"""
4218+
Ensure all external python packages have a python dependency
4219+
4220+
If another package in the DAG depends on python, we use that
4221+
python for the dependency of the external. If not, we assume
4222+
that the external PythonPackage is installed into the same
4223+
directory as the python it depends on.
4224+
"""
4225+
# TODO: Include this in the solve, rather than instantiating post-concretization
4226+
if "python" not in dependent_package.spec:
4227+
if extendee_spec:
4228+
python = extendee_spec
4229+
else:
4230+
python = _get_external_python_for_prefix(dependent_package)
4231+
if not python.concrete:
4232+
repo = spack.repo.PATH.repo_for_pkg(python)
4233+
python.namespace = repo.namespace
4234+
4235+
# Ensure architecture information is present
4236+
if not python.architecture:
4237+
host_platform = spack.platforms.host()
4238+
host_os = host_platform.default_operating_system()
4239+
host_target = host_platform.default_target()
4240+
python.architecture = spack.spec.ArchSpec(
4241+
(str(host_platform), str(host_os), str(host_target))
4242+
)
4243+
else:
4244+
if not python.architecture.platform:
4245+
python.architecture.platform = spack.platforms.host()
4246+
platform = spack.platforms.by_name(python.architecture.platform)
4247+
if not python.architecture.os:
4248+
python.architecture.os = platform.default_operating_system()
4249+
if not python.architecture.target:
4250+
python.architecture.target = _vendoring.archspec.cpu.host().family.name
4251+
4252+
python.external_path = dependent_package.spec.external_path
4253+
python._mark_concrete()
4254+
dependent_package.spec.add_dependency_edge(
4255+
python, depflag=dt.BUILD | dt.LINK | dt.RUN, virtuals=()
4256+
)
4257+
4258+
4259+
def _get_external_python_for_prefix(python_package):
4260+
"""
4261+
For an external package that extends python, find the most likely spec for the python
4262+
it depends on.
4263+
4264+
First search: an "installed" external that shares a prefix with this package
4265+
Second search: a configured external that shares a prefix with this package
4266+
Third search: search this prefix for a python package
4267+
4268+
Returns:
4269+
spack.spec.Spec: The external Spec for python most likely to be compatible with self.spec
4270+
"""
4271+
python_externals_installed = [
4272+
s
4273+
for s in spack.store.STORE.db.query("python")
4274+
if s.prefix == python_package.spec.external_path
4275+
]
4276+
if python_externals_installed:
4277+
return python_externals_installed[0]
4278+
4279+
python_external_config = spack.config.get("packages:python:externals", [])
4280+
python_externals_configured = [
4281+
spack.spec.parse_with_version_concrete(item["spec"])
4282+
for item in python_external_config
4283+
if item["prefix"] == python_package.spec.external_path
4284+
]
4285+
if python_externals_configured:
4286+
return python_externals_configured[0]
4287+
4288+
python_externals_detection = spack.detection.by_path(
4289+
["python"], path_hints=[python_package.spec.external_path]
4290+
)
4291+
4292+
python_externals_detected = [
4293+
spec
4294+
for spec in python_externals_detection.get("python", [])
4295+
if spec.external_path == python_package.spec.external_path
4296+
]
4297+
if python_externals_detected:
4298+
return python_externals_detected[0]
4299+
4300+
raise StopIteration(
4301+
"No external python could be detected for %s to depend on" % python_package.spec
4302+
)
4303+
4304+
42134305
def _inject_patches_variant(root: spack.spec.Spec) -> None:
42144306
# This dictionary will store object IDs rather than Specs as keys
42154307
# since the Spec __hash__ will change as patches are added to them

‎var/spack/repos/spack_repo/builtin/build_systems/python.py‎

Lines changed: 0 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,9 @@
1010
import stat
1111
from typing import Dict, Iterable, List, Mapping, Optional, Tuple
1212

13-
import _vendoring.archspec.cpu
14-
1513
import llnl.util.filesystem as fs
1614
from llnl.util.lang import ClassProperty, classproperty, match_predicate
1715

18-
import spack.config
19-
import spack.deptypes as dt
20-
import spack.detection
21-
import spack.platforms
22-
import spack.repo
23-
import spack.spec
24-
import spack.store
2516
from spack.package import (
2617
HeaderList,
2718
LibraryList,
@@ -250,89 +241,6 @@ def test_imports(self) -> None:
250241
):
251242
python("-c", f"import {module}")
252243

253-
def _update_external_dependencies(self, extendee_spec: Optional[Spec] = None) -> None:
254-
"""
255-
Ensure all external python packages have a python dependency
256-
257-
If another package in the DAG depends on python, we use that
258-
python for the dependency of the external. If not, we assume
259-
that the external PythonPackage is installed into the same
260-
directory as the python it depends on.
261-
"""
262-
# TODO: Include this in the solve, rather than instantiating post-concretization
263-
if "python" not in self.spec:
264-
if extendee_spec:
265-
python = extendee_spec
266-
elif "python" in self.spec.root:
267-
python = self.spec.root["python"]
268-
else:
269-
python = self.get_external_python_for_prefix()
270-
if not python.concrete:
271-
repo = spack.repo.PATH.repo_for_pkg(python)
272-
python.namespace = repo.namespace
273-
274-
# Ensure architecture information is present
275-
if not python.architecture:
276-
host_platform = spack.platforms.host()
277-
host_os = host_platform.default_operating_system()
278-
host_target = host_platform.default_target()
279-
python.architecture = spack.spec.ArchSpec(
280-
(str(host_platform), str(host_os), str(host_target))
281-
)
282-
else:
283-
if not python.architecture.platform:
284-
python.architecture.platform = spack.platforms.host()
285-
platform = spack.platforms.by_name(python.architecture.platform)
286-
if not python.architecture.os:
287-
python.architecture.os = platform.default_operating_system()
288-
if not python.architecture.target:
289-
python.architecture.target = _vendoring.archspec.cpu.host().family.name
290-
291-
python.external_path = self.spec.external_path
292-
python._mark_concrete()
293-
self.spec.add_dependency_edge(python, depflag=dt.BUILD | dt.LINK | dt.RUN, virtuals=())
294-
295-
def get_external_python_for_prefix(self):
296-
"""
297-
For an external package that extends python, find the most likely spec for the python
298-
it depends on.
299-
300-
First search: an "installed" external that shares a prefix with this package
301-
Second search: a configured external that shares a prefix with this package
302-
Third search: search this prefix for a python package
303-
304-
Returns:
305-
spack.spec.Spec: The external Spec for python most likely to be compatible with self.spec
306-
"""
307-
python_externals_installed = [
308-
s for s in spack.store.STORE.db.query("python") if s.prefix == self.spec.external_path
309-
]
310-
if python_externals_installed:
311-
return python_externals_installed[0]
312-
313-
python_external_config = spack.config.get("packages:python:externals", [])
314-
python_externals_configured = [
315-
spack.spec.parse_with_version_concrete(item["spec"])
316-
for item in python_external_config
317-
if item["prefix"] == self.spec.external_path
318-
]
319-
if python_externals_configured:
320-
return python_externals_configured[0]
321-
322-
python_externals_detection = spack.detection.by_path(
323-
["python"], path_hints=[self.spec.external_path]
324-
)
325-
326-
python_externals_detected = [
327-
spec
328-
for spec in python_externals_detection.get("python", [])
329-
if spec.external_path == self.spec.external_path
330-
]
331-
if python_externals_detected:
332-
return python_externals_detected[0]
333-
334-
raise StopIteration("No external python could be detected for %s to depend on" % self.spec)
335-
336244

337245
def _homepage(cls: "PythonPackage") -> Optional[str]:
338246
"""Get the homepage from PyPI if available."""

‎var/spack/test_repos/spack_repo/builtin_mock/build_systems/python.py‎

Lines changed: 0 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -10,24 +10,16 @@
1010
import stat
1111
from typing import Dict, Iterable, List, Mapping, Optional, Tuple
1212

13-
import _vendoring.archspec
14-
1513
import llnl.util.filesystem as fs
1614
import llnl.util.tty as tty
1715
from llnl.util.filesystem import HeaderList, LibraryList, join_path
1816
from llnl.util.lang import ClassProperty, classproperty, match_predicate
1917

2018
import spack.builder
21-
import spack.config
22-
import spack.deptypes as dt
23-
import spack.detection
2419
import spack.multimethod
2520
import spack.package_base
2621
import spack.phase_callbacks
27-
import spack.platforms
28-
import spack.repo
2922
import spack.spec
30-
import spack.store
3123
import spack.util.prefix
3224
from spack.directives import build_system, depends_on, extends
3325
from spack.error import NoHeadersError, NoLibrariesError
@@ -240,89 +232,6 @@ def test_imports(self) -> None:
240232
):
241233
python("-c", f"import {module}")
242234

243-
def _update_external_dependencies(self, extendee_spec=None):
244-
"""
245-
Ensure all external python packages have a python dependency
246-
247-
If another package in the DAG depends on python, we use that
248-
python for the dependency of the external. If not, we assume
249-
that the external PythonPackage is installed into the same
250-
directory as the python it depends on.
251-
"""
252-
# TODO: Include this in the solve, rather than instantiating post-concretization
253-
if "python" not in self.spec:
254-
if extendee_spec:
255-
python = extendee_spec
256-
elif "python" in self.spec.root:
257-
python = self.spec.root["python"]
258-
else:
259-
python = self.get_external_python_for_prefix()
260-
if not python.concrete:
261-
repo = spack.repo.PATH.repo_for_pkg(python)
262-
python.namespace = repo.namespace
263-
264-
# Ensure architecture information is present
265-
if not python.architecture:
266-
host_platform = spack.platforms.host()
267-
host_os = host_platform.default_operating_system()
268-
host_target = host_platform.default_target()
269-
python.architecture = spack.spec.ArchSpec(
270-
(str(host_platform), str(host_os), str(host_target))
271-
)
272-
else:
273-
if not python.architecture.platform:
274-
python.architecture.platform = spack.platforms.host()
275-
platform = spack.platforms.by_name(python.architecture.platform)
276-
if not python.architecture.os:
277-
python.architecture.os = platform.default_operating_system()
278-
if not python.architecture.target:
279-
python.architecture.target = _vendoring.archspec.cpu.host().family.name
280-
281-
python.external_path = self.spec.external_path
282-
python._mark_concrete()
283-
self.spec.add_dependency_edge(python, depflag=dt.BUILD | dt.LINK | dt.RUN, virtuals=())
284-
285-
def get_external_python_for_prefix(self):
286-
"""
287-
For an external package that extends python, find the most likely spec for the python
288-
it depends on.
289-
290-
First search: an "installed" external that shares a prefix with this package
291-
Second search: a configured external that shares a prefix with this package
292-
Third search: search this prefix for a python package
293-
294-
Returns:
295-
spack.spec.Spec: The external Spec for python most likely to be compatible with self.spec
296-
"""
297-
python_externals_installed = [
298-
s for s in spack.store.STORE.db.query("python") if s.prefix == self.spec.external_path
299-
]
300-
if python_externals_installed:
301-
return python_externals_installed[0]
302-
303-
python_external_config = spack.config.get("packages:python:externals", [])
304-
python_externals_configured = [
305-
spack.spec.parse_with_version_concrete(item["spec"])
306-
for item in python_external_config
307-
if item["prefix"] == self.spec.external_path
308-
]
309-
if python_externals_configured:
310-
return python_externals_configured[0]
311-
312-
python_externals_detection = spack.detection.by_path(
313-
["python"], path_hints=[self.spec.external_path]
314-
)
315-
316-
python_externals_detected = [
317-
spec
318-
for spec in python_externals_detection.get("python", [])
319-
if spec.external_path == self.spec.external_path
320-
]
321-
if python_externals_detected:
322-
return python_externals_detected[0]
323-
324-
raise StopIteration("No external python could be detected for %s to depend on" % self.spec)
325-
326235

327236
def _homepage(cls: "PythonPackage") -> Optional[str]:
328237
"""Get the homepage from PyPI if available."""

0 commit comments

Comments
 (0)