Skip to content

Commit a676f70

Browse files
haampiehaampie
authored
move depfile logic into its own module, separate traversal logic from model (#36911)
1 parent 2081ab8 commit a676f70

File tree

2 files changed

+264
-147
lines changed

2 files changed

+264
-147
lines changed

‎lib/spack/spack/cmd/env.py‎

Lines changed: 8 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
55

66
import argparse
7-
import io
87
import os
98
import shutil
109
import sys
@@ -24,10 +23,11 @@
2423
import spack.cmd.uninstall
2524
import spack.config
2625
import spack.environment as ev
26+
import spack.environment.depfile as depfile
2727
import spack.environment.shell
2828
import spack.schema.env
29+
import spack.spec
2930
import spack.tengine
30-
import spack.traverse as traverse
3131
import spack.util.string as string
3232
from spack.util.environment import EnvironmentModifications
3333

@@ -637,161 +637,22 @@ def env_depfile_setup_parser(subparser):
637637
)
638638

639639

640-
def _deptypes(use_buildcache):
641-
"""What edges should we follow for a given node? If it's a cache-only
642-
node, then we can drop build type deps."""
643-
return ("link", "run") if use_buildcache == "only" else ("build", "link", "run")
644-
645-
646-
class MakeTargetVisitor(object):
647-
"""This visitor produces an adjacency list of a (reduced) DAG, which
648-
is used to generate Makefile targets with their prerequisites."""
649-
650-
def __init__(self, target, pkg_buildcache, deps_buildcache):
651-
"""
652-
Args:
653-
target: function that maps dag_hash -> make target string
654-
pkg_buildcache (str): "only", "never", "auto": when "only",
655-
redundant build deps of roots are dropped
656-
deps_buildcache (str): same as pkg_buildcache, but for non-root specs.
657-
"""
658-
self.adjacency_list = []
659-
self.target = target
660-
self.pkg_buildcache = pkg_buildcache
661-
self.deps_buildcache = deps_buildcache
662-
self.deptypes_root = _deptypes(pkg_buildcache)
663-
self.deptypes_deps = _deptypes(deps_buildcache)
664-
665-
def neighbors(self, node):
666-
"""Produce a list of spec to follow from node"""
667-
deptypes = self.deptypes_root if node.depth == 0 else self.deptypes_deps
668-
return traverse.sort_edges(node.edge.spec.edges_to_dependencies(deptype=deptypes))
669-
670-
def build_cache_flag(self, depth):
671-
setting = self.pkg_buildcache if depth == 0 else self.deps_buildcache
672-
if setting == "only":
673-
return "--use-buildcache=only"
674-
elif setting == "never":
675-
return "--use-buildcache=never"
676-
return ""
677-
678-
def accept(self, node):
679-
fmt = "{name}-{version}-{hash}"
680-
tgt = node.edge.spec.format(fmt)
681-
spec_str = node.edge.spec.format(
682-
"{name}{@version}{%compiler}{variants}{arch=architecture}"
683-
)
684-
buildcache_flag = self.build_cache_flag(node.depth)
685-
prereqs = " ".join([self.target(dep.spec.format(fmt)) for dep in self.neighbors(node)])
686-
self.adjacency_list.append(
687-
(tgt, prereqs, node.edge.spec.dag_hash(), spec_str, buildcache_flag)
688-
)
689-
690-
# We already accepted this
691-
return True
692-
693-
694640
def env_depfile(args):
695641
# Currently only make is supported.
696642
spack.cmd.require_active_env(cmd_name="env depfile")
697643
env = ev.active_environment()
698644

699-
# Special make targets are useful when including a makefile in another, and you
700-
# need to "namespace" the targets to avoid conflicts.
701-
if args.make_prefix is None:
702-
prefix = os.path.join(env.env_subdir_path, "makedeps")
703-
else:
704-
prefix = args.make_prefix
705-
706-
def get_target(name):
707-
# The `all` and `clean` targets are phony. It doesn't make sense to
708-
# have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make
709-
# sense to have a prefix like `env/all`, `env/clean` when they are
710-
# supposed to be included
711-
if name in ("all", "clean") and os.path.isabs(prefix):
712-
return name
713-
else:
714-
return os.path.join(prefix, name)
715-
716-
def get_install_target(name):
717-
return os.path.join(prefix, "install", name)
718-
719-
def get_install_deps_target(name):
720-
return os.path.join(prefix, "install-deps", name)
721-
722645
# What things do we build when running make? By default, we build the
723646
# root specs. If specific specs are provided as input, we build those.
724-
if args.specs:
725-
abstract_specs = spack.cmd.parse_specs(args.specs)
726-
roots = [env.matching_spec(s) for s in abstract_specs]
727-
else:
728-
roots = [s for _, s in env.concretized_specs()]
647+
filter_specs = spack.cmd.parse_specs(args.specs) if args.specs else None
729648

730-
# We produce a sub-DAG from the DAG induced by roots, where we drop build
731-
# edges for those specs that are installed through a binary cache.
732-
pkg_buildcache, dep_buildcache = args.use_buildcache
733-
make_targets = MakeTargetVisitor(get_install_target, pkg_buildcache, dep_buildcache)
734-
traverse.traverse_breadth_first_with_visitor(
735-
roots, traverse.CoverNodesVisitor(make_targets, key=lambda s: s.dag_hash())
736-
)
737-
738-
# Root specs without deps are the prereqs for the environment target
739-
root_install_targets = [get_install_target(h.format("{name}-{version}-{hash}")) for h in roots]
740-
741-
all_pkg_identifiers = []
742-
743-
# The SPACK_PACKAGE_IDS variable is "exported", which can be used when including
744-
# generated makefiles to add post-install hooks, like pushing to a buildcache,
745-
# running tests, etc.
746-
# NOTE: GNU Make allows directory separators in variable names, so for consistency
747-
# we can namespace this variable with the same prefix as targets.
748-
if args.make_prefix is None:
749-
pkg_identifier_variable = "SPACK_PACKAGE_IDS"
750-
else:
751-
pkg_identifier_variable = os.path.join(prefix, "SPACK_PACKAGE_IDS")
752-
753-
# All install and install-deps targets
754-
all_install_related_targets = []
755-
756-
# Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers
757-
# <absolute path to env>/.spack-env/makedeps/install/pkg-version-hash in case
758-
# we don't have a custom make target prefix.
759-
phony_convenience_targets = []
760-
761-
for tgt, _, _, _, _ in make_targets.adjacency_list:
762-
all_pkg_identifiers.append(tgt)
763-
all_install_related_targets.append(get_install_target(tgt))
764-
all_install_related_targets.append(get_install_deps_target(tgt))
765-
if args.make_prefix is None:
766-
phony_convenience_targets.append(os.path.join("install", tgt))
767-
phony_convenience_targets.append(os.path.join("install-deps", tgt))
768-
769-
buf = io.StringIO()
649+
pkg_use_bc, dep_use_bc = args.use_buildcache
770650

771651
template = spack.tengine.make_environment().get_template(os.path.join("depfile", "Makefile"))
772-
773-
rendered = template.render(
774-
{
775-
"all_target": get_target("all"),
776-
"env_target": get_target("env"),
777-
"clean_target": get_target("clean"),
778-
"all_install_related_targets": " ".join(all_install_related_targets),
779-
"root_install_targets": " ".join(root_install_targets),
780-
"dirs_target": get_target("dirs"),
781-
"environment": env.path,
782-
"install_target": get_target("install"),
783-
"install_deps_target": get_target("install-deps"),
784-
"any_hash_target": get_target("%"),
785-
"jobserver_support": "+" if args.jobserver else "",
786-
"adjacency_list": make_targets.adjacency_list,
787-
"phony_convenience_targets": " ".join(phony_convenience_targets),
788-
"pkg_ids_variable": pkg_identifier_variable,
789-
"pkg_ids": " ".join(all_pkg_identifiers),
790-
}
791-
)
792-
793-
buf.write(rendered)
794-
makefile = buf.getvalue()
652+
model = depfile.MakefileModel.from_env(
653+
env, filter_specs, pkg_use_bc, dep_use_bc, args.make_prefix, args.jobserver
654+
)
655+
makefile = template.render(model.to_dict())
795656

796657
# Finally write to stdout/file.
797658
if args.output:

0 commit comments

Comments
 (0)