spack · becker33 · Mar 3, 2021 · Feb 3, 2021 · Jan 20, 2021 · Feb 2, 2021
diff --git a/.github/workflows/linux_unit_tests.yaml b/.github/workflows/linux_unit_tests.yaml
@@ -15,6 +15,7 @@ jobs:
     strategy:
       matrix:
         python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
+        concretizer: ['original', 'clingo']
 
     steps:
     - uses: actions/checkout@v2
@@ -50,16 +51,23 @@ jobs:
           mkdir -p ${KCOV_ROOT}/build
           cd ${KCOV_ROOT}/build && cmake -Wno-dev ${KCOV_ROOT}/kcov-${KCOV_VERSION} && cd -
           make -C ${KCOV_ROOT}/build && sudo  make -C ${KCOV_ROOT}/build install
+    - name: Bootstrap clingo from sources
+      if: ${{ matrix.concretizer == 'clingo' }}
+      run: |
+          . share/spack/setup-env.sh
+          spack external find --not-buildable cmake bison
+          spack -v solve zlib
     - name: Run unit tests
       env:
           COVERAGE: true
+          SPACK_TEST_SOLVER: ${{ matrix.concretizer }}
       run: |
           share/spack/qa/run-unit-tests
           coverage combine
           coverage xml
     - uses: codecov/codecov-action@v1
       with:
-        flags: unittests,linux
+        flags: unittests,linux,${{ matrix.concretizer }}
   shell:
     runs-on: ubuntu-latest
     steps:
@@ -143,28 +151,6 @@ jobs:
       run: |
           source share/spack/setup-env.sh
           spack unit-test -k 'not svn and not hg' -x --verbose
-
-  clingo:
-    # Test for the clingo based solver
-    runs-on: ubuntu-latest
-    container: spack/github-actions:clingo
-    steps:
-      - name: Run unit tests
-        run: |
-          whoami && echo PWD=$PWD && echo HOME=$HOME && echo SPACK_TEST_SOLVER=$SPACK_TEST_SOLVER
-          which clingo && clingo --version
-          git clone https://github.com/spack/spack.git && cd spack
-          git fetch origin ${{ github.ref }}:test-branch
-          git checkout test-branch
-          . share/spack/setup-env.sh
-          spack compiler find
-          spack solve mpileaks%gcc
-          coverage run $(which spack) unit-test -v
-          coverage combine
-          coverage xml
-      - uses: codecov/codecov-action@v1
-        with:
-          flags: unittests,linux,clingo
   clingo-cffi:
     # Test for the clingo based solver (using clingo-cffi)
     runs-on: ubuntu-latest

diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst
@@ -111,6 +111,53 @@ environment*, especially for ``PATH``.  Only software that comes with
 the system, or that you know you wish to use with Spack, should be
 included.  This procedure will avoid many strange build errors.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Optional: Bootstrapping clingo
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Spack supports using clingo as an external solver to compute which software
+needs to be installed. If you have a default compiler supporting C++14 Spack
+can automatically bootstrap this tool from sources the first time it is
+needed:
+
+.. code-block:: console
+
+   $ spack solve zlib
+   [+] /usr (external bison-3.0.4-wu5pgjchxzemk5ya2l3ddqug2d7jv6eb)
+   [+] /usr (external cmake-3.19.4-a4kmcfzxxy45mzku4ipmj5kdiiz5a57b)
+   [+] /usr (external python-3.6.9-x4fou4iqqlh5ydwddx3pvfcwznfrqztv)
+   ==> Installing re2c-1.2.1-e3x6nxtk3ahgd63ykgy44mpuva6jhtdt
+   [ ... ]
+   ==> Optimization: [0, 0, 0, 0, 0, 1, 0, 0, 0]
+   [email protected]%[email protected]+optimize+pic+shared arch=linux-ubuntu18.04-broadwell
+
+If you want to speed-up bootstrapping, you may try to search for ``cmake`` and ``bison``
+on your system:
+
+.. code-block:: console
+
+   $ spack external find cmake bison
+   ==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml
+   [email protected]  [email protected]
+
+All the tools Spack needs for its own functioning are installed in a separate store, which lives
+under the ``${HOME}/.spack`` directory. The software installed there can be queried with:
+
+.. code-block:: console
+
+   $ spack find --bootstrap
+   ==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store"
+   ==> 3 installed packages
+   -- linux-ubuntu18.04-x86_64 / [email protected] ------------------------
+   clingo-bootstrap@spack  [email protected]  [email protected]
+
+In case it's needed the bootstrap store can also be cleaned with:
+
+.. code-block:: console
+
+   $ spack clean -b
+   ==> Removing software in "/home/spack/.spack/bootstrap/store"
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 Optional: Alternate Prefix
 ^^^^^^^^^^^^^^^^^^^^^^^^^^

diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
@@ -0,0 +1,188 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import contextlib
+import os
+import sys
+
+import llnl.util.filesystem as fs
+import llnl.util.tty as tty
+
+import spack.architecture
+import spack.config
+import spack.paths
+import spack.repo
+import spack.spec
+import spack.store
+import spack.user_environment as uenv
+import spack.util.executable
+from spack.util.environment import EnvironmentModifications
+
+
+@contextlib.contextmanager
+def spack_python_interpreter():
+    """Override the current configuration to set the interpreter under
+    which Spack is currently running as the only Python external spec
+    available.
+    """
+    python_cls = type(spack.spec.Spec('python').package)
+    python_prefix = os.path.dirname(os.path.dirname(sys.executable))
+    externals = python_cls.determine_spec_details(
+        python_prefix, [os.path.basename(sys.executable)])
+    external_python = externals[0]
+
+    entry = {
+        'buildable': False,
+        'externals': [
+            {'prefix': python_prefix, 'spec': str(external_python)}
+        ]
+    }
+
+    with spack.config.override('packages:python::', entry):
+        yield
+
+
+def make_module_available(module, spec=None, install=False):
+    """Ensure module is importable"""
+    # If we already can import it, that's great
+    try:
+        __import__(module)
+        return
+    except ImportError:
+        pass
+
+    # If it's already installed, use it
+    # Search by spec
+    spec = spack.spec.Spec(spec or module)
+
+    # We have to run as part of this python
+    # We can constrain by a shortened version in place of a version range
+    # because this spec is only used for querying or as a placeholder to be
+    # replaced by an external that already has a concrete version. This syntax
+    # is not suffucient when concretizing without an external, as it will
+    # concretize to [email protected] instead of [email protected]
+    spec.constrain('^python@%d.%d' % sys.version_info[:2])
+    installed_specs = spack.store.db.query(spec, installed=True)
+
+    for ispec in installed_specs:
+        # TODO: make sure run-environment is appropriate
+        module_path = os.path.join(ispec.prefix,
+                                   ispec['python'].package.site_packages_dir)
+        module_path_64 = module_path.replace('/lib/', '/lib64/')
+        try:
+            sys.path.append(module_path)
+            sys.path.append(module_path_64)
+            __import__(module)
+            return
+        except ImportError:
+            tty.warn("Spec %s did not provide module %s" % (ispec, module))
+            sys.path = sys.path[:-2]
+
+    def _raise_error(module_name, module_spec):
+        error_msg = 'cannot import module "{0}"'.format(module_name)
+        if module_spec:
+            error_msg += ' from spec "{0}'.format(module_spec)
+        raise ImportError(error_msg)
+
+    if not install:
+        _raise_error(module, spec)
+
+    with spack_python_interpreter():
+        # We will install for ourselves, using this python if needed
+        # Concretize the spec
+        spec.concretize()
+    spec.package.do_install()
+
+    module_path = os.path.join(spec.prefix,
+                               spec['python'].package.site_packages_dir)
+    module_path_64 = module_path.replace('/lib/', '/lib64/')
+    try:
+        sys.path.append(module_path)
+        sys.path.append(module_path_64)
+        __import__(module)
+        return
+    except ImportError:
+        sys.path = sys.path[:-2]
+        _raise_error(module, spec)
+
+
+def get_executable(exe, spec=None, install=False):
+    """Find an executable named exe, either in PATH or in Spack
+
+    Args:
+        exe (str): needed executable name
+        spec (Spec or str): spec to search for exe in (default exe)
+        install (bool): install spec if not available
+
+    When ``install`` is True, Spack will use the python used to run Spack as an
+    external. The ``install`` option should only be used with packages that
+    install quickly (when using external python) or are guaranteed by Spack
+    organization to be in a binary mirror (clingo)."""
+    # Search the system first
+    runner = spack.util.executable.which(exe)
+    if runner:
+        return runner
+
+    # Check whether it's already installed
+    spec = spack.spec.Spec(spec or exe)
+    installed_specs = spack.store.db.query(spec, installed=True)
+    for ispec in installed_specs:
+        # filter out directories of the same name as the executable
+        exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe)
+                    if fs.is_exe(exe_p)]
+        if exe_path:
+            ret = spack.util.executable.Executable(exe_path[0])
+            envmod = EnvironmentModifications()
+            for dep in ispec.traverse(root=True, order='post'):
+                envmod.extend(uenv.environment_modifications_for_spec(dep))
+            ret.add_default_envmod(envmod)
+            return ret
+        else:
+            tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix))
+
+    def _raise_error(executable, exe_spec):
+        error_msg = 'cannot find the executable "{0}"'.format(executable)
+        if exe_spec:
+            error_msg += ' from spec "{0}'.format(exe_spec)
+        raise RuntimeError(error_msg)
+
+    # If we're not allowed to install this for ourselves, we can't find it
+    if not install:
+        _raise_error(exe, spec)
+
+    with spack_python_interpreter():
+        # We will install for ourselves, using this python if needed
+        # Concretize the spec
+        spec.concretize()
+
+    spec.package.do_install()
+    # filter out directories of the same name as the executable
+    exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe)
+                if fs.is_exe(exe_p)]
+    if exe_path:
+        ret = spack.util.executable.Executable(exe_path[0])
+        envmod = EnvironmentModifications()
+        for dep in spec.traverse(root=True, order='post'):
+            envmod.extend(uenv.environment_modifications_for_spec(dep))
+        ret.add_default_envmod(envmod)
+        return ret
+
+    _raise_error(exe, spec)
+
+
+@contextlib.contextmanager
+def ensure_bootstrap_configuration():
+    # Default configuration scopes excluding command
+    # line and builtin
+    config_scopes = [
+        spack.config.ConfigScope(name, path)
+        for name, path in spack.config.configuration_paths
+    ]
+
+    with spack.architecture.use_platform(spack.architecture.real_platform()):
+        with spack.config.use_configuration(*config_scopes):
+            with spack.repo.use_repositories(spack.paths.packages_path):
+                with spack.store.use_store(spack.paths.user_bootstrap_store):
+                    with spack_python_interpreter():
+                        yield
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
@@ -243,7 +243,28 @@ def install_args(self, spec, prefix):
         if ('py-setuptools' == spec.name or          # this is setuptools, or
             'py-setuptools' in spec._dependencies and  # it's an immediate dep
             'build' in spec._dependencies['py-setuptools'].deptypes):
-            args += ['--single-version-externally-managed', '--root=/']
+            args += ['--single-version-externally-managed']
+
+        # Get all relative paths since we set the root to `prefix`
+        # We query the python with which these will be used for the lib and inc
+        # directories. This ensures we use `lib`/`lib64` as expected by python.
+        python = spec['python'].package.command
+        command_start = 'print(distutils.sysconfig.'
+        commands = ';'.join([
+            'import distutils.sysconfig',
+            command_start + 'get_python_lib(plat_specific=False, prefix=""))',
+            command_start + 'get_python_lib(plat_specific=True, prefix=""))',
+            command_start + 'get_python_inc(plat_specific=True, prefix=""))'])
+        pure_site_packages_dir, plat_site_packages_dir, inc_dir = python(
+            '-c', commands, output=str, error=str).strip().split('\n')
+
+        args += ['--root=%s' % prefix,
+                 '--install-purelib=%s' % pure_site_packages_dir,
+                 '--install-platlib=%s' % plat_site_packages_dir,
+                 '--install-scripts=bin',
+                 '--install-data=""',
+                 '--install-headers=%s' % inc_dir
+                 ]
 
         return args
 

diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py
@@ -10,11 +10,12 @@
 import llnl.util.tty as tty
 
 import spack.caches
+import spack.config
 import spack.cmd.test
 import spack.cmd.common.arguments as arguments
+import spack.main
 import spack.repo
 import spack.stage
-import spack.config
 from spack.paths import lib_path, var_path
 
 
@@ -26,7 +27,7 @@
 class AllClean(argparse.Action):
     """Activates flags -s -d -f -m and -p simultaneously"""
     def __call__(self, parser, namespace, values, option_string=None):
-        parser.parse_args(['-sdfmp'], namespace=namespace)
+        parser.parse_args(['-sdfmpb'], namespace=namespace)
 
 
 def setup_parser(subparser):
@@ -46,15 +47,18 @@ def setup_parser(subparser):
         '-p', '--python-cache', action='store_true',
         help="remove .pyc, .pyo files and __pycache__ folders")
     subparser.add_argument(
-        '-a', '--all', action=AllClean, help="equivalent to -sdfmp", nargs=0
+        '-b', '--bootstrap', action='store_true',
+        help="remove software needed to bootstrap Spack")
+    subparser.add_argument(
+        '-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0
     )
     arguments.add_common_arguments(subparser, ['specs'])
 
 
 def clean(parser, args):
     # If nothing was set, activate the default
     if not any([args.specs, args.stage, args.downloads, args.failures,
-                args.misc_cache, args.python_cache]):
+                args.misc_cache, args.python_cache, args.bootstrap]):
         args.stage = True
 
     # Then do the cleaning falling through the cases
@@ -96,3 +100,10 @@ def clean(parser, args):
                         dname = os.path.join(root, d)
                         tty.debug('Removing {0}'.format(dname))
                         shutil.rmtree(dname)
+
+    if args.bootstrap:
+        msg = 'Removing software in "{0}"'
+        tty.msg(msg.format(spack.paths.user_bootstrap_store))
+        with spack.store.use_store(spack.paths.user_bootstrap_store):
+            uninstall = spack.main.SpackCommand('uninstall')
+            uninstall('-a', '-y')