-
-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Darwin chroot approach? #361
Description
As we know, OS X doesn't have bind mounts, but HFS+ does allow hardlinking directories.
So as long as the nix store and all desired impurities are on the same HFS+ filesystem, we could make a directory that hardlinks /nix and the impurities, and run all builds chroot-ed under there.
When I say chroot I'm not talking about the chroot build code (
Lines 1750 to 1854 in d98bfcb
| if (useChroot) { | |
| #if CHROOT_ENABLED | |
| /* Create a temporary directory in which we set up the chroot | |
| environment using bind-mounts. We put it in the Nix store | |
| to ensure that we can create hard-links to non-directory | |
| inputs in the fake Nix store in the chroot (see below). */ | |
| chrootRootDir = drvPath + ".chroot"; | |
| if (pathExists(chrootRootDir)) deletePath(chrootRootDir); | |
| /* Clean up the chroot directory automatically. */ | |
| autoDelChroot = std::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir)); | |
| printMsg(lvlChatty, format("setting up chroot environment in ‘%1%’") % chrootRootDir); | |
| /* Create a writable /tmp in the chroot. Many builders need | |
| this. (Of course they should really respect $TMPDIR | |
| instead.) */ | |
| Path chrootTmpDir = chrootRootDir + "/tmp"; | |
| createDirs(chrootTmpDir); | |
| chmod_(chrootTmpDir, 01777); | |
| /* Create a /etc/passwd with entries for the build user and the | |
| nobody account. The latter is kind of a hack to support | |
| Samba-in-QEMU. */ | |
| createDirs(chrootRootDir + "/etc"); | |
| writeFile(chrootRootDir + "/etc/passwd", | |
| (format( | |
| "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" | |
| "nobody:x:65534:65534:Nobody:/:/noshell\n") | |
| % (buildUser.enabled() ? buildUser.getUID() : getuid()) | |
| % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); | |
| /* Declare the build user's group so that programs get a consistent | |
| view of the system (e.g., "id -gn"). */ | |
| writeFile(chrootRootDir + "/etc/group", | |
| (format("nixbld:!:%1%:\n") | |
| % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); | |
| /* Create /etc/hosts with localhost entry. */ | |
| writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); | |
| /* Bind-mount a user-configurable set of directories from the | |
| host file system. */ | |
| PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", string(DEFAULT_CHROOT_DIRS))); | |
| PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string(""))); | |
| dirs.insert(dirs2.begin(), dirs2.end()); | |
| for (auto & i : dirs) { | |
| size_t p = i.find('='); | |
| if (p == string::npos) | |
| dirsInChroot[i] = i; | |
| else | |
| dirsInChroot[string(i, 0, p)] = string(i, p + 1); | |
| } | |
| dirsInChroot[tmpDir] = tmpDir; | |
| /* Make the closure of the inputs available in the chroot, | |
| rather than the whole Nix store. This prevents any access | |
| to undeclared dependencies. Directories are bind-mounted, | |
| while other inputs are hard-linked (since only directories | |
| can be bind-mounted). !!! As an extra security | |
| precaution, make the fake Nix store only writable by the | |
| build user. */ | |
| createDirs(chrootRootDir + settings.nixStore); | |
| chmod_(chrootRootDir + settings.nixStore, 01777); | |
| foreach (PathSet::iterator, i, inputPaths) { | |
| struct stat st; | |
| if (lstat(i->c_str(), &st)) | |
| throw SysError(format("getting attributes of path ‘%1%’") % *i); | |
| if (S_ISDIR(st.st_mode)) | |
| dirsInChroot[*i] = *i; | |
| else { | |
| Path p = chrootRootDir + *i; | |
| if (link(i->c_str(), p.c_str()) == -1) { | |
| /* Hard-linking fails if we exceed the maximum | |
| link count on a file (e.g. 32000 of ext3), | |
| which is quite possible after a `nix-store | |
| --optimise'. */ | |
| if (errno != EMLINK) | |
| throw SysError(format("linking ‘%1%’ to ‘%2%’") % p % *i); | |
| StringSink sink; | |
| dumpPath(*i, sink); | |
| StringSource source(sink.s); | |
| restorePath(p, source); | |
| } | |
| regularInputPaths.insert(*i); | |
| } | |
| } | |
| /* If we're repairing or checking, it's possible that we're | |
| rebuilding a path that is in settings.dirsInChroot | |
| (typically the dependencies of /bin/sh). Throw them | |
| out. */ | |
| if (buildMode != bmNormal) | |
| foreach (DerivationOutputs::iterator, i, drv.outputs) | |
| dirsInChroot.erase(i->second.path); | |
| #else | |
| throw Error("chroot builds are not supported on this platform"); | |
| #endif | |
| } | |
| else { |
To make a hardlink you need to call link(), there's an example CLI utility hlink at http://stackoverflow.com/a/805001/124416 and the very important counterpart hunlink at http://stackoverflow.com/a/4707231/124416 (if you delete files in hardlinked directories, they're gone everywhere - you need to unlink the hardlink).
I tried creating /nix/var/pure with a hardlinked nix/store and some libraries, and it does work, but I can't make chroot work yet 😅. I need a closure of all libraries I suppose, which I think I have but when I do a chroot it just exits without printing anything.
As to why this is needed, I built bash and it picked up libraries from an old homebrew install 😢.
A completely different approach would be to export root over nfs to localhost and use nfs instead of bind mounts.
More eyeballs appreciated.