Skip to content

Commit 061a608

Browse files
kwvgkwvg
committed
merge bitcoin#21778: LLD based macOS toolchain
1 parent ea891d4 commit 061a608

File tree

19 files changed

+72
-167
lines changed

19 files changed

+72
-167
lines changed

‎contrib/containers/ci/Dockerfile‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,20 +58,24 @@ RUN set -ex; \
5858
ARG LLVM_VERSION=16
5959
RUN set -ex; \
6060
echo "Installing LLVM and Clang ${LLVM_VERSION}..."; \
61-
curl -sL https://apt.llvm.org/llvm.sh | bash -s "${LLVM_VERSION}"; \
61+
curl -sL https://apt.llvm.org/llvm.sh | bash -s -- "${LLVM_VERSION}"; \
6262
echo "Installing additional packages..."; \
6363
apt-get update && apt-get install $APT_ARGS \
6464
"clang-format-${LLVM_VERSION}" \
6565
"clang-tidy-${LLVM_VERSION}" \
6666
"libc++-${LLVM_VERSION}-dev" \
6767
"libc++abi-${LLVM_VERSION}-dev" \
68-
"libclang-rt-${LLVM_VERSION}-dev"; \
68+
"libclang-rt-${LLVM_VERSION}-dev" \
69+
"lld-${LLVM_VERSION}"; \
6970
rm -rf /var/lib/apt/lists/*; \
7071
echo "Setting defaults..."; \
7172
lldbUpdAltArgs="update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${LLVM_VERSION} 100"; \
72-
for binName in clang clang++ clang-format clang-tidy clangd ld.lld lldb lldb-server; do \
73+
for binName in clang clang++ clang-format clang-tidy clangd dsymutil lld lldb lldb-server llvm-ar llvm-cov llvm-nm llvm-objdump llvm-ranlib llvm-strip; do \
7374
lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/${binName}-${LLVM_VERSION}"; \
7475
done; \
76+
for binName in ld64.lld ld.lld lld-link wasm-ld; do \
77+
lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/lld-${LLVM_VERSION}"; \
78+
done; \
7579
sh -c "${lldbUpdAltArgs}";
7680
# LD_LIBRARY_PATH is empty by default, this is the first entry
7781
ENV LD_LIBRARY_PATH="/usr/lib/llvm-${LLVM_VERSION}/lib"

‎contrib/devtools/symbol-check.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ def check_MACHO_sdk(binary) -> bool:
252252
return False
253253

254254
def check_MACHO_ld64(binary) -> bool:
255-
if binary.build_version.tools[0].version == [711, 0, 0]:
255+
if binary.build_version.tools[0].version == [17, 0, 6]:
256256
return True
257257
return False
258258

‎contrib/devtools/test-security-check.py‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -121,21 +121,15 @@ def test_MACHO(self):
121121
arch = get_arch(cc, source, executable)
122122

123123
if arch == lief.ARCHITECTURES.X86:
124-
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-no_fixup_chains']),
125-
(1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE NX CONTROL_FLOW'))
126-
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-fixup_chains']),
127-
(1, executable+': failed NOUNDEFS Canary PIE NX CONTROL_FLOW'))
128-
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all', '-Wl,-fixup_chains']),
129-
(1, executable+': failed NOUNDEFS PIE NX CONTROL_FLOW'))
130-
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']),
131-
(1, executable+': failed NOUNDEFS PIE CONTROL_FLOW'))
132-
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']),
133-
(1, executable+': failed PIE CONTROL_FLOW'))
134-
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']),
135-
(1, executable+': failed PIE CONTROL_FLOW'))
136-
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']),
137-
(1, executable+': failed PIE'))
138-
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']),
124+
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']),
125+
(1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE CONTROL_FLOW'))
126+
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains']),
127+
(1, executable+': failed NOUNDEFS Canary CONTROL_FLOW'))
128+
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']),
129+
(1, executable+': failed NOUNDEFS CONTROL_FLOW'))
130+
self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains']),
131+
(1, executable+': failed CONTROL_FLOW'))
132+
self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']),
139133
(0, ''))
140134
else:
141135
# arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks

‎contrib/guix/libexec/build.sh‎

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -135,18 +135,7 @@ for p in "${PATHS[@]}"; do
135135
done
136136

137137
# Disable Guix ld auto-rpath behavior
138-
case "$HOST" in
139-
*darwin*)
140-
# The auto-rpath behavior is necessary for darwin builds as some native
141-
# tools built by depends refer to and depend on Guix-built native
142-
# libraries
143-
#
144-
# After the native packages in depends are built, the ld wrapper should
145-
# no longer affect our build, as clang would instead reach for
146-
# x86_64-apple-darwin-ld from cctools
147-
;;
148-
*) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;;
149-
esac
138+
export GUIX_LD_WRAPPER_DISABLE_RPATH=yes
150139

151140
# Make /usr/bin if it doesn't exist
152141
[ -e /usr/bin ] || mkdir -p /usr/bin
@@ -175,16 +164,6 @@ esac
175164
# Environment variables for determinism
176165
export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name"
177166
export TZ="UTC"
178-
case "$HOST" in
179-
*darwin*)
180-
# cctools AR, unlike GNU binutils AR, does not have a deterministic mode
181-
# or a configure flag to enable determinism by default, it only
182-
# understands if this env-var is set or not. See:
183-
#
184-
# https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334
185-
export ZERO_AR_DATE=yes
186-
;;
187-
esac
188167

189168
####################
190169
# Depends Building #

‎contrib/guix/manifest.scm‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -607,8 +607,9 @@ inspecting signatures in Mach-O binaries.")
607607
((string-contains target "darwin")
608608
(list ;; Native GCC 11 toolchain
609609
gcc-toolchain-11
610-
binutils
611610
clang-toolchain-17
611+
lld-17
612+
(make-lld-wrapper lld-17 #:lld-as-ld? #t)
612613
python-signapple
613614
zip))
614615
(else '())))))

‎contrib/macdeploy/README.md‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -56,28 +56,13 @@ The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c28
5656

5757
## Deterministic macOS App Notes
5858

59-
macOS Applications are created in Linux by combining a recent `clang` and the Apple
60-
`binutils` (`ld`, `ar`, etc).
59+
macOS Applications are created in Linux using a recent LLVM.
6160

6261
Apple uses `clang` extensively for development and has upstreamed the necessary
6362
functionality so that a vanilla clang can take advantage. It supports the use of `-F`,
6463
`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when
6564
building for macOS.
6665

67-
Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the
68-
FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several
69-
other tools are needed as well. These do not build under Linux, so they have been patched to
70-
do so. The work here was used as a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4).
71-
72-
In order to build a working toolchain, the following source packages are needed from
73-
Apple: `cctools`, `dyld`, and `ld64`.
74-
75-
These tools inject timestamps by default, which produce non-deterministic binaries. The
76-
`ZERO_AR_DATE` environment variable is used to disable that.
77-
78-
This version of `cctools` has been patched to use the current version of `clang`'s headers
79-
and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`.
80-
8166
To complicate things further, all builds must target an Apple SDK. These SDKs are free to
8267
download, but not redistributable. See the SDK Extraction notes above for how to obtain it.
8368

‎depends/Makefile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,6 @@ all_packages = $(packages) $(native_packages)
178178

179179
meta_depends = Makefile config.guess config.sub funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
180180

181-
$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils)
182181
$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
183182

184183
include funcs.mk
@@ -210,9 +209,8 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
210209
# tool needs to be available in $PATH at all times.
211210
#
212211
# 2. If the tool is _**not**_ expected to be available in $PATH at all times
213-
# (such as is the case for our native_cctools binutils tools), it needs to
214-
# be referred to by its absolute path, such as would be output by the
215-
# AC_PATH_{PROG,TOOL} macros.
212+
# it needs to be referred to by its absolute path, such as would be output
213+
# by the AC_PATH_{PROG,TOOL} macros.
216214
#
217215
# Minor note: it is also okay to refer to tools by their absolute path even if
218216
# we expect them to be available in $PATH at all times, more specificity does

‎depends/builders/darwin.mk‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ darwin_STRIP:=$(shell xcrun -f strip)
1818
darwin_OBJDUMP:=$(shell xcrun -f objdump)
1919
darwin_NM:=$(shell xcrun -f nm)
2020
darwin_DSYMUTIL:=$(shell xcrun -f dsymutil)
21-
darwin_native_binutils=
2221
darwin_native_toolchain=
2322

2423
x86_64_darwin_CFLAGS += -arch x86_64

‎depends/funcs.mk‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ endef
4646

4747
define int_get_build_id
4848
$(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
49-
$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies)))
49+
$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies)))
5050
$(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
5151
$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id))
5252
$(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
@@ -299,4 +299,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$
299299
$(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package))))
300300

301301
#special exception: if a toolchain package exists, all non-native packages depend on it
302-
$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) ))
302+
$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) ))

‎depends/gen_id‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050
echo "BEGIN AR"
5151
bash -c "${AR} --version"
5252
env | grep '^AR_'
53-
echo "ZERO_AR_DATE=${ZERO_AR_DATE}"
5453
echo "END AR"
5554

5655
echo "BEGIN RANLIB"

0 commit comments

Comments
 (0)