1818// https://github.com/ClickHouse/ClickHouse/blob/master/src/base/phdr_cache.cpp
1919// and modified by Doris
2020
21+ #include " common/phdr_cache.h"
22+
2123#if defined(__clang__)
2224#pragma clang diagnostic ignored "-Wreserved-identifier"
2325#endif
4749#include < dlfcn.h>
4850#include < link.h>
4951
52+ #include < algorithm>
53+ #include < array>
5054#include < atomic>
5155#include < cstddef>
56+ #include < cstring>
57+ #include < limits>
58+ #include < memory>
59+ #include < mutex>
5260#include < stdexcept>
61+ #include < string>
62+ #include < utility>
5363#include < vector>
5464
65+ #if defined(USE_UNWIND) && USE_UNWIND
66+ #ifndef UNW_LOCAL_ONLY
67+ #define UNW_LOCAL_ONLY
68+ #endif
69+ #include < libunwind.h>
70+ #endif
71+
5572namespace {
5673
5774// This is adapted from
5875// https://github.com/scylladb/seastar/blob/master/core/exception_hacks.hh
5976// https://github.com/scylladb/seastar/blob/master/core/exception_hacks.cc
6077
78+ constexpr size_t MAX_PHDR_CACHE_LOADED_OBJECTS = 4096 ;
79+ constexpr size_t MAX_PHDR_CACHE_PROGRAM_HEADERS = 128 ;
80+ constexpr size_t MAX_PHDR_CACHE_OBJECT_NAME = 4096 ;
81+
6182using DLIterateFunction = int (*)(int (*callback)(dl_phdr_info* info, size_t size, void * data),
6283 void * data);
6384
@@ -69,74 +90,226 @@ DLIterateFunction getOriginalDLIteratePHDR() {
6990 return reinterpret_cast <DLIterateFunction>(func);
7091}
7192
72- using PHDRCache = std::vector<dl_phdr_info>;
93+ struct RawPHDRCacheEntry {
94+ dl_phdr_info info {};
95+ std::array<ElfW(Phdr), MAX_PHDR_CACHE_PROGRAM_HEADERS > phdrs {};
96+ size_t phdr_count = 0 ;
97+ std::array<char , MAX_PHDR_CACHE_OBJECT_NAME > name {};
98+ uintptr_t address_begin = std::numeric_limits<uintptr_t >::max();
99+ uintptr_t address_end = 0 ;
100+ };
101+
102+ struct RawPHDRCacheSnapshot {
103+ std::array<RawPHDRCacheEntry, MAX_PHDR_CACHE_LOADED_OBJECTS > entries {};
104+ size_t size = 0 ;
105+ bool overflow = false ;
106+ bool phdr_truncated = false ;
107+ bool name_truncated = false ;
108+ };
109+
110+ struct PHDRCacheEntry {
111+ dl_phdr_info info {};
112+ std::vector<ElfW(Phdr)> phdrs;
113+ std::string name;
114+ uintptr_t address_begin = std::numeric_limits<uintptr_t >::max();
115+ uintptr_t address_end = 0 ;
116+
117+ bool contains (uintptr_t ip) const {
118+ return ip == 0 || (address_begin <= ip && ip < address_end);
119+ }
120+ };
121+
122+ using PHDRCache = std::vector<PHDRCacheEntry>;
73123std::atomic<PHDRCache*> phdr_cache {};
124+ // This flag is flipped inside the stack-trace signal handler. Force a static TLS access model so
125+ // reading it from our dl_iterate_phdr interposer does not call into the dynamic loader's TLS path.
126+ __thread bool use_phdr_cache __attribute__ ((tls_model(" initial-exec" ))) = false;
74127
75- } // namespace
128+ uintptr_t saturated_segment_end (uintptr_t begin, uintptr_t size) {
129+ const uintptr_t max_address = std::numeric_limits<uintptr_t >::max ();
130+ return begin > max_address - size ? max_address : begin + size;
131+ }
76132
77- // Jemalloc heap profile follows libgcc's way of backtracing by default.
78- // rewrites dl_iterate_phdr will cause Jemalloc to fail to run after enable profile.
133+ void copy_object_name (const char * source, RawPHDRCacheEntry* entry,
134+ RawPHDRCacheSnapshot* snapshot) {
135+ if (source == nullptr ) {
136+ entry->name [0 ] = ' \0 ' ;
137+ return ;
138+ }
79139
80- // TODO, two solutions:
81- // 1. Jemalloc specifies GNU libunwind as the prof backtracing way, but my test failed,
82- // `--enable-prof-libunwind` not work: https://github.com/jemalloc/jemalloc/issues/2504
83- // 2. ClickHouse/libunwind solves Jemalloc profile backtracing, but the branch of ClickHouse/libunwind
84- // has been out of touch with GNU libunwind and LLVM libunwind, which will leave the fate to others.
85- /*
86- extern "C"
87- #ifndef __clang__
88- [[gnu::visibility("default")]] [[gnu::externally_visible]]
89- #endif
90- int
91- dl_iterate_phdr(int (*callback)(dl_phdr_info* info, size_t size, void* data), void* data) {
92- auto* current_phdr_cache = phdr_cache.load();
93- if (!current_phdr_cache) {
94- // Cache is not yet populated, pass through to the original function.
95- return getOriginalDLIteratePHDR()(callback, data);
140+ size_t length = 0 ;
141+ while (length + 1 < entry->name .size () && source[length] != ' \0 ' ) {
142+ entry->name [length] = source[length];
143+ ++length;
144+ }
145+ entry->name [length] = ' \0 ' ;
146+ if (source[length] != ' \0 ' ) {
147+ snapshot->name_truncated = true ;
148+ }
149+ }
150+
151+ int collectPHDRCacheEntry (dl_phdr_info* info, size_t /* size*/ , void * data) {
152+ auto * snapshot = reinterpret_cast <RawPHDRCacheSnapshot*>(data);
153+ if (snapshot->size >= snapshot->entries .size ()) {
154+ snapshot->overflow = true ;
155+ return 0 ;
156+ }
157+
158+ auto & entry = snapshot->entries [snapshot->size ++];
159+ entry.info = *info;
160+ copy_object_name (info->dlpi_name , &entry, snapshot);
161+
162+ const size_t phdr_count = std::min<size_t >(info->dlpi_phnum , entry.phdrs .size ());
163+ if (phdr_count < info->dlpi_phnum ) {
164+ snapshot->phdr_truncated = true ;
165+ }
166+ entry.phdr_count = phdr_count;
167+ entry.info .dlpi_phnum = static_cast <ElfW (Half)>(phdr_count);
168+ entry.info .dlpi_name = nullptr ;
169+ entry.info .dlpi_phdr = nullptr ;
170+
171+ if (info->dlpi_phdr == nullptr ) {
172+ return 0 ;
173+ }
174+ std::memcpy (entry.phdrs .data (), info->dlpi_phdr , phdr_count * sizeof (ElfW (Phdr)));
175+
176+ for (size_t i = 0 ; i < phdr_count; ++i) {
177+ const auto & phdr = entry.phdrs [i];
178+ if (phdr.p_type != PT_LOAD || phdr.p_memsz == 0 ) {
179+ continue ;
180+ }
181+ const auto begin = static_cast <uintptr_t >(info->dlpi_addr + phdr.p_vaddr );
182+ const auto end = saturated_segment_end (begin, static_cast <uintptr_t >(phdr.p_memsz ));
183+ entry.address_begin = std::min (entry.address_begin , begin);
184+ entry.address_end = std::max (entry.address_end , end);
185+ }
186+ return 0 ;
187+ }
188+
189+ PHDRCache* buildPHDRCache (const RawPHDRCacheSnapshot& snapshot) {
190+ auto * cache = new PHDRCache;
191+ cache->reserve (snapshot.size );
192+ for (size_t i = 0 ; i < snapshot.size ; ++i) {
193+ const auto & raw_entry = snapshot.entries [i];
194+ PHDRCacheEntry entry;
195+ entry.info = raw_entry.info ;
196+ entry.phdrs .assign (raw_entry.phdrs .begin (), raw_entry.phdrs .begin () + raw_entry.phdr_count );
197+ entry.name = raw_entry.name .data ();
198+ entry.address_begin = raw_entry.address_begin ;
199+ entry.address_end = raw_entry.address_end ;
200+ cache->emplace_back (std::move (entry));
201+ }
202+ for (auto & entry : *cache) {
203+ entry.info .dlpi_phdr = entry.phdrs .data ();
204+ entry.info .dlpi_name = entry.name .c_str ();
205+ }
206+ return cache;
207+ }
208+
209+ int iteratePHDRCache (int (*callback)(dl_phdr_info* info, size_t size, void * data), void* data,
210+ uintptr_t ip) {
211+ auto * current_phdr_cache = phdr_cache.load (std::memory_order_acquire);
212+ if (current_phdr_cache == nullptr ) {
213+ return 0 ;
96214 }
97215
98216 int result = 0 ;
99217 for (auto & entry : *current_phdr_cache) {
100- result = callback(&entry, offsetof(dl_phdr_info, dlpi_adds), data);
218+ if (!entry.contains (ip)) {
219+ continue ;
220+ }
221+ result = callback (&entry.info , sizeof (dl_phdr_info), data);
101222 if (result != 0 ) {
102223 break ;
103224 }
104225 }
105226 return result;
106227}
107- */
228+
229+ } // namespace
230+
231+ extern " C"
232+ #ifndef __clang__
233+ [[gnu::visibility("default")]] [[gnu::externally_visible]]
234+ #endif
235+ int
236+ dl_iterate_phdr (int (*callback)(dl_phdr_info* info, size_t size, void * data), void* data) {
237+ if (!use_phdr_cache) {
238+ return getOriginalDLIteratePHDR ()(callback, data);
239+ }
240+
241+ return iteratePHDRCache (callback, data, 0 );
242+ }
243+
244+ extern " C"
245+ #ifndef __clang__
246+ [[gnu::visibility("default")]] [[gnu::externally_visible]]
247+ #endif
248+ int
249+ doris_unwind_iterate_phdr (int (*callback)(dl_phdr_info* info, size_t size, void * data),
250+ void* data, uintptr_t ip) {
251+ return iteratePHDRCache (callback, data, ip);
252+ }
108253
109254#include " util/debug/leak_annotations.h"
110255
111256void updatePHDRCache () {
112257 // Fill out ELF header cache for access without locking.
113- // This assumes no dynamic object loading/unloading after this point
258+ // Old snapshots are intentionally kept alive because another thread may already be unwinding
259+ // through the previous cache when a Doris-controlled dlopen/dlclose refreshes this one.
114260
115- PHDRCache* new_phdr_cache = new PHDRCache ;
261+ auto raw_snapshot = std::make_unique<RawPHDRCacheSnapshot>() ;
116262 getOriginalDLIteratePHDR ()(
117- [](dl_phdr_info* info, size_t /* size*/ , void * data) {
263+ [](dl_phdr_info* info, size_t size, void * data) {
118264 // `info` is created by dl_iterate_phdr, which is a non-instrumented
119265 // libc function, so we have to unpoison it manually.
120266 __msan_unpoison (info, sizeof (*info));
121267
122- reinterpret_cast <PHDRCache*>(data)->push_back (*info);
123- return 0 ;
268+ return collectPHDRCacheEntry (info, size, data);
124269 },
125- new_phdr_cache);
126- phdr_cache.store (new_phdr_cache);
270+ raw_snapshot.get ());
271+
272+ PHDRCache* new_phdr_cache = buildPHDRCache (*raw_snapshot);
273+ phdr_cache.store (new_phdr_cache, std::memory_order_release);
127274
128275 // / Memory is intentionally leaked.
129276 ANNOTATE_LEAKING_OBJECT_PTR (new_phdr_cache);
130277}
131278
132279bool hasPHDRCache () {
133- return phdr_cache.load () != nullptr ;
280+ return phdr_cache.load (std::memory_order_acquire) != nullptr ;
281+ }
282+
283+ void configureLibunwindPHDRCache () {
284+ #if defined(USE_UNWIND) && USE_UNWIND
285+ static std::once_flag once;
286+ std::call_once (once, [] {
287+ unw_context_t context;
288+ unw_cursor_t cursor;
289+ (void )unw_getcontext (&context);
290+ (void )unw_init_local (&cursor, &context);
291+ // Doris-patched libunwind gets FDEs from the PHDR snapshot. Disable the global DWARF
292+ // register-state cache so a signal handler cannot self-deadlock on libunwind's cache mutex
293+ // after interrupting a thread that was already unwinding.
294+ (void )unw_set_caching_policy (unw_local_addr_space, UNW_CACHE_NONE );
295+ });
296+ #endif
297+ }
298+
299+ ScopedPHDRCacheRead::ScopedPHDRCacheRead () : _previous(use_phdr_cache) {
300+ use_phdr_cache = true ;
301+ }
302+
303+ ScopedPHDRCacheRead::~ScopedPHDRCacheRead () {
304+ use_phdr_cache = _previous;
134305}
135306
136307#else
137308
138309void updatePHDRCache () {}
139310
311+ void configureLibunwindPHDRCache () {}
312+
140313#if defined(USE_MUSL)
141314// / With statically linked with musl, dl_iterate_phdr is immutable.
142315bool hasPHDRCache () {
@@ -148,4 +321,8 @@ bool hasPHDRCache() {
148321}
149322#endif
150323
324+ ScopedPHDRCacheRead::ScopedPHDRCacheRead () = default ;
325+
326+ ScopedPHDRCacheRead::~ScopedPHDRCacheRead () = default ;
327+
151328#endif
0 commit comments