Modification of OBJ_NAME is not synchronized and can cause heap corruption during initialization in 1.1.0 #3505
Description
During initialization, the loads of ciphers and digests are guarded behind two separate OPENSSL_ONCEs: https://github.com/openssl/openssl/blob/master/crypto/init.c#L152-L184. However, both EVP_add_cipher and EVP_add_digest modify the global OBJ_NAME data structure, with no additional synchronization. Because callers can pick which bits of libcrypto they want to initialize, if two threads are in OPENSSL_init_crypto at the same time, one can be executing ossl_init_add_all_ciphers and the other can be executing ossl_init_add_all_digests, which then can cause heap corruption as they both end up in OBJ_NAME_add at the same time.
Here's a pair of stack traces taken from a core dump triggered off of an abort when malloc detected the heap corruption:
(gdb) bt
#0 0x00007fa88ab33a10 in raise () from /usr/lib/libc.so.6
#1 0x00007fa88ab3513a in abort () from /usr/lib/libc.so.6
#2 0x00007fa88ab722b0 in __libc_message () from /usr/lib/libc.so.6
#3 0x00007fa88ab7890e in malloc_printerr () from /usr/lib/libc.so.6
#4 0x00007fa88ab7911e in _int_free () from /usr/lib/libc.so.6
#5 0x00007fa88ab7b8a8 in _int_realloc () from /usr/lib/libc.so.6
#6 0x00007fa88ab7cc09 in realloc () from /usr/lib/libc.so.6
#7 0x00007fa88b84096c in expand (lh=0x7fa87c0009d0) at crypto/lhash/lhash.c:214
#8 OPENSSL_LH_insert (lh=0x7fa87c0009d0, data=data@entry=0x7fa884004180) at crypto/lhash/lhash.c:77
#9 0x00007fa88b85036d in lh_OBJ_NAME_insert (d=0x7fa884004180, lh=<optimized out>) at crypto/objects/obj_lcl.h:12
#10 OBJ_NAME_add (name=0x7fa88b8ecaf4 "RSA-RIPEMD160", type=1, type@entry=32769, data=data@entry=0x7fa88b8ecaf8 "RIPEMD160") at crypto/objects/o_names.c:208
#11 0x00007fa88b835bfb in EVP_add_digest (md=0x7fa88bb48b60 <ripemd160_md>) at crypto/evp/names.c:48
#12 0x00007fa88b821cca in openssl_add_all_digests_int () at crypto/evp/c_alld.c:34
#13 0x00007fa88b83e939 in ossl_init_add_all_digests () at crypto/init.c:176
#14 ossl_init_add_all_digests_ossl_ () at crypto/init.c:165
#15 0x00007fa88b0ca399 in __pthread_once_slow () from /usr/lib/libpthread.so.0
#16 0x00007fa88b89b9c9 in CRYPTO_THREAD_run_once (once=once@entry=0x7fa88bb6a114 <add_all_digests>, init=init@entry=0x7fa88b83e930 <ossl_init_add_all_digests_ossl_>) at crypto/threads_pthread.c:106
#17 0x00007fa88b83ef83 in OPENSSL_init_crypto (opts=opts@entry=8, settings=settings@entry=0x0) at crypto/init.c:538
#18 0x00007fa88b835c60 in EVP_get_digestbyname (name=0x7fa88b8ecac2 "SHA1") at crypto/evp/names.c:73
#19 0x00007fa88b85f2da in pkcs12_gen_mac (p12=0x7fa8840029a0, p12=0x7fa8840029a0, pkcs12_key_gen=<optimized out>, maclen=0x7fa88a5e91bc,
mac=0x7fa88a5e9210 "\336\067\371\246\211~\322o9Y\257\205|7~%\205h\032\036@\211\265\257sߊ\351|3\347\001\060\202\017\370\060\202\006/\006\t*\206H\206\367\r\006\025\006\t*\206H\206\367\r\001\a\001\060\034\006\223\316\\\240~\251\242\002\002\b", passlen=-1,
pass=0x7fa884003ba0 "mypass") at crypto/pkcs12/p12_mutl.c:103
#20 PKCS12_verify_mac (p12=p12@entry=0x7fa8840029a0, pass=pass@entry=0x7fa884003ba0 "mypass", passlen=passlen@entry=-1) at crypto/pkcs12/p12_mutl.c:156
#21 0x00007fa88b85edb0 in PKCS12_parse (p12=0x7fa8840029a0, pass=0x7fa884003ba0 "mypass", pkey=0x7fa88a5e9400, cert=0x7fa88a5e9408, ca=0x7fa88a5e9410) at crypto/pkcs12/p12_kiss.c:68
#22 0x00005558241ee296 in openssl::pkcs12::Pkcs12Ref::parse (self=0x7fa8840029a0, pass="mypass") at /home/sfackler/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.12/src/pkcs12.rs:37
#23 0x00005558241dc26c in native_tls::imp::Pkcs12::from_der (buf=&[u8](len: 4173) = {...}, pass="mypass") at /home/sfackler/.cargo/registry/src/github.com-1ecc6299db9ec823/native-tls-0.1.2/src/imp/openssl.rs:76
#24 0x00005558241dcbdb in native_tls::Pkcs12::from_der (der=&[u8](len: 4173) = {...}, password="mypass") at /home/sfackler/.cargo/registry/src/github.com-1ecc6299db9ec823/native-tls-0.1.2/src/lib.rs:181
#25 0x000055582410322e in hyper_native_tls::NativeTlsServer::new<&str> (identity="test/identity.p12", password="mypass") at src/lib.rs:168
#26 0x0000555824103d6c in hyper_native_tls::test::server () at src/lib.rs:257
#27 0x0000555824112572 in test::run_test::{{closure}} () at /checkout/src/libtest/lib.rs:1451
#28 core::ops::FnOnce::call_once<closure,(())> () at /checkout/src/libcore/ops.rs:2663
#29 test::{{impl}}::call_box<(),closure> () at /checkout/src/libtest/lib.rs:140
#30 0x0000555824212f2b in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#31 0x0000555824106774 in std::panicking::try<(),std::panic::AssertUnwindSafe<closure>> () at /checkout/src/libstd/panicking.rs:433
#32 std::panic::catch_unwind<std::panic::AssertUnwindSafe<closure>,()> () at /checkout/src/libstd/panic.rs:361
#33 test::run_test::run_test_inner::{{closure}} () at /checkout/src/libtest/lib.rs:1390
#34 std::sys_common::backtrace::__rust_begin_short_backtrace<closure,()> () at /checkout/src/libstd/sys_common/backtrace.rs:136
#35 0x0000555824107513 in std::thread::{{impl}}::spawn::{{closure}}::{{closure}}<closure,()> () at /checkout/src/libstd/thread/mod.rs:363
#36 std::panic::{{impl}}::call_once<(),closure> () at /checkout/src/libstd/panic.rs:296
#37 std::panicking::try::do_call<std::panic::AssertUnwindSafe<closure>,()> () at /checkout/src/libstd/panicking.rs:454
#38 0x0000555824212f2b in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#39 0x000055582410d37d in std::panicking::try<(),std::panic::AssertUnwindSafe<closure>> () at /checkout/src/libstd/panicking.rs:433
#40 std::panic::catch_unwind<std::panic::AssertUnwindSafe<closure>,()> () at /checkout/src/libstd/panic.rs:361
#41 std::thread::{{impl}}::spawn::{{closure}}<closure,()> () at /checkout/src/libstd/thread/mod.rs:362
#42 alloc::boxed::{{impl}}::call_box<(),closure> () at /checkout/src/liballoc/boxed.rs:648
#43 0x000055582420aca6 in alloc::boxed::{{impl}}::call_once<(),()> () at /checkout/src/liballoc/boxed.rs:658
#44 std::sys_common::thread::start_thread () at /checkout/src/libstd/sys_common/thread.rs:21
#45 std::sys::imp::thread::{{impl}}::new::thread_start () at /checkout/src/libstd/sys/unix/thread.rs:84
#46 0x00007fa88b0c22e7 in start_thread () from /usr/lib/libpthread.so.0
#47 0x00007fa88abec54f in clone () from /usr/lib/libc.so.6
(gdb) info threads
Id Target Id Frame
* 1 Thread 0x7fa88a5eb700 (LWP 95370) 0x00007fa88ab33a10 in raise () from /usr/lib/libc.so.6
2 Thread 0x7fa88bfdfc00 (LWP 95368) 0x00007fa88b0c8b63 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /usr/lib/libpthread.so.0
3 Thread 0x7fa88a7ec700 (LWP 95369) 0x00007fa88abf90eb in __lll_lock_wait_private () from /usr/lib/libc.so.6
(gdb) thread 3
[Switching to thread 3 (Thread 0x7fa88a7ec700 (LWP 95369))]
#0 0x00007fa88abf90eb in __lll_lock_wait_private () from /usr/lib/libc.so.6
(gdb) bt
#0 0x00007fa88abf90eb in __lll_lock_wait_private () from /usr/lib/libc.so.6
#1 0x00007fa88ab7c712 in malloc () from /usr/lib/libc.so.6
#2 0x00007fa88b8409d6 in OPENSSL_LH_insert (lh=0x7fa87c0009d0, data=data@entry=0x7fa87c0047d0) at crypto/lhash/lhash.c:83
#3 0x00007fa88b85036d in lh_OBJ_NAME_insert (d=0x7fa87c0047d0, lh=<optimized out>) at crypto/objects/obj_lcl.h:12
#4 OBJ_NAME_add (name=0x7fa88b8ef828 "camellia-256-ecb", type=type@entry=2, data=0x7fa88bb47a20 <camellia_256_ecb> "\364\002") at crypto/objects/o_names.c:208
#5 0x00007fa88b835b78 in EVP_add_cipher (c=<optimized out>) at crypto/evp/names.c:28
#6 0x00007fa88b821b6e in openssl_add_all_ciphers_int () at crypto/evp/c_allc.c:201
#7 0x00007fa88b83e959 in ossl_init_add_all_ciphers () at crypto/init.c:159
#8 ossl_init_add_all_ciphers_ossl_ () at crypto/init.c:148
#9 0x00007fa88b0ca399 in __pthread_once_slow () from /usr/lib/libpthread.so.0
#10 0x00007fa88b89b9c9 in CRYPTO_THREAD_run_once (once=once@entry=0x7fa88bb6a11c <add_all_ciphers>, init=init@entry=0x7fa88b83e950 <ossl_init_add_all_ciphers_ossl_>) at crypto/threads_pthread.c:106
#11 0x00007fa88b83ee93 in OPENSSL_init_crypto (opts=opts@entry=2097164, settings=0x0) at crypto/init.c:530
#12 0x00007fa88bb9a154 in OPENSSL_init_ssl (opts=opts@entry=2097152, settings=settings@entry=0x0) at ssl/ssl_init.c:194
#13 0x00007fa88bb9d73e in SSL_CTX_new (meth=0x7fa88bdd4680 <TLS_method_data.20701>) at ssl/ssl_lib.c:2358
#14 0x00005558241f0023 in openssl::ssl::SslContextBuilder::new (method=SslMethod = {...}) at /home/sfackler/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.12/src/ssl/mod.rs:599
#15 0x00005558241eea63 in openssl::ssl::connector::ctx (method=SslMethod = {...}) at /home/sfackler/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.12/src/ssl/connector.rs:23
#16 0x00005558241eee43 in openssl::ssl::connector::SslConnectorBuilder::new (method=SslMethod = {...}) at /home/sfackler/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.12/src/ssl/connector.rs:52
#17 0x00005558241dc563 in native_tls::imp::TlsConnector::builder () at /home/sfackler/.cargo/registry/src/github.com-1ecc6299db9ec823/native-tls-0.1.2/src/imp/openssl.rs:183
#18 0x00005558241dcfdb in native_tls::TlsConnector::builder () at /home/sfackler/.cargo/registry/src/github.com-1ecc6299db9ec823/native-tls-0.1.2/src/lib.rs:373
#19 0x0000555824102a7f in hyper_native_tls::NativeTlsClient::new () at src/lib.rs:117
#20 0x0000555824103b37 in hyper_native_tls::test::client () at src/lib.rs:245
#21 0x0000555824112572 in test::run_test::{{closure}} () at /checkout/src/libtest/lib.rs:1451
#22 core::ops::FnOnce::call_once<closure,(())> () at /checkout/src/libcore/ops.rs:2663
#23 test::{{impl}}::call_box<(),closure> () at /checkout/src/libtest/lib.rs:140
#24 0x0000555824212f2b in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#25 0x0000555824106774 in std::panicking::try<(),std::panic::AssertUnwindSafe<closure>> () at /checkout/src/libstd/panicking.rs:433
#26 std::panic::catch_unwind<std::panic::AssertUnwindSafe<closure>,()> () at /checkout/src/libstd/panic.rs:361
#27 test::run_test::run_test_inner::{{closure}} () at /checkout/src/libtest/lib.rs:1390
#28 std::sys_common::backtrace::__rust_begin_short_backtrace<closure,()> () at /checkout/src/libstd/sys_common/backtrace.rs:136
#29 0x0000555824107513 in std::thread::{{impl}}::spawn::{{closure}}::{{closure}}<closure,()> () at /checkout/src/libstd/thread/mod.rs:363
#30 std::panic::{{impl}}::call_once<(),closure> () at /checkout/src/libstd/panic.rs:296
#31 std::panicking::try::do_call<std::panic::AssertUnwindSafe<closure>,()> () at /checkout/src/libstd/panicking.rs:454
#32 0x0000555824212f2b in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#33 0x000055582410d37d in std::panicking::try<(),std::panic::AssertUnwindSafe<closure>> () at /checkout/src/libstd/panicking.rs:433
#34 std::panic::catch_unwind<std::panic::AssertUnwindSafe<closure>,()> () at /checkout/src/libstd/panic.rs:361
#35 std::thread::{{impl}}::spawn::{{closure}}<closure,()> () at /checkout/src/libstd/thread/mod.rs:362
#36 alloc::boxed::{{impl}}::call_box<(),closure> () at /checkout/src/liballoc/boxed.rs:648
#37 0x000055582420aca6 in alloc::boxed::{{impl}}::call_once<(),()> () at /checkout/src/liballoc/boxed.rs:658
#38 std::sys_common::thread::start_thread () at /checkout/src/libstd/sys_common/thread.rs:21
#39 std::sys::imp::thread::{{impl}}::new::thread_start () at /checkout/src/libstd/sys/unix/thread.rs:84
#40 0x00007fa88b0c22e7 in start_thread () from /usr/lib/libpthread.so.0
#41 0x00007fa88abec54f in clone () from /usr/lib/libc.so.6
I spent a couple of minutes trying to make a minimal reproduction, but it's a bit tricky since the race is very tight. The trace above can be somewhat reliably generated by running cargo test in https://github.com/sfackler/hyper-native-tls linked against OpenSSL 1.1.0e.