Skip to content

Commit 681ede6

Browse files
committed
ARROW-15604: [C++][CI] Sporadic ThreadSanitizer failure with OpenTracing
The particular cause of the failure was: ``` void InputReceived(ExecNode* input, ExecBatch batch) override { EVENT(span_, "InputReceived", {{"batch.length", batch.length}}); util::tracing::Span span; START_SPAN_WITH_PARENT(span, span_, "InputReceived", {{"node.label", label()}, {"batch.length", batch.length}}); DCHECK_EQ(input, inputs_[0]); bool did_push = producer_.Push(std::move(batch)); if (!did_push) return; // producer_ was Closed already if (input_counter_.Increment()) { // This will mark the exec plan finished Finish(); // At this point the main thread is generally free to exit but span hasn't been destructed yet } } ``` It could be fixed with scoping: ``` void InputReceived(ExecNode* input, ExecBatch batch) override { EVENT(span_, "InputReceived", {{"batch.length", batch.length}}); { util::tracing::Span span; START_SPAN_WITH_PARENT(span, span_, "InputReceived", {{"node.label", label()}, {"batch.length", batch.length}}); DCHECK_EQ(input, inputs_[0]); bool did_push = producer_.Push(std::move(batch)); if (!did_push) return; // producer_ was Closed already } if (input_counter_.Increment()) { // This will mark the exec plan finished Finish(); } } ``` However, I suspect this would quickly get tedious. Instead it seems we can control the lifetime of the OT runtime storage and bind it to our worker threads. In the future we may want to consider doing similar tricks to keep alive the memory pool, global thread pools, etc. Closes #12408 from westonpace/bugfix/ARROW-15604--sporadic-test-failure-with-ot Authored-by: Weston Pace <[email protected]> Signed-off-by: Weston Pace <[email protected]>
1 parent 6240eae commit 681ede6

10 files changed

Lines changed: 128 additions & 17 deletions

File tree

‎ci/docker/fedora-35-cpp.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ ENV ARROW_BUILD_TESTS=ON \
8585
ARROW_WITH_BROTLI=ON \
8686
ARROW_WITH_BZ2=ON \
8787
ARROW_WITH_LZ4=ON \
88-
ARROW_WITH_OPENTELEMETRY=OFF \
88+
ARROW_WITH_OPENTELEMETRY=ON \
8989
ARROW_WITH_SNAPPY=ON \
9090
ARROW_WITH_ZLIB=ON \
9191
ARROW_WITH_ZSTD=ON \

‎ci/docker/ubuntu-20.04-cpp.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ ENV ARROW_BUILD_TESTS=ON \
142142
ARROW_WITH_BROTLI=ON \
143143
ARROW_WITH_BZ2=ON \
144144
ARROW_WITH_LZ4=ON \
145-
ARROW_WITH_OPENTELEMETRY=OFF \
145+
ARROW_WITH_OPENTELEMETRY=ON \
146146
ARROW_WITH_SNAPPY=ON \
147147
ARROW_WITH_ZLIB=ON \
148148
ARROW_WITH_ZSTD=ON \

‎ci/docker/ubuntu-22.04-cpp.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ ENV ARROW_BUILD_TESTS=ON \
170170
ARROW_WITH_BROTLI=ON \
171171
ARROW_WITH_BZ2=ON \
172172
ARROW_WITH_LZ4=ON \
173-
ARROW_WITH_OPENTELEMETRY=OFF \
173+
ARROW_WITH_OPENTELEMETRY=ON \
174174
ARROW_WITH_SNAPPY=ON \
175175
ARROW_WITH_ZLIB=ON \
176176
ARROW_WITH_ZSTD=ON \

‎cpp/cmake_modules/ThirdpartyToolchain.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4096,7 +4096,7 @@ macro(build_opentelemetry)
40964096
# N.B. OTel targets and libraries don't follow any consistent naming scheme
40974097
if(_OPENTELEMETRY_LIB STREQUAL "http_client_curl")
40984098
set(_OPENTELEMETRY_STATIC_LIBRARY
4099-
"${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_OPENTELEMETRY_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}"
4099+
"${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_${_OPENTELEMETRY_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}"
41004100
)
41014101
elseif(_OPENTELEMETRY_LIB STREQUAL "ostream_span_exporter")
41024102
set(_OPENTELEMETRY_STATIC_LIBRARY

‎cpp/src/arrow/util/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ add_arrow_test(utility-test
6767
tdigest_test.cc
6868
test_common.cc
6969
time_test.cc
70+
tracing_test.cc
7071
trie_test.cc
7172
uri_test.cc
7273
utf8_util_test.cc

‎cpp/src/arrow/util/thread_pool.cc‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ namespace internal {
3737

3838
Executor::~Executor() = default;
3939

40+
// By default we do nothing here. Subclasses that expect to be allocated
41+
// with static storage duration should override this and ensure any threads respect the
42+
// lifetime of these resources.
43+
void Executor::KeepAlive(std::shared_ptr<Resource> resource) {}
44+
4045
namespace {
4146

4247
struct Task {
@@ -200,6 +205,8 @@ struct ThreadPool::State {
200205
// Are we shutting down?
201206
bool please_shutdown_ = false;
202207
bool quick_shutdown_ = false;
208+
209+
std::vector<std::shared_ptr<Resource>> kept_alive_resources_;
203210
};
204211

205212
// The worker loop is an independent function so that it can keep running
@@ -417,19 +424,11 @@ Status ThreadPool::SpawnReal(TaskHints hints, FnOnce<void()> task, StopToken sto
417424
StopCallback&& stop_callback) {
418425
{
419426
ProtectAgainstFork();
420-
std::lock_guard<std::mutex> lock(state_->mutex_);
421-
if (state_->please_shutdown_) {
422-
return Status::Invalid("operation forbidden during or after shutdown");
423-
}
424-
CollectFinishedWorkersUnlocked();
425-
state_->tasks_queued_or_running_++;
426-
if (static_cast<int>(state_->workers_.size()) < state_->tasks_queued_or_running_ &&
427-
state_->desired_capacity_ > static_cast<int>(state_->workers_.size())) {
428-
// We can still spin up more workers so spin up a new worker
429-
LaunchWorkersUnlocked(/*threads=*/1);
430-
}
431427
#ifdef ARROW_WITH_OPENTELEMETRY
432428
// Wrap the task to propagate a parent tracing span to it
429+
// This task-wrapping needs to be done before we grab the mutex because the
430+
// first call to OT (whatever that happens to be) will attempt to grab this mutex
431+
// when calling KeepAlive to keep the OT infrastructure alive.
433432
struct {
434433
void operator()() {
435434
auto scope = ::arrow::internal::tracing::GetTracer()->WithActiveSpan(activeSpan);
@@ -441,13 +440,30 @@ Status ThreadPool::SpawnReal(TaskHints hints, FnOnce<void()> task, StopToken sto
441440
::arrow::internal::tracing::GetTracer()->GetCurrentSpan()};
442441
task = std::move(wrapper);
443442
#endif
443+
std::lock_guard<std::mutex> lock(state_->mutex_);
444+
if (state_->please_shutdown_) {
445+
return Status::Invalid("operation forbidden during or after shutdown");
446+
}
447+
CollectFinishedWorkersUnlocked();
448+
state_->tasks_queued_or_running_++;
449+
if (static_cast<int>(state_->workers_.size()) < state_->tasks_queued_or_running_ &&
450+
state_->desired_capacity_ > static_cast<int>(state_->workers_.size())) {
451+
// We can still spin up more workers so spin up a new worker
452+
LaunchWorkersUnlocked(/*threads=*/1);
453+
}
444454
state_->pending_tasks_.push_back(
445455
{std::move(task), std::move(stop_token), std::move(stop_callback)});
446456
}
447457
state_->cv_.notify_one();
448458
return Status::OK();
449459
}
450460

461+
void ThreadPool::KeepAlive(std::shared_ptr<Executor::Resource> resource) {
462+
// Seems unlikely but we might as well guard against concurrent calls to KeepAlive
463+
std::lock_guard<std::mutex> lk(state_->mutex_);
464+
state_->kept_alive_resources_.push_back(std::move(resource));
465+
}
466+
451467
Result<std::shared_ptr<ThreadPool>> ThreadPool::Make(int threads) {
452468
auto pool = std::shared_ptr<ThreadPool>(new ThreadPool());
453469
RETURN_NOT_OK(pool->SetCapacity(threads));

‎cpp/src/arrow/util/thread_pool.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,27 @@ class ARROW_EXPORT Executor {
202202
// Executor. Returns false if this Executor does not support this property.
203203
virtual bool OwnsThisThread() { return false; }
204204

205+
/// \brief An interface to represent something with a custom destructor
206+
///
207+
/// \see KeepAlive
208+
class ARROW_EXPORT Resource {
209+
public:
210+
virtual ~Resource() = default;
211+
};
212+
213+
/// \brief Keep a resource alive until all executor threads have terminated
214+
///
215+
/// Executors may have static storage duration. In particular, the CPU and I/O
216+
/// executors are currently implemented this way. These threads may access other
217+
/// objects with static storage duration such as the OpenTelemetry runtime context
218+
/// the default memory pool, or other static executors.
219+
///
220+
/// The order in which these objects are destroyed is difficult to control. In order
221+
/// to ensure those objects remain alive until all threads have finished those objects
222+
/// should be wrapped in a Resource object and passed into this method. The given
223+
/// shared_ptr will be kept alive until all threads have finished their worker loops.
224+
virtual void KeepAlive(std::shared_ptr<Resource> resource);
225+
205226
protected:
206227
ARROW_DISALLOW_COPY_AND_ASSIGN(Executor);
207228

@@ -434,6 +455,8 @@ class ARROW_EXPORT ThreadPool : public Executor {
434455
// This is useful for sequencing tests
435456
void WaitForIdle();
436457

458+
void KeepAlive(std::shared_ptr<Executor::Resource> resource) override;
459+
437460
struct State;
438461

439462
protected:

‎cpp/src/arrow/util/tracing_internal.cc‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
// under the License.
1717

1818
#include "arrow/util/tracing_internal.h"
19+
#include "arrow/io/interfaces.h"
20+
#include "arrow/util/thread_pool.h"
1921
#include "arrow/util/tracing.h"
2022

2123
#include <iostream>
@@ -134,7 +136,23 @@ std::unique_ptr<sdktrace::SpanExporter> InitializeExporter() {
134136
return nullptr;
135137
}
136138

139+
struct StorageSingleton : public Executor::Resource {
140+
StorageSingleton()
141+
: storage_(otel::context::RuntimeContext::GetConstRuntimeContextStorage()) {}
142+
nostd::shared_ptr<const otel::context::RuntimeContextStorage> storage_;
143+
};
144+
145+
std::shared_ptr<Executor::Resource> GetStorageSingleton() {
146+
static std::shared_ptr<StorageSingleton> storage_singleton =
147+
std::make_shared<StorageSingleton>();
148+
return storage_singleton;
149+
}
150+
137151
nostd::shared_ptr<sdktrace::TracerProvider> InitializeSdkTracerProvider() {
152+
// Bind the lifetime of the OT runtime context to the CPU and I/O thread
153+
// pools. This will keep OT alive until all thread tasks have finished.
154+
internal::GetCpuThreadPool()->KeepAlive(GetStorageSingleton());
155+
io::default_io_context().executor()->KeepAlive(GetStorageSingleton());
138156
auto exporter = InitializeExporter();
139157
if (exporter) {
140158
sdktrace::BatchSpanProcessorOptions options;

‎cpp/src/arrow/util/tracing_test.cc‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include <thread>
19+
20+
#include <gtest/gtest.h>
21+
22+
#include "arrow/testing/gtest_util.h"
23+
#include "arrow/util/thread_pool.h"
24+
#include "arrow/util/tracing_internal.h"
25+
26+
namespace arrow {
27+
namespace util {
28+
namespace tracing {
29+
30+
#ifdef ARROW_WITH_OPENTELEMETRY
31+
32+
// This test is a regression for ARROW-15604. OT has some static state that
33+
// can be initialized after the CPU and I/O thread pools. We need to make
34+
// sure OT's state persists for the lifetime of any threads in those thread
35+
// pools.
36+
//
37+
// This test checks this by spawning a thread task that will outlive the unit
38+
// test's lifetime and so teardown of static state should begin before this
39+
// thread finishes.
40+
TEST(Tracing, OtLifetime) {
41+
ASSERT_OK(::arrow::internal::GetCpuThreadPool()->Spawn([] {
42+
// This thread will outlive the main test thread.
43+
Span span;
44+
START_SPAN(span, "Test");
45+
SleepFor(0.1);
46+
}));
47+
}
48+
49+
#endif
50+
51+
} // namespace tracing
52+
} // namespace util
53+
} // namespace arrow

‎cpp/thirdparty/versions.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,8 @@ ARROW_MIMALLOC_BUILD_VERSION=v1.7.3
6666
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=0f987bda01ca9df87ec90e9d98c63fa893ee61f3cca565e5ca5ed744fdcc5109
6767
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
6868
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
69-
ARROW_OPENTELEMETRY_BUILD_VERSION=v1.2.0
70-
ARROW_OPENTELEMETRY_BUILD_SHA256_CHECKSUM=7a6420f9e4fa44b81a5b06e30e5e116da71decc9086e5cc4f126e1efc8a397c2
69+
ARROW_OPENTELEMETRY_BUILD_VERSION=v1.3.0
70+
ARROW_OPENTELEMETRY_BUILD_SHA256_CHECKSUM=6a4c43b9c9f753841ebc0fe2717325271f02e2a1d5ddd0b52735c35243629ab3
7171
ARROW_OPENTELEMETRY_PROTO_BUILD_VERSION=v0.11.0
7272
ARROW_OPENTELEMETRY_PROTO_BUILD_SHA256_CHECKSUM=985367f8905e91018e636cbf0d83ab3f834b665c4f5899a27d10cae9657710e2
7373
ARROW_ORC_BUILD_VERSION=1.7.3

0 commit comments

Comments
 (0)