Skip to content

Commit deffec6

Browse files
jmarantzhtuch
jmarantz
authored and
htuch
committed
server: factor out MainCommon as a class, with a run method (#2568)
This is step 2 in the plan to improve startup performance. It makes it possible instantiate the full server stack without running the event loop, so the instantiation can be timed. This also simplifies the code required in main(), though a legacy path is retained allowing existing alternate main()s to continue to work. Risk Level: Medium: this has been extensively tested but deserves scrutiny as it is part of the startup path. Testing: //test/... including sanitizers. Added tests for the new MainCommon instantiation, as well as the legacy main_common() where some of the work was delegated to main(). This is another step toward addressing #2373 Signed-off-by: Joshua Marantz <[email protected]>
1 parent e2117cb commit deffec6

File tree

20 files changed

+383
-152
lines changed

20 files changed

+383
-152
lines changed

‎include/envoy/server/BUILD‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,10 @@ envoy_cc_library(
5757
envoy_cc_library(
5858
name = "hot_restart_interface",
5959
hdrs = ["hot_restart.h"],
60-
deps = ["//include/envoy/event:dispatcher_interface"],
60+
deps = [
61+
"//include/envoy/event:dispatcher_interface",
62+
"//include/envoy/thread:thread_interface",
63+
],
6164
)
6265

6366
envoy_cc_library(

‎include/envoy/server/hot_restart.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
#include "envoy/common/pure.h"
77
#include "envoy/event/dispatcher.h"
8+
#include "envoy/stats/stats.h"
9+
#include "envoy/thread/thread.h"
810

911
namespace Envoy {
1012
namespace Server {
@@ -78,6 +80,21 @@ class HotRestart {
7880
* perform a full or hot restart.
7981
*/
8082
virtual std::string version() PURE;
83+
84+
/**
85+
* @return Thread::BasicLockable& a lock for logging.
86+
*/
87+
virtual Thread::BasicLockable& logLock() PURE;
88+
89+
/**
90+
* @return Thread::BasicLockable& a lock for access logs.
91+
*/
92+
virtual Thread::BasicLockable& accessLogLock() PURE;
93+
94+
/**
95+
* @returns an allocator for stats.
96+
*/
97+
virtual Stats::RawStatDataAllocator& statsAllocator() PURE;
8198
};
8299

83100
} // namespace Server

‎include/envoy/stats/stats.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,5 +281,29 @@ class StoreRoot : public Store {
281281

282282
typedef std::unique_ptr<StoreRoot> StoreRootPtr;
283283

284+
struct RawStatData;
285+
286+
/**
287+
* Abstract interface for allocating a RawStatData.
288+
*/
289+
class RawStatDataAllocator {
290+
public:
291+
virtual ~RawStatDataAllocator() {}
292+
293+
/**
294+
* @return RawStatData* a raw stat data block for a given stat name or nullptr if there is no
295+
* more memory available for stats. The allocator should return a reference counted
296+
* data location by name if one already exists with the same name. This is used for
297+
* intra-process scope swapping as well as inter-process hot restart.
298+
*/
299+
virtual RawStatData* alloc(const std::string& name) PURE;
300+
301+
/**
302+
* Free a raw stat data block. The allocator should handle reference counting and only truly
303+
* free the block if it is no longer needed.
304+
*/
305+
virtual void free(RawStatData& data) PURE;
306+
};
307+
284308
} // namespace Stats
285309
} // namespace Envoy

‎source/common/stats/stats_impl.h‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -191,28 +191,6 @@ struct RawStatData {
191191
static size_t& initializeAndGetMutableMaxObjNameLength(size_t configured_size);
192192
};
193193

194-
/**
195-
* Abstract interface for allocating a RawStatData.
196-
*/
197-
class RawStatDataAllocator {
198-
public:
199-
virtual ~RawStatDataAllocator() {}
200-
201-
/**
202-
* @return RawStatData* a raw stat data block for a given stat name or nullptr if there is no more
203-
* memory available for stats. The allocator may return a reference counted data location
204-
* by name if one already exists with the same name. This is used for intra-process
205-
* scope swapping as well as inter-process hot restart.
206-
*/
207-
virtual RawStatData* alloc(const std::string& name) PURE;
208-
209-
/**
210-
* Free a raw stat data block. The allocator should handle reference counting and only truly
211-
* free the block if it is no longer needed.
212-
*/
213-
virtual void free(RawStatData& data) PURE;
214-
};
215-
216194
/**
217195
* Implementation of the Metric interface. Virtual inheritance is used because the interfaces that
218196
* will inherit from Metric will have other base classes that will also inherit from Metric.

‎source/exe/BUILD‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -79,14 +79,9 @@ envoy_cc_library(
7979
deps = [
8080
":envoy_main_common_lib",
8181
":extra_protocol_proxies_lib",
82-
"//source/server:hot_restart_lib",
83-
"//source/server:options_lib",
8482
"//source/server/config/http:lightstep_lib",
8583
"//source/server/config/http:zipkin_lib",
86-
] + select({
87-
"//bazel:disable_signal_trace": [],
88-
"//conditions:default": [":sigaction_lib"],
89-
}),
84+
],
9085
)
9186

9287
envoy_cc_library(
@@ -101,7 +96,10 @@ envoy_cc_library(
10196
"//source/server:hot_restart_nop_lib",
10297
"//source/server:proto_descriptors_lib",
10398
"//source/server/config_validation:server_lib",
104-
],
99+
] + select({
100+
"//bazel:disable_signal_trace": [],
101+
"//conditions:default": [":sigaction_lib"],
102+
}),
105103
)
106104

107105
envoy_cc_library(

‎source/exe/main.cc‎

Lines changed: 16 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,5 @@
1-
#include <iostream>
2-
#include <memory>
3-
41
#include "exe/main_common.h"
52

6-
#ifdef ENVOY_HANDLE_SIGNALS
7-
#include "exe/signal_action.h"
8-
#endif
9-
10-
#ifdef ENVOY_HOT_RESTART
11-
#include "server/hot_restart_impl.h"
12-
#endif
13-
14-
#include "server/options_impl.h"
15-
16-
#include "spdlog/spdlog.h"
17-
183
// NOLINT(namespace-envoy)
194

205
/**
@@ -25,31 +10,28 @@
2510
* after setting up command line options.
2611
*/
2712
int main(int argc, char** argv) {
28-
#ifdef ENVOY_HANDLE_SIGNALS
29-
// Enabled by default. Control with "bazel --define=signal_trace=disabled"
30-
Envoy::SignalAction handle_sigs;
31-
#endif
32-
3313
#ifdef ENVOY_HOT_RESTART
34-
// Enabled by default, except on OS X. Control with "bazel --define=hot_restart=disabled"
35-
const Envoy::OptionsImpl::HotRestartVersionCb hot_restart_version_cb =
36-
[](uint64_t max_num_stats, uint64_t max_stat_name_len) {
37-
return Envoy::Server::HotRestartImpl::hotRestartVersion(max_num_stats, max_stat_name_len);
38-
};
14+
constexpr bool enable_hot_restart = true;
3915
#else
40-
const Envoy::OptionsImpl::HotRestartVersionCb hot_restart_version_cb = [](uint64_t, uint64_t) {
41-
return "disabled";
42-
};
16+
constexpr bool enable_hot_restart = false;
4317
#endif
4418

45-
std::unique_ptr<Envoy::OptionsImpl> options;
19+
std::unique_ptr<Envoy::MainCommon> main_common;
20+
21+
// Initialize the server's main context under a try/catch loop and simply return EXIT_FAILURE
22+
// as needed. Whatever code in the initialization path that fails is expected to log an error
23+
// message so the user can diagnose.
4624
try {
47-
options = std::make_unique<Envoy::OptionsImpl>(argc, argv, hot_restart_version_cb,
48-
spdlog::level::info);
25+
main_common = std::make_unique<Envoy::MainCommon>(argc, argv, enable_hot_restart);
4926
} catch (const Envoy::NoServingException& e) {
50-
return 0;
27+
return EXIT_SUCCESS;
5128
} catch (const Envoy::MalformedArgvException& e) {
52-
return 1;
29+
return EXIT_FAILURE;
30+
} catch (const Envoy::EnvoyException& e) {
31+
return EXIT_FAILURE;
5332
}
54-
return Envoy::main_common(*options);
33+
34+
// Run the server listener loop outside try/catch blocks, so that unexpected exceptions
35+
// show up as a core-dumps for easier diagnostis.
36+
return main_common->run() ? EXIT_SUCCESS : EXIT_FAILURE;
5537
}

‎source/exe/main_common.cc‎

Lines changed: 89 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1+
#include "exe/main_common.h"
2+
13
#include <iostream>
24
#include <memory>
35

46
#include "common/common/compiler_requirements.h"
57
#include "common/event/libevent.h"
68
#include "common/network/utility.h"
79
#include "common/stats/stats_impl.h"
8-
#include "common/stats/thread_local_store.h"
910

1011
#include "server/config_validation/server.h"
1112
#include "server/drain_manager_impl.h"
@@ -22,79 +23,107 @@
2223
#include "ares.h"
2324

2425
namespace Envoy {
25-
namespace Server {
26-
27-
class ProdComponentFactory : public ComponentFactory {
28-
public:
29-
// Server::DrainManagerFactory
30-
DrainManagerPtr createDrainManager(Instance& server) override {
31-
return DrainManagerPtr{
32-
// The global drain manager only triggers on listener modification, which effectively is
33-
// hot restart at the global level. The per-listener drain managers decide whether to
34-
// to include /healthcheck/fail status.
35-
new DrainManagerImpl(server, envoy::api::v2::Listener_DrainType_MODIFY_ONLY)};
36-
}
3726

38-
Runtime::LoaderPtr createRuntime(Server::Instance& server,
39-
Server::Configuration::Initial& config) override {
40-
return Server::InstanceUtil::createRuntime(server, config);
41-
}
42-
};
27+
Server::DrainManagerPtr ProdComponentFactory::createDrainManager(Server::Instance& server) {
28+
// The global drain manager only triggers on listener modification, which effectively is
29+
// hot restart at the global level. The per-listener drain managers decide whether to
30+
// to include /healthcheck/fail status.
31+
return std::make_unique<Server::DrainManagerImpl>(server,
32+
envoy::api::v2::Listener_DrainType_MODIFY_ONLY);
33+
}
4334

44-
} // namespace Server
35+
Runtime::LoaderPtr ProdComponentFactory::createRuntime(Server::Instance& server,
36+
Server::Configuration::Initial& config) {
37+
return Server::InstanceUtil::createRuntime(server, config);
38+
}
4539

46-
int main_common(OptionsImpl& options) {
47-
Stats::RawStatData::configure(options);
40+
MainCommonBase::MainCommonBase(OptionsImpl& options, bool hot_restart) : options_(options) {
41+
ares_library_init(ARES_LIB_INIT_ALL);
42+
Event::Libevent::Global::initialize();
43+
RELEASE_ASSERT(Envoy::Server::validateProtoDescriptors());
4844

45+
switch (options_.mode()) {
46+
case Server::Mode::Serve: {
4947
#ifdef ENVOY_HOT_RESTART
50-
std::unique_ptr<Server::HotRestartImpl> restarter;
51-
try {
52-
restarter.reset(new Server::HotRestartImpl(options));
53-
} catch (Envoy::EnvoyException& e) {
54-
std::cerr << "unable to initialize hot restart: " << e.what() << std::endl;
55-
return 1;
48+
if (hot_restart) {
49+
restarter_.reset(new Server::HotRestartImpl(options_));
50+
}
51+
#endif
52+
if (!hot_restart) {
53+
restarter_.reset(new Server::HotRestartNopImpl());
54+
}
55+
56+
Stats::RawStatData::configure(options_);
57+
tls_.reset(new ThreadLocal::InstanceImpl);
58+
Thread::BasicLockable& log_lock = restarter_->logLock();
59+
Thread::BasicLockable& access_log_lock = restarter_->accessLogLock();
60+
auto local_address = Network::Utility::getLocalAddress(options_.localAddressIpVersion());
61+
Logger::Registry::initialize(options_.logLevel(), log_lock);
62+
63+
stats_store_.reset(new Stats::ThreadLocalStoreImpl(restarter_->statsAllocator()));
64+
server_.reset(new Server::InstanceImpl(options_, local_address, default_test_hooks_,
65+
*restarter_, *stats_store_, access_log_lock,
66+
component_factory_, *tls_));
67+
break;
5668
}
69+
case Server::Mode::Validate:
70+
break;
71+
}
72+
}
5773

58-
Thread::BasicLockable& log_lock = restarter->logLock();
59-
Thread::BasicLockable& access_log_lock = restarter->accessLogLock();
60-
Stats::RawStatDataAllocator& stats_allocator = *restarter;
61-
#else
62-
std::unique_ptr<Server::HotRestartNopImpl> restarter;
63-
restarter.reset(new Server::HotRestartNopImpl());
64-
65-
Thread::MutexBasicLockable log_lock, access_log_lock;
66-
Stats::HeapRawStatDataAllocator stats_allocator;
67-
#endif
74+
MainCommonBase::~MainCommonBase() { ares_library_cleanup(); }
6875

69-
RELEASE_ASSERT(Envoy::Server::validateProtoDescriptors());
70-
Event::Libevent::Global::initialize();
71-
Server::ProdComponentFactory component_factory;
72-
auto local_address = Network::Utility::getLocalAddress(options.localAddressIpVersion());
73-
switch (options.mode()) {
76+
bool MainCommonBase::run() {
77+
switch (options_.mode()) {
7478
case Server::Mode::Serve:
75-
break;
76-
case Server::Mode::Validate:
77-
Thread::MutexBasicLockable log_lock;
78-
Logger::Registry::initialize(options.logLevel(), log_lock);
79-
return Server::validateConfig(options, local_address, component_factory) ? 0 : 1;
79+
server_->run();
80+
return true;
81+
case Server::Mode::Validate: {
82+
auto local_address = Network::Utility::getLocalAddress(options_.localAddressIpVersion());
83+
return Server::validateConfig(options_, local_address, component_factory_);
8084
}
85+
}
86+
NOT_REACHED;
87+
}
8188

82-
ares_library_init(ARES_LIB_INIT_ALL);
89+
MainCommon::MainCommon(int argc, char** argv, bool hot_restart)
90+
: options_(computeOptions(argc, argv, hot_restart)), base_(*options_, hot_restart) {}
91+
92+
std::unique_ptr<OptionsImpl> MainCommon::computeOptions(int argc, char** argv, bool hot_restart) {
93+
OptionsImpl::HotRestartVersionCb hot_restart_version_cb = [](uint64_t, uint64_t) {
94+
return "disabled";
95+
};
8396

84-
Logger::Registry::initialize(options.logLevel(), log_lock);
85-
DefaultTestHooks default_test_hooks;
86-
ThreadLocal::InstanceImpl tls;
87-
Stats::ThreadLocalStoreImpl stats_store(stats_allocator);
97+
#ifdef ENVOY_HOT_RESTART
98+
if (hot_restart) {
99+
// Enabled by default, except on OS X. Control with "bazel --define=hot_restart=disabled"
100+
hot_restart_version_cb = [](uint64_t max_num_stats, uint64_t max_stat_name_len) {
101+
return Server::HotRestartImpl::hotRestartVersion(max_num_stats, max_stat_name_len);
102+
};
103+
}
104+
#else
105+
// Hot-restart should not be specified if the support is not compiled in.
106+
RELEASE_ASSERT(!hot_restart);
107+
#endif
108+
return std::make_unique<OptionsImpl>(argc, argv, hot_restart_version_cb, spdlog::level::info);
109+
}
110+
111+
// Legacy implementation of main_common.
112+
//
113+
// TODO(jmarantz): Remove this when all callers are removed. At that time, MainCommonBase
114+
// and MainCommon can be merged. The current theory is that only Google calls this.
115+
int main_common(OptionsImpl& options) {
88116
try {
89-
Server::InstanceImpl server(options, local_address, default_test_hooks, *restarter, stats_store,
90-
access_log_lock, component_factory, tls);
91-
server.run();
92-
} catch (const EnvoyException& e) {
93-
ares_library_cleanup();
94-
return 1;
117+
#if ENVOY_HOT_RESTART
118+
MainCommonBase main_common(options, true);
119+
#else
120+
MainCommonBase main_common(options, false);
121+
#endif
122+
return main_common.run() ? EXIT_SUCCESS : EXIT_FAILURE;
123+
} catch (EnvoyException& e) {
124+
return EXIT_FAILURE;
95125
}
96-
ares_library_cleanup();
97-
return 0;
126+
return EXIT_SUCCESS;
98127
}
99128

100129
} // namespace Envoy

0 commit comments

Comments
 (0)