Skip to content

Commit f4bfed9

Browse files
alexey-milovidovalexey-milovidov
authored
Merge pull request #29586 from evillique/log_levels_update
Add log levels updates
2 parents 00bb647 + f292617 commit f4bfed9

File tree

9 files changed

+167
-12
lines changed

9 files changed

+167
-12
lines changed

‎base/loggers/Loggers.cpp‎

Lines changed: 87 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
8484

8585
Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, log_file);
8686
log->setLevel(log_level);
87-
split->addChannel(log);
87+
split->addChannel(log, "log");
8888
}
8989

9090
const auto errorlog_path = config.getString("logger.errorlog", "");
@@ -116,7 +116,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
116116
Poco::AutoPtr<DB::OwnFormattingChannel> errorlog = new DB::OwnFormattingChannel(pf, error_log_file);
117117
errorlog->setLevel(errorlog_level);
118118
errorlog->open();
119-
split->addChannel(errorlog);
119+
split->addChannel(errorlog, "errorlog");
120120
}
121121

122122
if (config.getBool("logger.use_syslog", false))
@@ -155,7 +155,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
155155
Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, syslog_channel);
156156
log->setLevel(syslog_level);
157157

158-
split->addChannel(log);
158+
split->addChannel(log, "syslog");
159159
}
160160

161161
bool should_log_to_console = isatty(STDIN_FILENO) || isatty(STDERR_FILENO);
@@ -177,7 +177,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
177177
Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel);
178178
logger.warning("Logging " + console_log_level_string + " to console");
179179
log->setLevel(console_log_level);
180-
split->addChannel(log);
180+
split->addChannel(log, "console");
181181
}
182182

183183
split->open();
@@ -224,6 +224,89 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
224224
}
225225
}
226226

227+
void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger)
228+
{
229+
int max_log_level = 0;
230+
231+
const auto log_level_string = config.getString("logger.level", "trace");
232+
int log_level = Poco::Logger::parseLevel(log_level_string);
233+
if (log_level > max_log_level)
234+
max_log_level = log_level;
235+
236+
const auto log_path = config.getString("logger.log", "");
237+
if (!log_path.empty())
238+
split->setLevel("log", log_level);
239+
else
240+
split->setLevel("log", 0);
241+
242+
// Set level to console
243+
bool is_daemon = config.getBool("application.runAsDaemon", false);
244+
bool should_log_to_console = isatty(STDIN_FILENO) || isatty(STDERR_FILENO);
245+
if (config.getBool("logger.console", false)
246+
|| (!config.hasProperty("logger.console") && !is_daemon && should_log_to_console))
247+
split->setLevel("console", log_level);
248+
else
249+
split->setLevel("console", 0);
250+
251+
// Set level to errorlog
252+
int errorlog_level = 0;
253+
const auto errorlog_path = config.getString("logger.errorlog", "");
254+
if (!errorlog_path.empty())
255+
{
256+
errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice"));
257+
if (errorlog_level > max_log_level)
258+
max_log_level = errorlog_level;
259+
}
260+
split->setLevel("errorlog", errorlog_level);
261+
262+
// Set level to syslog
263+
int syslog_level = 0;
264+
if (config.getBool("logger.use_syslog", false))
265+
{
266+
syslog_level = Poco::Logger::parseLevel(config.getString("logger.syslog_level", log_level_string));
267+
if (syslog_level > max_log_level)
268+
max_log_level = syslog_level;
269+
}
270+
split->setLevel("syslog", syslog_level);
271+
272+
// Global logging level (it can be overridden for specific loggers).
273+
logger.setLevel(max_log_level);
274+
275+
// Set level to all already created loggers
276+
std::vector<std::string> names;
277+
278+
logger.root().names(names);
279+
for (const auto & name : names)
280+
logger.root().get(name).setLevel(max_log_level);
281+
282+
logger.root().setLevel(max_log_level);
283+
284+
// Explicitly specified log levels for specific loggers.
285+
{
286+
Poco::Util::AbstractConfiguration::Keys loggers_level;
287+
config.keys("logger.levels", loggers_level);
288+
289+
if (!loggers_level.empty())
290+
{
291+
for (const auto & key : loggers_level)
292+
{
293+
if (key == "logger" || key.starts_with("logger["))
294+
{
295+
const std::string name(config.getString("logger.levels." + key + ".name"));
296+
const std::string level(config.getString("logger.levels." + key + ".level"));
297+
logger.root().get(name).setLevel(level);
298+
}
299+
else
300+
{
301+
// Legacy syntax
302+
const std::string level(config.getString("logger.levels." + key, "trace"));
303+
logger.root().get(key).setLevel(level);
304+
}
305+
}
306+
}
307+
}
308+
}
309+
227310
void Loggers::closeLogs(Poco::Logger & logger)
228311
{
229312
if (log_file)

‎base/loggers/Loggers.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ class Loggers
1919
public:
2020
void buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger, const std::string & cmd_name = "");
2121

22+
void updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger);
23+
2224
/// Close log files. On next log write files will be reopened.
2325
void closeLogs(Poco::Logger & logger);
2426

‎base/loggers/OwnFormattingChannel.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#pragma once
2+
#include <atomic>
23
#include <Poco/AutoPtr.h>
34
#include <Poco/Channel.h>
45
#include <Poco/FormattingChannel.h>
@@ -14,7 +15,7 @@ class OwnFormattingChannel : public Poco::Channel, public ExtendedLogChannel
1415
public:
1516
explicit OwnFormattingChannel(
1617
Poco::AutoPtr<OwnPatternFormatter> pFormatter_ = nullptr, Poco::AutoPtr<Poco::Channel> pChannel_ = nullptr)
17-
: pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_))
18+
: pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_)), priority(Poco::Message::PRIO_TRACE)
1819
{
1920
}
2021

@@ -45,7 +46,7 @@ class OwnFormattingChannel : public Poco::Channel, public ExtendedLogChannel
4546
private:
4647
Poco::AutoPtr<OwnPatternFormatter> pFormatter;
4748
Poco::AutoPtr<Poco::Channel> pChannel;
48-
Poco::Message::Priority priority = Poco::Message::PRIO_TRACE;
49+
std::atomic<Poco::Message::Priority> priority;
4950
};
5051

5152
}

‎base/loggers/OwnSplitChannel.cpp‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "OwnSplitChannel.h"
2+
#include "OwnFormattingChannel.h"
23

34
#include <iostream>
45
#include <Core/Block.h>
@@ -75,7 +76,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
7576
ExtendedLogMessage msg_ext = ExtendedLogMessage::getFrom(msg);
7677

7778
/// Log data to child channels
78-
for (auto & channel : channels)
79+
for (auto & [name, channel] : channels)
7980
{
8081
if (channel.second)
8182
channel.second->logExtended(msg_ext); // extended child
@@ -137,9 +138,9 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
137138
}
138139

139140

140-
void OwnSplitChannel::addChannel(Poco::AutoPtr<Poco::Channel> channel)
141+
void OwnSplitChannel::addChannel(Poco::AutoPtr<Poco::Channel> channel, const std::string & name)
141142
{
142-
channels.emplace_back(std::move(channel), dynamic_cast<ExtendedLogChannel *>(channel.get()));
143+
channels.emplace(name, ExtendedChannelPtrPair(std::move(channel), dynamic_cast<ExtendedLogChannel *>(channel.get())));
143144
}
144145

145146
void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log, int max_priority)
@@ -149,4 +150,14 @@ void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log, int max_prior
149150
text_log_max_priority.store(max_priority, std::memory_order_relaxed);
150151
}
151152

153+
void OwnSplitChannel::setLevel(const std::string & name, int level)
154+
{
155+
auto it = channels.find(name);
156+
if (it != channels.end())
157+
{
158+
if (auto * channel = dynamic_cast<DB::OwnFormattingChannel *>(it->second.first.get()))
159+
channel->setLevel(level);
160+
}
161+
}
162+
152163
}

‎base/loggers/OwnSplitChannel.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,20 @@ class OwnSplitChannel : public Poco::Channel
1818
/// Makes an extended message from msg and passes it to the client logs queue and child (if possible)
1919
void log(const Poco::Message & msg) override;
2020
/// Adds a child channel
21-
void addChannel(Poco::AutoPtr<Poco::Channel> channel);
21+
void addChannel(Poco::AutoPtr<Poco::Channel> channel, const std::string & name);
2222

2323
void addTextLog(std::shared_ptr<DB::TextLog> log, int max_priority);
2424

25+
void setLevel(const std::string & name, int level);
26+
2527
private:
2628
void logSplit(const Poco::Message & msg);
2729
void tryLogSplit(const Poco::Message & msg);
2830

2931
using ChannelPtr = Poco::AutoPtr<Poco::Channel>;
3032
/// Handler and its pointer casted to extended interface
3133
using ExtendedChannelPtrPair = std::pair<ChannelPtr, ExtendedLogChannel *>;
32-
std::vector<ExtendedChannelPtrPair> channels;
34+
std::map<std::string, ExtendedChannelPtrPair> channels;
3335

3436
std::mutex text_log_mutex;
3537

‎programs/server/Server.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -844,7 +844,7 @@ if (ThreadFuzzer::instance().isEffective())
844844
// FIXME logging-related things need synchronization -- see the 'Logger * log' saved
845845
// in a lot of places. For now, disable updating log configuration without server restart.
846846
//setTextLog(global_context->getTextLog());
847-
//buildLoggers(*config, logger());
847+
updateLevels(*config, logger());
848848
global_context->setClustersConfig(config);
849849
global_context->setMacros(std::make_unique<Macros>(*config, "macros", log));
850850
global_context->setExternalAuthenticatorsConfig(*config);

‎tests/integration/test_log_levels_update/__init__.py‎

Whitespace-only changes.

‎tests/integration/test_log_levels_update/configs/log.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<yandex>
2+
<logger>
3+
<level>trace</level>
4+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
5+
</logger>
6+
</yandex>

‎tests/integration/test_log_levels_update/test.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import pytest
2+
import re
3+
4+
from helpers.cluster import ClickHouseCluster
5+
6+
cluster = ClickHouseCluster(__file__, name="log_quries_probability")
7+
node = cluster.add_instance('node', with_zookeeper=False)
8+
9+
config = '''<yandex>
10+
<logger>
11+
<level>information</level>
12+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
13+
</logger>
14+
</yandex>'''
15+
16+
17+
@pytest.fixture(scope="module")
18+
def start_cluster():
19+
try:
20+
cluster.start()
21+
yield cluster
22+
23+
finally:
24+
cluster.shutdown()
25+
26+
27+
def get_log(node):
28+
return node.exec_in_container(["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"])
29+
30+
def test_log_levels_update(start_cluster):
31+
# Make sure that there are enough log messages for the test
32+
for i in range(5):
33+
node.query("SELECT 1")
34+
35+
log = get_log(node)
36+
assert re.search("(<Trace>|<Debug>)", log)
37+
38+
node.replace_config("/etc/clickhouse-server/config.d/log.xml", config)
39+
node.query("SYSTEM RELOAD CONFIG;")
40+
node.exec_in_container(["bash", "-c", "> /var/log/clickhouse-server/clickhouse-server.log"])
41+
42+
for i in range(5):
43+
node.query("SELECT 1")
44+
45+
log = get_log(node)
46+
assert len(log) > 0
47+
assert not re.search("(<Trace>|<Debug>)", log)
48+
49+
50+

0 commit comments

Comments
 (0)