Skip to content

Commit 3d7befe

Browse files
copperybeancopperybean
authored
Merge branch 'master' into gcmaster-parquet
2 parents 4b1c9ad + 823afac commit 3d7befe

File tree

77 files changed

+904
-151
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+904
-151
lines changed

‎CMakeLists.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,9 @@ if (ENABLE_CHECK_HEAVY_BUILDS)
6868
set (RLIMIT_AS 20000000000)
6969
endif()
7070

71-
# For some files currently building RISCV64 might be too slow. TODO: Improve compilation times per file
72-
if (ARCH_RISCV64)
71+
# For some files currently building RISCV64/LOONGARCH64 might be too slow.
72+
# TODO: Improve compilation times per file
73+
if (ARCH_RISCV64 OR ARCH_LOONGARCH64)
7374
set (RLIMIT_CPU 1800)
7475
endif()
7576

‎docs/en/interfaces/formats.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ SELECT * FROM nestedt FORMAT TSV
197197
- [input_format_tsv_enum_as_number](/docs/en/operations/settings/settings-formats.md/#input_format_tsv_enum_as_number) - treat inserted enum values in TSV formats as enum indices. Default value - `false`.
198198
- [input_format_tsv_use_best_effort_in_schema_inference](/docs/en/operations/settings/settings-formats.md/#input_format_tsv_use_best_effort_in_schema_inference) - use some tweaks and heuristics to infer schema in TSV format. If disabled, all fields will be inferred as Strings. Default value - `true`.
199199
- [output_format_tsv_crlf_end_of_line](/docs/en/operations/settings/settings-formats.md/#output_format_tsv_crlf_end_of_line) - if it is set true, end of line in TSV output format will be `\r\n` instead of `\n`. Default value - `false`.
200+
- [input_format_tsv_crlf_end_of_line](/docs/en/operations/settings/settings-formats.md/#input_format_tsv_crlf_end_of_line) - if it is set true, end of line in TSV input format will be `\r\n` instead of `\n`. Default value - `false`.
200201
- [input_format_tsv_skip_first_lines](/docs/en/operations/settings/settings-formats.md/#input_format_tsv_skip_first_lines) - skip specified number of lines at the beginning of data. Default value - `0`.
201202
- [input_format_tsv_detect_header](/docs/en/operations/settings/settings-formats.md/#input_format_tsv_detect_header) - automatically detect header with names and types in TSV format. Default value - `true`.
202203
- [input_format_tsv_skip_trailing_empty_lines](/docs/en/operations/settings/settings-formats.md/#input_format_tsv_skip_trailing_empty_lines) - skip trailing empty lines at the end of data. Default value - `false`.

‎docs/en/operations/settings/settings-formats.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -831,7 +831,13 @@ Default value: `0`.
831831

832832
### output_format_tsv_crlf_end_of_line {#output_format_tsv_crlf_end_of_line}
833833

834-
Use DOC/Windows-style line separator (CRLF) in TSV instead of Unix style (LF).
834+
Use DOS/Windows-style line separator (CRLF) in TSV instead of Unix style (LF).
835+
836+
Disabled by default.
837+
838+
### input_format_tsv_crlf_end_of_line {#input_format_tsv_crlf_end_of_line}
839+
840+
Use DOS/Windows-style line separator (CRLF) for TSV input files instead of Unix style (LF).
835841

836842
Disabled by default.
837843

‎docs/ru/interfaces/formats.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ Hello\nworld
119119
Hello\
120120
world
121121
```
122+
`\n\r` (CRLF) поддерживается с помощью настройки `input_format_tsv_crlf_end_of_line`.
122123

123124
Второй вариант поддерживается, так как его использует MySQL при записи tab-separated дампа.
124125

‎programs/client/Client.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1178,7 +1178,7 @@ void Client::processConfig()
11781178

11791179
pager = config().getString("pager", "");
11801180

1181-
setDefaultFormatsFromConfiguration();
1181+
setDefaultFormatsAndCompressionFromConfiguration();
11821182

11831183
global_context->setClientName(std::string(DEFAULT_CLIENT_NAME));
11841184
global_context->setQueryKindInitial();

‎programs/local/LocalServer.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -607,7 +607,7 @@ void LocalServer::processConfig()
607607
if (config().has("macros"))
608608
global_context->setMacros(std::make_unique<Macros>(config(), "macros", log));
609609

610-
setDefaultFormatsFromConfiguration();
610+
setDefaultFormatsAndCompressionFromConfiguration();
611611

612612
/// Sets external authenticators config (LDAP, Kerberos).
613613
global_context->setExternalAuthenticatorsConfig(config());

‎src/AggregateFunctions/AggregateFunctionsArgMinArgMax.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ struct Settings;
1414
namespace ErrorCodes
1515
{
1616

17-
extern const int CORRUPTED_DATA;
17+
extern const int INCORRECT_DATA;
1818
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
1919
extern const int LOGICAL_ERROR;
2020
}
@@ -198,7 +198,7 @@ class AggregateFunctionArgMinMax final
198198
this->data(place).value().read(buf, *serialization_val, arena);
199199
if (unlikely(this->data(place).value().has() != this->data(place).result().has()))
200200
throw Exception(
201-
ErrorCodes::CORRUPTED_DATA,
201+
ErrorCodes::INCORRECT_DATA,
202202
"Invalid state of the aggregate function {}: has_value ({}) != has_result ({})",
203203
getName(),
204204
this->data(place).value().has(),

‎src/Analyzer/Passes/AutoFinalOnQueryPass.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ class AutoFinalOnQueryPassVisitor : public InDepthQueryTreeVisitorWithContext<Au
4242
return;
4343

4444
const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage();
45-
bool is_final_supported = storage && storage->supportsFinal();
45+
bool is_final_supported = storage && !storage->isRemote() && storage->supportsFinal();
4646
if (!is_final_supported)
4747
return;
4848

‎src/Analyzer/QueryTreePassManager.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ void QueryTreePassManager::run(QueryTreeNodePtr query_tree_node)
192192
void QueryTreePassManager::runOnlyResolve(QueryTreeNodePtr query_tree_node)
193193
{
194194
// Run only QueryAnalysisPass and GroupingFunctionsResolvePass passes.
195-
run(query_tree_node, 2);
195+
run(query_tree_node, 3);
196196
}
197197

198198
void QueryTreePassManager::run(QueryTreeNodePtr query_tree_node, size_t up_to_pass_index)
@@ -249,6 +249,7 @@ void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze)
249249
{
250250
manager.addPass(std::make_unique<QueryAnalysisPass>(only_analyze));
251251
manager.addPass(std::make_unique<GroupingFunctionsResolvePass>());
252+
manager.addPass(std::make_unique<AutoFinalOnQueryPass>());
252253

253254
manager.addPass(std::make_unique<RemoveUnusedProjectionColumnsPass>());
254255
manager.addPass(std::make_unique<FunctionToSubcolumnsPass>());
@@ -294,7 +295,6 @@ void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze)
294295

295296
manager.addPass(std::make_unique<LogicalExpressionOptimizerPass>());
296297

297-
manager.addPass(std::make_unique<AutoFinalOnQueryPass>());
298298
manager.addPass(std::make_unique<CrossToInnerJoinPass>());
299299
manager.addPass(std::make_unique<ShardNumColumnToFunctionPass>());
300300

‎src/Client/ClientBase.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include <Common/StringUtils.h>
2222
#include <Common/filesystemHelpers.h>
2323
#include <Common/NetException.h>
24+
#include <Common/tryGetFileNameByFileDescriptor.h>
2425
#include <Columns/ColumnString.h>
2526
#include <Columns/ColumnsNumber.h>
2627
#include <Formats/FormatFactory.h>
@@ -643,6 +644,9 @@ try
643644
bool extras_into_stdout = need_render_progress || logs_into_stdout;
644645
bool select_only_into_file = select_into_file && !select_into_file_and_stdout;
645646

647+
if (!out_file_buf && default_output_compression_method != CompressionMethod::None)
648+
out_file_buf = wrapWriteBufferWithCompressionMethod(out_buf, default_output_compression_method, 3, 0);
649+
646650
/// It is not clear how to write progress and logs
647651
/// intermixed with data with parallel formatting.
648652
/// It may increase code complexity significantly.
@@ -735,7 +739,7 @@ bool ClientBase::isRegularFile(int fd)
735739
return fstat(fd, &file_stat) == 0 && S_ISREG(file_stat.st_mode);
736740
}
737741

738-
void ClientBase::setDefaultFormatsFromConfiguration()
742+
void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
739743
{
740744
if (config().has("output-format"))
741745
{
@@ -759,6 +763,10 @@ void ClientBase::setDefaultFormatsFromConfiguration()
759763
default_output_format = *format_from_file_name;
760764
else
761765
default_output_format = "TSV";
766+
767+
std::optional<String> file_name = tryGetFileNameFromFileDescriptor(STDOUT_FILENO);
768+
if (file_name)
769+
default_output_compression_method = chooseCompressionMethod(*file_name, "");
762770
}
763771
else if (is_interactive)
764772
{

0 commit comments

Comments
 (0)