add compression method for files: Xz #16578
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -36,6 +36,7 @@ add_subdirectory (murmurhash) | |
| add_subdirectory (replxx-cmake) | ||
| add_subdirectory (ryu-cmake) | ||
| add_subdirectory (unixodbc-cmake) | ||
| add_subdirectory (xz) | ||
|
Member
There was a problem hiding this comment. Looks like you've forget to add submodule
Contributor
Author
There was a problem hiding this comment.
Member
There was a problem hiding this comment. Not really, this is just to know where to fetch the submodule from. You need to run smth like this: And this will create a special file
Member
There was a problem hiding this comment. Also to make fasttest update your submodule you need to add ClickHouse/docker/test/fasttest/run.sh Line 130 in f10a520 Without fasttest other tests won't be run
Member
There was a problem hiding this comment. Also it's possible to disable |
||
|
|
||
| add_subdirectory (poco-cmake) | ||
| add_subdirectory (croaring-cmake) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -330,6 +330,13 @@ if (ZSTD_LIBRARY) | |
| endif () | ||
| endif() | ||
|
|
||
| set (LZMA_LIBRARY liblzma) | ||
| set (LZMA_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/xz/src/liblzma/api) | ||
|
Comment on lines
+333
to
+334
Member
There was a problem hiding this comment. And I guess it is better to move this out into separate
Contributor
Author
There was a problem hiding this comment. What should I do to pass description check?
Member
There was a problem hiding this comment.
Update description of PR and include changelog entry using specified format. Just copy it and modify the template, but leave the format.
Member
There was a problem hiding this comment. Yes, it's much better to provide our own CMakeLists. CMake often cannot be safely reused (without building unneeded targets, polluting build options). |
||
| if (LZMA_LIBRARY) | ||
| target_link_libraries (clickhouse_common_io PUBLIC ${LZMA_LIBRARY}) | ||
| target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${LZMA_INCLUDE_DIR}) | ||
| endif() | ||
|
|
||
| if (USE_ICU) | ||
| dbms_target_link_libraries (PRIVATE ${ICU_LIBRARIES}) | ||
| dbms_target_include_directories (SYSTEM PRIVATE ${ICU_INCLUDE_DIRS}) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| #include <IO/LZMADeflatingWriteBuffer.h> | ||
|
|
||
|
|
||
| namespace DB | ||
| { | ||
| namespace ErrorCodes | ||
| { | ||
| extern const int LZMA_STREAM_ENCODER_FAILED; | ||
| } | ||
|
|
||
| LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer( | ||
| std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment) | ||
| : BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment), out(std::move(out_)) | ||
| { | ||
|
|
||
| lstr = LZMA_STREAM_INIT; | ||
| lstr.allocator = nullptr; | ||
| lstr.next_in = nullptr; | ||
| lstr.avail_in = 0; | ||
| lstr.next_out = nullptr; | ||
| lstr.avail_out = 0; | ||
|
|
||
| // options for further compression | ||
| lzma_options_lzma opt_lzma2; | ||
| if (lzma_lzma_preset(&opt_lzma2, compression_level)) | ||
| throw Exception(ErrorCodes::LZMA_STREAM_ENCODER_FAILED, "lzma preset failed: lzma version: {}", LZMA_VERSION_STRING); | ||
|
|
||
|
|
||
| // LZMA_FILTER_X86 - | ||
| // LZMA2 - codec for *.xz files compression; LZMA is not suitable for this purpose | ||
| // VLI - variable length integer (in *.xz most integers encoded as VLI) | ||
| // LZMA_VLI_UNKNOWN (UINT64_MAX) - VLI value to denote that the value is unknown | ||
| lzma_filter filters[] = { | ||
| {.id = LZMA_FILTER_X86, .options = nullptr}, | ||
| {.id = LZMA_FILTER_LZMA2, .options = &opt_lzma2}, | ||
| {.id = LZMA_VLI_UNKNOWN, .options = nullptr}, | ||
| }; | ||
| lzma_ret ret = lzma_stream_encoder(&lstr, filters, LZMA_CHECK_CRC64); | ||
|
|
||
| if (ret != LZMA_OK) | ||
| throw Exception( | ||
| ErrorCodes::LZMA_STREAM_ENCODER_FAILED, | ||
| "lzma stream encoder init failed: error code: {} lzma version: {}", | ||
| ret, | ||
| LZMA_VERSION_STRING); | ||
| } | ||
|
|
||
| LZMADeflatingWriteBuffer::~LZMADeflatingWriteBuffer() | ||
| { | ||
| try | ||
| { | ||
| finish(); | ||
|
|
||
| lzma_end(&lstr); | ||
| } | ||
| catch (...) | ||
| { | ||
| tryLogCurrentException(__PRETTY_FUNCTION__); | ||
| } | ||
| } | ||
|
|
||
| void LZMADeflatingWriteBuffer::nextImpl() | ||
| { | ||
| if (!offset()) | ||
| return; | ||
|
|
||
| lstr.next_in = reinterpret_cast<unsigned char *>(working_buffer.begin()); | ||
| lstr.avail_in = offset(); | ||
|
|
||
| lzma_action action = LZMA_RUN; | ||
| do | ||
| { | ||
| out->nextIfAtEnd(); | ||
| lstr.next_out = reinterpret_cast<unsigned char *>(out->position()); | ||
| lstr.avail_out = out->buffer().end() - out->position(); | ||
|
|
||
| lzma_ret ret = lzma_code(&lstr, action); | ||
| out->position() = out->buffer().end() - lstr.avail_out; | ||
|
|
||
| if (ret == LZMA_STREAM_END) | ||
| return; | ||
|
|
||
| if (ret != LZMA_OK) | ||
| throw Exception( | ||
| ErrorCodes::LZMA_STREAM_ENCODER_FAILED, | ||
| "lzma stream encoding failed: error code: {}; lzma_version: {}", | ||
| ret, | ||
| LZMA_VERSION_STRING); | ||
|
|
||
| } while (lstr.avail_in > 0 || lstr.avail_out == 0); | ||
| } | ||
|
|
||
|
|
||
| void LZMADeflatingWriteBuffer::finish() | ||
| { | ||
| if (finished) | ||
| return; | ||
|
|
||
| next(); | ||
|
|
||
| do | ||
| { | ||
| out->nextIfAtEnd(); | ||
| lstr.next_out = reinterpret_cast<unsigned char *>(out->position()); | ||
| lstr.avail_out = out->buffer().end() - out->position(); | ||
|
|
||
| lzma_ret ret = lzma_code(&lstr, LZMA_FINISH); | ||
| out->position() = out->buffer().end() - lstr.avail_out; | ||
|
|
||
| if (ret == LZMA_STREAM_END) | ||
| { | ||
| finished = true; | ||
| return; | ||
| } | ||
|
|
||
| if (ret != LZMA_OK) | ||
| throw Exception( | ||
| ErrorCodes::LZMA_STREAM_ENCODER_FAILED, | ||
| "lzma stream encoding failed: error code: {}; lzma version: {}", | ||
| ret, | ||
| LZMA_VERSION_STRING); | ||
|
|
||
| } while (lstr.avail_out == 0); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| #pragma once | ||
|
|
||
| #include <IO/BufferWithOwnMemory.h> | ||
| #include <IO/WriteBuffer.h> | ||
|
|
||
| #include <lzma.h> | ||
|
|
||
| namespace DB | ||
| { | ||
| /// Performs compression using lzma library and writes compressed data to out_ WriteBuffer. | ||
| class LZMADeflatingWriteBuffer : public BufferWithOwnMemory<WriteBuffer> | ||
| { | ||
| public: | ||
| LZMADeflatingWriteBuffer( | ||
| std::unique_ptr<WriteBuffer> out_, | ||
| int compression_level, | ||
| size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, | ||
| char * existing_memory = nullptr, | ||
| size_t alignment = 0); | ||
|
|
||
| void finish(); | ||
|
|
||
| ~LZMADeflatingWriteBuffer() override; | ||
|
|
||
| private: | ||
| void nextImpl() override; | ||
|
|
||
| std::unique_ptr<WriteBuffer> out; | ||
| lzma_stream lstr; | ||
| bool finished = false; | ||
| }; | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Uses spaces for indent, while all other lines uses tabs (since
git submodule adduses them)