Skip to content

Commit f3ce0d4

Browse files
yangzhgyangzhengguo01
authored andcommitted
add compress and decompress when serialize (apache#6)
* add compress and decompress when serialize Change-Id: If89c1177ed2b1f1b070c88a9211baaccd1c207c4 * add black line Change-Id: I76f9663c5872248fdb7a6ec107ee5cc31f5f6501 * add black line Change-Id: I7eaeccf3a9ba205fb6ed0addfd48ab8a44f9b8b8 Co-authored-by: yangzhengguo01 <[email protected]>
1 parent be8472a commit f3ce0d4

9 files changed

Lines changed: 49 additions & 46 deletions

File tree

‎be/src/runtime/types.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,9 @@
2727
#include "olap/hll.h"
2828
#include "runtime/primitive_type.h"
2929
#include "thrift/protocol/TDebugProtocol.h"
30-
31-
#include "vec/data_types/data_types_number.h"
3230
#include "vec/data_types/data_type_string.h"
3331
#include "vec/data_types/data_types_decimal.h"
32+
#include "vec/data_types/data_types_number.h"
3433

3534
namespace doris {
3635

@@ -76,6 +75,10 @@ struct TypeDescriptor {
7675
DCHECK_NE(type, TYPE_ARRAY);
7776
DCHECK_NE(type, TYPE_MAP);
7877
#endif
78+
if (type == TYPE_DECIMAL || type == TYPE_DECIMALV2) {
79+
precision = 27;
80+
scale = 9;
81+
}
7982
}
8083

8184
static TypeDescriptor create_char_type(int len) {

‎be/src/vec/data_types/data_type_number_base.cpp‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,12 @@
1919

2020
#include <type_traits>
2121

22+
#include "gen_cpp/data.pb.h"
2223
#include "vec/columns/column_const.h"
2324
#include "vec/columns/column_vector.h"
24-
// #include <IO/ReadHelpers.h>
25-
// #include <IO/WriteHelpers.h>
2625
#include "vec/common/assert_cast.h"
2726
#include "vec/common/nan_utils.h"
2827
#include "vec/common/typeid_cast.h"
29-
// #include <Formats/FormatSettings.h>
30-
// #include <Formats/ProtobufReader.h>
31-
// #include <Formats/ProtobufWriter.h>
32-
#include "gen_cpp/data.pb.h"
3328
#include "vec/io/io_helper.h"
3429

3530
namespace doris::vectorized {
@@ -264,7 +259,7 @@ void DataTypeNumberBase<T>::serialize(const IColumn& column, PColumn* pcolumn) c
264259
const FieldType& x = assert_cast<const ColumnVector<T>&>(column).getData()[i];
265260
writeBinary(x, buf);
266261
}
267-
pcolumn->mutable_binary()->append(buf.str());
262+
write_binary(buf, pcolumn);
268263
}
269264

270265
template <typename T>
@@ -277,7 +272,9 @@ void DataTypeNumberBase<T>::serialize(const IColumn& column, size_t row_num,
277272

278273
template <typename T>
279274
void DataTypeNumberBase<T>::deserialize(const PColumn& pcolumn, IColumn* column) const {
280-
std::istringstream istr(pcolumn.binary());
275+
std::string uncompressed;
276+
read_binary(pcolumn, &uncompressed);
277+
std::istringstream istr(uncompressed);
281278
while (istr.peek() != EOF) {
282279
typename ColumnVector<T>::value_type x;
283280
readBinary(x, istr);

‎be/src/vec/data_types/data_type_string.cpp‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
#include "vec/data_types/data_type_string.h"
19+
1820
#include "gen_cpp/data.pb.h"
1921
#include "vec/columns/column_const.h"
2022
#include "vec/columns/column_string.h"
@@ -25,16 +27,6 @@
2527
#include "vec/core/field.h"
2628
#include "vec/io/io_helper.h"
2729

28-
//#include <Formats/FormatSettings.h>
29-
//#include <Formats/ProtobufReader.h>
30-
//#include <Formats/ProtobufWriter.h>
31-
#include "vec/data_types/data_type_string.h"
32-
//#include <vec/DataTypes/DataTypeFactory.h>
33-
//
34-
//#include <IO/ReadHelpers.h>
35-
//#include <IO/WriteHelpers.h>
36-
//#include <IO/VarInt.h>
37-
3830
#ifdef __SSE2__
3931
#include <emmintrin.h>
4032
#endif
@@ -261,7 +253,7 @@ void DataTypeString::serialize(const IColumn& column, PColumn* pcolumn) const {
261253
const auto& s = assert_cast<const ColumnString&>(column).getDataAt(i);
262254
writeStringBinary(s, buf);
263255
}
264-
pcolumn->mutable_binary()->append(buf.str());
256+
write_binary(buf, pcolumn);
265257
}
266258

267259
void DataTypeString::serialize(const IColumn& column, size_t row_num, PColumn* pcolumn) const {
@@ -276,7 +268,9 @@ void DataTypeString::deserialize(const PColumn& pcolumn, IColumn* column) const
276268
ColumnString::Chars& data = column_string->getChars();
277269
ColumnString::Offsets& offsets = column_string->getOffsets();
278270
size_t offset = 0;
279-
std::istringstream istr(pcolumn.binary());
271+
std::string uncompressed;
272+
read_binary(pcolumn, &uncompressed);
273+
std::istringstream istr(uncompressed);
280274
while (istr.peek() != EOF) {
281275
std::string s;
282276
readBinary(s, istr);

‎be/src/vec/data_types/data_types_decimal.cpp‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,6 @@
2424
#include "vec/common/int_exp.h"
2525
#include "vec/common/typeid_cast.h"
2626
#include "vec/io/io_helper.h"
27-
//#include <DataTypes/DataTypeFactory.h>
28-
//#include <Formats/ProtobufReader.h>
29-
//#include <Formats/ProtobufWriter.h>
30-
//#include <IO/ReadHelpers.h>
31-
//#include <IO/WriteHelpers.h>
32-
//#include <IO/readDecimalText.h>
33-
//#include <Parsers/IAST.h>
34-
//#include <Parsers/ASTLiteral.h>
35-
//#include <Interpreters/Context.h>
36-
#include "gen_cpp/data.pb.h"
37-
3827

3928
namespace doris::vectorized {
4029

@@ -211,13 +200,16 @@ void DataTypeDecimal<T>::serialize(const IColumn& column, PColumn* pcolumn) cons
211200
const FieldType& x = assert_cast<const ColumnType&>(column).getElement(i);
212201
writeBinary(x, buf);
213202
}
214-
pcolumn->mutable_binary()->append(buf.str());
203+
204+
write_binary(buf, pcolumn);
215205
pcolumn->mutable_decimal_param()->set_precision(precision);
216206
pcolumn->mutable_decimal_param()->set_scale(scale);
217207
}
218208
template <typename T>
219209
void DataTypeDecimal<T>::deserialize(const PColumn& pcolumn, IColumn* column) const {
220-
std::istringstream istr(pcolumn.binary());
210+
std::string uncompressed;
211+
read_binary(pcolumn, &uncompressed);
212+
std::istringstream istr(uncompressed);
221213
while (istr.peek() != EOF) {
222214
typename FieldType::NativeType x;
223215
readBinary(x, istr);

‎be/src/vec/io/io_helper.h‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,11 @@
1717

1818
#pragma once
1919

20+
#include <snappy/snappy.h>
21+
2022
#include <iostream>
2123

24+
#include "gen_cpp/data.pb.h"
2225
#include "vec/common/exception.h"
2326
#include "vec/io/var_int.h"
2427

@@ -54,7 +57,7 @@ inline void writeStringBinary(const StringRef& s, std::ostream& buf) {
5457
}
5558

5659
inline void writeStringBinary(const char* s, std::ostream& buf) {
57-
writeStringBinary(StringRef {s}, buf);
60+
writeStringBinary(StringRef{s}, buf);
5861
}
5962

6063
template <typename Type>
@@ -93,6 +96,13 @@ inline void writeBinary(const Decimal64& x, std::ostream& buf) {
9396
inline void writeBinary(const Decimal128& x, std::ostream& buf) {
9497
writePODBinary(x, buf);
9598
}
99+
inline void write_binary(const std::ostringstream& buf, PColumn* pcolumn) {
100+
std::string uncompressed = buf.str();
101+
std::string compressed;
102+
snappy::Compress(uncompressed.data(), uncompressed.size(), &compressed);
103+
pcolumn->set_compressed(true);
104+
pcolumn->mutable_binary()->append(compressed);
105+
}
96106

97107
/// Read POD-type in native format
98108
template <typename Type>
@@ -166,4 +176,11 @@ inline void readBinary(Decimal64& x, std::istream& buf) {
166176
inline void readBinary(Decimal128& x, std::istream& buf) {
167177
readPODBinary(x, buf);
168178
}
169-
} // namespace doris::vectorized
179+
inline void read_binary(const PColumn& pcolumn, std::string* data) {
180+
if (pcolumn.compressed()) {
181+
snappy::Uncompress(pcolumn.binary().data(), pcolumn.binary().size(), data);
182+
} else {
183+
*data = pcolumn.binary();
184+
}
185+
}
186+
} // namespace doris::vectorized

‎be/test/vec/aggregate_functions/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# under the License.
1717

1818
# where to put generated libraries
19-
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test/vec/AggregateFunctions")
19+
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test/vec/aggregate_functions")
2020

2121
ADD_BE_TEST(agg_test)
2222

‎be/test/vec/core/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# under the License.
1717

1818
# where to put generated libraries
19-
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test/vec/Core")
19+
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test/vec/core")
2020

2121
ADD_BE_TEST(block_test)
2222

‎be/test/vec/function/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# under the License.
1717

1818
# where to put generated libraries
19-
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test/vec/Function")
19+
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test/vec/function")
2020

2121
ADD_BE_TEST(function_abs_test)
2222
ADD_BE_TEST(function_comparison_test)

‎gensrc/proto/data.proto‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,13 @@ message PColumn {
7676
required DataType type = 2;
7777
// serialized data type
7878
// required DataType data_type = 3;
79-
repeated bool is_null = 4;
79+
repeated bool is_null = 3;
8080
// optional bool nested = 5 [default = false];
8181
// optional bool plain = 6 [default = false];
8282
// store in plain text or in binary data
83-
optional bytes binary = 7;
83+
optional bytes binary = 4;
84+
optional bool compressed = 5 [default = false];
85+
optional Decimal decimal_param = 6;
8486
// the following is used for store plain data
8587
// repeated string type_params = 8;
8688
// repeated int32 int32_data = 9;
@@ -92,11 +94,9 @@ message PColumn {
9294
// repeated bool bool_data = 15;
9395
// repeated string string_data = 16;
9496
// repeated bytes bytes_data = 17;
95-
optional Decimal decimal_param = 18;
9697
}
9798
message PBlock {
9899
optional uint64 num_rows = 1;
99100
optional uint32 num_columns = 2;
100-
optional bool compressed = 3 [default = false];
101-
repeated PColumn columns = 4;
102-
}
101+
repeated PColumn columns = 3;
102+
}

0 commit comments

Comments
 (0)