Skip to content

Commit c8a9ccd

Browse files
qidayexiaokang
qidaye
authored and
xiaokang
committed
[fix](es catalog) fix issue with select and insert from es catalog core (#24318)
Issue Number: close #24315 The root cause of this issue is that Elasticsearch's long type allows inserting floats and strings. Doris did not handle these cases when doing type conversion. The current strategy is to take the integer before the decimal point if a float or string is found.
1 parent adf9ecc commit c8a9ccd

14 files changed

Lines changed: 70 additions & 36 deletions

File tree

‎be/src/exec/es/es_scroll_parser.cpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,16 @@ template <typename T>
347347
Status insert_int_value(const rapidjson::Value& col, PrimitiveType type,
348348
vectorized::IColumn* col_ptr, bool pure_doc_value, bool nullable) {
349349
if (col.IsNumber()) {
350-
T value = (T)(sizeof(T) < 8 ? col.GetInt() : col.GetInt64());
350+
T value;
351+
// ES allows inserting float and double in int/long types.
352+
// To parse these numbers in Doris, we direct cast them to int types.
353+
if (col.IsDouble()) {
354+
value = static_cast<T>(col.GetDouble());
355+
} else if (col.IsFloat()) {
356+
value = static_cast<T>(col.GetFloat());
357+
} else {
358+
value = (T)(sizeof(T) < 8 ? col.GetInt() : col.GetInt64());
359+
}
351360
col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&value)), 0);
352361
return Status::OK();
353362
}
@@ -363,8 +372,14 @@ Status insert_int_value(const rapidjson::Value& col, PrimitiveType type,
363372
RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type);
364373

365374
StringParser::ParseResult result;
366-
const std::string& val = col.GetString();
367-
size_t len = col.GetStringLength();
375+
std::string val = col.GetString();
376+
// ES allows inserting numbers and characters containing decimals in numeric types.
377+
// To parse these numbers in Doris, we remove the decimals here.
378+
size_t pos = val.find(".");
379+
if (pos != std::string::npos) {
380+
val = val.substr(0, pos);
381+
}
382+
size_t len = val.length();
368383
T v = StringParser::string_to_int<T>(val.c_str(), len, &result);
369384
RETURN_ERROR_IF_PARSING_FAILED(result, col, type);
370385

‎docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"test6": 1659931810000,
88
"test7": 1659931810000,
99
"test8": "2022-08-08T12:10:10Z",
10+
"test9": 12345,
1011
"c_bool": [true, false, true, true],
1112
"c_byte": [1, -2, -3, 4],
1213
"c_short": [128, 129, -129, -130],
@@ -26,4 +27,4 @@
2627
{"name": "Andy", "age": 18},
2728
{"name": "Tim", "age": 28}
2829
]
29-
}
30+
}

‎docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1_es6.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"test2": "text#1",
44
"test3": 3.14,
55
"test4": "2022-08-08",
6+
"test5": 12345,
67
"c_bool": [true, false, true, true],
78
"c_byte": [1, -2, -3, 4],
89
"c_short": [128, 129, -129, -130],
@@ -22,4 +23,4 @@
2223
{"name": "Andy", "age": 18},
2324
{"name": "Tim", "age": 28}
2425
]
25-
}
26+
}

‎docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"test6": 1660018210000,
88
"test7": "2022-08-09 12:10:10",
99
"test8": 1660018210000,
10+
"test9": 2222.2,
1011
"c_bool": [true, false, true, true],
1112
"c_byte": [1, -2, -3, 4],
1213
"c_short": [128, 129, -129, -130],
@@ -26,4 +27,4 @@
2627
{"name": "Andy", "age": 18},
2728
{"name": "Tim", "age": 28}
2829
]
29-
}
30+
}

‎docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"test2": "text2",
44
"test3": 4,
55
"test4": "2022-08-08",
6+
"test5": 2222.2,
67
"c_bool": [true, false, true, true],
78
"c_byte": [1, -2, -3, 4],
89
"c_short": [128, 129, -129, -130],
@@ -22,4 +23,4 @@
2223
{"name": "Andy", "age": 18},
2324
{"name": "Tim", "age": 28}
2425
]
25-
}
26+
}

‎docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"test6": 1660104610000,
88
"test7": 1660104610000,
99
"test8": "2022-08-10T12:10:10",
10+
"test9": 3333.22,
1011
"c_bool": [true, false, true, true],
1112
"c_byte": [1, -2, -3, 4],
1213
"c_short": [128, 129, -129, -130],
@@ -26,4 +27,4 @@
2627
{"name": "Andy", "age": 18},
2728
{"name": "Tim", "age": 28}
2829
]
29-
}
30+
}

‎docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"test2": "text3_4*5",
44
"test3": 5.0,
55
"test4": "2022-08-08",
6+
"test5": "3333.22",
67
"c_bool": [true, false, true, true],
78
"c_byte": [1, -2, -3, 4],
89
"c_short": [128, 129, -129, -130],
@@ -22,4 +23,4 @@
2223
{"name": "Andy", "age": 18},
2324
{"name": "Tim", "age": 28}
2425
]
25-
}
26+
}

‎docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"test6": 1660191010000,
88
"test7": "2022-08-11 12:10:10",
99
"test8": "2022-08-11T12:10:10+09:00",
10+
"test9": "4444.22",
1011
"c_bool": [true, false, true, true],
1112
"c_byte": [1, -2, -3, 4],
1213
"c_short": [128, 129, -129, -130],
@@ -26,4 +27,4 @@
2627
{"name": "Andy", "age": 18},
2728
{"name": "Tim", "age": 28}
2829
]
29-
}
30+
}

‎docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
"test4": {
2525
"type": "date"
2626
},
27+
"test5": {
28+
"type": "long"
29+
},
2730
"c_bool": {
2831
"type": "boolean"
2932
},
@@ -85,4 +88,4 @@
8588
}
8689
}
8790
}
88-
}
91+
}

‎docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727
"test4": {
2828
"type": "date"
2929
},
30+
"test5": {
31+
"type": "long"
32+
},
3033
"c_bool": {
3134
"type": "boolean"
3235
},
@@ -88,4 +91,4 @@
8891
}
8992
}
9093
}
91-
}
94+
}

0 commit comments

Comments
 (0)