Skip to content

Commit ad80db5

Browse files
committed
fix apache#9351 can't load parquet file with column name case sensitive with Doris column
1 parent ac3f981 commit ad80db5

3 files changed

Lines changed: 17 additions & 6 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/load/Load.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1045,12 +1045,23 @@ private static void initColumns(Table tbl, List<ImportColumnDesc> columnExprs,
10451045
return;
10461046
}
10471047

1048+
Set<String> tmpSet = Sets.newHashSet();
1049+
for (ImportColumnDesc importColumnDesc : copiedColumnExprs) {
1050+
if (importColumnDesc.getExpr() == null) {
1051+
tmpSet.add(importColumnDesc.getColumnName());
1052+
}
1053+
}
1054+
10481055
// init slot desc add expr map, also transform hadoop functions
10491056
for (ImportColumnDesc importColumnDesc : copiedColumnExprs) {
10501057
// make column name case match with real column name
10511058
String columnName = importColumnDesc.getColumnName();
1052-
String realColName = tbl.getColumn(columnName) == null ? columnName
1053-
: tbl.getColumn(columnName).getName();
1059+
String realColName;
1060+
if (tbl.getColumn(columnName) == null || tmpSet.contains(columnName) ){
1061+
realColName = columnName;
1062+
} else {
1063+
realColName = tbl.getColumn(columnName).getName();
1064+
}
10541065
if (importColumnDesc.getExpr() != null) {
10551066
Expr expr = transformHadoopFunctionExpr(tbl, realColName, importColumnDesc.getExpr());
10561067
exprsByName.put(realColName, expr);

‎fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,8 @@ private void initParams(ParamCreateContext context)
247247
*/
248248
private void initColumns(ParamCreateContext context) throws UserException {
249249
context.srcTupleDescriptor = analyzer.getDescTbl().createTupleDescriptor();
250-
context.slotDescByName = Maps.newHashMap();
251-
context.exprMap = Maps.newHashMap();
250+
context.slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
251+
context.exprMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
252252

253253
// for load job, column exprs is got from file group
254254
// for query, there is no column exprs, they will be got from table's schema in "Load.initColumns"

‎fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,8 @@ public class StreamLoadScanNode extends LoadScanNode {
6666
private TupleDescriptor srcTupleDesc;
6767
private TBrokerScanRange brokerScanRange;
6868

69-
private Map<String, SlotDescriptor> slotDescByName = Maps.newHashMap();
70-
private Map<String, Expr> exprsByName = Maps.newHashMap();
69+
private Map<String, SlotDescriptor> slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
70+
private Map<String, Expr> exprsByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
7171

7272
// used to construct for streaming loading
7373
public StreamLoadScanNode(

0 commit comments

Comments
 (0)