Skip to content

Commit 1706699

Browse files
authored
[fix](multi-catalog)support the max compute partition prune (#27154)
1. max compute partition prune, we just support filter mc partitions by '=',it can filter just one partition to support multiple partition filter and range operator('>','<', '>='..), the partition prune should be supported. 2. add max compute row count cache and partitionValues cache 3. add max compute regression case
1 parent f4afcae commit 1706699

25 files changed

Lines changed: 683 additions & 152 deletions

‎be/src/runtime/descriptors.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,6 @@ MaxComputeTableDescriptor::MaxComputeTableDescriptor(const TTableDescriptor& tde
195195
_table(tdesc.mcTable.table),
196196
_access_key(tdesc.mcTable.access_key),
197197
_secret_key(tdesc.mcTable.secret_key),
198-
_partition_spec(tdesc.mcTable.partition_spec),
199198
_public_access(tdesc.mcTable.public_access) {}
200199

201200
MaxComputeTableDescriptor::~MaxComputeTableDescriptor() = default;

‎be/src/runtime/descriptors.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,6 @@ class MaxComputeTableDescriptor : public TableDescriptor {
240240
const std::string table() const { return _table; }
241241
const std::string access_key() const { return _access_key; }
242242
const std::string secret_key() const { return _secret_key; }
243-
const std::string partition_spec() const { return _partition_spec; }
244243
const std::string public_access() const { return _public_access; }
245244

246245
private:
@@ -249,7 +248,6 @@ class MaxComputeTableDescriptor : public TableDescriptor {
249248
std::string _table;
250249
std::string _access_key;
251250
std::string _secret_key;
252-
std::string _partition_spec;
253251
std::string _public_access;
254252
};
255253

‎be/src/vec/exec/format/table/max_compute_jni_reader.cpp‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,15 @@ class Block;
3838
namespace doris::vectorized {
3939

4040
MaxComputeJniReader::MaxComputeJniReader(const MaxComputeTableDescriptor* mc_desc,
41+
const TMaxComputeFileDesc& max_compute_params,
4142
const std::vector<SlotDescriptor*>& file_slot_descs,
4243
const TFileRangeDesc& range, RuntimeState* state,
4344
RuntimeProfile* profile)
44-
: _file_slot_descs(file_slot_descs), _range(range), _state(state), _profile(profile) {
45+
: _max_compute_params(max_compute_params),
46+
_file_slot_descs(file_slot_descs),
47+
_range(range),
48+
_state(state),
49+
_profile(profile) {
4550
_table_desc = mc_desc;
4651
std::ostringstream required_fields;
4752
std::ostringstream columns_types;
@@ -64,7 +69,7 @@ MaxComputeJniReader::MaxComputeJniReader(const MaxComputeTableDescriptor* mc_des
6469
{"access_key", _table_desc->access_key()},
6570
{"secret_key", _table_desc->secret_key()},
6671
{"project", _table_desc->project()},
67-
{"partition_spec", _table_desc->partition_spec()},
72+
{"partition_spec", _max_compute_params.partition_spec},
6873
{"table", _table_desc->table()},
6974
{"public_access", _table_desc->public_access()},
7075
{"start_offset", std::to_string(_range.start_offset)},

‎be/src/vec/exec/format/table/max_compute_jni_reader.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class MaxComputeJniReader : public GenericReader {
5454

5555
public:
5656
MaxComputeJniReader(const MaxComputeTableDescriptor* mc_desc,
57+
const TMaxComputeFileDesc& max_compute_params,
5758
const std::vector<SlotDescriptor*>& file_slot_descs,
5859
const TFileRangeDesc& range, RuntimeState* state, RuntimeProfile* profile);
5960

@@ -69,6 +70,7 @@ class MaxComputeJniReader : public GenericReader {
6970

7071
private:
7172
const MaxComputeTableDescriptor* _table_desc = nullptr;
73+
const TMaxComputeFileDesc& _max_compute_params;
7274
const std::vector<SlotDescriptor*>& _file_slot_descs;
7375
const TFileRangeDesc& _range;
7476
RuntimeState* _state = nullptr;

‎be/src/vec/exec/scan/vfile_scanner.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -745,13 +745,13 @@ Status VFileScanner::_get_next_reader() {
745745
bool need_to_get_parsed_schema = false;
746746
switch (format_type) {
747747
case TFileFormatType::FORMAT_JNI: {
748-
if (_real_tuple_desc->table_desc()->table_type() ==
749-
::doris::TTableType::type::MAX_COMPUTE_TABLE) {
750-
const MaxComputeTableDescriptor* mc_desc =
751-
static_cast<const MaxComputeTableDescriptor*>(
752-
_real_tuple_desc->table_desc());
748+
if (range.__isset.table_format_params &&
749+
range.table_format_params.table_format_type == "max_compute") {
750+
const auto* mc_desc = static_cast<const MaxComputeTableDescriptor*>(
751+
_real_tuple_desc->table_desc());
753752
std::unique_ptr<MaxComputeJniReader> mc_reader = MaxComputeJniReader::create_unique(
754-
mc_desc, _file_slot_descs, range, _state, _profile);
753+
mc_desc, range.table_format_params.max_compute_params, _file_slot_descs,
754+
range, _state, _profile);
755755
init_status = mc_reader->init_reader(_colname_to_value_range);
756756
_cur_reader = std::move(mc_reader);
757757
} else if (range.__isset.table_format_params &&

‎fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,11 @@ public class MaxComputeJniScanner extends JniScanner {
6161
private static final String START_OFFSET = "start_offset";
6262
private static final String SPLIT_SIZE = "split_size";
6363
private static final String PUBLIC_ACCESS = "public_access";
64-
private final RootAllocator arrowAllocator = new RootAllocator(Integer.MAX_VALUE);
6564
private final Map<String, MaxComputeTableScan> tableScans = new ConcurrentHashMap<>();
6665
private final String region;
6766
private final String project;
6867
private final String table;
68+
private RootAllocator arrowAllocator;
6969
private PartitionSpec partitionSpec;
7070
private Set<String> partitionColumns;
7171
private MaxComputeTableScan curTableScan;
@@ -171,9 +171,14 @@ public void open() throws IOException {
171171
partitionColumns = session.getSchema().getPartitionColumns().stream()
172172
.map(Column::getName)
173173
.collect(Collectors.toSet());
174-
List<Column> maxComputeColumns = new ArrayList<>(readColumns);
175-
maxComputeColumns.removeIf(e -> partitionColumns.contains(e.getName()));
176-
curReader = session.openArrowRecordReader(start, totalRows, maxComputeColumns, arrowAllocator);
174+
List<Column> pushDownColumns = new ArrayList<>(readColumns);
175+
pushDownColumns.removeIf(e -> partitionColumns.contains(e.getName()));
176+
if (pushDownColumns.isEmpty() && !partitionColumns.isEmpty()) {
177+
// query columns required non-null, when query partition table
178+
pushDownColumns.add(session.getSchema().getColumn(0));
179+
}
180+
arrowAllocator = new RootAllocator(Integer.MAX_VALUE);
181+
curReader = session.openArrowRecordReader(start, totalRows, pushDownColumns, arrowAllocator);
177182
remainBatchRows = totalRows;
178183
} catch (TunnelException e) {
179184
if (retryCount > 0 && e.getErrorMsg().contains("TableModified")) {
@@ -256,7 +261,8 @@ public void close() throws IOException {
256261
startOffset = -1;
257262
splitSize = -1;
258263
if (curReader != null) {
259-
arrowAllocator.releaseBytes(arrowAllocator.getAllocatedMemory());
264+
arrowAllocator.close();
265+
arrowAllocator = null;
260266
curReader.close();
261267
curReader = null;
262268
}
@@ -281,15 +287,25 @@ protected int getNext() throws IOException {
281287
private int readVectors(int expectedRows) throws IOException {
282288
VectorSchemaRoot batch;
283289
int curReadRows = 0;
284-
while (curReadRows < expectedRows && (batch = curReader.read()) != null) {
290+
while (curReadRows < expectedRows) {
291+
batch = curReader.read();
292+
if (batch == null) {
293+
break;
294+
}
285295
try {
286296
List<FieldVector> fieldVectors = batch.getFieldVectors();
287297
int batchRows = 0;
288298
for (FieldVector column : fieldVectors) {
299+
Integer readColumnId = readColumnsToId.get(column.getName());
300+
if (readColumnId == null) {
301+
// use for partition if no column need to read.
302+
batchRows = column.getValueCount();
303+
continue;
304+
}
289305
columnValue.reset(column);
290306
batchRows = column.getValueCount();
291307
for (int j = 0; j < batchRows; j++) {
292-
appendData(readColumnsToId.get(column.getName()), columnValue);
308+
appendData(readColumnId, columnValue);
293309
}
294310
}
295311
if (partitionSpec != null) {
@@ -305,6 +321,8 @@ private int readVectors(int expectedRows) throws IOException {
305321
}
306322
}
307323
curReadRows += batchRows;
324+
} catch (Exception e) {
325+
throw new RuntimeException("Fail to read arrow data, reason: " + e.getMessage(), e);
308326
} finally {
309327
batch.close();
310328
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.doris.catalog.TableIf.TableType;
2727
import org.apache.doris.catalog.Type;
2828
import org.apache.doris.catalog.external.HMSExternalTable;
29+
import org.apache.doris.catalog.external.MaxComputeExternalTable;
2930
import org.apache.doris.common.AnalysisException;
3031
import org.apache.doris.common.ErrorCode;
3132
import org.apache.doris.common.ErrorReport;
@@ -37,6 +38,7 @@
3738
import org.apache.doris.common.util.OrderByPair;
3839
import org.apache.doris.datasource.CatalogIf;
3940
import org.apache.doris.datasource.HMSExternalCatalog;
41+
import org.apache.doris.datasource.MaxComputeExternalCatalog;
4042
import org.apache.doris.mysql.privilege.PrivPredicate;
4143
import org.apache.doris.qe.ConnectContext;
4244
import org.apache.doris.qe.ShowResultSetMetaData;
@@ -125,7 +127,8 @@ public void analyze(Analyzer analyzer) throws UserException {
125127
}
126128

127129
DatabaseIf db = catalog.getDbOrAnalysisException(dbName);
128-
TableIf table = db.getTableOrMetaException(tblName, Table.TableType.OLAP, TableType.HMS_EXTERNAL_TABLE);
130+
TableIf table = db.getTableOrMetaException(tblName, Table.TableType.OLAP,
131+
TableType.HMS_EXTERNAL_TABLE, TableType.MAX_COMPUTE_EXTERNAL_TABLE);
129132

130133
if (table instanceof HMSExternalTable) {
131134
if (((HMSExternalTable) table).isView()) {
@@ -137,6 +140,13 @@ public void analyze(Analyzer analyzer) throws UserException {
137140
return;
138141
}
139142

143+
if (table instanceof MaxComputeExternalTable) {
144+
if (((MaxComputeExternalTable) table).getOdpsTable().getPartitions().isEmpty()) {
145+
throw new AnalysisException("Table " + tblName + " is not a partitioned table");
146+
}
147+
return;
148+
}
149+
140150
table.readLock();
141151
try {
142152
// build proc path
@@ -169,7 +179,8 @@ public void analyzeImpl(Analyzer analyzer) throws UserException {
169179
}
170180

171181
// disallow unsupported catalog
172-
if (!(catalog.isInternalCatalog() || catalog instanceof HMSExternalCatalog)) {
182+
if (!(catalog.isInternalCatalog() || catalog instanceof HMSExternalCatalog
183+
|| catalog instanceof MaxComputeExternalCatalog)) {
173184
throw new AnalysisException(String.format("Catalog of type '%s' is not allowed in ShowPartitionsStmt",
174185
catalog.getType()));
175186
}

0 commit comments

Comments
 (0)