|
23 | 23 | import org.apache.doris.analysis.SlotDescriptor; |
24 | 24 | import org.apache.doris.analysis.SlotId; |
25 | 25 | import org.apache.doris.analysis.SlotRef; |
| 26 | +import org.apache.doris.analysis.StatementBase; |
26 | 27 | import org.apache.doris.analysis.TableName; |
27 | 28 | import org.apache.doris.analysis.TableScanParams; |
28 | 29 | import org.apache.doris.analysis.TableSnapshot; |
29 | 30 | import org.apache.doris.analysis.TupleDescriptor; |
30 | 31 | import org.apache.doris.analysis.TupleId; |
| 32 | +import org.apache.doris.analysis.UserIdentity; |
| 33 | +import org.apache.doris.blockrule.SqlBlockRule; |
| 34 | +import org.apache.doris.blockrule.SqlBlockRuleMgr; |
31 | 35 | import org.apache.doris.catalog.Column; |
32 | 36 | import org.apache.doris.catalog.DatabaseIf; |
33 | 37 | import org.apache.doris.catalog.Env; |
|
65 | 69 | import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; |
66 | 70 | import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper; |
67 | 71 | import org.apache.doris.datasource.mvcc.MvccTableInfo; |
| 72 | +import org.apache.doris.mysql.privilege.Auth; |
68 | 73 | import org.apache.doris.nereids.StatementContext; |
69 | 74 | import org.apache.doris.planner.PlanNodeId; |
70 | 75 | import org.apache.doris.planner.ScanContext; |
|
103 | 108 | import org.apache.iceberg.FileFormat; |
104 | 109 | import org.apache.iceberg.FileMetadata; |
105 | 110 | import org.apache.iceberg.FileScanTask; |
| 111 | +import org.apache.iceberg.ManifestFile; |
106 | 112 | import org.apache.iceberg.ManifestFiles; |
107 | 113 | import org.apache.iceberg.MetadataColumns; |
108 | 114 | import org.apache.iceberg.MetadataTableType; |
|
123 | 129 | import org.apache.iceberg.expressions.Expressions; |
124 | 130 | import org.apache.iceberg.hadoop.HadoopTables; |
125 | 131 | import org.apache.iceberg.io.CloseableIterable; |
| 132 | +import org.apache.iceberg.io.FileIO; |
126 | 133 | import org.apache.iceberg.types.Types; |
127 | 134 | import org.apache.iceberg.util.ScanTaskUtil; |
128 | 135 | import org.apache.thrift.TDeserializer; |
@@ -169,6 +176,76 @@ public class IcebergScanNodeTest { |
169 | 176 | @Rule |
170 | 177 | public final TemporaryFolder temporaryFolder = new TemporaryFolder(); |
171 | 178 |
|
| 179 | + @Test |
| 180 | + public void testExplainRequiresExactFilePlanning() throws Exception { |
| 181 | + assertBatchPlanningAllowed(true, false, 0, 0, true, false); |
| 182 | + } |
| 183 | + |
| 184 | + @Test |
| 185 | + public void testSqlScanLimitsRequireExactFilePlanning() throws Exception { |
| 186 | + assertBatchPlanningAllowed(false, false, 1, 0, true, false); |
| 187 | + assertBatchPlanningAllowed(false, false, 0, 1, true, false); |
| 188 | + assertBatchPlanningAllowed(false, true, 1, 0, true, false); |
| 189 | + assertBatchPlanningAllowed(false, true, 0, 1, true, false); |
| 190 | + } |
| 191 | + |
| 192 | + @Test |
| 193 | + public void testUnrestrictedQueriesKeepBatchPlanning() throws Exception { |
| 194 | + assertBatchPlanningAllowed(false, false, 0, 0, true, true); |
| 195 | + assertBatchPlanningAllowed(false, false, 1, 1, false, true); |
| 196 | + } |
| 197 | + |
| 198 | + private void assertBatchPlanningAllowed(boolean explain, boolean global, long partitions, long splits, |
| 199 | + boolean enabled, boolean expected) throws Exception { |
| 200 | + SessionVariable variables = new SessionVariable(); |
| 201 | + variables.enableExternalTableBatchMode = true; |
| 202 | + variables.numFilesInBatchMode = 1; |
| 203 | + TestIcebergScanNode node = new TestIcebergScanNode(variables); |
| 204 | + Table table = Mockito.mock(Table.class); |
| 205 | + FileIO io = Mockito.mock(FileIO.class); |
| 206 | + TableScan scan = Mockito.mock(TableScan.class); |
| 207 | + Snapshot snapshot = Mockito.mock(Snapshot.class); |
| 208 | + ManifestFile manifest = Mockito.mock(ManifestFile.class); |
| 209 | + Mockito.when(table.io()).thenReturn(io); |
| 210 | + Mockito.when(table.specs()).thenReturn(Collections.singletonMap(0, PartitionSpec.unpartitioned())); |
| 211 | + Mockito.when(scan.table()).thenReturn(table); |
| 212 | + Mockito.when(scan.schema()).thenReturn(PartitionSpec.unpartitioned().schema()); |
| 213 | + Mockito.when(scan.snapshot()).thenReturn(snapshot); |
| 214 | + Mockito.when(scan.filter()).thenReturn(Expressions.alwaysTrue()); |
| 215 | + Mockito.when(snapshot.schemaId()).thenReturn(0); |
| 216 | + Mockito.when(snapshot.dataManifests(io)).thenReturn(Collections.singletonList(manifest)); |
| 217 | + Mockito.when(manifest.partitionSpecId()).thenReturn(0); |
| 218 | + Mockito.when(manifest.hasAddedFiles()).thenReturn(true); |
| 219 | + Mockito.when(manifest.addedFilesCount()).thenReturn(2); |
| 220 | + Mockito.when(manifest.existingFilesCount()).thenReturn(0); |
| 221 | + node.setTableScan(scan); |
| 222 | + setIcebergTable(node, table); |
| 223 | + setPreExecutionAuthenticator(node, new ExecutionAuthenticator() {}); |
| 224 | + |
| 225 | + ConnectContext context = new ConnectContext(); |
| 226 | + context.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp("batch_rule_user", "%")); |
| 227 | + StatementContext statementContext = new StatementContext(); |
| 228 | + StatementBase statement = Mockito.mock(StatementBase.class); |
| 229 | + Mockito.when(statement.isExplain()).thenReturn(explain); |
| 230 | + statementContext.setParsedStatement(statement); |
| 231 | + context.setStatementContext(statementContext); |
| 232 | + context.setThreadLocalInfo(); |
| 233 | + Env env = Mockito.mock(Env.class); |
| 234 | + Auth auth = Mockito.mock(Auth.class); |
| 235 | + SqlBlockRuleMgr rules = new SqlBlockRuleMgr(); |
| 236 | + rules.getNameToSqlBlockRuleMap().put("scan_limit", new SqlBlockRule( |
| 237 | + "scan_limit", "NULL", "NULL", partitions, splits, 0L, false, global, enabled)); |
| 238 | + Mockito.when(env.getAuth()).thenReturn(auth); |
| 239 | + Mockito.when(env.getSqlBlockRuleMgr()).thenReturn(rules); |
| 240 | + Mockito.when(auth.getSqlBlockRules("batch_rule_user")).thenReturn(new String[] {"scan_limit"}); |
| 241 | + try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) { |
| 242 | + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); |
| 243 | + Assert.assertEquals(expected, node.isRealBatchMode()); |
| 244 | + } finally { |
| 245 | + ConnectContext.remove(); |
| 246 | + } |
| 247 | + } |
| 248 | + |
172 | 249 | @Test |
173 | 250 | public void testDoesNotUseHiveParquetInt96TimeZone() { |
174 | 251 | IcebergScanNode node = Mockito.mock(IcebergScanNode.class, Mockito.CALLS_REAL_METHODS); |
|
0 commit comments