Skip to content

Commit 711db43

Browse files
authored
[fix](fe) Disable LOCAL TVF (branch-4.2) (#69592)
### What problem does this PR solve? Issue Number: N/A Related PR: #69591, #69527 Problem Summary: LOCAL TVF can discover files on a backend during analysis. Reject LOCAL unconditionally before property parsing, backend selection and file discovery. FILE TVF with local storage delegates to the same entry point. Explicit LOCAL INSERT and analyzed EXPLAIN INSERT use the same rejection before sink property validation. No configuration option enables these paths, including for ADMIN/OPERATOR. The underlying storage implementation is retained. Tests assert rejection before backend access and preserve remote FILE/sink behavior where supported. Existing LOCAL-dependent success suites explicitly return while the feature is unavailable. Export completion/error checks remain active; LOCAL export-readback comparisons are retired. Remote HDFS/S3 partition cases, the existing UUID S3 fixture branch, and ORC value checks through S3 TVF using the original uploaded fixtures remain active where present on this branch. ### Release note LOCAL TVF and FILE TVF with local storage are disabled for all users. There is no configuration option to re-enable them. Explicit LOCAL TVF INSERT is also disabled. ### Check List (For Author) - Test - [x] Unit Test — 8 targeted FE tests passed; zero failures, errors or skips. - [x] Regression test — rejection coverage and affected-suite adaptations included; suite bodies have not been executed locally. - [ ] Manual test - [ ] No need to test Validation: Checkstyle passed for all 5 changed Java files. Changed Groovy suites passed syntax parsing with Groovy 4.0.19. Whitespace checks passed. Runtime regression and formal local code review have not been performed. - Behavior changed: - [ ] No. - [x] Yes. LOCAL admission always fails; no startup opt-in is available. - Does this need documentation? - [ ] No. - [x] Yes. Release note included; operator documentation should reflect disabled LOCAL support. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label
1 parent 0698e3e commit 711db43

38 files changed

Lines changed: 522 additions & 928 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@
117117
import org.apache.doris.qe.AutoCloseSessionVariable;
118118
import org.apache.doris.qe.ConnectContext;
119119
import org.apache.doris.qe.SessionVariable;
120+
import org.apache.doris.tablefunction.LocalTableValuedFunction;
120121
import org.apache.doris.thrift.TPartialUpdateNewRowPolicy;
121122

122123
import com.google.common.base.Preconditions;
@@ -662,6 +663,10 @@ private Plan bindTVFTableSink(MatchingContext<UnboundTVFTableSink<Plan>> ctx) {
662663
"INSERT INTO TVF only supports local/s3/hdfs, but got: " + tvfName);
663664
}
664665

666+
if (tvfName.equals(LocalTableValuedFunction.NAME)) {
667+
LocalTableValuedFunction.checkAccess();
668+
}
669+
665670
// Validate required properties
666671
if (!properties.containsKey("file_path")) {
667672
throw new AnalysisException("TVF sink requires 'file_path' property");

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ public class LocalTableValuedFunction extends ExternalFileTableValuedFunction {
6161
private boolean sharedStorage = false;
6262

6363
public LocalTableValuedFunction(Map<String, String> properties) throws AnalysisException {
64+
checkAccess();
65+
6466
// 1. analyze common properties
6567
Map<String, String> props = super.parseCommonProperties(properties);
6668

@@ -135,6 +137,11 @@ private void getFileListFromBackend() throws AnalysisException {
135137
}
136138
}
137139

140+
/** Reject LOCAL before property parsing, file discovery or sink binding. */
141+
public static void checkAccess() {
142+
throw new org.apache.doris.nereids.exceptions.AnalysisException("LOCAL TVF is disabled.");
143+
}
144+
138145
@Override
139146
public TFileType getTFileType() {
140147
return TFileType.FILE_LOCAL;

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,8 @@ class BindTVFTableSinkTest {
3838
void duplicateOutputNamesKeepDistinctPositions() {
3939
ConnectContext context = MemoTestUtils.createConnectContext();
4040
InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(
41-
"INSERT INTO local("
41+
"INSERT INTO s3("
4242
+ "'file_path'='/tmp/tvf_duplicate_columns_',"
43-
+ "'backend_id'='1',"
4443
+ "'format'='csv') "
4544
+ "SELECT 1 AS x, 2 AS x");
4645

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.mysql.privilege.AccessControllerManager;
22+
import org.apache.doris.mysql.privilege.PrivPredicate;
23+
import org.apache.doris.nereids.exceptions.AnalysisException;
24+
import org.apache.doris.nereids.parser.NereidsParser;
25+
import org.apache.doris.nereids.trees.plans.Plan;
26+
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
27+
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand;
28+
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
29+
import org.apache.doris.nereids.util.MemoTestUtils;
30+
import org.apache.doris.nereids.util.PlanChecker;
31+
import org.apache.doris.qe.ConnectContext;
32+
33+
import org.junit.jupiter.api.Assertions;
34+
import org.junit.jupiter.api.Test;
35+
import org.mockito.MockedStatic;
36+
import org.mockito.Mockito;
37+
38+
class LocalTvfSinkDisabledTest {
39+
@Test
40+
void disabledLocalRejectsInsertAndExplainBeforeProperties() {
41+
for (boolean authorized : new boolean[] {false, true}) {
42+
for (String prefix : new String[] {"", "EXPLAIN "}) {
43+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
44+
() -> analyzeInsert(prefix + "INSERT INTO local('format'='csv') SELECT 1", authorized));
45+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
46+
}
47+
}
48+
}
49+
50+
@Test
51+
void disabledLocalKeepsRemoteSinkValidation() {
52+
for (String name : new String[] {"s3", "hdfs"}) {
53+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
54+
() -> analyzeInsert("INSERT INTO " + name + "('format'='csv') SELECT 1", false));
55+
Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'"));
56+
}
57+
}
58+
59+
private Plan analyzeInsert(String sql, boolean authorized) {
60+
ConnectContext context = MemoTestUtils.createConnectContext();
61+
Env env = Mockito.mock(Env.class);
62+
AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
63+
Mockito.when(env.getAccessManager()).thenReturn(accessManager);
64+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(authorized);
65+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) {
66+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
67+
LogicalPlan parsed = new NereidsParser().parseSingle(sql);
68+
if (parsed instanceof ExplainCommand) {
69+
parsed = ((ExplainCommand) parsed).getLogicalPlan();
70+
}
71+
InsertIntoTVFCommand command = (InsertIntoTVFCommand) parsed;
72+
return PlanChecker.from(context).analyze(command.getExplainPlan(context)).getPlan();
73+
}
74+
}
75+
}
Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.tablefunction;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.mysql.privilege.AccessControllerManager;
22+
import org.apache.doris.mysql.privilege.PrivPredicate;
23+
import org.apache.doris.nereids.exceptions.AnalysisException;
24+
import org.apache.doris.qe.ConnectContext;
25+
import org.apache.doris.rpc.BackendServiceProxy;
26+
import org.apache.doris.system.SystemInfoService;
27+
28+
import com.google.common.collect.ImmutableMap;
29+
import org.junit.jupiter.api.AfterEach;
30+
import org.junit.jupiter.api.Assertions;
31+
import org.junit.jupiter.api.BeforeEach;
32+
import org.junit.jupiter.api.Test;
33+
import org.mockito.MockedConstruction;
34+
import org.mockito.MockedStatic;
35+
import org.mockito.Mockito;
36+
37+
import java.util.Map;
38+
39+
public class LocalTableValuedFunctionDisabledTest {
40+
private static final Map<String, String> PROPERTIES = ImmutableMap.of(
41+
"file_path", "local_disabled_test.csv", "backend_id", "1", "format", "csv");
42+
43+
private final Env env = Mockito.mock(Env.class);
44+
private final ConnectContext context = Mockito.mock(ConnectContext.class);
45+
private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
46+
private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class);
47+
private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class);
48+
private MockedStatic<Env> mockedEnv;
49+
private MockedStatic<ConnectContext> mockedContext;
50+
private MockedStatic<BackendServiceProxy> mockedProxy;
51+
52+
@BeforeEach
53+
public void setUp() {
54+
mockedEnv = Mockito.mockStatic(Env.class);
55+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
56+
mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo);
57+
mockedContext = Mockito.mockStatic(ConnectContext.class);
58+
mockedContext.when(ConnectContext::get).thenReturn(context);
59+
mockedProxy = Mockito.mockStatic(BackendServiceProxy.class);
60+
mockedProxy.when(BackendServiceProxy::getInstance).thenReturn(proxy);
61+
Mockito.when(env.getAccessManager()).thenReturn(accessManager);
62+
}
63+
64+
@AfterEach
65+
public void tearDown() {
66+
mockedProxy.close();
67+
mockedContext.close();
68+
mockedEnv.close();
69+
}
70+
71+
@Test
72+
public void testLocalDisabledBeforePropertyParsing() {
73+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
74+
() -> new LocalTableValuedFunction(ImmutableMap.of()));
75+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
76+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
77+
}
78+
79+
@Test
80+
public void testFormatsDisabledBeforeFileAccess() {
81+
for (String format : new String[] {"csv", "json", "lance"}) {
82+
Map<String, String> properties = ImmutableMap.of(
83+
"file_path", "local_disabled_test", "backend_id", "1", "format", format);
84+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
85+
() -> new LocalTableValuedFunction(properties));
86+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
87+
}
88+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
89+
}
90+
91+
@Test
92+
public void testFileLocalDisabledBeforeFileAccess() {
93+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
94+
() -> new FileTableValuedFunction(PROPERTIES));
95+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
96+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
97+
}
98+
99+
@Test
100+
public void testPrivilegedUsersCannotAccessLocalFiles() {
101+
for (PrivPredicate privilege : new PrivPredicate[] {PrivPredicate.ADMIN, PrivPredicate.OPERATOR}) {
102+
Mockito.when(accessManager.checkGlobalPriv(context, privilege)).thenReturn(true);
103+
Assertions.assertThrows(AnalysisException.class, () -> new LocalTableValuedFunction(PROPERTIES));
104+
Assertions.assertThrows(AnalysisException.class, () -> new FileTableValuedFunction(PROPERTIES));
105+
}
106+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
107+
}
108+
109+
@Test
110+
public void testFileS3RemainsAvailable() {
111+
Map<String, String> properties = ImmutableMap.<String, String>builder()
112+
.put("uri", "s3://test-bucket/local_disabled_test.csv")
113+
.put("s3.endpoint", "s3.us-east-1.amazonaws.com")
114+
.put("s3.region", "us-east-1")
115+
.put("s3.access_key", "test-access-key")
116+
.put("s3.secret_key", "test-secret-key")
117+
.put("format", "csv")
118+
.build();
119+
try (MockedConstruction<S3TableValuedFunction> mockedS3 =
120+
Mockito.mockConstruction(S3TableValuedFunction.class)) {
121+
Assertions.assertDoesNotThrow(() -> new FileTableValuedFunction(properties));
122+
Assertions.assertEquals(1, mockedS3.constructed().size());
123+
}
124+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
125+
}
126+
}
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_local_tvf_disabled", "p0") {
19+
for (def functionName : ["LOCAL", "FILE"]) {
20+
for (def prefix : ["", "EXPLAIN "]) {
21+
test {
22+
sql """${prefix}SELECT * FROM ${functionName}(
23+
"file_path" = "local_enable_test.csv",
24+
"backend_id" = "-1",
25+
"format" = "csv"
26+
)"""
27+
exception "LOCAL TVF is disabled"
28+
}
29+
}
30+
}
31+
}

‎regression-test/suites/datatype_p0/uuid/test_uuid_csv_roundtrip.groovy‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ import java.nio.file.Paths
1919

2020
// Checklist: G02 G11 G13.
2121
suite("test_uuid_csv_roundtrip") {
22+
logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
23+
return
24+
2225
if (!getFeConfig("enable_outfile_to_local").equalsIgnoreCase("true")) {
2326
logger.warn("Please set enable_outfile_to_local to true to run test_uuid_csv_roundtrip")
2427
return

‎regression-test/suites/datatype_p0/uuid/test_uuid_csv_schema.groovy‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,30 +16,41 @@
1616
// under the License.
1717

1818
// External checklist: T04, R02, R09, Q01-Q03, W06, V01-V02.
19+
import com.azure.storage.blob.BlobClientBuilder
20+
import com.azure.storage.common.StorageSharedKeyCredential
21+
1922
suite("test_uuid_csv_schema", "p0,external") {
2023
sql "SET enable_sql_cache = false"
2124
sql "SET enable_query_cache = false"
22-
String localPath = context.config.otherConfigs.get('uuidLocalExportPath')
25+
String provider = getS3Provider()
26+
String endpoint = getS3Endpoint()
27+
if (provider.equalsIgnoreCase("AZURE")) {
28+
if (endpoint == "azure-endpoint") {
29+
endpoint = "https://${getS3AK()}.blob.core.windows.net"
30+
} else if (!endpoint.contains("://")) {
31+
endpoint = "https://${endpoint}"
32+
}
33+
}
2334
String token = UUID.randomUUID().toString()
2435
def sources = [:]
2536
for (String kind : ['values', 'invalid']) {
2637
File fixture = new File(context.dataPath, "test_data/uuid_external_${kind}.csv")
27-
if (localPath != null) {
28-
File target = new File("${localPath}_${token}_${kind}.csv")
29-
java.nio.file.Files.copy(fixture.toPath(), target.toPath())
30-
String path = "${context.config.otherConfigs.get('uuidLocalTvfPath')}_${token}_${kind}.csv"
31-
sources[kind] = """LOCAL('file_path'='${path}',
32-
'backend_id'='${(sql 'SHOW BACKENDS')[0][0]}','format'='csv',
33-
'column_separator'=',','csv_schema'='id:int;u:uuid')"""
38+
String bucket = context.config.otherConfigs.get('s3BucketName')
39+
String key = "uuid_csv_schema/${token}/${kind}.csv"
40+
if (provider.equalsIgnoreCase("AZURE")) {
41+
new BlobClientBuilder()
42+
.endpoint(endpoint)
43+
.containerName(bucket)
44+
.blobName(key)
45+
.credential(new StorageSharedKeyCredential(getS3AK(), getS3SK()))
46+
.buildClient().uploadFromFile(fixture.absolutePath)
3447
} else {
35-
String bucket = context.config.otherConfigs.get('s3BucketName')
36-
String key = "uuid_csv_schema/${token}/${kind}.csv"
3748
getS3Client().putObject(bucket, key, fixture)
38-
sources[kind] = """S3('uri'='s3://${bucket}/${key}','format'='csv',
39-
'column_separator'=',','csv_schema'='id:int;u:uuid',
40-
's3.endpoint'='${getS3Endpoint()}','s3.region'='${getS3Region()}',
41-
's3.access_key'='${getS3AK()}','s3.secret_key'='${getS3SK()}')"""
4249
}
50+
sources[kind] = """S3('uri'='s3://${bucket}/${key}','format'='csv',
51+
'column_separator'=',','csv_schema'='id:int;u:uuid',
52+
'provider'='${provider}','s3.endpoint'='${endpoint}','s3.region'='${getS3Region()}',
53+
's3.access_key'='${getS3AK()}','s3.secret_key'='${getS3SK()}')"""
4354
}
4455
for (boolean scannerV2 : [false, true]) {
4556
sql "SET enable_file_scanner_v2 = ${scannerV2}"

0 commit comments

Comments
 (0)