Skip to content

Commit d376a62

Browse files
committed
[fix](fe) Disable LOCAL TVF by default
### What problem does this PR solve? Issue Number: N/A Related PR: #69526 Problem Summary: Add startup-only enable_local_tvf=false. Reject LOCAL TVF and FILE-local at FE admission before discovery. Explicit opt-in retains the existing storage implementation and requires ADMIN/OPERATOR. Supported explicit LOCAL sinks use the same FE admission check. ### Release note LOCAL TVF and FILE TVF with local storage require enable_local_tvf=true in fe.conf on every query-serving FE and an FE restart. This change provides default-disable mitigation; enabled storage-path sandbox hardening remains separate work. ### Check List (For Author) - Test: Focused FE unit and regression cases added but not executed. Changed Java files pass Checkstyle; diff whitespace checks pass. - Behavior changed: Yes. LOCAL access requires startup opt-in and ADMIN/OPERATOR admission. - Does this need documentation: Yes. Configuration description and release note included; user-documentation follow-up remains pending.
1 parent 7679a4f commit d376a62

14 files changed

Lines changed: 336 additions & 2 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2815,6 +2815,10 @@ public static void validateMtmvCacheConfig() throws ConfigException {
28152815
+ "configuration to prevent command injection attacks.")
28162816
public static boolean enable_java_udf = true;
28172817

2818+
@ConfField(description = "Whether to enable LOCAL TVF and FILE TVF with local storage. Disabled by default. "
2819+
+ "Set this in fe.conf and restart every FE to enable it. Existing access checks still apply.")
2820+
public static boolean enable_local_tvf = false;
2821+
28182822
@ConfField(mutable = true, masterOnly = true, description = "When enabled, data can be processed using the "
28192823
+ "globally created Java UDF function during import. " + "The default setting is false.")
28202824
public static boolean enable_udf_in_load = false;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
import org.apache.doris.qe.AutoCloseSessionVariable;
105105
import org.apache.doris.qe.ConnectContext;
106106
import org.apache.doris.qe.SessionVariable;
107+
import org.apache.doris.tablefunction.LocalTableValuedFunction;
107108
import org.apache.doris.thrift.TPartialUpdateNewRowPolicy;
108109

109110
import com.google.common.annotations.VisibleForTesting;
@@ -701,6 +702,10 @@ private Plan bindTVFTableSink(MatchingContext<UnboundTVFTableSink<Plan>> ctx) {
701702
"INSERT INTO TVF only supports local/s3/hdfs, but got: " + tvfName);
702703
}
703704

705+
if (tvfName.equals(LocalTableValuedFunction.NAME)) {
706+
LocalTableValuedFunction.checkAccess(ctx.connectContext);
707+
}
708+
704709
// Validate required properties
705710
if (!properties.containsKey("file_path")) {
706711
throw new AnalysisException("TVF sink requires 'file_path' property");

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.analysis.StorageBackend.StorageType;
2222
import org.apache.doris.catalog.Env;
2323
import org.apache.doris.common.AnalysisException;
24+
import org.apache.doris.common.Config;
2425
import org.apache.doris.common.ErrorCode;
2526
import org.apache.doris.datasource.storage.StorageAdapter;
2627
import org.apache.doris.mysql.privilege.PrivPredicate;
@@ -137,13 +138,22 @@ private void getFileListFromBackend() throws AnalysisException {
137138
}
138139
}
139140

140-
@Override
141-
public void checkAuth(ConnectContext ctx) {
141+
/** Check LOCAL admission before property parsing, discovery or sink binding. */
142+
public static void checkAccess(ConnectContext ctx) {
143+
if (!Config.enable_local_tvf) {
144+
throw new org.apache.doris.nereids.exceptions.AnalysisException(
145+
"LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it.");
146+
}
142147
if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN)
143148
&& !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.OPERATOR)) {
144149
String message = ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR.formatErrorMsg("ADMIN/OPERATOR");
145150
throw new org.apache.doris.nereids.exceptions.AnalysisException(message);
146151
}
152+
}
153+
154+
@Override
155+
public void checkAuth(ConnectContext ctx) {
156+
checkAccess(ctx);
147157
super.checkAuth(ctx);
148158
}
149159

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.common.Config;
22+
import org.apache.doris.mysql.privilege.AccessControllerManager;
23+
import org.apache.doris.mysql.privilege.PrivPredicate;
24+
import org.apache.doris.nereids.exceptions.AnalysisException;
25+
import org.apache.doris.nereids.parser.NereidsParser;
26+
import org.apache.doris.nereids.trees.plans.Plan;
27+
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
28+
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand;
29+
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
30+
import org.apache.doris.nereids.util.MemoTestUtils;
31+
import org.apache.doris.nereids.util.PlanChecker;
32+
import org.apache.doris.qe.ConnectContext;
33+
34+
import org.junit.jupiter.api.AfterEach;
35+
import org.junit.jupiter.api.Assertions;
36+
import org.junit.jupiter.api.BeforeEach;
37+
import org.junit.jupiter.api.Test;
38+
import org.mockito.MockedStatic;
39+
import org.mockito.Mockito;
40+
41+
class LocalTvfSinkEnableTest {
42+
private boolean originalEnableLocalTvf;
43+
44+
@BeforeEach
45+
void setUp() {
46+
originalEnableLocalTvf = Config.enable_local_tvf;
47+
Config.enable_local_tvf = false;
48+
}
49+
50+
@AfterEach
51+
void tearDown() {
52+
Config.enable_local_tvf = originalEnableLocalTvf;
53+
}
54+
55+
@Test
56+
void disabledLocalRejectsInsertAndExplainBeforeProperties() {
57+
for (String prefix : new String[] {"", "EXPLAIN "}) {
58+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
59+
() -> analyzeInsert(prefix + "INSERT INTO local('format'='csv') SELECT 1", true));
60+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
61+
}
62+
}
63+
64+
@Test
65+
void enabledLocalKeepsPropertyValidation() {
66+
Config.enable_local_tvf = true;
67+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
68+
() -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1", true));
69+
Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'"));
70+
}
71+
72+
@Test
73+
void enabledLocalChecksPrivilegesBeforeProperties() {
74+
Config.enable_local_tvf = true;
75+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
76+
() -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1", false));
77+
Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR"));
78+
}
79+
80+
@Test
81+
void disabledLocalKeepsRemoteSinkValidation() {
82+
for (String name : new String[] {"s3", "hdfs"}) {
83+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
84+
() -> analyzeInsert("INSERT INTO " + name + "('format'='csv') SELECT 1", false));
85+
Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'"));
86+
}
87+
}
88+
89+
private Plan analyzeInsert(String sql, boolean authorized) {
90+
ConnectContext context = MemoTestUtils.createConnectContext();
91+
Env env = Mockito.mock(Env.class);
92+
AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
93+
Mockito.when(env.getAccessManager()).thenReturn(accessManager);
94+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(authorized);
95+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) {
96+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
97+
LogicalPlan parsed = new NereidsParser().parseSingle(sql);
98+
if (parsed instanceof ExplainCommand) {
99+
parsed = ((ExplainCommand) parsed).getLogicalPlan();
100+
}
101+
InsertIntoTVFCommand command = (InsertIntoTVFCommand) parsed;
102+
return PlanChecker.from(context).analyze(command.getExplainPlan(context)).getPlan();
103+
}
104+
}
105+
}

‎fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionAuthTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.tablefunction;
1919

2020
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.common.Config;
2122
import org.apache.doris.mysql.privilege.AccessControllerManager;
2223
import org.apache.doris.mysql.privilege.PrivPredicate;
2324
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -50,12 +51,15 @@ public class LocalTableValuedFunctionAuthTest {
5051
private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
5152
private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class);
5253
private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class);
54+
private boolean originalEnableLocalTvf;
5355
private MockedStatic<Env> mockedEnv;
5456
private MockedStatic<ConnectContext> mockedContext;
5557
private MockedStatic<BackendServiceProxy> mockedProxy;
5658

5759
@BeforeEach
5860
public void setUp() throws Exception {
61+
originalEnableLocalTvf = Config.enable_local_tvf;
62+
Config.enable_local_tvf = true;
5963
mockedEnv = Mockito.mockStatic(Env.class);
6064
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
6165
mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo);
@@ -76,6 +80,7 @@ public void setUp() throws Exception {
7680

7781
@AfterEach
7882
public void tearDown() {
83+
Config.enable_local_tvf = originalEnableLocalTvf;
7984
mockedProxy.close();
8085
mockedContext.close();
8186
mockedEnv.close();
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.tablefunction;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.common.Config;
22+
import org.apache.doris.mysql.privilege.AccessControllerManager;
23+
import org.apache.doris.mysql.privilege.PrivPredicate;
24+
import org.apache.doris.nereids.exceptions.AnalysisException;
25+
import org.apache.doris.proto.InternalService.PGlobResponse;
26+
import org.apache.doris.proto.Types.PStatus;
27+
import org.apache.doris.qe.ConnectContext;
28+
import org.apache.doris.qe.SessionVariable;
29+
import org.apache.doris.rpc.BackendServiceProxy;
30+
import org.apache.doris.system.Backend;
31+
import org.apache.doris.system.SystemInfoService;
32+
import org.apache.doris.thrift.TNetworkAddress;
33+
34+
import com.google.common.collect.ImmutableMap;
35+
import org.junit.jupiter.api.AfterEach;
36+
import org.junit.jupiter.api.Assertions;
37+
import org.junit.jupiter.api.BeforeEach;
38+
import org.junit.jupiter.api.Test;
39+
import org.mockito.MockedConstruction;
40+
import org.mockito.MockedStatic;
41+
import org.mockito.Mockito;
42+
43+
import java.util.Map;
44+
import java.util.concurrent.CompletableFuture;
45+
46+
public class LocalTableValuedFunctionEnableTest {
47+
private static final Map<String, String> PROPERTIES = ImmutableMap.of(
48+
"file_path", "local_privilege_test.csv", "backend_id", "1", "format", "csv");
49+
50+
private final Env env = Mockito.mock(Env.class);
51+
private final ConnectContext context = Mockito.mock(ConnectContext.class);
52+
private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
53+
private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class);
54+
private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class);
55+
private boolean originalEnableLocalTvf;
56+
private MockedStatic<Env> mockedEnv;
57+
private MockedStatic<ConnectContext> mockedContext;
58+
private MockedStatic<BackendServiceProxy> mockedProxy;
59+
60+
@BeforeEach
61+
public void setUp() throws Exception {
62+
originalEnableLocalTvf = Config.enable_local_tvf;
63+
Config.enable_local_tvf = false;
64+
mockedEnv = Mockito.mockStatic(Env.class);
65+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
66+
mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo);
67+
mockedContext = Mockito.mockStatic(ConnectContext.class);
68+
mockedContext.when(ConnectContext::get).thenReturn(context);
69+
mockedProxy = Mockito.mockStatic(BackendServiceProxy.class);
70+
mockedProxy.when(BackendServiceProxy::getInstance).thenReturn(proxy);
71+
Mockito.when(env.getAccessManager()).thenReturn(accessManager);
72+
Mockito.when(context.getSessionVariable()).thenReturn(new SessionVariable());
73+
Mockito.when(context.getQueryTimeoutS()).thenReturn(60);
74+
75+
Backend backend = Mockito.mock(Backend.class);
76+
Mockito.when(systemInfo.getBackend(1L)).thenReturn(backend);
77+
Mockito.when(backend.getBrpcAddress()).thenReturn(new TNetworkAddress("127.0.0.1", 8060));
78+
Mockito.when(proxy.glob(Mockito.any(), Mockito.any())).thenReturn(CompletableFuture.completedFuture(
79+
PGlobResponse.newBuilder().setStatus(PStatus.newBuilder().setStatusCode(0)).build()));
80+
}
81+
82+
@AfterEach
83+
public void tearDown() {
84+
Config.enable_local_tvf = originalEnableLocalTvf;
85+
mockedProxy.close();
86+
mockedContext.close();
87+
mockedEnv.close();
88+
}
89+
90+
@Test
91+
public void testDisabledRejectsLocalBeforePropertyParsing() {
92+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
93+
() -> new LocalTableValuedFunction(ImmutableMap.of()));
94+
Assertions.assertTrue(exception.getMessage().contains("enable_local_tvf=true"));
95+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
96+
}
97+
98+
@Test
99+
public void testDisabledRejectsCsvAndLanceBeforeFileAccess() {
100+
for (String format : new String[] {"csv", "lance"}) {
101+
Map<String, String> properties = ImmutableMap.of(
102+
"file_path", "local_enable_test.csv", "backend_id", "1", "format", format);
103+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
104+
() -> new LocalTableValuedFunction(properties));
105+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
106+
}
107+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
108+
}
109+
110+
@Test
111+
public void testEnabledLocalReachesExistingFileAccess() throws Exception {
112+
Config.enable_local_tvf = true;
113+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true);
114+
new LocalTableValuedFunction(PROPERTIES);
115+
Mockito.verify(proxy).glob(Mockito.any(), Mockito.any());
116+
}
117+
118+
@Test
119+
public void testDisabledRejectsFileLocalBeforeFileAccess() {
120+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
121+
() -> new FileTableValuedFunction(PROPERTIES));
122+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
123+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
124+
}
125+
126+
@Test
127+
public void testEnabledFileLocalReachesExistingFileAccess() throws Exception {
128+
Config.enable_local_tvf = true;
129+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true);
130+
new FileTableValuedFunction(PROPERTIES);
131+
Mockito.verify(proxy).glob(Mockito.any(), Mockito.any());
132+
}
133+
134+
@Test
135+
public void testDisabledLocalDoesNotBlockFileS3() {
136+
Map<String, String> properties = ImmutableMap.<String, String>builder()
137+
.put("uri", "s3://test-bucket/local_enable_test.csv")
138+
.put("s3.endpoint", "s3.us-east-1.amazonaws.com")
139+
.put("s3.region", "us-east-1")
140+
.put("s3.access_key", "test-access-key")
141+
.put("s3.secret_key", "test-secret-key")
142+
.put("format", "csv")
143+
.build();
144+
try (MockedConstruction<S3TableValuedFunction> mockedS3 =
145+
Mockito.mockConstruction(S3TableValuedFunction.class)) {
146+
Assertions.assertDoesNotThrow(() -> new FileTableValuedFunction(properties));
147+
Assertions.assertEquals(1, mockedS3.constructed().size());
148+
}
149+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
150+
}
151+
}

‎regression-test/pipeline/cloud_p0/conf/fe_custom.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,6 @@ max_bucket_num_per_partition=512
5353
enable_table_stream=true
5454
enable_feature_binlog=true
5555
enable_feature_changelog=true
56+
57+
# Exercise LOCAL TVF explicitly; production defaults to disabled.
58+
enable_local_tvf = true

‎regression-test/pipeline/cloud_p1/conf/fe_custom.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,6 @@ arrow_flight_sql_port = 8081
3939
enable_job_schedule_second_for_test = true
4040

4141
enable_notify_be_after_load_txn_commit=true
42+
43+
# Exercise LOCAL TVF explicitly; production defaults to disabled.
44+
enable_local_tvf = true

‎regression-test/pipeline/external/conf/fe.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,6 @@ hms_events_polling_interval_ms=700
105105

106106

107107
KRB5_CONFIG=/keytabs/krb5.conf
108+
109+
# Exercise LOCAL TVF explicitly; production defaults to disabled.
110+
enable_local_tvf = true

‎regression-test/pipeline/nonConcurrent/conf/fe.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,3 +94,6 @@ max_spilled_profile_num = 2000
9494

9595
check_table_lock_leaky=true
9696
enable_table_stream=true
97+
98+
# Exercise LOCAL TVF explicitly; production defaults to disabled.
99+
enable_local_tvf = true

0 commit comments

Comments
 (0)