Skip to content

Commit 42c3453

Browse files
committed
[fix](fe) Disable LOCAL TVF by default
### What problem does this PR solve? Issue Number: N/A Related PR: #69527 Problem Summary: Add startup-only enable_local_tvf=false. Reject LOCAL TVF and FILE-local at FE admission before discovery. Explicit opt-in retains the existing storage implementation and requires ADMIN/OPERATOR. Supported explicit LOCAL sinks use the same FE admission check. ### Release note LOCAL TVF and FILE TVF with local storage require enable_local_tvf=true in fe.conf on every query-serving FE and an FE restart. This change provides default-disable mitigation; enabled storage-path sandbox hardening remains separate work. ### Check List (For Author) - Test: Focused FE unit and regression cases added but not executed. Changed Java files pass Checkstyle; diff whitespace checks pass. - Behavior changed: Yes. LOCAL access requires startup opt-in and ADMIN/OPERATOR admission. - Does this need documentation: Yes. Configuration description and release note included; user-documentation follow-up remains pending.
1 parent a69f73d commit 42c3453

13 files changed

Lines changed: 381 additions & 1 deletion

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3103,6 +3103,10 @@ public class Config extends ConfigBase {
31033103
})
31043104
public static boolean enable_java_udf = true;
31053105

3106+
@ConfField(description = "Whether to enable LOCAL TVF and FILE TVF with local storage. Disabled by default. "
3107+
+ "Set this in fe.conf and restart every FE to enable it. Existing access checks still apply.")
3108+
public static boolean enable_local_tvf = false;
3109+
31063110
@ConfField(mutable = true, masterOnly = true, description = {
31073111
"开启后,可以在导入时,利用创建的全局 java_udf 函数处理数据,默认为 false。",
31083112
"When enabled, data can be processed using the globally created java_udf function during import."

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@
117117
import org.apache.doris.qe.AutoCloseSessionVariable;
118118
import org.apache.doris.qe.ConnectContext;
119119
import org.apache.doris.qe.SessionVariable;
120+
import org.apache.doris.tablefunction.LocalTableValuedFunction;
120121
import org.apache.doris.thrift.TPartialUpdateNewRowPolicy;
121122

122123
import com.google.common.base.Preconditions;
@@ -662,6 +663,10 @@ private Plan bindTVFTableSink(MatchingContext<UnboundTVFTableSink<Plan>> ctx) {
662663
"INSERT INTO TVF only supports local/s3/hdfs, but got: " + tvfName);
663664
}
664665

666+
if (tvfName.equals(LocalTableValuedFunction.NAME)) {
667+
LocalTableValuedFunction.checkAccess(ctx.connectContext);
668+
}
669+
665670
// Validate required properties
666671
if (!properties.containsKey("file_path")) {
667672
throw new AnalysisException("TVF sink requires 'file_path' property");

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,12 @@
2121
import org.apache.doris.analysis.StorageBackend.StorageType;
2222
import org.apache.doris.catalog.Env;
2323
import org.apache.doris.common.AnalysisException;
24+
import org.apache.doris.common.Config;
25+
import org.apache.doris.common.ErrorCode;
2426
import org.apache.doris.common.UserException;
2527
import org.apache.doris.datasource.FederationBackendPolicy;
2628
import org.apache.doris.datasource.property.storage.StorageProperties;
29+
import org.apache.doris.mysql.privilege.PrivPredicate;
2730
import org.apache.doris.proto.InternalService;
2831
import org.apache.doris.proto.InternalService.PGlobResponse;
2932
import org.apache.doris.proto.InternalService.PGlobResponse.PFileInfo;
@@ -61,6 +64,9 @@ public class LocalTableValuedFunction extends ExternalFileTableValuedFunction {
6164
private boolean sharedStorage = false;
6265

6366
public LocalTableValuedFunction(Map<String, String> properties) throws AnalysisException {
67+
// Check admission before properties, backend selection or file discovery.
68+
checkAuth(ConnectContext.get());
69+
6470
// 1. analyze common properties
6571
Map<String, String> props = super.parseCommonProperties(properties);
6672

@@ -135,6 +141,25 @@ private void getFileListFromBackend() throws AnalysisException {
135141
}
136142
}
137143

144+
/** Check LOCAL admission before property parsing, discovery or sink binding. */
145+
public static void checkAccess(ConnectContext ctx) {
146+
if (!Config.enable_local_tvf) {
147+
throw new org.apache.doris.nereids.exceptions.AnalysisException(
148+
"LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it.");
149+
}
150+
if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN)
151+
&& !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.OPERATOR)) {
152+
String message = ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR.formatErrorMsg("ADMIN/OPERATOR");
153+
throw new org.apache.doris.nereids.exceptions.AnalysisException(message);
154+
}
155+
}
156+
157+
@Override
158+
public void checkAuth(ConnectContext ctx) {
159+
checkAccess(ctx);
160+
super.checkAuth(ctx);
161+
}
162+
138163
@Override
139164
public TFileType getTFileType() {
140165
return TFileType.FILE_LOCAL;
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.common.Config;
22+
import org.apache.doris.mysql.privilege.AccessControllerManager;
23+
import org.apache.doris.mysql.privilege.PrivPredicate;
24+
import org.apache.doris.nereids.exceptions.AnalysisException;
25+
import org.apache.doris.nereids.parser.NereidsParser;
26+
import org.apache.doris.nereids.trees.plans.Plan;
27+
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
28+
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand;
29+
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
30+
import org.apache.doris.nereids.util.MemoTestUtils;
31+
import org.apache.doris.nereids.util.PlanChecker;
32+
import org.apache.doris.qe.ConnectContext;
33+
34+
import org.junit.jupiter.api.AfterEach;
35+
import org.junit.jupiter.api.Assertions;
36+
import org.junit.jupiter.api.BeforeEach;
37+
import org.junit.jupiter.api.Test;
38+
import org.mockito.MockedStatic;
39+
import org.mockito.Mockito;
40+
41+
class LocalTvfSinkEnableTest {
42+
private boolean originalEnableLocalTvf;
43+
44+
@BeforeEach
45+
void setUp() {
46+
originalEnableLocalTvf = Config.enable_local_tvf;
47+
Config.enable_local_tvf = false;
48+
}
49+
50+
@AfterEach
51+
void tearDown() {
52+
Config.enable_local_tvf = originalEnableLocalTvf;
53+
}
54+
55+
@Test
56+
void disabledLocalRejectsInsertAndExplainBeforeProperties() {
57+
for (String prefix : new String[] {"", "EXPLAIN "}) {
58+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
59+
() -> analyzeInsert(prefix + "INSERT INTO local('format'='csv') SELECT 1", true));
60+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
61+
}
62+
}
63+
64+
@Test
65+
void enabledLocalKeepsPropertyValidation() {
66+
Config.enable_local_tvf = true;
67+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
68+
() -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1", true));
69+
Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'"));
70+
}
71+
72+
@Test
73+
void enabledLocalChecksPrivilegesBeforeProperties() {
74+
Config.enable_local_tvf = true;
75+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
76+
() -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1", false));
77+
Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR"));
78+
}
79+
80+
@Test
81+
void disabledLocalKeepsRemoteSinkValidation() {
82+
for (String name : new String[] {"s3", "hdfs"}) {
83+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
84+
() -> analyzeInsert("INSERT INTO " + name + "('format'='csv') SELECT 1", false));
85+
Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'"));
86+
}
87+
}
88+
89+
private Plan analyzeInsert(String sql, boolean authorized) {
90+
ConnectContext context = MemoTestUtils.createConnectContext();
91+
Env env = Mockito.mock(Env.class);
92+
AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
93+
Mockito.when(env.getAccessManager()).thenReturn(accessManager);
94+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(authorized);
95+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) {
96+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
97+
LogicalPlan parsed = new NereidsParser().parseSingle(sql);
98+
if (parsed instanceof ExplainCommand) {
99+
parsed = ((ExplainCommand) parsed).getLogicalPlan();
100+
}
101+
InsertIntoTVFCommand command = (InsertIntoTVFCommand) parsed;
102+
return PlanChecker.from(context).analyze(command.getExplainPlan(context)).getPlan();
103+
}
104+
}
105+
}
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.tablefunction;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.common.Config;
22+
import org.apache.doris.mysql.privilege.AccessControllerManager;
23+
import org.apache.doris.mysql.privilege.PrivPredicate;
24+
import org.apache.doris.nereids.exceptions.AnalysisException;
25+
import org.apache.doris.proto.InternalService.PGlobResponse;
26+
import org.apache.doris.proto.Types.PStatus;
27+
import org.apache.doris.qe.ConnectContext;
28+
import org.apache.doris.qe.SessionVariable;
29+
import org.apache.doris.resource.computegroup.ComputeGroup;
30+
import org.apache.doris.rpc.BackendServiceProxy;
31+
import org.apache.doris.system.Backend;
32+
import org.apache.doris.system.SystemInfoService;
33+
import org.apache.doris.thrift.TNetworkAddress;
34+
35+
import com.google.common.collect.ImmutableMap;
36+
import org.junit.jupiter.api.AfterEach;
37+
import org.junit.jupiter.api.Assertions;
38+
import org.junit.jupiter.api.BeforeEach;
39+
import org.junit.jupiter.api.Test;
40+
import org.mockito.MockedConstruction;
41+
import org.mockito.MockedStatic;
42+
import org.mockito.Mockito;
43+
44+
import java.util.Collections;
45+
import java.util.Map;
46+
import java.util.concurrent.CompletableFuture;
47+
48+
public class LocalTableValuedFunctionEnableTest {
49+
private static final Map<String, String> PROPERTIES = ImmutableMap.of(
50+
"file_path", "local_privilege_test.csv", "backend_id", "1", "format", "csv");
51+
52+
private final Env env = Mockito.mock(Env.class);
53+
private final ConnectContext context = Mockito.mock(ConnectContext.class);
54+
private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
55+
private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class);
56+
private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class);
57+
private boolean originalEnableLocalTvf;
58+
private MockedStatic<Env> mockedEnv;
59+
private MockedStatic<ConnectContext> mockedContext;
60+
private MockedStatic<BackendServiceProxy> mockedProxy;
61+
62+
@BeforeEach
63+
public void setUp() throws Exception {
64+
originalEnableLocalTvf = Config.enable_local_tvf;
65+
Config.enable_local_tvf = false;
66+
mockedEnv = Mockito.mockStatic(Env.class);
67+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
68+
mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo);
69+
mockedContext = Mockito.mockStatic(ConnectContext.class);
70+
mockedContext.when(ConnectContext::get).thenReturn(context);
71+
mockedProxy = Mockito.mockStatic(BackendServiceProxy.class);
72+
mockedProxy.when(BackendServiceProxy::getInstance).thenReturn(proxy);
73+
Mockito.when(env.getAccessManager()).thenReturn(accessManager);
74+
Mockito.when(context.getSessionVariable()).thenReturn(new SessionVariable());
75+
Mockito.when(context.getQueryTimeoutS()).thenReturn(60);
76+
77+
Backend backend = Mockito.mock(Backend.class);
78+
Mockito.when(backend.getId()).thenReturn(1L);
79+
Mockito.when(backend.getHost()).thenReturn("127.0.0.1");
80+
Mockito.when(backend.isAlive()).thenReturn(true);
81+
Mockito.when(backend.isQueryAvailable()).thenReturn(true);
82+
Mockito.when(backend.isLoadAvailable()).thenReturn(true);
83+
ComputeGroup computeGroup = Mockito.mock(ComputeGroup.class);
84+
Mockito.when(context.getComputeGroup()).thenReturn(computeGroup);
85+
Mockito.when(computeGroup.getBackendList()).thenReturn(Collections.singletonList(backend));
86+
Mockito.when(systemInfo.getBackend(1L)).thenReturn(backend);
87+
Mockito.when(backend.getBrpcAddress()).thenReturn(new TNetworkAddress("127.0.0.1", 8060));
88+
Mockito.when(proxy.glob(Mockito.any(), Mockito.any())).thenReturn(CompletableFuture.completedFuture(
89+
PGlobResponse.newBuilder().setStatus(PStatus.newBuilder().setStatusCode(0)).build()));
90+
}
91+
92+
@AfterEach
93+
public void tearDown() {
94+
Config.enable_local_tvf = originalEnableLocalTvf;
95+
mockedProxy.close();
96+
mockedContext.close();
97+
mockedEnv.close();
98+
}
99+
100+
@Test
101+
public void testDisabledRejectsLocalBeforePropertyParsing() {
102+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
103+
() -> new LocalTableValuedFunction(ImmutableMap.of()));
104+
Assertions.assertTrue(exception.getMessage().contains("enable_local_tvf=true"));
105+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
106+
}
107+
108+
@Test
109+
public void testDisabledRejectsCsvAndLanceBeforeFileAccess() {
110+
for (String format : new String[] {"csv", "lance"}) {
111+
Map<String, String> properties = ImmutableMap.of(
112+
"file_path", "local_enable_test.csv", "backend_id", "1", "format", format);
113+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
114+
() -> new LocalTableValuedFunction(properties));
115+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
116+
}
117+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
118+
}
119+
120+
@Test
121+
public void testEnabledLocalReachesExistingFileAccess() throws Exception {
122+
Config.enable_local_tvf = true;
123+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true);
124+
new LocalTableValuedFunction(PROPERTIES);
125+
Mockito.verify(proxy).glob(Mockito.any(), Mockito.any());
126+
}
127+
128+
@Test
129+
public void testDisabledRejectsFileLocalBeforeFileAccess() {
130+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
131+
() -> new FileTableValuedFunction(PROPERTIES));
132+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
133+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
134+
}
135+
136+
@Test
137+
public void testEnabledFileLocalReachesExistingFileAccess() throws Exception {
138+
Config.enable_local_tvf = true;
139+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true);
140+
new FileTableValuedFunction(PROPERTIES);
141+
Mockito.verify(proxy).glob(Mockito.any(), Mockito.any());
142+
}
143+
144+
@Test
145+
public void testDisabledLocalDoesNotBlockFileS3() {
146+
Map<String, String> properties = ImmutableMap.<String, String>builder()
147+
.put("uri", "s3://test-bucket/local_enable_test.csv")
148+
.put("s3.endpoint", "s3.us-east-1.amazonaws.com")
149+
.put("s3.region", "us-east-1")
150+
.put("s3.access_key", "test-access-key")
151+
.put("s3.secret_key", "test-secret-key")
152+
.put("format", "csv")
153+
.build();
154+
try (MockedConstruction<S3TableValuedFunction> mockedS3 =
155+
Mockito.mockConstruction(S3TableValuedFunction.class)) {
156+
Assertions.assertDoesNotThrow(() -> new FileTableValuedFunction(properties));
157+
Assertions.assertEquals(1, mockedS3.constructed().size());
158+
}
159+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
160+
}
161+
162+
@Test
163+
public void testEnabledLocalRejectsUnprivilegedUserBeforeFileAccess() {
164+
Config.enable_local_tvf = true;
165+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
166+
() -> new LocalTableValuedFunction(PROPERTIES));
167+
Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR"));
168+
Mockito.verifyNoInteractions(systemInfo, proxy);
169+
}
170+
171+
@Test
172+
public void testOperatorCanAccessLocalFiles() throws Exception {
173+
Config.enable_local_tvf = true;
174+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.OPERATOR)).thenReturn(true);
175+
new LocalTableValuedFunction(PROPERTIES);
176+
Mockito.verify(proxy).glob(Mockito.any(), Mockito.any());
177+
}
178+
179+
@Test
180+
public void testEnabledLancePreservesExistingBackendSelection() throws Exception {
181+
Config.enable_local_tvf = true;
182+
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true);
183+
new LocalTableValuedFunction(ImmutableMap.of(
184+
"file_path", "local_enable_test.lance", "backend_id", "1", "format", "lance"));
185+
Mockito.verifyNoInteractions(proxy);
186+
}
187+
}

‎regression-test/pipeline/cloud_p0/conf/fe_custom.conf‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,7 @@ check_table_lock_leaky = true
4949
enable_outfile_to_local=true
5050

5151
enable_notify_be_after_load_txn_commit=true
52-
max_bucket_num_per_partition=512
52+
max_bucket_num_per_partition=512
53+
54+
# Exercise LOCAL TVF explicitly; production defaults to disabled.
55+
enable_local_tvf = true

‎regression-test/pipeline/cloud_p1/conf/fe_custom.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,6 @@ arrow_flight_sql_port = 8081
3939
enable_job_schedule_second_for_test = true
4040

4141
enable_notify_be_after_load_txn_commit=true
42+
43+
# Exercise LOCAL TVF explicitly; production defaults to disabled.
44+
enable_local_tvf = true

‎regression-test/pipeline/external/conf/fe.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,6 @@ hms_events_polling_interval_ms=700
101101

102102

103103
KRB5_CONFIG=/keytabs/krb5.conf
104+
105+
# Exercise LOCAL TVF explicitly; production defaults to disabled.
106+
enable_local_tvf = true

0 commit comments

Comments
 (0)