Skip to content

Commit 531cd94

Browse files
committed
[fix](fe) Disable LOCAL TVF (branch-4.2)
### What problem does this PR solve? Issue Number: N/A Related PR: #69591, #69527 Problem Summary: Disable LOCAL and FILE-local admission without an opt-in setting, before file discovery. Explicit LOCAL INSERT and analyzed EXPLAIN INSERT use the same rejection before sink property validation. Retain storage implementation and adapt only directly affected fixtures and suites. ### Release note LOCAL TVF and FILE TVF local storage are unavailable for all users, with no enable setting. LOCAL TVF INSERT is also unavailable. ### Check List (For Author) - Test: 8 targeted FE unit tests passed without failures/skips; changed Java Checkstyle, Groovy syntax and whitespace checks passed. Regression suite bodies not executed. - Behavior changed: Yes. Unconditional LOCAL rejection. - Does this need documentation: Yes. Release note included; operator documentation follow-up.
1 parent 018f794 commit 531cd94

44 files changed

Lines changed: 158 additions & 1086 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3103,10 +3103,6 @@ public class Config extends ConfigBase {
31033103
})
31043104
public static boolean enable_java_udf = true;
31053105

3106-
@ConfField(description = "Whether to enable LOCAL TVF and FILE TVF with local storage. Disabled by default. "
3107-
+ "Set this in fe.conf and restart every FE to enable it. Existing access checks still apply.")
3108-
public static boolean enable_local_tvf = false;
3109-
31103106
@ConfField(mutable = true, masterOnly = true, description = {
31113107
"开启后,可以在导入时,利用创建的全局 java_udf 函数处理数据,默认为 false。",
31123108
"When enabled, data can be processed using the globally created java_udf function during import."

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -664,7 +664,7 @@ private Plan bindTVFTableSink(MatchingContext<UnboundTVFTableSink<Plan>> ctx) {
664664
}
665665

666666
if (tvfName.equals(LocalTableValuedFunction.NAME)) {
667-
LocalTableValuedFunction.checkAccess(ctx.connectContext);
667+
LocalTableValuedFunction.checkAccess();
668668
}
669669

670670
// Validate required properties

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,9 @@
2121
import org.apache.doris.analysis.StorageBackend.StorageType;
2222
import org.apache.doris.catalog.Env;
2323
import org.apache.doris.common.AnalysisException;
24-
import org.apache.doris.common.Config;
25-
import org.apache.doris.common.ErrorCode;
2624
import org.apache.doris.common.UserException;
2725
import org.apache.doris.datasource.FederationBackendPolicy;
2826
import org.apache.doris.datasource.property.storage.StorageProperties;
29-
import org.apache.doris.mysql.privilege.PrivPredicate;
3027
import org.apache.doris.proto.InternalService;
3128
import org.apache.doris.proto.InternalService.PGlobResponse;
3229
import org.apache.doris.proto.InternalService.PGlobResponse.PFileInfo;
@@ -64,8 +61,7 @@ public class LocalTableValuedFunction extends ExternalFileTableValuedFunction {
6461
private boolean sharedStorage = false;
6562

6663
public LocalTableValuedFunction(Map<String, String> properties) throws AnalysisException {
67-
// Check admission before properties, backend selection or file discovery.
68-
checkAuth(ConnectContext.get());
64+
checkAccess();
6965

7066
// 1. analyze common properties
7167
Map<String, String> props = super.parseCommonProperties(properties);
@@ -141,23 +137,9 @@ private void getFileListFromBackend() throws AnalysisException {
141137
}
142138
}
143139

144-
/** Check LOCAL admission before property parsing, discovery or sink binding. */
145-
public static void checkAccess(ConnectContext ctx) {
146-
if (!Config.enable_local_tvf) {
147-
throw new org.apache.doris.nereids.exceptions.AnalysisException(
148-
"LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it.");
149-
}
150-
if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN)
151-
&& !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.OPERATOR)) {
152-
String message = ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR.formatErrorMsg("ADMIN/OPERATOR");
153-
throw new org.apache.doris.nereids.exceptions.AnalysisException(message);
154-
}
155-
}
156-
157-
@Override
158-
public void checkAuth(ConnectContext ctx) {
159-
checkAccess(ctx);
160-
super.checkAuth(ctx);
140+
/** Reject LOCAL before property parsing, file discovery or sink binding. */
141+
public static void checkAccess() {
142+
throw new org.apache.doris.nereids.exceptions.AnalysisException("LOCAL TVF is disabled.");
161143
}
162144

163145
@Override

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
package org.apache.doris.nereids.rules.analysis;
1919

20-
import org.apache.doris.common.Config;
2120
import org.apache.doris.nereids.parser.NereidsParser;
2221
import org.apache.doris.nereids.trees.expressions.NamedExpression;
2322
import org.apache.doris.nereids.trees.plans.Plan;
@@ -28,34 +27,19 @@
2827
import org.apache.doris.nereids.util.PlanChecker;
2928
import org.apache.doris.qe.ConnectContext;
3029

31-
import org.junit.jupiter.api.AfterEach;
3230
import org.junit.jupiter.api.Assertions;
33-
import org.junit.jupiter.api.BeforeEach;
3431
import org.junit.jupiter.api.Test;
3532

3633
import java.util.List;
3734

3835
class BindTVFTableSinkTest {
39-
private boolean originalEnableLocalTvf;
40-
41-
@BeforeEach
42-
void setUp() {
43-
originalEnableLocalTvf = Config.enable_local_tvf;
44-
Config.enable_local_tvf = true;
45-
}
46-
47-
@AfterEach
48-
void tearDown() {
49-
Config.enable_local_tvf = originalEnableLocalTvf;
50-
}
5136

5237
@Test
5338
void duplicateOutputNamesKeepDistinctPositions() {
5439
ConnectContext context = MemoTestUtils.createConnectContext();
5540
InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(
56-
"INSERT INTO local("
41+
"INSERT INTO s3("
5742
+ "'file_path'='/tmp/tvf_duplicate_columns_',"
58-
+ "'backend_id'='1',"
5943
+ "'format'='csv') "
6044
+ "SELECT 1 AS x, 2 AS x");
6145

fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkEnableTest.java renamed to fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java

Lines changed: 7 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
package org.apache.doris.nereids.rules.analysis;
1919

2020
import org.apache.doris.catalog.Env;
21-
import org.apache.doris.common.Config;
2221
import org.apache.doris.mysql.privilege.AccessControllerManager;
2322
import org.apache.doris.mysql.privilege.PrivPredicate;
2423
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -31,52 +30,23 @@
3130
import org.apache.doris.nereids.util.PlanChecker;
3231
import org.apache.doris.qe.ConnectContext;
3332

34-
import org.junit.jupiter.api.AfterEach;
3533
import org.junit.jupiter.api.Assertions;
36-
import org.junit.jupiter.api.BeforeEach;
3734
import org.junit.jupiter.api.Test;
3835
import org.mockito.MockedStatic;
3936
import org.mockito.Mockito;
4037

41-
class LocalTvfSinkEnableTest {
42-
private boolean originalEnableLocalTvf;
43-
44-
@BeforeEach
45-
void setUp() {
46-
originalEnableLocalTvf = Config.enable_local_tvf;
47-
Config.enable_local_tvf = false;
48-
}
49-
50-
@AfterEach
51-
void tearDown() {
52-
Config.enable_local_tvf = originalEnableLocalTvf;
53-
}
54-
38+
class LocalTvfSinkDisabledTest {
5539
@Test
5640
void disabledLocalRejectsInsertAndExplainBeforeProperties() {
57-
for (String prefix : new String[] {"", "EXPLAIN "}) {
58-
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
59-
() -> analyzeInsert(prefix + "INSERT INTO local('format'='csv') SELECT 1", true));
60-
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
41+
for (boolean authorized : new boolean[] {false, true}) {
42+
for (String prefix : new String[] {"", "EXPLAIN "}) {
43+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
44+
() -> analyzeInsert(prefix + "INSERT INTO local('format'='csv') SELECT 1", authorized));
45+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
46+
}
6147
}
6248
}
6349

64-
@Test
65-
void enabledLocalKeepsPropertyValidation() {
66-
Config.enable_local_tvf = true;
67-
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
68-
() -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1", true));
69-
Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'"));
70-
}
71-
72-
@Test
73-
void enabledLocalChecksPrivilegesBeforeProperties() {
74-
Config.enable_local_tvf = true;
75-
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
76-
() -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1", false));
77-
Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR"));
78-
}
79-
8050
@Test
8151
void disabledLocalKeepsRemoteSinkValidation() {
8252
for (String name : new String[] {"s3", "hdfs"}) {

fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java renamed to fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionDisabledTest.java

Lines changed: 18 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,12 @@
1818
package org.apache.doris.tablefunction;
1919

2020
import org.apache.doris.catalog.Env;
21-
import org.apache.doris.common.Config;
2221
import org.apache.doris.mysql.privilege.AccessControllerManager;
2322
import org.apache.doris.mysql.privilege.PrivPredicate;
2423
import org.apache.doris.nereids.exceptions.AnalysisException;
25-
import org.apache.doris.proto.InternalService.PGlobResponse;
26-
import org.apache.doris.proto.Types.PStatus;
2724
import org.apache.doris.qe.ConnectContext;
28-
import org.apache.doris.qe.SessionVariable;
29-
import org.apache.doris.resource.computegroup.ComputeGroup;
3025
import org.apache.doris.rpc.BackendServiceProxy;
31-
import org.apache.doris.system.Backend;
3226
import org.apache.doris.system.SystemInfoService;
33-
import org.apache.doris.thrift.TNetworkAddress;
3427

3528
import com.google.common.collect.ImmutableMap;
3629
import org.junit.jupiter.api.AfterEach;
@@ -41,28 +34,23 @@
4134
import org.mockito.MockedStatic;
4235
import org.mockito.Mockito;
4336

44-
import java.util.Collections;
4537
import java.util.Map;
46-
import java.util.concurrent.CompletableFuture;
4738

48-
public class LocalTableValuedFunctionEnableTest {
39+
public class LocalTableValuedFunctionDisabledTest {
4940
private static final Map<String, String> PROPERTIES = ImmutableMap.of(
50-
"file_path", "local_privilege_test.csv", "backend_id", "1", "format", "csv");
41+
"file_path", "local_disabled_test.csv", "backend_id", "1", "format", "csv");
5142

5243
private final Env env = Mockito.mock(Env.class);
5344
private final ConnectContext context = Mockito.mock(ConnectContext.class);
5445
private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
5546
private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class);
5647
private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class);
57-
private boolean originalEnableLocalTvf;
5848
private MockedStatic<Env> mockedEnv;
5949
private MockedStatic<ConnectContext> mockedContext;
6050
private MockedStatic<BackendServiceProxy> mockedProxy;
6151

6252
@BeforeEach
63-
public void setUp() throws Exception {
64-
originalEnableLocalTvf = Config.enable_local_tvf;
65-
Config.enable_local_tvf = false;
53+
public void setUp() {
6654
mockedEnv = Mockito.mockStatic(Env.class);
6755
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
6856
mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo);
@@ -71,45 +59,28 @@ public void setUp() throws Exception {
7159
mockedProxy = Mockito.mockStatic(BackendServiceProxy.class);
7260
mockedProxy.when(BackendServiceProxy::getInstance).thenReturn(proxy);
7361
Mockito.when(env.getAccessManager()).thenReturn(accessManager);
74-
Mockito.when(context.getSessionVariable()).thenReturn(new SessionVariable());
75-
Mockito.when(context.getQueryTimeoutS()).thenReturn(60);
76-
77-
Backend backend = Mockito.mock(Backend.class);
78-
Mockito.when(backend.getId()).thenReturn(1L);
79-
Mockito.when(backend.getHost()).thenReturn("127.0.0.1");
80-
Mockito.when(backend.isAlive()).thenReturn(true);
81-
Mockito.when(backend.isQueryAvailable()).thenReturn(true);
82-
Mockito.when(backend.isLoadAvailable()).thenReturn(true);
83-
ComputeGroup computeGroup = Mockito.mock(ComputeGroup.class);
84-
Mockito.when(context.getComputeGroup()).thenReturn(computeGroup);
85-
Mockito.when(computeGroup.getBackendList()).thenReturn(Collections.singletonList(backend));
86-
Mockito.when(systemInfo.getBackend(1L)).thenReturn(backend);
87-
Mockito.when(backend.getBrpcAddress()).thenReturn(new TNetworkAddress("127.0.0.1", 8060));
88-
Mockito.when(proxy.glob(Mockito.any(), Mockito.any())).thenReturn(CompletableFuture.completedFuture(
89-
PGlobResponse.newBuilder().setStatus(PStatus.newBuilder().setStatusCode(0)).build()));
9062
}
9163

9264
@AfterEach
9365
public void tearDown() {
94-
Config.enable_local_tvf = originalEnableLocalTvf;
9566
mockedProxy.close();
9667
mockedContext.close();
9768
mockedEnv.close();
9869
}
9970

10071
@Test
101-
public void testDisabledRejectsLocalBeforePropertyParsing() {
72+
public void testLocalDisabledBeforePropertyParsing() {
10273
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
10374
() -> new LocalTableValuedFunction(ImmutableMap.of()));
104-
Assertions.assertTrue(exception.getMessage().contains("enable_local_tvf=true"));
75+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
10576
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
10677
}
10778

10879
@Test
109-
public void testDisabledRejectsCsvAndLanceBeforeFileAccess() {
110-
for (String format : new String[] {"csv", "lance"}) {
80+
public void testFormatsDisabledBeforeFileAccess() {
81+
for (String format : new String[] {"csv", "json", "lance"}) {
11182
Map<String, String> properties = ImmutableMap.of(
112-
"file_path", "local_enable_test.csv", "backend_id", "1", "format", format);
83+
"file_path", "local_disabled_test", "backend_id", "1", "format", format);
11384
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
11485
() -> new LocalTableValuedFunction(properties));
11586
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
@@ -118,33 +89,27 @@ public void testDisabledRejectsCsvAndLanceBeforeFileAccess() {
11889
}
11990

12091
@Test
121-
public void testEnabledLocalReachesExistingFileAccess() throws Exception {
122-
Config.enable_local_tvf = true;
123-
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true);
124-
new LocalTableValuedFunction(PROPERTIES);
125-
Mockito.verify(proxy).glob(Mockito.any(), Mockito.any());
126-
}
127-
128-
@Test
129-
public void testDisabledRejectsFileLocalBeforeFileAccess() {
92+
public void testFileLocalDisabledBeforeFileAccess() {
13093
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
13194
() -> new FileTableValuedFunction(PROPERTIES));
13295
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
13396
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
13497
}
13598

13699
@Test
137-
public void testEnabledFileLocalReachesExistingFileAccess() throws Exception {
138-
Config.enable_local_tvf = true;
139-
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true);
140-
new FileTableValuedFunction(PROPERTIES);
141-
Mockito.verify(proxy).glob(Mockito.any(), Mockito.any());
100+
public void testPrivilegedUsersCannotAccessLocalFiles() {
101+
for (PrivPredicate privilege : new PrivPredicate[] {PrivPredicate.ADMIN, PrivPredicate.OPERATOR}) {
102+
Mockito.when(accessManager.checkGlobalPriv(context, privilege)).thenReturn(true);
103+
Assertions.assertThrows(AnalysisException.class, () -> new LocalTableValuedFunction(PROPERTIES));
104+
Assertions.assertThrows(AnalysisException.class, () -> new FileTableValuedFunction(PROPERTIES));
105+
}
106+
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
142107
}
143108

144109
@Test
145-
public void testDisabledLocalDoesNotBlockFileS3() {
110+
public void testFileS3RemainsAvailable() {
146111
Map<String, String> properties = ImmutableMap.<String, String>builder()
147-
.put("uri", "s3://test-bucket/local_enable_test.csv")
112+
.put("uri", "s3://test-bucket/local_disabled_test.csv")
148113
.put("s3.endpoint", "s3.us-east-1.amazonaws.com")
149114
.put("s3.region", "us-east-1")
150115
.put("s3.access_key", "test-access-key")
@@ -158,30 +123,4 @@ public void testDisabledLocalDoesNotBlockFileS3() {
158123
}
159124
Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
160125
}
161-
162-
@Test
163-
public void testEnabledLocalRejectsUnprivilegedUserBeforeFileAccess() {
164-
Config.enable_local_tvf = true;
165-
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
166-
() -> new LocalTableValuedFunction(PROPERTIES));
167-
Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR"));
168-
Mockito.verifyNoInteractions(systemInfo, proxy);
169-
}
170-
171-
@Test
172-
public void testOperatorCanAccessLocalFiles() throws Exception {
173-
Config.enable_local_tvf = true;
174-
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.OPERATOR)).thenReturn(true);
175-
new LocalTableValuedFunction(PROPERTIES);
176-
Mockito.verify(proxy).glob(Mockito.any(), Mockito.any());
177-
}
178-
179-
@Test
180-
public void testEnabledLancePreservesExistingBackendSelection() throws Exception {
181-
Config.enable_local_tvf = true;
182-
Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true);
183-
new LocalTableValuedFunction(ImmutableMap.of(
184-
"file_path", "local_enable_test.lance", "backend_id", "1", "format", "lance"));
185-
Mockito.verifyNoInteractions(proxy);
186-
}
187126
}

‎regression-test/pipeline/cloud_p0/conf/fe_custom.conf‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,4 @@ check_table_lock_leaky = true
4949
enable_outfile_to_local=true
5050

5151
enable_notify_be_after_load_txn_commit=true
52-
max_bucket_num_per_partition=512
53-
54-
# Exercise LOCAL TVF explicitly; production defaults to disabled.
55-
enable_local_tvf = true
52+
max_bucket_num_per_partition=512

‎regression-test/pipeline/cloud_p1/conf/fe_custom.conf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,3 @@ arrow_flight_sql_port = 8081
3939
enable_job_schedule_second_for_test = true
4040

4141
enable_notify_be_after_load_txn_commit=true
42-
43-
# Exercise LOCAL TVF explicitly; production defaults to disabled.
44-
enable_local_tvf = true

‎regression-test/pipeline/external/conf/fe.conf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,3 @@ hms_events_polling_interval_ms=700
101101

102102

103103
KRB5_CONFIG=/keytabs/krb5.conf
104-
105-
# Exercise LOCAL TVF explicitly; production defaults to disabled.
106-
enable_local_tvf = true

‎regression-test/pipeline/nonConcurrent/conf/fe.conf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,3 @@ max_query_profile_num = 2000
9292
max_spilled_profile_num = 2000
9393

9494
check_table_lock_leaky=true
95-
96-
# Exercise LOCAL TVF explicitly; production defaults to disabled.
97-
enable_local_tvf = true

0 commit comments

Comments
 (0)