Skip to content

Commit 14272ec

Browse files
morningmanmorningman
committed
[refactor](catalog) P3b: consolidate kerberos authentication into fe-kerberos (#64655)
**P3b: consolidate the drifted Kerberos/Hadoop authentication implementations into the new top-level neutral leaf module `fe-kerberos`** as the single source of truth. Done as 3 commits: 1. **trino → JDK** (`4a740e1`) — replace the only external dependency in the auth path, trino's `KerberosTicketUtils`, with a JDK-only (`javax.security.auth.kerberos`) byte-for-byte equivalent, so the kerberos path is trino-free. 2. **relocate** (`8898e15`) — move the 13 `fe-common` `security.authentication.*` classes to `org.apache.doris.kerberos.*` in `fe-kerberos`; retarget all consumer imports (fe-core + 3 be-java-extensions scanners); merge the duplicate `AuthType`. 3. **unify interface** (`5e3e896`) — merge the two competing `HadoopAuthenticator` interfaces (fe-common's `PrivilegedExceptionAction` variant vs fe-filesystem-spi's `IOCallable` variant) into the single fe-kerberos one, and delete fe-filesystem-hdfs's own `KerberosHadoopAuthenticator`/`SimpleHadoopAuthenticator` copies (which had drifted from the canonical impls). `DFSFileSystem` now routes through the shared authenticators. `fe-kerberos` remains a top-level neutral leaf (no dependency cycle). HDFS filesystem access now uses the same authenticators as the HMS path (restoring parity). Two intentional behavior changes in fe-filesystem-hdfs: simple / no-`hadoop.username` now runs as remote user `hadoop` (was: FE process user, direct); kerberos uses the shared `LoginContext` + 80%-lifetime refresh. fe-filesystem-hdfs 79/0/0 (+fe-kerberos/spi), checkstyle 0, connector import-gate clean, whole-repo grep for the removed symbols = 0. > ⚠️ docker kerberos e2e (HDFS kerberized + HMS) NOT yet run — the real gate; UGI login can't be exercised in unit tests.
1 parent 7c88f49 commit 14272ec

83 files changed

Lines changed: 590 additions & 838 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
import org.apache.doris.common.classloader.ThreadClassLoaderContext;
2121
import org.apache.doris.common.jni.JniScanner;
2222
import org.apache.doris.common.jni.vec.ColumnType;
23-
import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
24-
import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache;
23+
import org.apache.doris.kerberos.PreExecutionAuthenticator;
24+
import org.apache.doris.kerberos.PreExecutionAuthenticatorCache;
2525

2626
import com.google.common.base.Joiner;
2727
import com.google.common.base.Preconditions;

‎fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java‎

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,22 @@
2121
import org.apache.doris.common.jni.JniScanner;
2222
import org.apache.doris.common.jni.vec.ColumnType;
2323
import org.apache.doris.common.jni.vec.ColumnValue;
24-
import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
25-
import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache;
24+
import org.apache.doris.kerberos.PreExecutionAuthenticator;
25+
import org.apache.doris.kerberos.PreExecutionAuthenticatorCache;
2626

2727
import com.google.common.base.Preconditions;
2828
import org.apache.iceberg.FileScanTask;
2929
import org.apache.iceberg.StructLike;
3030
import org.apache.iceberg.io.CloseableIterator;
31+
import org.apache.iceberg.types.Types.NestedField;
32+
import org.apache.iceberg.types.Types.StructType;
3133
import org.apache.iceberg.util.SerializationUtil;
3234
import org.slf4j.Logger;
3335
import org.slf4j.LoggerFactory;
3436

3537
import java.io.IOException;
38+
import java.util.ArrayList;
39+
import java.util.List;
3640
import java.util.Map;
3741
import java.util.TimeZone;
3842
import java.util.stream.Collectors;
@@ -46,7 +50,7 @@ public class IcebergSysTableJniScanner extends JniScanner {
4650
private final ClassLoader classLoader;
4751
private final PreExecutionAuthenticator preExecutionAuthenticator;
4852
private final FileScanTask scanTask;
49-
private final int requiredFieldCount;
53+
private final List<SelectedField> fields;
5054
private final String timezone;
5155
private CloseableIterator<StructLike> reader;
5256

@@ -56,22 +60,15 @@ public IcebergSysTableJniScanner(int batchSize, Map<String, String> params) {
5660
Preconditions.checkArgument(serializedSplitParams != null && !serializedSplitParams.isEmpty(),
5761
"serialized_split should not be empty");
5862
this.scanTask = SerializationUtil.deserializeFromBase64(serializedSplitParams);
59-
String requiredFieldsParam = params.get("required_fields");
60-
Preconditions.checkArgument(requiredFieldsParam != null && !requiredFieldsParam.isEmpty(),
61-
"required_fields should not be empty");
62-
String[] requiredFields = requiredFieldsParam.split(",");
63-
this.requiredFieldCount = requiredFields.length;
63+
String[] requiredFields = params.get("required_fields").split(",");
64+
this.fields = selectSchema(scanTask.schema().asStruct(), requiredFields);
6465
this.timezone = params.getOrDefault("time_zone", TimeZone.getDefault().getID());
6566
Map<String, String> hadoopOptionParams = params.entrySet().stream()
6667
.filter(kv -> kv.getKey().startsWith(HADOOP_OPTION_PREFIX))
6768
.collect(Collectors
6869
.toMap(kv1 -> kv1.getKey().substring(HADOOP_OPTION_PREFIX.length()), kv1 -> kv1.getValue()));
6970
this.preExecutionAuthenticator = PreExecutionAuthenticatorCache.getAuthenticator(hadoopOptionParams);
70-
String requiredTypesParam = params.get("required_types");
71-
Preconditions.checkArgument(requiredTypesParam != null && !requiredTypesParam.isEmpty(),
72-
"required_types should not be empty");
73-
String[] requiredTypeStrings = requiredTypesParam.split("#");
74-
ColumnType[] requiredTypes = parseRequiredTypes(requiredTypeStrings, requiredFields);
71+
ColumnType[] requiredTypes = parseRequiredTypes(params.get("required_types").split("#"), requiredFields);
7572
initTableInfo(requiredTypes, requiredFields, batchSize);
7673
}
7774

@@ -109,10 +106,9 @@ protected int getNext() throws IOException {
109106
break;
110107
}
111108
StructLike row = reader.next();
112-
for (int i = 0; i < requiredFieldCount; i++) {
113-
// FE keeps the fields requested by BE at the start of the Iceberg projection.
114-
// FileScanTask.schema() is not the row schema for every DataTask implementation.
115-
Object value = row.get(i, Object.class);
109+
for (int i = 0; i < fields.size(); i++) {
110+
SelectedField field = fields.get(i);
111+
Object value = row.get(field.sourceIndex, field.field.type().typeId().javaClass());
116112
ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone);
117113
appendData(i, columnValue);
118114
}
@@ -133,10 +129,35 @@ public void close() throws IOException {
133129
}
134130
}
135131

132+
private static List<SelectedField> selectSchema(StructType schema, String[] requiredFields) {
133+
List<NestedField> schemaFields = schema.fields();
134+
List<SelectedField> selectedFields = new ArrayList<>();
135+
for (String requiredField : requiredFields) {
136+
NestedField field = schema.field(requiredField);
137+
if (field == null) {
138+
throw new IllegalArgumentException("RequiredField " + requiredField + " not found in schema");
139+
}
140+
int sourceIndex = schemaFields.indexOf(field);
141+
if (sourceIndex < 0) {
142+
throw new IllegalArgumentException(
143+
"RequiredField " + requiredField + " not found in source schema fields");
144+
}
145+
selectedFields.add(new SelectedField(sourceIndex, field));
146+
}
147+
return selectedFields;
148+
}
149+
150+
private static final class SelectedField {
151+
private final int sourceIndex;
152+
private final NestedField field;
153+
154+
private SelectedField(int sourceIndex, NestedField field) {
155+
this.sourceIndex = sourceIndex;
156+
this.field = field;
157+
}
158+
}
159+
136160
private static ColumnType[] parseRequiredTypes(String[] typeStrings, String[] requiredFields) {
137-
Preconditions.checkArgument(typeStrings.length == requiredFields.length,
138-
"required_types size %s does not match required_fields size %s",
139-
typeStrings.length, requiredFields.length);
140161
ColumnType[] requiredTypes = new ColumnType[typeStrings.length];
141162
for (int i = 0; i < typeStrings.length; i++) {
142163
String type = typeStrings[i];

‎fe/be-java-extensions/java-common/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ under the License.
3434
</properties>
3535

3636
<dependencies>
37+
<dependency>
38+
<groupId>org.apache.doris</groupId>
39+
<artifactId>fe-kerberos</artifactId>
40+
<version>${project.version}</version>
41+
</dependency>
3742
<dependency>
3843
<groupId>org.apache.doris</groupId>
3944
<artifactId>fe-common</artifactId>

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
import org.apache.doris.common.jni.JniScanner;
2121
import org.apache.doris.common.jni.vec.ColumnType;
2222
import org.apache.doris.common.jni.vec.TableSchema;
23-
import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
24-
import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache;
23+
import org.apache.doris.kerberos.PreExecutionAuthenticator;
24+
import org.apache.doris.kerberos.PreExecutionAuthenticatorCache;
2525

2626
import com.google.common.base.Preconditions;
2727
import org.apache.paimon.CoreOptions;

‎fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthType.java‎

Lines changed: 0 additions & 60 deletions
This file was deleted.

‎fe/fe-core/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,11 @@ under the License.
6464
<groupId>${project.groupId}</groupId>
6565
<artifactId>fe-foundation</artifactId>
6666
</dependency>
67+
<dependency>
68+
<groupId>${project.groupId}</groupId>
69+
<artifactId>fe-kerberos</artifactId>
70+
<version>${project.version}</version>
71+
</dependency>
6772
<dependency>
6873
<groupId>${project.groupId}</groupId>
6974
<artifactId>fe-common</artifactId>

‎fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
import org.apache.doris.common.DdlException;
2121
import org.apache.doris.common.proc.BaseProcResult;
22-
import org.apache.doris.common.security.authentication.AuthenticationConfig;
22+
import org.apache.doris.kerberos.AuthenticationConfig;
2323
import org.apache.doris.thrift.THdfsConf;
2424
import org.apache.doris.thrift.THdfsParams;
2525

‎fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@
1919

2020
import org.apache.doris.cloud.proto.Cloud;
2121
import org.apache.doris.common.DdlException;
22-
import org.apache.doris.common.security.authentication.AuthenticationConfig;
2322
import org.apache.doris.common.util.DatasourcePrintableMap;
2423
import org.apache.doris.datasource.property.storage.StorageProperties;
2524
import org.apache.doris.filesystem.FileSystem;
2625
import org.apache.doris.filesystem.Location;
2726
import org.apache.doris.fs.FileSystemFactory;
27+
import org.apache.doris.kerberos.AuthenticationConfig;
2828

2929
import com.google.common.base.Preconditions;
3030
import com.google.common.base.Strings;

‎fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@
1818
package org.apache.doris.catalog;
1919

2020
import org.apache.doris.common.DdlException;
21-
import org.apache.doris.common.security.authentication.AuthType;
22-
import org.apache.doris.common.security.authentication.AuthenticationConfig;
2321
import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
2422
import org.apache.doris.datasource.property.storage.S3Properties;
23+
import org.apache.doris.kerberos.AuthType;
24+
import org.apache.doris.kerberos.AuthenticationConfig;
2525
import org.apache.doris.thrift.THiveTable;
2626
import org.apache.doris.thrift.TTableDescriptor;
2727
import org.apache.doris.thrift.TTableType;

‎fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
package org.apache.doris.common;
1919

2020

21-
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
21+
import org.apache.doris.kerberos.ExecutionAuthenticator;
2222
import org.apache.doris.metric.Metric;
2323
import org.apache.doris.metric.Metric.MetricUnit;
2424
import org.apache.doris.metric.MetricLabel;

0 commit comments

Comments
 (0)