Skip to content

Commit a08f593

Browse files
authored
[fix](hive) Preserve simple HMS user identity (apache#66890)
### What problem does this PR solve? Issue Number: N/A Related PR: apache#64304 Problem Summary: After external catalogs became plugin-driven, simple-auth Hive Metastore RPCs no longer ran under the catalog configured Hadoop user. HMS-created directories could therefore use the FE process user while data writes used the configured user, causing permission failures. Restore a plugin-side simple UGI for HMS RPCs and preserve the metastore username alias and legacy default user. ### Release note Fix Hive Metastore writes failing when the configured Hadoop user differs from the FE process user. ### Check List (For Author) - Test: Unit Test - HiveConnectorPluginAuthenticatorTest - Behavior changed: Yes. Simple-auth HMS RPCs again run as the configured Hadoop user. - Does this need documentation: No
1 parent 5202d06 commit a08f593

25 files changed

Lines changed: 1686 additions & 220 deletions

File tree

‎fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java‎

Lines changed: 43 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@
2121
import org.apache.doris.connector.hms.CachingHmsClient;
2222
import org.apache.doris.connector.hms.HmsClient;
2323
import org.apache.doris.connector.hms.HmsClientConfig;
24+
import org.apache.doris.connector.hms.HmsConfHelper;
2425
import org.apache.doris.connector.hms.ThriftHmsClient;
2526
import org.apache.doris.connector.hms.event.HmsEventSource;
2627
import org.apache.doris.connector.metastore.HmsMetaStoreProperties;
28+
import org.apache.doris.connector.metastore.spi.AbstractHmsMetaStoreProperties;
2729
import org.apache.doris.connector.metastore.spi.MetaStoreProviders;
2830
import org.apache.doris.connector.spi.Connector;
2931
import org.apache.doris.connector.spi.ConnectorCapability;
@@ -38,6 +40,8 @@
3840
import org.apache.doris.connector.spi.procedure.ConnectorProcedureOps;
3941
import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider;
4042
import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider;
43+
import org.apache.doris.kerberos.AuthType;
44+
import org.apache.doris.kerberos.AuthenticationConfig;
4145
import org.apache.doris.kerberos.HadoopAuthenticator;
4246
import org.apache.doris.kerberos.KerberosAuthSpec;
4347
import org.apache.doris.kerberos.KerberosAuthenticationConfig;
@@ -65,9 +69,6 @@ public class HiveConnector implements Connector {
6569

6670
private static final Logger LOG = LogManager.getLogger(HiveConnector.class);
6771

68-
// Catalog property key gating the plugin-side Kerberos authenticator (value matches AuthType.KERBEROS).
69-
private static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication";
70-
7172
// The sibling connector type a flipped hms gateway delegates iceberg-on-HMS tables to. A string literal
7273
// (not the iceberg plugin's own type constant, which is child-first and invisible from the hive loader);
7374
// matches the type name IcebergConnectorProvider registers.
@@ -130,6 +131,7 @@ public class HiveConnector implements Connector {
130131
private volatile Connector hudiSibling;
131132

132133
public HiveConnector(Map<String, String> properties, ConnectorContext context) {
134+
HmsConfHelper.initializeHadoopConfigDir(context);
133135
this.props = HiveCatalogProperties.of(properties);
134136
this.properties = props.getRaw();
135137
this.context = context;
@@ -587,18 +589,15 @@ private HmsClient createClient() {
587589
// connector's constructor built. A catalog created before the type was removed, or one carrying no
588590
// metastore URI, fails when the connector is (re)built — including on the lazy build an FE does for a
589591
// catalog it loaded from the image — with the same messages this method used to produce.
590-
int poolSize = props.getHmsClientPoolSize();
591-
592-
// getHmsClientProperties(), not the raw map: the metastore URI must reach HiveConf under its canonical
593-
// key even when the catalog spells it with the "uri" short form.
594-
HmsClientConfig config = new HmsClientConfig(props.getHmsClientProperties(), poolSize);
592+
HmsClientConfig config = buildHmsClientConfig();
593+
int poolSize = config.getPoolSize();
595594
LOG.info("Creating Hive connector client for catalog='{}', uri={}, type={}, poolSize={}",
596595
context.getCatalogName(), config.getMetastoreUri(),
597596
config.getMetastoreType(), poolSize);
598597

599-
// For a Kerberos catalog run the metastore RPC under the PLUGIN's UGI doAs (buildPluginAuthenticator),
600-
// NOT the FE-injected context: after the catalog flip that context resolves to NOOP (SIMPLE) auth, which
601-
// would silently downgrade a Kerberos HMS. AuthAction.execute is a generic method (<T> T execute(...)),
598+
// Run the metastore RPC under the PLUGIN's UGI doAs (buildPluginAuthenticator), NOT the FE-injected
599+
// context: after the catalog flip that context resolves to NOOP auth and loses both the configured simple
600+
// user and Kerberos login. AuthAction.execute is a generic method (<T> T execute(...)),
602601
// so it cannot be a lambda — use an anonymous class. ThriftHmsClient.doAs pins the RPC's TCCL to the
603602
// plugin (child-first) classloader (so SecurityUtil.<clinit> resolves hadoop from the plugin copy, not a
604603
// split-brain against fe-core's copy); the plugin's HadoopAuthenticator only wraps it in a UGI doAs.
@@ -623,6 +622,16 @@ public <T> T execute(Callable<T> callable) throws Exception {
623622
HiveConnectorMetadata.buildTypeMappingOptions(props)));
624623
}
625624

625+
HmsClientConfig buildHmsClientConfig() {
626+
// Use canonical properties so the URI short alias and deployment timeout both reach HiveConf.
627+
AbstractHmsMetaStoreProperties hms = (AbstractHmsMetaStoreProperties) MetaStoreProviders.bindForType(
628+
HmsClientConfig.METASTORE_TYPE_HMS, props.getHmsClientProperties(), Collections.emptyMap());
629+
return new HmsClientConfig(hms.getConfResources(), HmsConfHelper.mergeCatalogProperties(
630+
props.getHmsClientProperties(),
631+
hms.toHiveConfOverrides(HmsConfHelper.metastoreClientTimeoutSecond(context))),
632+
props.getHmsClientPoolSize());
633+
}
634+
626635
/**
627636
* Wraps the raw pooled metastore client in the connector-owned {@link CachingHmsClient} so this connector
628637
* keeps caching its scan-side metastore reads AFTER the hms cutover. Once a hive catalog becomes plugin-driven
@@ -649,11 +658,10 @@ HmsClient wrapWithCache(HmsClient raw) {
649658
}
650659

651660
/**
652-
* Lazily builds and memoizes the plugin-side Kerberos authenticator that {@link #createClient()} wraps the
661+
* Lazily builds and memoizes the plugin-side authenticator that {@link #createClient()} wraps the
653662
* metastore RPC under, so the RPC uses the PLUGIN's own {@code UserGroupInformation} copy (hadoop +
654-
* fe-kerberos are bundled child-first in the hive plugin). Returns {@code null} for a non-Kerberos catalog
655-
* so the FE-injected auth path is preserved unchanged. Construction is cheap — the keytab login is lazy in
656-
* {@code getUGI()} on the first {@code doAs}.
663+
* fe-kerberos are bundled child-first in the hive plugin). Construction is cheap — the keytab login is lazy
664+
* in {@code getUGI()} on the first {@code doAs}.
657665
*/
658666
private HadoopAuthenticator pluginAuthenticator() {
659667
if (!pluginAuthComputed) {
@@ -668,19 +676,16 @@ private HadoopAuthenticator pluginAuthenticator() {
668676
}
669677

670678
/**
671-
* Resolves the plugin-side Kerberos authenticator for the catalog, or {@code null} for a non-Kerberos
672-
* catalog. Two Kerberos sources are covered, in precedence order (mirroring the legacy
673-
* {@code HMSBaseProperties.initHadoopAuthenticator}):
679+
* Resolves the plugin-side authenticator for the catalog. Authentication modes are covered in precedence
680+
* order, mirroring the legacy {@code HMSBaseProperties.initHadoopAuthenticator}:
674681
* <ol>
675-
* <li><b>Storage</b> Kerberos — the raw {@code hadoop.security.authentication=kerberos} passthrough
676-
* (HDFS login), built from the catalog Hadoop configuration. When storage is Kerberos this single
677-
* login also carries the HMS metastore RPC (same UGI).</li>
678-
* <li><b>HMS-metastore</b> Kerberos with non-Kerberos storage — a secured Hive Metastore whose data
679-
* storage is simple (e.g. a Kerberized HMS over S3). The HMS client principal/keytab facts
682+
* <li><b>Explicit HMS mode</b> — SIMPLE selects the configured metastore user even when storage is
683+
* Kerberos; KERBEROS uses the HMS client principal/keytab facts
680684
* ({@link HmsMetaStoreProperties#kerberos()}, resolved through the shared metastore-spi parser) feed a
681685
* {@link KerberosAuthenticationConfig}, so the {@code doAs} logs in the same client identity fe-core
682-
* used. The HMS <em>service</em> principal / SASL settings ride the catalog's own HiveConf, not the
683-
* login.</li>
686+
* used.</li>
687+
* <li><b>Storage fallback</b> — only when HMS mode is absent, storage Kerberos may supply the HMS login;
688+
* otherwise the configured simple user or legacy {@code hadoop} default drives {@code set_ugi}.</li>
684689
* </ol>
685690
* Package-visible + static for KDC-free unit testing.
686691
*/
@@ -691,26 +696,30 @@ static HadoopAuthenticator buildPluginAuthenticator(Map<String, String> properti
691696
// SecurityUtil.<clinit>, whose internal `new Configuration()` captures the current TCCL. This runs on the
692697
// (unpinned) createClient thread, so without the pin it would resolve hadoop's DNSDomainNameResolver from
693698
// fe-core's system-loader copy and split-brain-poison SecurityUtil against the plugin copy (the same
694-
// failure ThriftHmsClient.doAs guards). buildHadoopConf pins only the OUTER conf's loader, not the TCCL
699+
// failure ThriftHmsClient.doAs guards). buildHmsConf pins only the OUTER conf's loader, not the TCCL
695700
// that SecurityUtil's own Configuration reads — so the thread pin is required here too.
696701
// (HadoopKerberosAuthenticator's keytab login is lazy in getUGI, already under doAs's pin.)
697702
ClassLoader previous = Thread.currentThread().getContextClassLoader();
698703
try {
699704
Thread.currentThread().setContextClassLoader(HiveConnector.class.getClassLoader());
700-
if ("kerberos".equalsIgnoreCase(properties.get(HADOOP_SECURITY_AUTHENTICATION))) {
701-
return HadoopAuthenticator.getHadoopAuthenticator(buildHadoopConf(properties));
702-
}
703-
HmsMetaStoreProperties hms = (HmsMetaStoreProperties) MetaStoreProviders.bindForType(
705+
AbstractHmsMetaStoreProperties hms = (AbstractHmsMetaStoreProperties) MetaStoreProviders.bindForType(
704706
HmsClientConfig.METASTORE_TYPE_HMS, properties, Collections.emptyMap());
705707
Optional<KerberosAuthSpec> spec = hms.kerberos();
706708
if (spec.isPresent() && spec.get().hasCredentials()) {
707-
Configuration conf = buildHadoopConf(properties);
709+
Configuration conf = buildHmsConf(hms);
708710
conf.set("hadoop.security.authentication", "kerberos");
709711
conf.set("hive.metastore.sasl.enabled", "true");
710712
return HadoopAuthenticator.getHadoopAuthenticator(
711713
new KerberosAuthenticationConfig(spec.get().getPrincipal(), spec.get().getKeytab(), conf));
712714
}
713-
return null;
715+
if (hms.getAuthType() == AuthType.KERBEROS) {
716+
return null;
717+
}
718+
Configuration conf = buildHmsConf(hms);
719+
// HMS set_ugi reads the current UGI; resolve it from the HMS configuration without changing the
720+
// separate storage identity used by the connector's file operations.
721+
return HadoopAuthenticator.getHadoopAuthenticator(
722+
AuthenticationConfig.getSimpleAuthenticationConfig(conf));
714723
} finally {
715724
Thread.currentThread().setContextClassLoader(previous);
716725
}
@@ -722,11 +731,8 @@ static HadoopAuthenticator buildPluginAuthenticator(Map<String, String> properti
722731
* hadoop-mapreduce onto the unit-test classpath. The classloader is pinned to the plugin loader so the
723732
* child-first (plugin) copy of the auth classes is resolved.
724733
*/
725-
private static Configuration buildHadoopConf(Map<String, String> properties) {
726-
Configuration conf = new Configuration();
727-
conf.setClassLoader(HiveConnector.class.getClassLoader());
728-
properties.forEach(conf::set);
729-
return conf;
734+
private static Configuration buildHmsConf(AbstractHmsMetaStoreProperties hms) {
735+
return HmsConfHelper.createHadoopConfWithResources(hms.getConfResources(), hms.toHiveConfOverrides(""));
730736
}
731737

732738
@Override

‎fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPluginAuthenticatorTest.java‎

Lines changed: 115 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,25 +21,31 @@
2121

2222
import org.junit.jupiter.api.Assertions;
2323
import org.junit.jupiter.api.Test;
24+
import org.junit.jupiter.api.io.TempDir;
2425

26+
import java.nio.file.Files;
27+
import java.nio.file.Path;
2528
import java.util.HashMap;
2629
import java.util.Map;
2730

2831
/**
2932
* Unit tests for {@link HiveConnector#buildPluginAuthenticator(Map)} — the connector-owned plugin-side
30-
* Kerberos authenticator resolution.
33+
* authenticator resolution.
3134
*
3235
* <p>The load-bearing case is <b>HMS-metastore Kerberos with simple (non-Kerberos) storage</b>
3336
* (e.g. a Kerberized Hive Metastore over S3). After the catalog flip the FE-injected
3437
* {@code ConnectorContext.executeAuthenticated} resolves to NOOP (SIMPLE) auth, so a Kerberos HMS would be
3538
* silently downgraded unless the connector owns the login itself. These tests pin that the connector builds a
36-
* plugin authenticator from the HMS client principal/keytab facts, and does NOT build one when the metastore
37-
* is simple-auth (which would force needless SIMPLE-vs-Kerberos churn).
39+
* plugin authenticator from the HMS client principal/keytab facts. Simple-auth cases also pin the HMS UGI to
40+
* the configured Hadoop user, including the metastore alias and legacy default.
3841
*
3942
* <p>The actual keytab login is lazy (on first {@code doAs}), so these assertions never touch a KDC.
4043
*/
4144
public class HiveConnectorPluginAuthenticatorTest {
4245

46+
@TempDir
47+
Path tempDir;
48+
4349
private static Map<String, String> props(String... kv) {
4450
Map<String, String> m = new HashMap<>();
4551
for (int i = 0; i < kv.length; i += 2) {
@@ -75,21 +81,88 @@ public void hmsMetastoreKerberosWithSimpleStorageBuildsAuthenticator() {
7581
"HMS-metastore kerberos with simple storage must yield a plugin authenticator");
7682
}
7783

78-
/** A simple-auth HMS builds no authenticator (a spurious one would force needless SIMPLE-vs-Kerberos churn). */
7984
@Test
80-
public void hmsSimpleAuthReturnsNull() {
85+
public void hmsSimpleAuthUsesConfiguredHadoopUser() throws Exception {
86+
HadoopAuthenticator auth = HiveConnector.buildPluginAuthenticator(
87+
props("hive.metastore.uris", "thrift://hms:9083",
88+
"hive.metastore.authentication.type", "simple",
89+
"hadoop.username", "hive"));
90+
91+
Assertions.assertNotNull(auth, "simple-auth HMS must yield a plugin authenticator");
92+
Assertions.assertEquals("hive", auth.getUGI().getUserName());
93+
}
94+
95+
@Test
96+
public void hmsSimpleAuthUsesMetastoreUsernameAlias() throws Exception {
97+
HadoopAuthenticator auth = HiveConnector.buildPluginAuthenticator(
98+
props("hive.metastore.uris", "thrift://hms:9083",
99+
"hive.metastore.authentication.type", "simple",
100+
"hive.metastore.username", "hive"));
101+
102+
Assertions.assertNotNull(auth, "simple-auth HMS must support the metastore username alias");
103+
Assertions.assertEquals("hive", auth.getUGI().getUserName());
104+
}
105+
106+
@Test
107+
public void explicitSimpleHmsWinsOverKerberosStorage() throws Exception {
108+
HadoopAuthenticator auth = HiveConnector.buildPluginAuthenticator(
109+
props("hive.metastore.uris", "thrift://hms:9083",
110+
"hive.metastore.authentication.type", "simple",
111+
"hive.metastore.username", "metastore-user",
112+
"hadoop.security.authentication", "kerberos",
113+
"hadoop.kerberos.principal", "[email protected]",
114+
"hadoop.kerberos.keytab", "/etc/security/storage.keytab"));
115+
116+
Assertions.assertEquals("metastore-user", auth.getUGI().getUserName());
117+
}
118+
119+
@Test
120+
public void blankHadoopUserFallsBackToLegacyDefault() throws Exception {
121+
for (String blank : new String[] {"", " "}) {
122+
HadoopAuthenticator auth = HiveConnector.buildPluginAuthenticator(
123+
props("hive.metastore.uris", "thrift://hms:9083",
124+
"hive.metastore.authentication.type", "simple",
125+
"hadoop.username", blank));
126+
Assertions.assertEquals("hadoop", auth.getUGI().getUserName());
127+
}
128+
}
129+
130+
@Test
131+
public void simpleHmsUserCanComeFromHiveConfResource() throws Exception {
132+
Path resource = tempDir.resolve("hive-site.xml");
133+
Files.writeString(resource, "<configuration><property><name>hadoop.username</name>"
134+
+ "<value>resource-user</value></property></configuration>");
135+
String previous = System.setProperty("doris.hadoop.config.dir", tempDir + "/");
136+
try {
137+
HadoopAuthenticator auth = HiveConnector.buildPluginAuthenticator(
138+
props("hive.metastore.uris", "thrift://hms:9083",
139+
"hive.metastore.authentication.type", "simple",
140+
"hive.conf.resources", resource.getFileName().toString()));
141+
Assertions.assertEquals("resource-user", auth.getUGI().getUserName());
142+
} finally {
143+
if (previous == null) {
144+
System.clearProperty("doris.hadoop.config.dir");
145+
} else {
146+
System.setProperty("doris.hadoop.config.dir", previous);
147+
}
148+
}
149+
}
150+
151+
@Test
152+
public void hmsSimpleAuthUsesLegacyDefaultUser() throws Exception {
81153
HadoopAuthenticator auth = HiveConnector.buildPluginAuthenticator(
82154
props("hive.metastore.uris", "thrift://hms:9083",
83155
"hive.metastore.authentication.type", "simple"));
84-
Assertions.assertNull(auth, "simple-auth HMS must not build a plugin authenticator");
156+
Assertions.assertNotNull(auth, "simple-auth HMS must yield a plugin authenticator");
157+
Assertions.assertEquals("hadoop", auth.getUGI().getUserName());
85158
}
86159

87-
/** A plain HMS with no auth configured builds no authenticator. */
88160
@Test
89-
public void plainHmsWithoutKerberosReturnsNull() {
161+
public void plainHmsUsesLegacyDefaultUser() throws Exception {
90162
HadoopAuthenticator auth = HiveConnector.buildPluginAuthenticator(
91163
props("hive.metastore.uris", "thrift://hms:9083"));
92-
Assertions.assertNull(auth, "plain HMS without kerberos must not build an authenticator");
164+
Assertions.assertNotNull(auth, "plain HMS must yield a plugin authenticator");
165+
Assertions.assertEquals("hadoop", auth.getUGI().getUserName());
93166
}
94167

95168
/**
@@ -103,4 +176,37 @@ public void hmsKerberosWithBlankCredsReturnsNull() {
103176
"hive.metastore.authentication.type", "kerberos"));
104177
Assertions.assertNull(auth, "kerberos HMS without a client principal/keytab pair must not build one");
105178
}
179+
180+
@Test
181+
public void hmsClientUsesFeConfiguredSocketTimeout() {
182+
HiveConnector connector = new HiveConnector(HiveTestProperties.minimalMap(),
183+
new FakeConnectorContext(Map.of("hive_metastore_client_timeout_second", "47")));
184+
Assertions.assertEquals("47", connector.buildHmsClientConfig().getProperties()
185+
.get("hive.metastore.client.socket.timeout"));
186+
}
187+
188+
@Test
189+
public void firstConnectorInitializesConfiguredHadoopResourceDirectory() throws Exception {
190+
Path resource = tempDir.resolve("hive-site.xml");
191+
Files.writeString(resource, "<configuration><property><name>hadoop.username</name>"
192+
+ "<value>first-hive-user</value></property></configuration>");
193+
Map<String, String> properties = props(
194+
"hive.metastore.uris", "thrift://hms:9083",
195+
"hive.metastore.authentication.type", "simple",
196+
"hive.conf.resources", resource.getFileName().toString());
197+
String previous = System.getProperty("doris.hadoop.config.dir");
198+
System.clearProperty("doris.hadoop.config.dir");
199+
try {
200+
new HiveConnector(properties,
201+
new FakeConnectorContext(Map.of("hadoop_config_dir", tempDir.toString())));
202+
Assertions.assertEquals("first-hive-user",
203+
HiveConnector.buildPluginAuthenticator(properties).getUGI().getUserName());
204+
} finally {
205+
if (previous == null) {
206+
System.clearProperty("doris.hadoop.config.dir");
207+
} else {
208+
System.setProperty("doris.hadoop.config.dir", previous);
209+
}
210+
}
211+
}
106212
}

0 commit comments

Comments
 (0)