Skip to content

Commit 52e1545

Browse files
[improvement](regression) Reduce Kerberos test environment memory (apache#65564)
Problem Summary: The external regression pipeline currently runs two heavyweight Kerberos containers. Each container starts HiveServer2, YARN ResourceManager/NodeManager, MariaDB, and other processes that the Doris Kerberos cases do not use, resulting in roughly 4.3 GB of combined memory usage. This PR replaces those images with a minimal reusable environment based on Apache Hive 3.1.3. Each realm now runs only: - a native MIT Kerberos KDC - one HDFS NameNode and DataNode - one Kerberos-enabled Hive Metastore backed by embedded Derby The existing realm names, ports, service principals, client principals, and keytab filenames remain unchanged. JVM heaps are explicitly bounded, and each container has a 1.5 GB memory limit. HDFS listeners bind independently from their advertised Kerberos hostnames, and startup now waits until each DataNode is registered with its NameNode. The container no longer owns Hive test fixtures. `test_single_hive_kerberos` and `test_two_hive_kerberos` now create their test database, Hive table, and four rows through Doris. `test_non_catalog_kerberos` also creates its database so it no longer depends on suite execution order. Existing expected output files remain unchanged.
1 parent 8b081ea commit 52e1545

34 files changed

Lines changed: 706 additions & 739 deletions
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Generated by run-thirdparties-docker.sh.
2+
/conf/kerberos1/*.conf
3+
/conf/kerberos1/*.xml
4+
/conf/kerberos2/*.conf
5+
/conf/kerberos2/*.xml
6+
/data/
7+
/two-kerberos-hives/*.conf
8+
/two-kerberos-hives/*.keytab

docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh renamed to docker/thirdparties/docker-compose/kerberos/Dockerfile

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
#!/usr/bin/env bash
21
# Licensed to the Apache Software Foundation (ASF) under one
32
# or more contributor license agreements. See the NOTICE file
43
# distributed with this work for additional information
@@ -16,25 +15,20 @@
1615
# specific language governing permissions and limitations
1716
# under the License.
1817

19-
set -euo pipefail
18+
FROM apache/hive:3.1.3
2019

21-
if test $# -gt 0; then
22-
echo "$0 does not accept arguments" >&2
23-
exit 32
24-
fi
20+
USER root
2521

26-
# Supervisord is not running
27-
if ! test -f /tmp/supervisor.sock; then
28-
exit 0
29-
fi
22+
RUN apt-get -o Acquire::Retries=3 -o APT::Update::Error-Mode=any update \
23+
&& DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
24+
krb5-admin-server \
25+
krb5-kdc \
26+
krb5-user \
27+
&& rm -rf /var/lib/apt/lists/*
3028

31-
# Check if all Hadoop services are running
32-
FAILED=$(supervisorctl status | grep -v RUNNING || true)
29+
COPY entrypoint-hive-master.sh /opt/doris/entrypoint.sh
30+
COPY health-checks/health.sh /opt/doris/health.sh
3331

34-
if [ "$FAILED" == "" ]; then
35-
echo "All services are running"
36-
exit 0
37-
else
38-
echo "Some of the services are failing: ${FAILED}"
39-
exit 1
40-
fi
32+
RUN chmod 755 /opt/doris/entrypoint.sh /opt/doris/health.sh
33+
34+
ENTRYPOINT ["/opt/doris/entrypoint.sh"]

‎docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,9 @@
3333
[realms]
3434
LABS.TERADATA.COM = {
3535
kdc = hadoop-master:5588
36-
admin_server = hadoop-master:5749
37-
}
38-
OTHERLABS.TERADATA.COM = {
39-
kdc = hadoop-master:5589
40-
admin_server = hadoop-master:5750
4136
}
4237
OTHERREALM.COM = {
4338
kdc = hadoop-master-2:6688
44-
admin_server = hadoop-master-2:6749
4539
}
4640

4741
[domain_realm]

‎docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml renamed to docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<?xml version="1.0" encoding="UTF-8"?>
1+
<?xml version="1.0"?>
22
<!--
33
Licensed to the Apache Software Foundation (ASF) under one
44
or more contributor license agreements. See the NOTICE file
@@ -18,12 +18,37 @@ specific language governing permissions and limitations
1818
under the License.
1919
-->
2020
<configuration>
21+
<property>
22+
<name>fs.defaultFS</name>
23+
<value>hdfs://${HOST}:${FS_PORT}</value>
24+
</property>
25+
<property>
26+
<name>hadoop.security.authentication</name>
27+
<value>kerberos</value>
28+
</property>
29+
<property>
30+
<name>hadoop.security.authorization</name>
31+
<value>true</value>
32+
</property>
2133
<property>
2234
<name>hadoop.security.auth_to_local</name>
2335
<value>
24-
RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*//
36+
RULE:[2:$1@$0](.*@LABS.TERADATA.COM)s/@.*//
2537
RULE:[2:$1@$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
38+
RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*//
2639
DEFAULT
2740
</value>
2841
</property>
42+
<property>
43+
<name>hadoop.proxyuser.hive.hosts</name>
44+
<value>*</value>
45+
</property>
46+
<property>
47+
<name>hadoop.proxyuser.hive.users</name>
48+
<value>*</value>
49+
</property>
50+
<property>
51+
<name>hadoop.security.token.service.use_ip</name>
52+
<value>false</value>
53+
</property>
2954
</configuration>
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<configuration>
21+
<property>
22+
<name>dfs.namenode.name.dir</name>
23+
<value>file:///data/hdfs/name</value>
24+
</property>
25+
<property>
26+
<name>dfs.datanode.data.dir</name>
27+
<value>file:///data/hdfs/data</value>
28+
</property>
29+
<property>
30+
<name>dfs.replication</name>
31+
<value>1</value>
32+
</property>
33+
<property>
34+
<name>dfs.permissions.enabled</name>
35+
<value>false</value>
36+
</property>
37+
<property>
38+
<name>dfs.namenode.datanode.registration.ip-hostname-check</name>
39+
<value>false</value>
40+
</property>
41+
<property>
42+
<name>dfs.block.access.token.enable</name>
43+
<value>true</value>
44+
</property>
45+
<property>
46+
<name>dfs.namenode.kerberos.principal</name>
47+
<value>hdfs/${HOST}@${REALM}</value>
48+
</property>
49+
<property>
50+
<name>dfs.namenode.rpc-bind-host</name>
51+
<value>0.0.0.0</value>
52+
</property>
53+
<property>
54+
<name>dfs.namenode.keytab.file</name>
55+
<value>/data/keytabs/hdfs.keytab</value>
56+
</property>
57+
<property>
58+
<name>dfs.datanode.kerberos.principal</name>
59+
<value>hdfs/${HOST}@${REALM}</value>
60+
</property>
61+
<property>
62+
<name>dfs.datanode.keytab.file</name>
63+
<value>/data/keytabs/hdfs.keytab</value>
64+
</property>
65+
<property>
66+
<name>dfs.namenode.kerberos.internal.spnego.principal</name>
67+
<value>HTTP/${HOST}@${REALM}</value>
68+
</property>
69+
<property>
70+
<name>dfs.web.authentication.kerberos.principal</name>
71+
<value>HTTP/${HOST}@${REALM}</value>
72+
</property>
73+
<property>
74+
<name>dfs.web.authentication.kerberos.keytab</name>
75+
<value>/data/keytabs/spnego.keytab</value>
76+
</property>
77+
<property>
78+
<name>dfs.data.transfer.protection</name>
79+
<value>authentication</value>
80+
</property>
81+
<property>
82+
<name>dfs.http.policy</name>
83+
<value>HTTP_ONLY</value>
84+
</property>
85+
<property>
86+
<name>ignore.secure.ports.for.testing</name>
87+
<value>true</value>
88+
</property>
89+
<property>
90+
<name>dfs.datanode.address</name>
91+
<value>0.0.0.0:${DFS_DN_PORT}</value>
92+
</property>
93+
<property>
94+
<name>dfs.datanode.http.address</name>
95+
<value>0.0.0.0:${DFS_DN_HTTP_PORT}</value>
96+
</property>
97+
<property>
98+
<name>dfs.datanode.ipc.address</name>
99+
<value>0.0.0.0:${DFS_DN_IPC_PORT}</value>
100+
</property>
101+
<property>
102+
<name>dfs.datanode.hostname</name>
103+
<value>${HOST}</value>
104+
</property>
105+
<property>
106+
<name>dfs.client.use.datanode.hostname</name>
107+
<value>true</value>
108+
</property>
109+
<property>
110+
<name>dfs.namenode.http-address</name>
111+
<value>0.0.0.0:${DFS_NN_HTTP_PORT}</value>
112+
</property>
113+
</configuration>
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<configuration>
21+
<property>
22+
<name>javax.jdo.option.ConnectionURL</name>
23+
<value>jdbc:derby:;databaseName=/data/metastore/metastore_db;create=true</value>
24+
</property>
25+
<property>
26+
<name>javax.jdo.option.ConnectionDriverName</name>
27+
<value>org.apache.derby.jdbc.EmbeddedDriver</value>
28+
</property>
29+
<property>
30+
<name>datanucleus.autoCreateSchema</name>
31+
<value>false</value>
32+
</property>
33+
<property>
34+
<name>hive.metastore.schema.verification</name>
35+
<value>true</value>
36+
</property>
37+
<property>
38+
<name>hive.metastore.uris</name>
39+
<value>thrift://${HOST}:${HMS_PORT}</value>
40+
</property>
41+
<property>
42+
<name>hive.metastore.warehouse.dir</name>
43+
<value>hdfs://${HOST}:${FS_PORT}/user/hive/warehouse</value>
44+
</property>
45+
<property>
46+
<name>hive.metastore.sasl.enabled</name>
47+
<value>true</value>
48+
</property>
49+
<property>
50+
<name>hive.metastore.kerberos.principal</name>
51+
<value>hive/${HOST}@${REALM}</value>
52+
</property>
53+
<property>
54+
<name>hive.metastore.kerberos.keytab.file</name>
55+
<value>/data/keytabs/hive.keytab</value>
56+
</property>
57+
<property>
58+
<name>hive.metastore.execute.setugi</name>
59+
<value>true</value>
60+
</property>
61+
<property>
62+
<name>metastore.storage.schema.reader.impl</name>
63+
<value>org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader</value>
64+
</property>
65+
</configuration>

docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh renamed to docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
#!/usr/bin/env bash
21
# Licensed to the Apache Software Foundation (ASF) under one
32
# or more contributor license agreements. See the NOTICE file
43
# distributed with this work for additional information
@@ -16,5 +15,15 @@
1615
# specific language governing permissions and limitations
1716
# under the License.
1817

19-
kinit -kt /etc/hive/conf/hive.keytab hive/[email protected]
20-
beeline -u "jdbc:hive2://localhost:16000/default;principal=hive/[email protected]" -e "show databases;"
18+
[kdcdefaults]
19+
kdc_ports = ${KDC_PORT}
20+
kdc_tcp_ports = ${KDC_PORT}
21+
22+
[realms]
23+
${REALM} = {
24+
database_name = /data/kdc/principal
25+
key_stash_file = /data/kdc/.k5.${REALM}
26+
max_life = 24h
27+
max_renewable_life = 7d
28+
supported_enctypes = aes128-cts-hmac-sha1-96:normal
29+
}

0 commit comments

Comments
 (0)