Skip to content

Commit 4aa55ba

Browse files
kevinjqliuclaude
andauthored
fix(hms): use get_table_req for Hive 4 metastore compatibility (apache#3187)
fix(hms): use get_table_req for Hive 4 metastore compatibility Hive 4.0.1 removed the `get_table` thrift method, so the HMS catalog fails with `Invalid method name: 'get_table'` against any Hive 4 metastore. Switch `load_table`, `table_exists`, and `rename_table` to `get_table_req`, which exists in every Hive release since 2.3 (the IDL the `hive_metastore` crate is generated from), so older metastores keep working. Move the integration test metastore from `apache/hive:3.1.3` (Debian Bullseye, EOL) to `apache/hive:4.2.1` so CI exercises a Hive 4 server, mirroring apache/iceberg-python#3924. Co-authored-by: Claude Fable 5.1 <[email protected]>
1 parent d6c2eb4 commit 4aa55ba

2 files changed

Lines changed: 31 additions & 19 deletions

File tree

‎crates/catalog/hms/src/catalog.rs‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ use std::sync::Arc;
2323
use anyhow::anyhow;
2424
use async_trait::async_trait;
2525
use hive_metastore::{
26-
ThriftHiveMetastoreClient, ThriftHiveMetastoreClientBuilder,
27-
ThriftHiveMetastoreGetDatabaseException, ThriftHiveMetastoreGetTableException,
26+
GetTableRequest, ThriftHiveMetastoreClient, ThriftHiveMetastoreClientBuilder,
27+
ThriftHiveMetastoreGetDatabaseException, ThriftHiveMetastoreGetTableReqException,
2828
};
2929
use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory};
3030
use iceberg::io::{FileIO, FileIOBuilder, StorageFactory};
@@ -583,10 +583,15 @@ impl Catalog for HmsCatalog {
583583
let hive_table = self
584584
.client
585585
.0
586-
.get_table(db_name.clone().into(), table.name.clone().into())
586+
.get_table_req(GetTableRequest {
587+
db_name: db_name.clone().into(),
588+
tbl_name: table.name.clone().into(),
589+
..Default::default()
590+
})
587591
.await
588592
.map(from_thrift_exception)
589-
.map_err(from_thrift_error)??;
593+
.map_err(from_thrift_error)??
594+
.table;
590595

591596
let metadata_location = get_metadata_location(&hive_table.parameters)?;
592597

@@ -661,12 +666,18 @@ impl Catalog for HmsCatalog {
661666
let resp = self
662667
.client
663668
.0
664-
.get_table(db_name.into(), table_name.into())
669+
.get_table_req(GetTableRequest {
670+
db_name: db_name.into(),
671+
tbl_name: table_name.into(),
672+
..Default::default()
673+
})
665674
.await;
666675

667676
match resp {
668677
Ok(MaybeException::Ok(_)) => Ok(true),
669-
Ok(MaybeException::Exception(ThriftHiveMetastoreGetTableException::O2(_))) => Ok(false),
678+
Ok(MaybeException::Exception(ThriftHiveMetastoreGetTableReqException::O2(_))) => {
679+
Ok(false)
680+
}
670681
Ok(MaybeException::Exception(exception)) => Err(Error::new(
671682
ErrorKind::Unexpected,
672683
"Operation failed for hitting thrift error".to_string(),
@@ -698,10 +709,15 @@ impl Catalog for HmsCatalog {
698709
let mut tbl = self
699710
.client
700711
.0
701-
.get_table(src_dbname.clone().into(), src_tbl_name.clone().into())
712+
.get_table_req(GetTableRequest {
713+
db_name: src_dbname.clone().into(),
714+
tbl_name: src_tbl_name.clone().into(),
715+
..Default::default()
716+
})
702717
.await
703718
.map(from_thrift_exception)
704-
.map_err(from_thrift_error)??;
719+
.map_err(from_thrift_error)??
720+
.table;
705721

706722
tbl.db_name = Some(dest_dbname.into());
707723
tbl.table_name = Some(dest_tbl_name.into());

‎dev/hms/Dockerfile‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,16 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
FROM apache/hive:3.1.3
17-
18-
ENV AWSSDK_VERSION=2.20.18
19-
ENV HADOOP_VERSION=3.1.0
16+
FROM apache/hive:4.2.1
2017

2118
USER root
2219

23-
ADD --chmod=644 --checksum=sha256:a18508b9348af095ea41301e439354dbd449e304ac44c6885b2b4fe78de88126 \
24-
https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \
25-
/opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar
26-
ADD --chmod=644 --checksum=sha256:faf78ac4880f56cf52791d84ec1068ce7c66acc4295d580a726104b734c01fcd \
27-
https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar \
28-
/opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar
20+
# Link the hadoop-aws and AWS SDK jars that the image ships into the metastore classpath
21+
RUN ln -s /opt/hadoop/share/hadoop/tools/lib/hadoop-aws-*.jar /opt/hive/lib/ && \
22+
ln -s /opt/hadoop/share/hadoop/tools/lib/bundle-*.jar /opt/hive/lib/
2923

30-
COPY core-site.xml /opt/hadoop/etc/hadoop/core-site.xml
24+
# The entrypoint links this directory into the Hive config directory, over its own core-site.xml
25+
ENV HIVE_CUSTOM_CONF_DIR=/opt/hive/custom-conf
26+
COPY core-site.xml ${HIVE_CUSTOM_CONF_DIR}/core-site.xml
3127

3228
USER hive

0 commit comments

Comments
 (0)