Skip to content

Instantly share code, notes, and snippets.

@eedduuar
Last active August 1, 2024 14:16
Show Gist options
  • Select an option

  • Save eedduuar/ab8e2f4ea1934bd2a15f595b0c6da8c5 to your computer and use it in GitHub Desktop.

Select an option

Save eedduuar/ab8e2f4ea1934bd2a15f595b0c6da8c5 to your computer and use it in GitHub Desktop.
pyiceberg - hive catalog example
# poetry add "pyiceberg[s3fs,hive,pyarrow]"
import os
import pandas as pd
import pyarrow as pa
from pyiceberg.catalog.hive import HiveCatalog
class Iceberg:
def __init__(self):
self.namespace = "db"
self.catalog = HiveCatalog(
"iceberg",
**{
"uri": "thrift://localhost:9083",
"s3.endpoint": "https://s3.us-west-2.amazonaws.com",
"py-io-impl": "pyiceberg.io.pyarrow.PyArrowFileIO",
"s3.access-key": os.getenv("AWS_ACCESS_KEY_ID"),
"s3.secret-key": os.getenv("AWS_SECRET_ACCESS_KEY"),
"s3.region": "us-west-2",
}
)
def insertData(self, df: pd.DataFrame, table_name: str):
table_full_name = self.namespace + "." + table_name
pyArrowTable = pa.Table.from_pandas(df)
if not self.catalog.table_exists(table_full_name):
table = self.catalog.create_table(
table_full_name,
schema=pyArrowTable.schema,
)
else:
table = self.catalog.load_table(table_full_name)
table.append(pyArrowTable)
if __name__ == "__main__":
engine = IcebergEngine()
print(engine.catalog.list_tables("db"))
df = pd.read_csv("/tmp/1.csv")
engine.insertData(df, "test_table")
"""
➜ ./trino
trino> use iceberg.id;
USE
trino:idb> select * from test_table;
position | key | uniqueid | value >
----------+------------------------------------------------------------------+---------------------------------------------------------+------------->
1| HouseholdIncome.Household.HouseholdMembers | HouseholdHouseholdMembersAlienNumber | 123456789 >
1 | HouseholdIncome.Household.HouseholdMembers | HouseholdHouseholdMembersValue | 2500 >
1 | HouseholdIncome.Household.HouseholdMembers | HouseholdHouseholdMembersFirstName | first >
1 | HouseholdIncome.Household.HouseholdMembers | HouseholdHouseholdMembersLastName | member >
1 | HouseholdIncome.Household.HouseholdMembers | HouseholdHouseholdMembersDateOfBirth | 1991-01-01 >
"""
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment