Ankit Malik
28 lines
575 B
Python
28 lines
575 B
Python
import polars as pl
|
|
from clickhouse_connect.driver import Client
|
|
from log import log
|
|
|
|
|
|
def load_to_clickhouse(
|
|
client: Client,
|
|
table_name: str,
|
|
df: pl.DataFrame,
|
|
) -> None:
|
|
"""
|
|
Load a Polars DataFrame into ClickHouse using Arrow.
|
|
"""
|
|
|
|
if df.is_empty():
|
|
log.warning(f"{table_name}: DataFrame is empty. Skipping.")
|
|
return
|
|
|
|
arrow_table = df.to_arrow()
|
|
|
|
client.insert_arrow(
|
|
table=table_name,
|
|
arrow_table=arrow_table,
|
|
)
|
|
|
|
log.info(
|
|
f"{table_name}: inserted {len(df):,} rows into ClickHouse"
|
|
) |