Ankit Malik
32 lines
667 B
Python
32 lines
667 B
Python
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.engine import URL, Engine
|
|
import os
|
|
import clickhouse_connect
|
|
import polars as pd
|
|
import pyarrow
|
|
from log import log
|
|
|
|
|
|
def load_to_clickhouse(
|
|
client: Client,
|
|
table_name: str,
|
|
df: pl.DataFrame,
|
|
) -> None:
|
|
"""
|
|
Load a Polars DataFrame into ClickHouse using Arrow.
|
|
"""
|
|
|
|
if df.is_empty():
|
|
log.warning(f"{table_name}: DataFrame is empty. Skipping.")
|
|
return
|
|
|
|
arrow_table = df.to_arrow()
|
|
|
|
client.insert_arrow(
|
|
table=table_name,
|
|
arrow_table=arrow_table,
|
|
)
|
|
|
|
log.info(
|
|
f"{table_name}: inserted {len(df):,} rows into ClickHouse"
|
|
) |