3rd commit

clickhouse_task/load_table.py

@@ -0,0 +1,28 @@
+import polars as pl
+from clickhouse_connect.driver import Client
+from log import log
+
+
+def load_to_clickhouse(
+    client: Client,
+    table_name: str,
+    df: pl.DataFrame,
+) -> None:
+    """
+    Load a Polars DataFrame into ClickHouse using Arrow.
+    """
+
+    if df.is_empty():
+        log.warning(f"{table_name}: DataFrame is empty. Skipping.")
+        return
+
+    arrow_table = df.to_arrow()
+
+    client.insert_arrow(
+        table=table_name,
+        arrow_table=arrow_table,
+    )
+
+    log.info(
+        f"{table_name}: inserted {len(df):,} rows into ClickHouse"
+    )