ankitm/data_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

14-06-26 1st comit

Browse Source
This commit is contained in:
Ankit Malik
2026-06-15 12:44:13 +05:30
parent 418a8847b2
commit 166a2a1d19
14 changed files with 1981 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clickhouse_task/delete_task.py
+174 -14
View File
@@ -1,31 +1,191 @@
from db_con.connection import * from __future__ import annotations
from log import * import polars as pl
from log import log
def truncate_table(client, table_name: str) -> None: def truncate_table(
client,
table_name: str,
) -> None:
""" """
Truncate a ClickHouse table. Full refresh tables.
""" """
query = f"TRUNCATE TABLE {table_name}"
print(f"Truncating table: {table_name}") client.command(
client.command(query) f"TRUNCATE TABLE {table_name}"
)
log.info(f"Table {table_name} truncated successfully.") log.info(
"Truncated table %s",
table_name,
)
def delete_rows(
def delete_rows(client, table_name: str, condition: str) -> None: client,
table_name: str,
where_clause: str,
) -> None:
""" """
Delete rows from a ClickHouse table based on a condition. Generic ClickHouse delete.
""" """
query = f""" query = f"""
ALTER TABLE {table_name} ALTER TABLE {table_name}
DELETE WHERE {condition} DELETE
WHERE {where_clause}
""" """
print(f"Deleting rows from {table_name} where {condition}") log.info(
"Deleting from %s",
table_name,
)
client.command(query) client.command(query)
log.info("Delete command submitted successfully.")
def delete_existing_data(
client,
table_name: str,
run_date,
mids: list[int],
emp_visit_df: pl.DataFrame,
) -> None:
"""
Incremental delete logic.
Matches the old SQL procedure.
"""
# --------------------------------------------------
# MID based tables
# --------------------------------------------------
mid_tables = {
"additional_visibility",
"Coverage",
"Survey",
"Promotion",
"PaidVisibility",
"SOS_OneApp",
"Stock_Details",
"Login",
"coverage_remarks",
}
if table_name in mid_tables and mids:
mids_str = ",".join(
map(str, mids)
)
delete_rows(
client,
table_name,
f"MID IN ({mids_str})",
)
return
# --------------------------------------------------
# Journey Plan
# --------------------------------------------------
if table_name == "Journey_Plan":
delete_rows(
client,
table_name,
f"""
toMonth(visit_date) = {run_date.month}
AND toYear(visit_date) = {run_date.year}
""",
)
return
# --------------------------------------------------
# Logins
# --------------------------------------------------
if table_name == "Login":
delete_rows(
client,
table_name,
f"""
project_id = 40148
AND toDate(login_date) = toDate('{run_date}')
"""
)
# --------------------------------------------------
# Web Logins
# --------------------------------------------------
if table_name == "Web_Logins":
delete_rows(
client,
table_name,
f"""
toDate(date)
= toDate('{run_date}')
""",
)
return
# --------------------------------------------------
# Attendance
# --------------------------------------------------
if table_name == "Attendance":
delete_rows(
client,
table_name,
f"""
toDate(attendance_date)
= toDate('{run_date}')
""",
)
return
# --------------------------------------------------
# OQaD
# --------------------------------------------------
if (
table_name == "OQaD"
and not emp_visit_df.is_empty()
):
conditions = [
(
f"(employee_id={row['EmpId']} "
f"AND toDate(visit_date)="
f"toDate('{row['VisitDate']}'))"
)
for row in emp_visit_df.iter_rows(
named=True
)
]
delete_rows(
client,
table_name,
" OR ".join(conditions),
)
return
log.info(
"No delete logic required for %s",
table_name,
)
clickhouse_task/load_table.py
+1
View File
@@ -3,6 +3,7 @@ from sqlalchemy.engine import URL, Engine
import os import os
import clickhouse_connect import clickhouse_connect
import polars as pd import polars as pd
import pyarrow
from log import log from log import log
db_con/.env
+1 -1
View File
@@ -9,4 +9,4 @@ CH_HOST=172.188.12.194
CH_PORT=8123 CH_PORT=8123
CH_USER=default CH_USER=default
CH_PASS=dipanshu_k CH_PASS=dipanshu_k
CH_DB=kelloggs CH_DB=kelloggs_1
kpi/facts.py
+28 -26
View File
@@ -140,23 +140,27 @@ def fetch_SOS_OneApp(engine: Engine, mids: list[int]) -> pl.DataFrame:
return df return df
def fetch_OQaD(
def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame: engine: Engine,
run_date: date,
if not mids: ) -> pl.DataFrame:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
mid_list = ",".join(map(str, mids))
sql = f""" sql = f"""
WITH MID_TABLE_COV1 AS WITH MID_TABLE_COV1 AS
( (
SELECT DISTINCT SELECT
EmpId, EmpId,
CAST(VisitDate AS DATE) AS VisitDate CAST(VisitDate AS DATE) AS VisitDate
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE MID IN ({mid_list}) WHERE CAST(CreateDate AS DATE) = '{run_date}'
UNION
SELECT
EmpId,
CAST(VisitDate AS DATE) AS VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CAST(UpdateDate AS DATE) = '{run_date}'
), ),
QUIZ AS QUIZ AS
@@ -180,12 +184,12 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame:
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
ON QU.QuestionCategoryId = QC.QuestionCategoryId ON QU.QuestionCategoryId = QC.QuestionCategoryId
WHERE E.EmpName NOT LIKE 'test%' WHERE E.RightId = 6
AND E.EmpName NOT LIKE 'test%'
AND E.EmpName NOT LIKE '%TEST%' AND E.EmpName NOT LIKE '%TEST%'
AND E.RightId = 6
AND ( AND (
E.ResignDate IS NULL E.ResignDate IS NULL
OR E.ResignDate >= DQ.VisitDate OR E.ResignDate >= '{run_date}'
) )
AND EXISTS AND EXISTS
( (
@@ -197,6 +201,7 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame:
) )
SELECT SELECT
40148 AS project_id,
Q.EmpId AS employee_id, Q.EmpId AS employee_id,
0 AS process_id, 0 AS process_id,
Q.VisitDate AS visit_date, Q.VisitDate AS visit_date,
@@ -204,16 +209,18 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame:
Q.QuestionCategory AS question_category, Q.QuestionCategory AS question_category,
QM.QuestionId AS question_id, QM.QuestionId AS question_id,
QM.Question AS question, QM.Question AS question,
ISNULL(QA.AnswerId, 0) AS answer_id,
ISNULL(QA.AnswerId,0) AS answer_id, ISNULL(QA.Answer, '') AS answer,
ISNULL(QA.Answer,'') AS answer,
CASE CASE
WHEN QA.AnswerId IS NULL THEN 'Not Answer' WHEN QA.AnswerId IS NULL THEN 'Not Answer'
WHEN QA.RightAnswer = 1 THEN 'Y' WHEN QA.RightAnswer = 1 THEN 'Y'
WHEN QA.RightAnswer IS NULL THEN 'Not Answer' WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
ELSE 'N' ELSE 'N'
END AS correct_answer END AS correct_answer,
GETDATE() AS update_date,
'SP-Pius' AS update_by
FROM QUIZ Q FROM QUIZ Q
@@ -224,23 +231,18 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame:
ON Q.AnswerId = QA.AnswerId ON Q.AnswerId = QA.AnswerId
""" """
log.info(f"Fetching OQaD data for {len(mids):,} MIDs") log.info("Fetching OQaD data for run_date=%s", run_date)
df = pl.read_database( df = pl.read_database(
query=sql, query=sql,
connection=engine connection=engine,
) )
log.info("Fetched %s rows", len(df))
log.info(f"Fetched {len(df):,} rows")
return df return df
def fetch_Survey(engine: Engine, mids: list[int]) -> pl.DataFrame: def fetch_Survey(engine: Engine, mids: list[int]) -> pl.DataFrame:
if not mids: if not mids:
log.warning("No MIDs — nothing to fetch.") log.warning("No MIDs — nothing to fetch.")
logs/etl_20260612.log
+40
View File
@@ -982,3 +982,43 @@
2026-06-12 19:04:53 | INFO | Fetched 5,984 stores 2026-06-12 19:04:53 | INFO | Fetched 5,984 stores
2026-06-12 19:04:53 | INFO | Table Store_Master truncated successfully. 2026-06-12 19:04:53 | INFO | Table Store_Master truncated successfully.
2026-06-12 19:04:53 | INFO | Truncate a ClickHouse table - Store_Master 2026-06-12 19:04:53 | INFO | Truncate a ClickHouse table - Store_Master
2026-06-12 19:07:43 | INFO | ================================================================================
2026-06-12 19:07:43 | INFO | Hello from data-move Python data pipeline !
2026-06-12 19:07:43 | INFO | Data-pipeline running Date is -:2026-06-11
2026-06-12 19:07:43 | INFO | connecting with both db servers sql-serveras well as clickhouse DB
2026-06-12 19:07:44 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001148C85BE30>
2026-06-12 19:07:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001148DA15BD0>
2026-06-12 19:07:46 | INFO | Both databases connected successfully
2026-06-12 19:07:46 | INFO | Collecting MIDs for: 2026-06-11
2026-06-12 19:07:46 | INFO | Found 818 MIDs
2026-06-12 19:07:46 | INFO | ================================================================================
2026-06-12 19:07:46 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT
2026-06-12 19:07:46 | INFO | Fetching data for 818 MIDs
2026-06-12 19:07:47 | INFO | Fetched 3,677 rows from SQL Server
2026-06-12 19:07:47 | INFO | Delete command submitted successfully.
2026-06-12 19:07:47 | INFO | SOS_OneApp: inserted 3,677 rows into ClickHouse
2026-06-12 19:07:47 | INFO | ================================================================================
2026-06-12 19:07:47 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT
2026-06-12 19:07:47 | INFO | Fetching OQaD data for 818 MIDs
2026-06-12 19:07:50 | INFO | Fetched 464 rows
2026-06-12 19:09:36 | INFO | ================================================================================
2026-06-12 19:09:36 | INFO | Hello from data-move Python data pipeline !
2026-06-12 19:09:36 | INFO | Data-pipeline running Date is -:2026-06-11
2026-06-12 19:09:36 | INFO | connecting with both db servers sql-serveras well as clickhouse DB
2026-06-12 19:09:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001732452BCD0>
2026-06-12 19:09:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000173256E1BD0>
2026-06-12 19:09:39 | INFO | Both databases connected successfully
2026-06-12 19:09:39 | INFO | Collecting MIDs for: 2026-06-11
2026-06-12 19:09:39 | INFO | Found 818 MIDs
2026-06-12 19:09:39 | INFO | ================================================================================
2026-06-12 19:09:39 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT
2026-06-12 19:09:39 | INFO | Fetching data for 818 MIDs
2026-06-12 19:09:40 | INFO | Fetched 3,677 rows from SQL Server
2026-06-12 19:09:40 | INFO | Deleting rows from SOS_OneApp
2026-06-12 19:09:40 | INFO | Delete command submitted successfully.
2026-06-12 19:09:41 | INFO | SOS_OneApp: inserted 3,677 rows into ClickHouse
2026-06-12 19:09:41 | INFO | ================================================================================
2026-06-12 19:09:41 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT
2026-06-12 19:09:41 | INFO | Fetching OQaD data for 818 MIDs
2026-06-12 19:09:44 | INFO | Fetched 464 rows
2026-06-12 19:09:44 | INFO | Deleting rows from OQaD
logs/etl_20260615.log
+1103
View File
File diff suppressed because one or more lines are too long
main.py
+44 -1
View File
@@ -71,7 +71,50 @@ def main():
log.info("Both databases connected successfully") log.info("Both databases connected successfully")
mids=collect_mids(sql_engine , run_date) mids = MID_TABLE_COV(sql_engine, run_date)
emp_visit_df = MID_TABLE_COV1(
sql_engine,
run_date
)
delete_existing_data(
client=client,
run_date=run_date,
mids=mids,
emp_visit_df=emp_visit_df,
)
mid_list = ",".join(map(str, mids)) mid_list = ",".join(map(str, mids))
conditions = { conditions = {
"mids": f"MID IN ({mid_list})", "mids": f"MID IN ({mid_list})",
masters/dimensions.py
+1 -1
View File
@@ -30,7 +30,7 @@ p=40148
def fetch_Store_Master(engine: Engine) -> pl.DataFrame: def fetch_Store_Master(engine: Engine) -> pl.DataFrame:
sql = """ sql = """
SELECT SELECT
RegionId AS region_id,
RegionName AS region, RegionName AS region,
StateId AS state_id, StateId AS state_id,
StateName AS state, StateName AS state,
mids.py
+27 -1
View File
@@ -19,7 +19,7 @@ from db_con.connection import *
def collect_mids(engine: Engine, target_date: date) -> list[int]: def MID_TABLE_COV(engine: Engine, target_date: date) -> list[int]:
sql = text(""" sql = text("""
SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
@@ -36,3 +36,29 @@ def collect_mids(engine: Engine, target_date: date) -> list[int]:
mids = [row[0] for row in result.fetchall()] mids = [row[0] for row in result.fetchall()]
log.info(f"Found {len(mids):,} MIDs") log.info(f"Found {len(mids):,} MIDs")
return mids return mids
def MID_TABLE_COV1(
engine: Engine,
target_date: date,
) -> pl.DataFrame:
query = f"""
SELECT
EmpId,
CAST(VisitDate AS DATE) AS VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CAST(CreateDate AS DATE) = '{target_date}'
UNION
SELECT
EmpId,
CAST(VisitDate AS DATE) AS VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CAST(UpdateDate AS DATE) = '{target_date}'
"""
return pl.read_database(
query=query,
connection=engine,
)
pyproject.toml
+1
View File
@@ -9,6 +9,7 @@ dependencies = [
"clickhouse-sqlalchemy>=0.3.2", "clickhouse-sqlalchemy>=0.3.2",
"dotenv>=0.9.9", "dotenv>=0.9.9",
"polars>=1.41.2", "polars>=1.41.2",
"pyarrow>=24.0.0",
"pyodbc>=5.3.0", "pyodbc>=5.3.0",
"pyyaml>=6.0.3", "pyyaml>=6.0.3",
] ]
tables.yml
+1 -1
View File
@@ -8,7 +8,7 @@ tables:
- name: OQaD - name: OQaD
type: FACT type: FACT
operation: INSERT operation: INSERT
fetch_by: mids fetch_by: run_date
condition: mids condition: mids
- name: Survey - name: Survey
test.py
+257 -31
View File
@@ -1,41 +1,267 @@
from __future__ import annotations
import sys
from datetime import date, datetime, timedelta
import polars as pl
import yaml import yaml
from datetime import date, timedelta
mids_list=[1,3] from log import log
run_date=date.today() from clickhouse_task.create_table import create_clickhouse_table
conditions = { from clickhouse_task.delete_task import delete_existing_data , truncate_table
"mids": f"MID IN ({','.join(map(str, mids_list))})", from clickhouse_task.load_table import load_to_clickhouse
"j_plan": ( from db_con.connection import (
f"MONTH(VisitDate) = {run_date.month} " build_sql_server_engine,
f"AND YEAR(VisitDate) = {run_date.year}" build_clickhouse_engine,
), get_clickhouse_client,
)
"mapping": ( from mids import MID_TABLE_COV, MID_TABLE_COV1
f"CAST(Z.FromDate AS DATE) <= '{run_date}' "
f"AND CAST(Z.ToDate AS DATE) >= '{run_date}'"
),
"web": ( from masters.dimensions import *
f"CAST(login_date AS DATE) = '{run_date}'" from masters.bridge import *
), from kpi.facts import *
"none": None,
}
with open("tables.yml", "r") as file: # ============================================================
config = yaml.safe_load(file) # Helpers
# ============================================================
for table in config["tables"]: def get_dataframe(
fn_name: str,
fetch_by: str,
sql_engine,
mids,
run_date,
) -> pl.DataFrame:
table_name=table["name"] fn = globals()[fn_name]
table_type=table["type"]
operation=table["operation"] if fetch_by == "mids":
condition=table["condition"] return fn(sql_engine, mids)
c = conditions[condition]
print(table_name) if fetch_by == "run_date":
print(table_type) return fn(sql_engine, run_date)
print(operation)
print(c) return fn(sql_engine)
def ensure_table_exists(
client,
clickhouse_engine,
table_name: str,
df: pl.DataFrame,
) -> bool:
exists = client.command(
f"EXISTS TABLE {table_name}"
)
if not exists:
log.info(
"Creating ClickHouse table: %s",
table_name,
)
create_clickhouse_table(
df=df,
table_name=table_name,
clickhouse_engine=clickhouse_engine,
)
return bool(exists)
# ============================================================
# Main
# ============================================================
def main():
log.info("=" * 80)
log.info("Hello from data-move Python data pipeline !")
# --------------------------------------------------------
# Run Date
# --------------------------------------------------------
if len(sys.argv) > 1:
run_date = datetime.strptime(
sys.argv[1],
"%Y-%m-%d",
).date()
else:
run_date = date.today() - timedelta(days=1)
log.info(
"Pipeline Run Date : %s",
run_date,
)
# --------------------------------------------------------
# Connections
# --------------------------------------------------------
log.info(
"Connecting to SQL Server and ClickHouse"
)
sql_engine = build_sql_server_engine()
clickhouse_engine = (
build_clickhouse_engine()
)
client = get_clickhouse_client()
log.info(
"Database connections established"
)
# --------------------------------------------------------
# Collect Delete Keys
# --------------------------------------------------------
mids = MID_TABLE_COV(
sql_engine,
run_date,
)
emp_visit_df = MID_TABLE_COV1(
sql_engine,
run_date,
)
# --------------------------------------------------------
# Delete Existing Data
# --------------------------------------------------------
delete_existing_data(
client=client,
run_date=run_date,
mids=mids,
emp_visit_df=emp_visit_df,
)
# --------------------------------------------------------
# Load Config
# --------------------------------------------------------
with open(
"tables.yml",
"r",
) as file:
config = yaml.safe_load(file)
# --------------------------------------------------------
# Process Tables
# --------------------------------------------------------
for table in config["tables"]:
table_name = table["name"]
table_type = table["type"]
operation = table["operation"]
fetch_by = table["fetch_by"]
log.info("=" * 80)
log.info(
"TABLE=%s | TYPE=%s | OPERATION=%s",
table_name,
table_type,
operation,
)
try:
# --------------------------------------------
# Fetch
# --------------------------------------------
fn_name = f"fetch_{table_name}"
df = get_dataframe(
fn_name=fn_name,
fetch_by=fetch_by,
sql_engine=sql_engine,
mids=mids,
run_date=run_date,
)
if df.is_empty():
log.warning(
"%s returned no rows",
table_name,
)
continue
log.info(
"Fetched %s rows",
len(df),
)
# --------------------------------------------
# Create Table if Missing
# --------------------------------------------
exists = ensure_table_exists(
client=client,
clickhouse_engine=clickhouse_engine,
table_name=table_name,
df=df,
)
# --------------------------------------------
# Full Refresh Tables
# --------------------------------------------
if exists and operation == "DELETE+INSERT":
truncate_table(
client,
table_name,
)
log.info(
"Truncated %s",
table_name,
)
# --------------------------------------------
# Load
# --------------------------------------------
load_to_clickhouse(
client=client,
table_name=table_name,
df=df,
)
log.info(
"%s loaded successfully (%s rows)",
table_name,
len(df),
)
except Exception as e:
log.exception(
"Failed processing table %s",
table_name,
)
raise
log.info("=" * 80)
log.info("Pipeline Completed Successfully")
log.info("=" * 80)
if __name__ == "__main__":
main()
test2.py
+275
View File
@@ -0,0 +1,275 @@
# /// script
# requires-python = ">=3.11"
# dependencies = [
# "polars>=0.20.0",
# "pyarrow>=18.0.0",
# "sqlalchemy>=2.0.0",
# "pyodbc>=5.0.0",
# "clickhouse-connect>=0.7.0",
# "clickhouse-sqlalchemy>=0.3.2",
# "pyyaml>=6.0.3",
# "python-dotenv>=1.0.0",
# ]
# ///
from __future__ import annotations
import sys
from datetime import date, datetime, timedelta
import polars as pl
import yaml
from log import log
from clickhouse_task.create_table import create_clickhouse_table
from clickhouse_task.delete_task import (
delete_existing_data,
truncate_table,
)
from clickhouse_task.load_table import load_to_clickhouse
from db_con.connection import (
build_sql_server_engine,
build_clickhouse_engine,
get_clickhouse_client,
)
from mids import (
MID_TABLE_COV,
MID_TABLE_COV1,
)
from masters.dimensions import *
from masters.bridge import *
from kpi.facts import *
# ==========================================================
# Helpers
# ==========================================================
def table_exists(
client,
table_name: str,
) -> bool:
return bool(
client.command(
f"EXISTS TABLE {table_name}"
)
)
def get_dataframe(
fn_name: str,
fetch_by: str,
sql_engine,
mids,
run_date,
):
fn = globals()[fn_name]
if fetch_by == "mids":
return fn(sql_engine, mids)
if fetch_by == "run_date":
return fn(sql_engine, run_date)
return fn(sql_engine)
# ==========================================================
# Main
# ==========================================================
def main():
log.info("=" * 80)
log.info("Hello from data-move Python data pipeline!")
# ------------------------------------------------------
# Run Date
# ------------------------------------------------------
if len(sys.argv) > 1:
run_date = datetime.strptime(
sys.argv[1],
"%Y-%m-%d",
).date()
else:
run_date = date.today() - timedelta(days=1)
log.info(
"Pipeline Run Date: %s",
run_date,
)
# ------------------------------------------------------
# Connections
# ------------------------------------------------------
log.info(
"Connecting to databases..."
)
sql_engine = build_sql_server_engine()
clickhouse_engine = build_clickhouse_engine()
client = get_clickhouse_client()
log.info(
"Database connections established"
)
# ------------------------------------------------------
# Delete Keys
# ------------------------------------------------------
mids = MID_TABLE_COV(
sql_engine,
run_date,
)
emp_visit_df = MID_TABLE_COV1(
sql_engine,
run_date,
)
# ------------------------------------------------------
# Config
# ------------------------------------------------------
with open(
"tables.yml",
"r",
) as file:
config = yaml.safe_load(file)
# ------------------------------------------------------
# Process Tables
# ------------------------------------------------------
for table in config["tables"]:
table_name = table["name"]
operation = table["operation"]
fetch_by = table["fetch_by"]
log.info("=" * 80)
log.info(
"Processing Table: %s",
table_name,
)
try:
# ------------------------------------------
# Fetch Data
# ------------------------------------------
fn_name = f"fetch_{table_name}"
df = get_dataframe(
fn_name=fn_name,
fetch_by=fetch_by,
sql_engine=sql_engine,
mids=mids,
run_date=run_date,
)
if df.is_empty():
log.warning(
"%s returned no rows",
table_name,
)
continue
log.info(
"Fetched %s rows",
len(df),
)
# ------------------------------------------
# Create Table If Missing
# ------------------------------------------
exists = table_exists(
client,
table_name,
)
if not exists:
log.info(
"Creating table %s",
table_name,
)
create_clickhouse_table(
df=df,
table_name=table_name,
clickhouse_engine=clickhouse_engine,
)
# ------------------------------------------
# Existing Table Logic
# ------------------------------------------
else:
if operation == "DELETE+INSERT":
truncate_table(
client,
table_name,
)
else:
delete_existing_data(
client=client,
table_name=table_name,
run_date=run_date,
mids=mids,
emp_visit_df=emp_visit_df,
)
# ------------------------------------------
# Load Data
# ------------------------------------------
load_to_clickhouse(
client=client,
table_name=table_name,
df=df,
)
log.info(
"%s loaded successfully (%s rows)",
table_name,
len(df),
)
except Exception:
log.exception(
"Failed processing table %s",
table_name,
)
raise
log.info("=" * 80)
log.info("Pipeline Completed Successfully")
log.info("=" * 80)
if __name__ == "__main__":
main()
uv.lock
Generated
+24
View File
@@ -163,6 +163,7 @@ dependencies = [
{ name = "clickhouse-sqlalchemy" }, { name = "clickhouse-sqlalchemy" },
{ name = "dotenv" }, { name = "dotenv" },
{ name = "polars" }, { name = "polars" },
{ name = "pyarrow" },
{ name = "pyodbc" }, { name = "pyodbc" },
{ name = "pyyaml" }, { name = "pyyaml" },
] ]
@@ -173,6 +174,7 @@ requires-dist = [
{ name = "clickhouse-sqlalchemy", specifier = ">=0.3.2" }, { name = "clickhouse-sqlalchemy", specifier = ">=0.3.2" },
{ name = "dotenv", specifier = ">=0.9.9" }, { name = "dotenv", specifier = ">=0.9.9" },
{ name = "polars", specifier = ">=1.41.2" }, { name = "polars", specifier = ">=1.41.2" },
{ name = "pyarrow", specifier = ">=24.0.0" },
{ name = "pyodbc", specifier = ">=5.3.0" }, { name = "pyodbc", specifier = ">=5.3.0" },
{ name = "pyyaml", specifier = ">=6.0.3" }, { name = "pyyaml", specifier = ">=6.0.3" },
] ]
@@ -289,6 +291,28 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cd/c6/79e9f3f270270d7ed5575d92b7bfef49f01abd9275447161275b23b553a8/polars_runtime_32-1.41.2-cp310-abi3-win_arm64.whl", hash = "sha256:843d96f69d18eca53429c1198e58891db7f18111f83b9c419bb45ad9d73eaed5", size = 46006901, upload-time = "2026-05-29T17:37:42.522Z" }, { url = "https://files.pythonhosted.org/packages/cd/c6/79e9f3f270270d7ed5575d92b7bfef49f01abd9275447161275b23b553a8/polars_runtime_32-1.41.2-cp310-abi3-win_arm64.whl", hash = "sha256:843d96f69d18eca53429c1198e58891db7f18111f83b9c419bb45ad9d73eaed5", size = 46006901, upload-time = "2026-05-29T17:37:42.522Z" },
] ]
[[package]]
name = "pyarrow"
version = "24.0.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" },
{ url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" },
{ url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" },
{ url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" },
{ url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" },
{ url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" },
{ url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" },
{ url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" },
{ url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" },
{ url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" },
{ url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" },
{ url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" },
{ url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" },
{ url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" },
]
[[package]] [[package]]
name = "pyodbc" name = "pyodbc"
version = "5.3.0" version = "5.3.0"