14-06-26 1st comit

clickhouse_task/delete_task.py

@@ -1,31 +1,191 @@
-from db_con.connection import *
+from __future__ import annotations
 
-from log import *
+import polars as pl
+
+from log import log
 
 
-def truncate_table(client, table_name: str) -> None:
+def truncate_table(
+    client,
+    table_name: str,
+) -> None:
     """
-    Truncate a ClickHouse table.
+    Full refresh tables.
     """
-    query = f"TRUNCATE TABLE {table_name}"
 
-    print(f"Truncating table: {table_name}")
-    client.command(query)
+    client.command(
+        f"TRUNCATE TABLE {table_name}"
+    )
 
-    log.info(f"Table {table_name} truncated successfully.")
+    log.info(
+        "Truncated table %s",
+        table_name,
+    )
 
 
-
-def delete_rows(client, table_name: str, condition: str) -> None:
+def delete_rows(
+    client,
+    table_name: str,
+    where_clause: str,
+) -> None:
     """
-    Delete rows from a ClickHouse table based on a condition.
+    Generic ClickHouse delete.
     """
+
     query = f"""
     ALTER TABLE {table_name}
-    DELETE WHERE {condition}
+    DELETE
+    WHERE {where_clause}
     """
 
-    print(f"Deleting rows from {table_name} where {condition}")
+    log.info(
+        "Deleting from %s",
+        table_name,
+    )
+
     client.command(query)
 
-    log.info("Delete command submitted successfully.")    
+
+def delete_existing_data(
+    client,
+    table_name: str,
+    run_date,
+    mids: list[int],
+    emp_visit_df: pl.DataFrame,
+) -> None:
+    """
+    Incremental delete logic.
+    Matches the old SQL procedure.
+    """
+
+    # --------------------------------------------------
+    # MID based tables
+    # --------------------------------------------------
+
+    mid_tables = {
+        "additional_visibility",
+        "Coverage",
+        "Survey",
+        "Promotion",
+        "PaidVisibility",
+        "SOS_OneApp",
+        "Stock_Details",
+        "Login",
+        "coverage_remarks",
+    }
+
+    if table_name in mid_tables and mids:
+
+        mids_str = ",".join(
+            map(str, mids)
+        )
+
+        delete_rows(
+            client,
+            table_name,
+            f"MID IN ({mids_str})",
+        )
+
+        return
+
+    # --------------------------------------------------
+    # Journey Plan
+    # --------------------------------------------------
+
+    if table_name == "Journey_Plan":
+
+        delete_rows(
+            client,
+            table_name,
+            f"""
+            toMonth(visit_date) = {run_date.month}
+            AND toYear(visit_date) = {run_date.year}
+            """,
+        )
+
+        return
+
+    # --------------------------------------------------
+    # Logins
+    # --------------------------------------------------
+    if table_name == "Login":
+        delete_rows(
+        client,
+        table_name,
+        f"""
+        project_id = 40148
+        AND toDate(login_date) = toDate('{run_date}')
+        """
+    )
+
+
+    # --------------------------------------------------
+    # Web Logins
+    # --------------------------------------------------
+
+
+
+
+
+    if table_name == "Web_Logins":
+
+        delete_rows(
+            client,
+            table_name,
+            f"""
+            toDate(date)
+            = toDate('{run_date}')
+            """,
+        )
+
+        return
+
+    # --------------------------------------------------
+    # Attendance
+    # --------------------------------------------------
+
+    if table_name == "Attendance":
+
+        delete_rows(
+            client,
+            table_name,
+            f"""
+            toDate(attendance_date)
+            = toDate('{run_date}')
+            """,
+        )
+
+        return
+
+    # --------------------------------------------------
+    # OQaD
+    # --------------------------------------------------
+
+    if (
+        table_name == "OQaD"
+        and not emp_visit_df.is_empty()
+    ):
+
+        conditions = [
+            (
+                f"(employee_id={row['EmpId']} "
+                f"AND toDate(visit_date)="
+                f"toDate('{row['VisitDate']}'))"
+            )
+            for row in emp_visit_df.iter_rows(
+                named=True
+            )
+        ]
+
+        delete_rows(
+            client,
+            table_name,
+            " OR ".join(conditions),
+        )
+
+        return
+
+    log.info(
+        "No delete logic required for %s",
+        table_name,
+    )