first commit

mids.py

@@ -0,0 +1,36 @@
+import os
+import pyarrow
+import sys
+import logging
+from datetime import date, timedelta
+import polars as pl
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine, URL
+import clickhouse_connect
+from dotenv import load_dotenv
+
+from log import log 
+from clickhouse_task.create_table import create_clickhouse_table , check
+from db_con.connection import *
+
+
+
+
+
+
+
+def collect_mids(engine: Engine, target_date: date) -> list[int]:
+
+    sql = text("""
+        SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
+        WHERE CONVERT(date, CreateDate) = :target_date
+        UNION
+        SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
+        WHERE CONVERT(date, UpdateDate) = :target_date
+    """)
+    log.info(f"Collecting MIDs for: {target_date}")
+    with engine.connect() as conn:
+        result = conn.execute(sql, {"target_date": target_date})
+        mids = [row[0] for row in result.fetchall()]
+    log.info(f"Found {len(mids):,} MIDs")
+    return mids