Ankit Malik
36 lines
989 B
Python
36 lines
989 B
Python
import os
|
|
import pyarrow
|
|
import sys
|
|
import logging
|
|
from datetime import date, timedelta
|
|
import polars as pl
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.engine import Engine, URL
|
|
import clickhouse_connect
|
|
from dotenv import load_dotenv
|
|
|
|
from log import log
|
|
from clickhouse_task.create_table import create_clickhouse_table , check
|
|
from db_con.connection import *
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def collect_mids(engine: Engine, target_date: date) -> list[int]:
|
|
|
|
sql = text("""
|
|
SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
|
|
WHERE CONVERT(date, CreateDate) = :target_date
|
|
UNION
|
|
SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
|
|
WHERE CONVERT(date, UpdateDate) = :target_date
|
|
""")
|
|
log.info(f"Collecting MIDs for: {target_date}")
|
|
with engine.connect() as conn:
|
|
result = conn.execute(sql, {"target_date": target_date})
|
|
mids = [row[0] for row in result.fetchall()]
|
|
log.info(f"Found {len(mids):,} MIDs")
|
|
return mids |