16-06-2026 1st commit

This commit is contained in:
Ankit Malik
2026-06-16 13:16:51 +05:30
parent 8c4b3b736e
commit 822b4d2fdf
20 changed files with 850 additions and 78 deletions
+41
View File
@@ -0,0 +1,41 @@
from pathlib import Path
import polars as pl
from sqlalchemy import Engine
from datetime import date
from log import log
def fetch_data(
engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
mid_list = ",".join(str(mid) for mid in mids)
sql_file = Path("sql") / table_type / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df