import os # import pyarrow import sys import logging from datetime import date, timedelta import polars as pl from sqlalchemy import create_engine, text from sqlalchemy.engine import Engine, URL import clickhouse_connect from dotenv import load_dotenv from log import log from clickhouse_task.create_table import * from db_con.connection import * def MID_TABLE_COV(engine: Engine, target_date: date) -> list[int]: sql = text(""" SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage WHERE CONVERT(date, CreateDate) = :target_date UNION SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage WHERE CONVERT(date, UpdateDate) = :target_date """) log.info(f"Collecting MIDs for: {target_date}") with engine.connect() as conn: result = conn.execute(sql, {"target_date": target_date}) mids = [row[0] for row in result.fetchall()] log.info(f"Found {len(mids):,} MIDs") return mids def MID_TABLE_COV1( engine: Engine, target_date: date, ) -> pl.DataFrame: query = f""" SELECT EmpId, CAST(VisitDate AS DATE) AS VisitDate FROM OneApp_KelloggsMT.dbo.T_OQAD WHERE CAST(CreateDate AS DATE) = '{target_date}' UNION SELECT EmpId, CAST(VisitDate AS DATE) AS VisitDate FROM OneApp_KelloggsMT.dbo.T_OQAD WHERE CAST(UpdateDate AS DATE) = '{target_date}' """ return pl.read_database( query=query, connection=engine, ) def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame: sql_template = f""" WITH MID_TABLE_COV1 AS ( SELECT EmpId, VisitDate FROM OneApp_KelloggsMT.dbo.T_OQAD WHERE CreateDate >= {run_date} AND CreateDate < DATEADD(DAY,1,'{run_date}') UNION ALL SELECT EmpId, VisitDate FROM OneApp_KelloggsMT.dbo.T_OQAD WHERE UpdateDate >= {run_date} AND UpdateDate < DATEADD(DAY,1, '{run_date}') ), QUIZ AS ( SELECT Distinct E.EmpId as empid , CONVERT(VARCHAR,DQ.VisitDate) AS VisitDate FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId where e.EmpName not like 'test%' and e.RightId in (6) and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%' AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) ) ) select * from quiz """ sql = sql_template.format( run_date=run_date.strftime("%Y-%m-%d") ) log.info(f"Fetching quiz_empids data for EMPID and Visitid") df = pl.read_database( query=sql, connection=engine ) log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server") return df