Ankit Malik
123 lines
3.4 KiB
Python
123 lines
3.4 KiB
Python
import os
|
|
# import pyarrow
|
|
import sys
|
|
import logging
|
|
from datetime import date, timedelta
|
|
import polars as pl
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.engine import Engine, URL
|
|
import clickhouse_connect
|
|
from dotenv import load_dotenv
|
|
|
|
from log import log
|
|
from clickhouse_task.create_table import *
|
|
from db_con.connection import *
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def MID_TABLE_COV(engine: Engine, target_date: date) -> list[int]:
|
|
|
|
sql = text("""
|
|
SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
|
|
WHERE CONVERT(date, CreateDate) = :target_date
|
|
UNION
|
|
SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
|
|
WHERE CONVERT(date, UpdateDate) = :target_date
|
|
""")
|
|
|
|
log.info(f"Collecting MIDs for: {target_date}")
|
|
|
|
with engine.connect() as conn:
|
|
result = conn.execute(sql, {"target_date": target_date})
|
|
mids = [row[0] for row in result.fetchall()]
|
|
log.info(f"Found {len(mids):,} MIDs")
|
|
return mids
|
|
|
|
def MID_TABLE_COV1(
|
|
engine: Engine,
|
|
target_date: date,
|
|
) -> pl.DataFrame:
|
|
|
|
query = f"""
|
|
SELECT
|
|
EmpId,
|
|
CAST(VisitDate AS DATE) AS VisitDate
|
|
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
|
WHERE CAST(CreateDate AS DATE) = '{target_date}'
|
|
|
|
UNION
|
|
|
|
SELECT
|
|
EmpId,
|
|
CAST(VisitDate AS DATE) AS VisitDate
|
|
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
|
WHERE CAST(UpdateDate AS DATE) = '{target_date}'
|
|
"""
|
|
|
|
return pl.read_database(
|
|
query=query,
|
|
connection=engine,
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame:
|
|
|
|
sql_template = f"""
|
|
WITH MID_TABLE_COV1 AS
|
|
(
|
|
SELECT EmpId, VisitDate
|
|
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
|
WHERE CreateDate >= {run_date}
|
|
AND CreateDate < DATEADD(DAY,1,'{run_date}')
|
|
|
|
UNION ALL
|
|
|
|
SELECT EmpId, VisitDate
|
|
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
|
WHERE UpdateDate >= {run_date}
|
|
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
|
|
),
|
|
|
|
QUIZ AS
|
|
(
|
|
SELECT Distinct E.EmpId as empid
|
|
, CONVERT(VARCHAR,DQ.VisitDate) AS VisitDate
|
|
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN
|
|
OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join
|
|
OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join
|
|
OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId
|
|
where e.EmpName not like 'test%' and e.RightId in (6)
|
|
and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%'
|
|
AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE
|
|
DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) )
|
|
) select * from quiz
|
|
"""
|
|
sql = sql_template.format(
|
|
run_date=run_date.strftime("%Y-%m-%d")
|
|
)
|
|
|
|
log.info(f"Fetching quiz_empids data for EMPID and Visitid")
|
|
|
|
df = pl.read_database(
|
|
query=sql,
|
|
connection=engine
|
|
)
|
|
|
|
|
|
log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server")
|
|
|
|
return df
|