first commit
This commit is contained in:
Ankit Malik
+876
@@ -0,0 +1,876 @@
|
||||
import os
|
||||
import pyarrow
|
||||
import sys
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
import polars as pl
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import Engine, URL
|
||||
import clickhouse_connect
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from log import log
|
||||
from clickhouse_task.create_table import create_clickhouse_table , check
|
||||
from db_con.connection import *
|
||||
from mids import *
|
||||
|
||||
|
||||
|
||||
#PROJECT_ID = 40148
|
||||
p=40148
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_SOS_OneApp(engine: Engine, mids: list[int]) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
|
||||
sql = f""" SELECT
|
||||
MID,
|
||||
EmpId AS employee_id,
|
||||
StoreId AS store_id,
|
||||
VisitDate AS visit_date,
|
||||
StoreTypeid AS storetype_id,
|
||||
ChannelId AS channel_id,
|
||||
SOSDefinitionName,
|
||||
SOSHeaderDeatils,
|
||||
SOSHeaderName,
|
||||
SOSHeaderID,
|
||||
SOSChildDeatils,
|
||||
SOSChildName,
|
||||
SOSChildID,
|
||||
SOSHeaderFacing,
|
||||
ChildTotalFacing,
|
||||
ChildSelfFacing,
|
||||
SOSTarget
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
sm.CountryName,
|
||||
sc.MID,
|
||||
sm.RegionName,
|
||||
sm.StateName,
|
||||
sm.CityName,
|
||||
Em.SupervisorName,
|
||||
Em.EmpId,
|
||||
Em.EmpName AS EmployeeName,
|
||||
Em.DesignationName AS Designation,
|
||||
sm.StoreId,
|
||||
CAST(sc.VisitDate AS DATE) AS VisitDate,
|
||||
sm.StoreCode,
|
||||
sm.StoreName,
|
||||
sm.Address,
|
||||
sm.StoreTypeid,
|
||||
sm.ChannelId,
|
||||
sm.ChainName,
|
||||
|
||||
MSD.SOSDefinitionName,
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand'
|
||||
END AS SOSHeaderDeatils,
|
||||
|
||||
TS.SOSHeaderName,
|
||||
TS.SOSHeaderValue AS SOSHeaderID,
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand'
|
||||
END AS SOSChildDeatils,
|
||||
|
||||
TSC.SOSChildName,
|
||||
TSC.SOSChildValue AS SOSChildID,
|
||||
TSC.ChildTotalFacing,
|
||||
TS.SOSHeaderFacing,
|
||||
TSC.ChildSelfFacing,
|
||||
|
||||
(
|
||||
SELECT TOP 1 SOSTarget
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a
|
||||
WHERE a.SOSDefinitionId = MSD.SOSDefinitionId
|
||||
AND a.StoreId = sm.StoreId
|
||||
AND a.FromDate <= sc.VisitDate
|
||||
AND a.ToDate >= sc.VisitDate
|
||||
) AS SOSTarget
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader TS
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
ON TS.MID = SC.MID
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
|
||||
ON SC.StoreId = SM.StoreId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
|
||||
ON SC.EmpId = EM.EmpId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild TSC
|
||||
ON TS.SOSId = TSC.SOSId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition MSD
|
||||
ON MSD.SOSDefinitionId = TSC.SOSDefinitionId
|
||||
|
||||
WHERE EM.EmpName NOT LIKE 'test%'
|
||||
AND SC.MID IN ({mid_list})
|
||||
) A
|
||||
"""
|
||||
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame:
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(map(str, mids))
|
||||
|
||||
sql = f"""
|
||||
WITH MID_TABLE_COV1 AS
|
||||
(
|
||||
SELECT DISTINCT
|
||||
EmpId,
|
||||
CAST(VisitDate AS DATE) AS VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
|
||||
WHERE MID IN ({mid_list})
|
||||
),
|
||||
|
||||
QUIZ AS
|
||||
(
|
||||
SELECT DISTINCT
|
||||
E.EmpId,
|
||||
CAST(DQ.VisitDate AS DATE) AS VisitDate,
|
||||
DQ.QuestionId,
|
||||
DQ.AnswerId,
|
||||
QC.QuestionCategoryId,
|
||||
QC.QuestionCategory
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
|
||||
ON DQ.EmpId = E.EmpId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
|
||||
ON DQ.QuestionId = QU.QuestionId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
|
||||
ON QU.QuestionCategoryId = QC.QuestionCategoryId
|
||||
|
||||
WHERE E.EmpName NOT LIKE 'test%'
|
||||
AND E.EmpName NOT LIKE '%TEST%'
|
||||
AND E.RightId = 6
|
||||
AND (
|
||||
E.ResignDate IS NULL
|
||||
OR E.ResignDate >= DQ.VisitDate
|
||||
)
|
||||
AND EXISTS
|
||||
(
|
||||
SELECT 1
|
||||
FROM MID_TABLE_COV1 A
|
||||
WHERE A.EmpId = DQ.EmpId
|
||||
AND A.VisitDate = CAST(DQ.VisitDate AS DATE)
|
||||
)
|
||||
)
|
||||
|
||||
SELECT
|
||||
Q.EmpId AS employee_id,
|
||||
0 AS process_id,
|
||||
Q.VisitDate AS visit_date,
|
||||
Q.QuestionCategoryId AS question_category_id,
|
||||
Q.QuestionCategory AS question_category,
|
||||
QM.QuestionId AS question_id,
|
||||
QM.Question AS question,
|
||||
|
||||
ISNULL(QA.AnswerId,0) AS answer_id,
|
||||
ISNULL(QA.Answer,'') AS answer,
|
||||
|
||||
CASE
|
||||
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
|
||||
WHEN QA.RightAnswer = 1 THEN 'Y'
|
||||
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
|
||||
ELSE 'N'
|
||||
END AS correct_answer
|
||||
|
||||
FROM QUIZ Q
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
|
||||
ON Q.QuestionId = QM.QuestionId
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
|
||||
ON Q.AnswerId = QA.AnswerId
|
||||
"""
|
||||
|
||||
log.info(f"Fetching OQaD data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} rows")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_Survey(engine: Engine, mids: list[int]) -> pl.DataFrame:
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(map(str, mids))
|
||||
|
||||
sql = f"""
|
||||
WITH QUIZ AS
|
||||
(
|
||||
SELECT
|
||||
SC.MID,
|
||||
SC.StoreId,
|
||||
EM.EmpId,
|
||||
EM.SupervisorId,
|
||||
CAST(SC.VisitDate AS DATE) AS VisitDate,
|
||||
|
||||
DQ.QuestionId,
|
||||
QU.Question,
|
||||
DQ.Answer,
|
||||
|
||||
QC.CategoryId,
|
||||
QC.Category,
|
||||
|
||||
MSS.SubCategoryId,
|
||||
MSS.SubCategory
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_SURVEY DQ
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
ON DQ.MID = SC.MID
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyQuestion QU
|
||||
ON DQ.QuestionId = QU.QuestionId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveySubCategory MSS
|
||||
ON QU.SubCategoryId = MSS.SubCategoryId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyCategory QC
|
||||
ON MSS.CategoryId = QC.CategoryId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
|
||||
ON SC.EmpId = EM.EmpId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_Survey MS
|
||||
ON DQ.SurveyId = MS.SurveyId
|
||||
AND QU.SurveyId = MS.SurveyId
|
||||
|
||||
WHERE EM.EmpName NOT LIKE 'test%'
|
||||
AND SC.MID IN ({mid_list})
|
||||
)
|
||||
|
||||
SELECT
|
||||
Q.MID,
|
||||
Q.SupervisorId,
|
||||
Q.EmpId,
|
||||
SM.StoreId,
|
||||
Q.VisitDate,
|
||||
|
||||
SM.StoreTypeId,
|
||||
SM.ChainId,
|
||||
|
||||
Q.CategoryId,
|
||||
Q.Category,
|
||||
|
||||
Q.SubCategoryId,
|
||||
Q.SubCategory,
|
||||
|
||||
Q.QuestionId,
|
||||
Q.Question,
|
||||
Q.Answer
|
||||
|
||||
FROM QUIZ Q
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
|
||||
ON Q.StoreId = SM.StoreId
|
||||
"""
|
||||
|
||||
log.info(f"Fetching Survey data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} Survey rows")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_additional_visibility(
|
||||
engine: Engine,
|
||||
mids: list[int]
|
||||
) -> pl.DataFrame:
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(map(str, mids))
|
||||
|
||||
sql = f"""
|
||||
SELECT
|
||||
TS.MID,
|
||||
|
||||
EM.EmpId AS emp_id,
|
||||
|
||||
SM.StoreId AS store_id,
|
||||
SM.StoreTypeId AS storetype_id,
|
||||
SM.ChannelId AS channel_id,
|
||||
SM.ChainId AS chain_id,
|
||||
|
||||
CASE
|
||||
WHEN SM.CameraAllow = 1 THEN 'Y'
|
||||
ELSE 'N'
|
||||
END AS camera_allowed,
|
||||
|
||||
CAST(SC.VisitDate AS DATE) AS visit_date,
|
||||
|
||||
CASE
|
||||
WHEN TS.Present = 0 THEN 'N'
|
||||
ELSE 'Y'
|
||||
END AS is_present,
|
||||
|
||||
ISNULL(MB.BrandId, 0) AS brand_id,
|
||||
ISNULL(MD.DisplayId, 0) AS display_id,
|
||||
|
||||
ISNULL(TS.Remark, '') AS remarks,
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(TS.ImageUrl, '') = '' THEN ''
|
||||
ELSE CONCAT(
|
||||
'https://kimt1.parinaam.in/Upload/VisibilityImages/',
|
||||
TS.ImageUrl
|
||||
)
|
||||
END AS image_url
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_AdditionalVisibility TS
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
ON TS.MID = SC.MID
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
|
||||
ON SC.StoreId = SM.StoreId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
|
||||
ON SC.EmpId = EM.EmpId
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_Brand MB
|
||||
ON TS.BrandId = MB.BrandId
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_Display MD
|
||||
ON TS.DisplayId = MD.DisplayId
|
||||
|
||||
WHERE EM.EmpName NOT LIKE 'test%'
|
||||
AND TS.Present = 1
|
||||
AND SC.MID IN ({mid_list})
|
||||
"""
|
||||
|
||||
log.info(
|
||||
f"Fetching Additional Visibility data for {len(mids):,} MIDs"
|
||||
)
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Additional Visibility rows"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_Coverage(engine: Engine, mids: list[int]) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
|
||||
sql = f"""
|
||||
SELECT
|
||||
{p} AS project_id,
|
||||
JP.MID,
|
||||
sm.StoreId AS store_id,
|
||||
JP.EmpId AS employee_id,
|
||||
CONVERT(date, JP.VisitDate) AS visit_date,
|
||||
JP.InTime AS in_time,
|
||||
JP.OutTime AS out_time,
|
||||
CASE
|
||||
WHEN JP.OutTime IS NULL OR JP.InTime IS NULL THEN NULL
|
||||
WHEN JP.OutTime < JP.InTime THEN 0
|
||||
ELSE DATEDIFF(SECOND, JP.InTime, JP.OutTime) / 60
|
||||
END AS duration_minutes,
|
||||
CASE
|
||||
WHEN (
|
||||
SELECT TOP 1 EmpId
|
||||
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
WHERE SC.EmpId = JP.EmpId
|
||||
AND SC.StoreId = JP.StoreId
|
||||
AND SC.VisitDate = JP.VisitDate
|
||||
AND SC.ReasonId IN (0,1,3,9,10,19,20)
|
||||
) > 0
|
||||
THEN 'Y' ELSE 'N'
|
||||
END AS is_covered,
|
||||
CASE JP.Deviation
|
||||
WHEN 0 THEN 'Planned'
|
||||
WHEN 1 THEN 'Adhoc'
|
||||
WHEN 2 THEN 'Beat Plan'
|
||||
WHEN 3 THEN 'Non Merchandised'
|
||||
WHEN 4 THEN 'Add New Store'
|
||||
WHEN 5 THEN 'Non Program'
|
||||
ELSE ''
|
||||
END AS coverage_type,
|
||||
sm.StoreTypeId AS storetype_id,
|
||||
Em.SupervisorId AS supervisor_id,
|
||||
ISNULL(JP.ReasonId, 0) AS reason_id,
|
||||
sm.CameraAllow AS camera_allow,
|
||||
CAST(
|
||||
CASE
|
||||
WHEN sm.Latitude IS NULL OR sm.Latitude = 0 THEN 0
|
||||
WHEN sm.Longitude IS NULL OR sm.Longitude = 0 THEN 0
|
||||
WHEN JP.Latitude IS NULL OR JP.Latitude = 0 THEN 0
|
||||
WHEN JP.Longitude IS NULL OR JP.Longitude = 0 THEN 0
|
||||
ELSE SQRT(
|
||||
POWER(69.1 * (JP.Latitude - sm.Latitude), 2) +
|
||||
POWER(69.1 * (sm.Longitude - JP.Longitude)
|
||||
* COS(JP.Latitude / 57.3), 2)
|
||||
) * 1000
|
||||
END AS FLOAT
|
||||
) AS distance_meters,
|
||||
GETDATE() AS update_date,
|
||||
'ETL-SQLAlchemy' AS update_by
|
||||
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP WITH (NOLOCK)
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm
|
||||
ON JP.StoreId = sm.StoreId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em
|
||||
ON JP.EmpId = Em.EmpId
|
||||
WHERE JP.MID IN ({mid_list})
|
||||
AND Em.UserName NOT LIKE 'test%'
|
||||
"""
|
||||
|
||||
log.info(f"Fetching coverage data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_stock_details(
|
||||
engine: Engine,
|
||||
mids: list[int]
|
||||
) -> pl.DataFrame:
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(map(str, mids))
|
||||
|
||||
sql = f"""
|
||||
SELECT
|
||||
SC.MID,
|
||||
|
||||
EM.SupervisorId AS supervisor_id,
|
||||
EM.EmpId AS employee_id,
|
||||
|
||||
SM.StoreId AS store_id,
|
||||
|
||||
CAST(SC.VisitDate AS DATE) AS visitdate,
|
||||
|
||||
SC.StoreTypeId,
|
||||
SM.StoreCategoryId,
|
||||
|
||||
VP.ProductId AS product_id,
|
||||
|
||||
TS.MSL,
|
||||
MP.MBQ,
|
||||
|
||||
ISNULL(TS.OpeningStock, 0)
|
||||
+ ISNULL(TS.MidDayStock, 0) AS stock_qty,
|
||||
|
||||
0 AS damagedstock,
|
||||
0 AS loststock,
|
||||
0 AS expirystock,
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(TS.OpeningStock, 0)
|
||||
+ ISNULL(TS.MidDayStock, 0) >= 1
|
||||
THEN 'Y'
|
||||
ELSE 'N'
|
||||
END AS skuavailability,
|
||||
|
||||
'Parinaam' AS stocktype
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_Stock TS
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
ON TS.MID = SC.MID
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
|
||||
ON SC.StoreId = SM.StoreId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
|
||||
ON SC.EmpId = EM.EmpId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Product VP
|
||||
ON TS.ProductId = VP.ProductId
|
||||
|
||||
INNER JOIN
|
||||
(
|
||||
SELECT
|
||||
MBQ,
|
||||
ProductId,
|
||||
StateId,
|
||||
ChainId,
|
||||
StoreTypeId,
|
||||
StoreCategoryId,
|
||||
StoreClassId
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_ProductAssortment
|
||||
WHERE FromDate <= CAST(GETDATE() AS DATE)
|
||||
AND ToDate >= CAST(GETDATE() AS DATE)
|
||||
) MP
|
||||
ON MP.StateId = SM.StateId
|
||||
AND MP.ChainId = SM.ChainId
|
||||
AND MP.StoreTypeId = SM.StoreTypeId
|
||||
AND MP.StoreCategoryId = SM.StoreCategoryId
|
||||
AND MP.StoreClassId = SM.StoreClassId
|
||||
AND MP.ProductId = TS.ProductId
|
||||
|
||||
WHERE EM.EmpName NOT LIKE 'test%'
|
||||
AND SC.MID IN ({mid_list})
|
||||
"""
|
||||
|
||||
log.info(
|
||||
f"Fetching Stock Details data for {len(mids):,} MIDs"
|
||||
)
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Stock Details rows"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_Attendance(
|
||||
engine: Engine,
|
||||
end_date: date | None = None,
|
||||
days_back: int = 15
|
||||
) -> pl.DataFrame:
|
||||
"""
|
||||
Fetch attendance source data.
|
||||
|
||||
Default:
|
||||
end_date = yesterday
|
||||
start_date = yesterday - 15 days
|
||||
"""
|
||||
|
||||
if end_date is None:
|
||||
end_date = date.today() - timedelta(days=1)
|
||||
|
||||
start_date = end_date - timedelta(days=days_back)
|
||||
|
||||
sql = f"""
|
||||
SELECT
|
||||
JP.EmpId AS employee_id,
|
||||
JP.StoreId AS store_id,
|
||||
CAST(JP.VisitDate AS DATE) AS visit_date,
|
||||
|
||||
EM.ManagerId AS supervisor_id,
|
||||
EM.JoinDate AS date_of_join,
|
||||
EM.ResignDate AS date_of_resign,
|
||||
EM.LegacyCode,
|
||||
|
||||
SC.ReasonId,
|
||||
SC.InTime,
|
||||
SC.OutTime,
|
||||
|
||||
MPU.PositionId,
|
||||
MP.PositionCode
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan JP
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM
|
||||
ON JP.EmpId = EM.Id
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
ON JP.EmpId = SC.EmpId
|
||||
AND JP.StoreId = SC.StoreId
|
||||
AND JP.VisitDate = SC.VisitDate
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Mapping_PositionUser MPU
|
||||
ON JP.EmpId = MPU.EmpId
|
||||
AND JP.VisitDate BETWEEN MPU.FromDate AND MPU.ToDate
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_Position MP
|
||||
ON MPU.PositionId = MP.PositionId
|
||||
|
||||
WHERE JP.VisitDate BETWEEN '{start_date}'
|
||||
AND '{end_date}'
|
||||
AND EM.RightId = 6
|
||||
AND EM.EmployeeName NOT LIKE '%test%'
|
||||
"""
|
||||
|
||||
log.info(
|
||||
f"Fetching Attendance data from {start_date} to {end_date}"
|
||||
)
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} attendance rows "
|
||||
f"for {df['employee_id'].n_unique():,} employees"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
def fetch_login(engine: Engine , mids: list[int]) -> pl.DataFrame:
|
||||
|
||||
sql = """
|
||||
WITH login_data AS
|
||||
(
|
||||
SELECT
|
||||
UD.EmpId,
|
||||
CAST(UD.LoginDate AS DATE) AS LoginDate,
|
||||
CONVERT(VARCHAR(8), UD.InTime, 108) AS LoginTime,
|
||||
|
||||
ROW_NUMBER() OVER
|
||||
(
|
||||
PARTITION BY
|
||||
UD.EmpId,
|
||||
CAST(UD.LoginDate AS DATE)
|
||||
ORDER BY UD.LoginDate
|
||||
) AS rn
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_DeviceLogin UD
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
|
||||
ON UD.EmpId = EM.EmpId
|
||||
|
||||
WHERE CAST(UD.LoginDate AS DATE) =
|
||||
CAST(DATEADD(DAY,-1,GETDATE()) AS DATE)
|
||||
|
||||
AND EM.RightId = 6
|
||||
AND EM.EmpName NOT LIKE '%test%'
|
||||
AND (
|
||||
EM.ResignDate IS NULL
|
||||
OR CAST(EM.ResignDate AS DATE) >=
|
||||
CAST(DATEADD(DAY,-1,GETDATE()) AS DATE)
|
||||
)
|
||||
)
|
||||
|
||||
SELECT
|
||||
LD.EmpId AS employee_id,
|
||||
LD.LoginDate AS login_date,
|
||||
LD.LoginTime AS login_time,
|
||||
|
||||
(
|
||||
SELECT MIN(CONVERT(VARCHAR(8), SC.InTime, 108))
|
||||
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
WHERE SC.IsDel = 0
|
||||
AND SC.EmpId = LD.EmpId
|
||||
AND CAST(SC.VisitDate AS DATE) = LD.LoginDate
|
||||
) AS first_store_in_time,
|
||||
|
||||
(
|
||||
SELECT MAX(CONVERT(VARCHAR(8), SC.OutTime, 108))
|
||||
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
WHERE SC.IsDel = 0
|
||||
AND SC.EmpId = LD.EmpId
|
||||
AND CAST(SC.VisitDate AS DATE) = LD.LoginDate
|
||||
) AS last_store_out_time,
|
||||
|
||||
CONCAT('40148','_',CAST(LD.EmpId AS VARCHAR(50))) AS unique_id
|
||||
|
||||
FROM login_data LD
|
||||
WHERE LD.rn = 1
|
||||
"""
|
||||
|
||||
|
||||
log.info("Fetching Login data for yesterday")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} Login rows")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def fetch_Promotion(engine: Engine, mids: list[int]) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
|
||||
sql = f"""
|
||||
SELECT
|
||||
sc.MID AS mid,
|
||||
{p} AS project_id,
|
||||
sc.StoreId AS store_id,
|
||||
Em.EmpId AS employee_id,
|
||||
CONVERT(date, sc.VisitDate) AS visit_date,
|
||||
Em.SupervisorId AS supervisor_id,
|
||||
sm.ChannelId AS channel_id,
|
||||
sm.ChainId AS chain_id,
|
||||
sm.StoreTypeId AS storetype_id,
|
||||
msd.PromoDefinitionId AS promo_definition_id,
|
||||
msd.PromoDefinitionName AS promo_definition_name,
|
||||
|
||||
-- Resolve which dimension level the promo targets
|
||||
-- (Category / SubCategory / Brand / SubBrand)
|
||||
CASE
|
||||
WHEN ISNULL(ts.PromoTable,'') = 'Master_Category' THEN 'Category'
|
||||
WHEN ISNULL(ts.PromoTable,'') = 'Master_SubCategory' THEN 'SubCategory'
|
||||
WHEN ISNULL(ts.PromoTable,'') = 'Master_Brand' THEN 'Brand'
|
||||
WHEN ISNULL(ts.PromoTable,'') = 'Master_SubBrand' THEN 'SubBrand'
|
||||
ELSE ''
|
||||
END AS promotion_details,
|
||||
|
||||
ts.PromoValue AS promotion_details_id,
|
||||
|
||||
-- Resolve the actual name of that dimension item
|
||||
-- SQL Server evaluates exactly ONE of these subqueries per row
|
||||
CASE
|
||||
WHEN ISNULL(ts.PromoTable,'') = 'Master_Category'
|
||||
THEN (SELECT a.CategoryName
|
||||
FROM OneApp_KelloggsMT.dbo.Master_Category a
|
||||
WHERE a.CategoryId = ts.PromoValue)
|
||||
WHEN ISNULL(ts.PromoTable,'') = 'Master_SubCategory'
|
||||
THEN (SELECT a.SubCategoryName
|
||||
FROM OneApp_KelloggsMT.dbo.Master_SubCategory a
|
||||
WHERE a.SubCategoryId = ts.PromoValue)
|
||||
WHEN ISNULL(ts.PromoTable,'') = 'Master_Brand'
|
||||
THEN (SELECT a.BrandName
|
||||
FROM OneApp_KelloggsMT.dbo.Master_Brand a
|
||||
WHERE a.BrandId = ts.PromoValue)
|
||||
WHEN ISNULL(ts.PromoTable,'') = 'Master_SubBrand'
|
||||
THEN (SELECT a.SubBrandName
|
||||
FROM OneApp_KelloggsMT.dbo.Master_SubBrand a
|
||||
WHERE a.SubBrandId = ts.PromoValue)
|
||||
END AS promo_value_name,
|
||||
|
||||
-- Present flag: 0 → 'N', anything else → 'Y'
|
||||
CASE WHEN ts.Present = 0 THEN 'N' ELSE 'Y' END AS is_present,
|
||||
|
||||
-- Reason only populated when promo is ABSENT
|
||||
CASE
|
||||
WHEN ts.Present = 1 THEN ''
|
||||
ELSE ISNULL(mnp.PromoReason, '')
|
||||
END AS reason,
|
||||
|
||||
ISNULL(mpq.PromoQuestionName, '') AS question,
|
||||
ISNULL(tpq.PromoAnswerName, '') AS answer,
|
||||
|
||||
-- Image URLs — empty string when no image
|
||||
CASE
|
||||
WHEN ISNULL(SHI.PromoImage1,'') = '' THEN ''
|
||||
ELSE 'https://kimt1.parinaam.in/Upload/PromotionImages/'
|
||||
+ SHI.PromoImage1
|
||||
END AS image1,
|
||||
CASE
|
||||
WHEN ISNULL(SHI.PromoImage2,'') = '' THEN ''
|
||||
ELSE 'https://kimt1.parinaam.in/Upload/PromotionImages/'
|
||||
+ SHI.PromoImage2
|
||||
END AS image2,
|
||||
|
||||
GETDATE() AS update_date,
|
||||
'ETL-SQLAlchemy' AS update_by
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_Promotion ts
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc
|
||||
ON ts.mid = sc.mid
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm
|
||||
ON sc.StoreId = sm.StoreId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em
|
||||
ON sc.EmpId = Em.EmpId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_PromotionDefinition msd
|
||||
ON msd.PromoDefinitionId = ts.PromoDefinitionId
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_PromotionImages SHI
|
||||
ON ts.PId = SHI.PId
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_PromotionReason mnp
|
||||
ON ts.PromoReasonId = mnp.PromoReasonId
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.t_promotionquestion tpq
|
||||
ON ts.PId = tpq.PId
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq
|
||||
ON tpq.PromoQuestionId = mpq.PromoQuestionId
|
||||
|
||||
WHERE Em.EmpName NOT LIKE 'test%'
|
||||
AND sc.MID IN ({mid_list})
|
||||
"""
|
||||
|
||||
log.info(f"Fetching Coverage data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user