data_pipeline/kpi/facts.py

import os
import pyarrow
import sys
import logging
from datetime import date, timedelta
import polars as pl
from sqlalchemy import create_engine, text
from sqlalchemy.engine import Engine, URL
import clickhouse_connect
from dotenv import load_dotenv

from log import log 
from clickhouse_task.create_table import create_clickhouse_table , check
from db_con.connection import *
from mids import *



#PROJECT_ID = 40148
p=40148




def fetch_SOS_OneApp(engine: Engine, mids: list[int]) -> pl.DataFrame:
    
    
    
    if not mids:
        log.warning("No MIDs — nothing to fetch.")
        return pl.DataFrame()

    mid_list = ",".join(str(mid) for mid in mids)

    sql = f"""   SELECT
        MID,
        EmpId AS employee_id,
        StoreId AS store_id,
        VisitDate AS visit_date,
        StoreTypeid AS storetype_id,
        ChannelId AS channel_id,
        SOSDefinitionName,
        SOSHeaderDeatils,
        SOSHeaderName,
        SOSHeaderID,
        SOSChildDeatils,
        SOSChildName,
        SOSChildID,
        SOSHeaderFacing,
        ChildTotalFacing,
        ChildSelfFacing,
        SOSTarget
    FROM
    (
        SELECT
            sm.CountryName,
            sc.MID,
            sm.RegionName,
            sm.StateName,
            sm.CityName,
            Em.SupervisorName,
            Em.EmpId,
            Em.EmpName AS EmployeeName,
            Em.DesignationName AS Designation,
            sm.StoreId,
            CAST(sc.VisitDate AS DATE) AS VisitDate,
            sm.StoreCode,
            sm.StoreName,
            sm.Address,
            sm.StoreTypeid,
            sm.ChannelId,
            sm.ChainName,

            MSD.SOSDefinitionName,

            CASE
                WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category'
                WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory'
                WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand'
                WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand'
            END AS SOSHeaderDeatils,

            TS.SOSHeaderName,
            TS.SOSHeaderValue AS SOSHeaderID,

            CASE
                WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category'
                WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory'
                WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand'
                WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand'
            END AS SOSChildDeatils,

            TSC.SOSChildName,
            TSC.SOSChildValue AS SOSChildID,
            TSC.ChildTotalFacing,
            TS.SOSHeaderFacing,
            TSC.ChildSelfFacing,

            (
                SELECT TOP 1 SOSTarget
                FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a
                WHERE a.SOSDefinitionId = MSD.SOSDefinitionId
                    AND a.StoreId = sm.StoreId
                    AND a.FromDate <= sc.VisitDate
                    AND a.ToDate >= sc.VisitDate
            ) AS SOSTarget

        FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader TS

        INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
            ON TS.MID = SC.MID

        INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
            ON SC.StoreId = SM.StoreId

        INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
            ON SC.EmpId = EM.EmpId

        INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild TSC
            ON TS.SOSId = TSC.SOSId

        INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition MSD
            ON MSD.SOSDefinitionId = TSC.SOSDefinitionId

        WHERE EM.EmpName NOT LIKE 'test%'
            AND SC.MID IN ({mid_list})
    ) A
        """
    
    log.info(f"Fetching  data for {len(mids):,} MIDs")

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(f"Fetched {len(df):,} rows from SQL Server")

    return df


def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame:

    if not mids:
        log.warning("No MIDs — nothing to fetch.")
        return pl.DataFrame()

    mid_list = ",".join(map(str, mids))

    sql = f"""
    WITH MID_TABLE_COV1 AS
    (
        SELECT DISTINCT
            EmpId,
            CAST(VisitDate AS DATE) AS VisitDate
        FROM OneApp_KelloggsMT.dbo.T_StoreCoverage
        WHERE MID IN ({mid_list})
    ),

    QUIZ AS
    (
        SELECT DISTINCT
            E.EmpId,
            CAST(DQ.VisitDate AS DATE) AS VisitDate,
            DQ.QuestionId,
            DQ.AnswerId,
            QC.QuestionCategoryId,
            QC.QuestionCategory

        FROM OneApp_KelloggsMT.dbo.T_OQAD DQ

        INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
            ON DQ.EmpId = E.EmpId

        INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
            ON DQ.QuestionId = QU.QuestionId

        INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
            ON QU.QuestionCategoryId = QC.QuestionCategoryId

        WHERE E.EmpName NOT LIKE 'test%'
          AND E.EmpName NOT LIKE '%TEST%'
          AND E.RightId = 6
          AND (
                E.ResignDate IS NULL
                OR E.ResignDate >= DQ.VisitDate
              )
          AND EXISTS
          (
              SELECT 1
              FROM MID_TABLE_COV1 A
              WHERE A.EmpId = DQ.EmpId
                AND A.VisitDate = CAST(DQ.VisitDate AS DATE)
          )
    )

    SELECT
        Q.EmpId AS employee_id,
        0 AS process_id,
        Q.VisitDate AS visit_date,
        Q.QuestionCategoryId AS question_category_id,
        Q.QuestionCategory AS question_category,
        QM.QuestionId AS question_id,
        QM.Question AS question,

        ISNULL(QA.AnswerId,0) AS answer_id,
        ISNULL(QA.Answer,'') AS answer,

        CASE
            WHEN QA.AnswerId IS NULL THEN 'Not Answer'
            WHEN QA.RightAnswer = 1 THEN 'Y'
            WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
            ELSE 'N'
        END AS correct_answer

    FROM QUIZ Q

    INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
        ON Q.QuestionId = QM.QuestionId

    LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
        ON Q.AnswerId = QA.AnswerId
    """
    
    log.info(f"Fetching OQaD data for {len(mids):,} MIDs")

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(f"Fetched {len(df):,} rows")

    return df






def fetch_Survey(engine: Engine, mids: list[int]) -> pl.DataFrame:
    if not mids:
        log.warning("No MIDs — nothing to fetch.")
        return pl.DataFrame()

    mid_list = ",".join(map(str, mids))

    sql = f"""
    WITH QUIZ AS
    (
        SELECT
            SC.MID,
            SC.StoreId,
            EM.EmpId,
            EM.SupervisorId,
            CAST(SC.VisitDate AS DATE) AS VisitDate,

            DQ.QuestionId,
            QU.Question,
            DQ.Answer,

            QC.CategoryId,
            QC.Category,

            MSS.SubCategoryId,
            MSS.SubCategory

        FROM OneApp_KelloggsMT.dbo.T_SURVEY DQ

        INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
            ON DQ.MID = SC.MID

        INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyQuestion QU
            ON DQ.QuestionId = QU.QuestionId

        INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveySubCategory MSS
            ON QU.SubCategoryId = MSS.SubCategoryId

        INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyCategory QC
            ON MSS.CategoryId = QC.CategoryId

        INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
            ON SC.EmpId = EM.EmpId

        INNER JOIN OneApp_KelloggsMT.dbo.Master_Survey MS
            ON DQ.SurveyId = MS.SurveyId
           AND QU.SurveyId = MS.SurveyId

        WHERE EM.EmpName NOT LIKE 'test%'
          AND SC.MID IN ({mid_list})
    )

    SELECT
        Q.MID,
        Q.SupervisorId,
        Q.EmpId,
        SM.StoreId,
        Q.VisitDate,

        SM.StoreTypeId,
        SM.ChainId,

        Q.CategoryId,
        Q.Category,

        Q.SubCategoryId,
        Q.SubCategory,

        Q.QuestionId,
        Q.Question,
        Q.Answer

    FROM QUIZ Q

    INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
        ON Q.StoreId = SM.StoreId
    """

    log.info(f"Fetching Survey data for {len(mids):,} MIDs")

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(f"Fetched {len(df):,} Survey rows")

    return df


def fetch_additional_visibility(
    engine: Engine,
    mids: list[int]
) -> pl.DataFrame:

    if not mids:
        log.warning("No MIDs — nothing to fetch.")
        return pl.DataFrame()

    mid_list = ",".join(map(str, mids))

    sql = f"""
    SELECT
        TS.MID,

        EM.EmpId AS emp_id,

        SM.StoreId AS store_id,
        SM.StoreTypeId AS storetype_id,
        SM.ChannelId AS channel_id,
        SM.ChainId AS chain_id,

        CASE
            WHEN SM.CameraAllow = 1 THEN 'Y'
            ELSE 'N'
        END AS camera_allowed,

        CAST(SC.VisitDate AS DATE) AS visit_date,

        CASE
            WHEN TS.Present = 0 THEN 'N'
            ELSE 'Y'
        END AS is_present,

        ISNULL(MB.BrandId, 0) AS brand_id,
        ISNULL(MD.DisplayId, 0) AS display_id,

        ISNULL(TS.Remark, '') AS remarks,

        CASE
            WHEN ISNULL(TS.ImageUrl, '') = '' THEN ''
            ELSE CONCAT(
                'https://kimt1.parinaam.in/Upload/VisibilityImages/',
                TS.ImageUrl
            )
        END AS image_url

    FROM OneApp_KelloggsMT.dbo.T_AdditionalVisibility TS

    INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
        ON TS.MID = SC.MID

    INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
        ON SC.StoreId = SM.StoreId

    INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
        ON SC.EmpId = EM.EmpId

    LEFT JOIN OneApp_KelloggsMT.dbo.Master_Brand MB
        ON TS.BrandId = MB.BrandId

    LEFT JOIN OneApp_KelloggsMT.dbo.Master_Display MD
        ON TS.DisplayId = MD.DisplayId

    WHERE EM.EmpName NOT LIKE 'test%'
      AND TS.Present = 1
      AND SC.MID IN ({mid_list})
    """

    log.info(
        f"Fetching Additional Visibility data for {len(mids):,} MIDs"
    )

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(
        f"Fetched {len(df):,} Additional Visibility rows"
    )

    return df


def fetch_Coverage(engine: Engine, mids: list[int]) -> pl.DataFrame:
    
    
    
    if not mids:
        log.warning("No MIDs — nothing to fetch.")
        return pl.DataFrame()

    mid_list = ",".join(str(mid) for mid in mids)

    sql = f"""
        SELECT
            {p}                        AS project_id,
            JP.MID,
            sm.StoreId                          AS store_id,
            JP.EmpId                            AS employee_id,
            CONVERT(date, JP.VisitDate)         AS visit_date,
            JP.InTime                           AS in_time,
            JP.OutTime                          AS out_time,
            CASE
                WHEN JP.OutTime IS NULL OR JP.InTime IS NULL THEN NULL
                WHEN JP.OutTime < JP.InTime THEN 0
                ELSE DATEDIFF(SECOND, JP.InTime, JP.OutTime) / 60
            END                                 AS duration_minutes,
            CASE
                WHEN (
                    SELECT TOP 1 EmpId
                    FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
                    WHERE SC.EmpId = JP.EmpId
                      AND SC.StoreId = JP.StoreId
                      AND SC.VisitDate = JP.VisitDate
                      AND SC.ReasonId IN (0,1,3,9,10,19,20)
                ) > 0
                THEN 'Y' ELSE 'N'
            END                                 AS is_covered,
            CASE JP.Deviation
                WHEN 0 THEN 'Planned'
                WHEN 1 THEN 'Adhoc'
                WHEN 2 THEN 'Beat Plan'
                WHEN 3 THEN 'Non Merchandised'
                WHEN 4 THEN 'Add New Store'
                WHEN 5 THEN 'Non Program'
                ELSE ''
            END                                 AS coverage_type,
            sm.StoreTypeId                      AS storetype_id,
            Em.SupervisorId                     AS supervisor_id,
            ISNULL(JP.ReasonId, 0)              AS reason_id,
            sm.CameraAllow                      AS camera_allow,
            CAST(
                CASE
                    WHEN sm.Latitude IS NULL OR sm.Latitude = 0 THEN 0
                    WHEN sm.Longitude IS NULL OR sm.Longitude = 0 THEN 0
                    WHEN JP.Latitude IS NULL OR JP.Latitude = 0 THEN 0
                    WHEN JP.Longitude IS NULL OR JP.Longitude = 0 THEN 0
                    ELSE SQRT(
                        POWER(69.1 * (JP.Latitude - sm.Latitude), 2) +
                        POWER(69.1 * (sm.Longitude - JP.Longitude)
                              * COS(JP.Latitude / 57.3), 2)
                    ) * 1000
                END AS FLOAT
            )                                   AS distance_meters,
            GETDATE()                           AS update_date,
            'ETL-SQLAlchemy'                    AS update_by
        FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP WITH (NOLOCK)
        INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm
            ON JP.StoreId = sm.StoreId
        INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em
            ON JP.EmpId = Em.EmpId
        WHERE JP.MID IN ({mid_list})
          AND Em.UserName NOT LIKE 'test%'
    """
    
    log.info(f"Fetching coverage data for {len(mids):,} MIDs")

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(f"Fetched {len(df):,} rows from SQL Server")

    return df




def fetch_stock_details(
    engine: Engine,
    mids: list[int]
) -> pl.DataFrame:

    if not mids:
        log.warning("No MIDs — nothing to fetch.")
        return pl.DataFrame()

    mid_list = ",".join(map(str, mids))

    sql = f"""
    SELECT
        SC.MID,

        EM.SupervisorId AS supervisor_id,
        EM.EmpId AS employee_id,

        SM.StoreId AS store_id,

        CAST(SC.VisitDate AS DATE) AS visitdate,

        SC.StoreTypeId,
        SM.StoreCategoryId,

        VP.ProductId AS product_id,

        TS.MSL,
        MP.MBQ,

        ISNULL(TS.OpeningStock, 0)
        + ISNULL(TS.MidDayStock, 0) AS stock_qty,

        0 AS damagedstock,
        0 AS loststock,
        0 AS expirystock,

        CASE
            WHEN ISNULL(TS.OpeningStock, 0)
               + ISNULL(TS.MidDayStock, 0) >= 1
            THEN 'Y'
            ELSE 'N'
        END AS skuavailability,

        'Parinaam' AS stocktype

    FROM OneApp_KelloggsMT.dbo.T_Stock TS

    INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
        ON TS.MID = SC.MID

    INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
        ON SC.StoreId = SM.StoreId

    INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
        ON SC.EmpId = EM.EmpId

    INNER JOIN OneApp_KelloggsMT.dbo.vw_Product VP
        ON TS.ProductId = VP.ProductId

    INNER JOIN
    (
        SELECT
            MBQ,
            ProductId,
            StateId,
            ChainId,
            StoreTypeId,
            StoreCategoryId,
            StoreClassId
        FROM OneApp_KelloggsMT.dbo.Mapping_ProductAssortment
        WHERE FromDate <= CAST(GETDATE() AS DATE)
          AND ToDate >= CAST(GETDATE() AS DATE)
    ) MP
        ON MP.StateId = SM.StateId
       AND MP.ChainId = SM.ChainId
       AND MP.StoreTypeId = SM.StoreTypeId
       AND MP.StoreCategoryId = SM.StoreCategoryId
       AND MP.StoreClassId = SM.StoreClassId
       AND MP.ProductId = TS.ProductId

    WHERE EM.EmpName NOT LIKE 'test%'
      AND SC.MID IN ({mid_list})
    """

    log.info(
        f"Fetching Stock Details data for {len(mids):,} MIDs"
    )

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(
        f"Fetched {len(df):,} Stock Details rows"
    )

    return df





def fetch_Attendance(
    engine: Engine,
    end_date: date | None = None,
    days_back: int = 15
) -> pl.DataFrame:
    """
    Fetch attendance source data.

    Default:
        end_date = yesterday
        start_date = yesterday - 15 days
    """

    if end_date is None:
        end_date = date.today() - timedelta(days=1)

    start_date = end_date - timedelta(days=days_back)

    sql = f"""
    SELECT
        JP.EmpId AS employee_id,
        JP.StoreId AS store_id,
        CAST(JP.VisitDate AS DATE) AS visit_date,

        EM.ManagerId AS supervisor_id,
        EM.JoinDate AS date_of_join,
        EM.ResignDate AS date_of_resign,
        EM.LegacyCode,

        SC.ReasonId,
        SC.InTime,
        SC.OutTime,

        MPU.PositionId,
        MP.PositionCode

    FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan JP

    INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM
        ON JP.EmpId = EM.Id

    LEFT JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
        ON JP.EmpId = SC.EmpId
       AND JP.StoreId = SC.StoreId
       AND JP.VisitDate = SC.VisitDate

    LEFT JOIN OneApp_KelloggsMT.dbo.Mapping_PositionUser MPU
        ON JP.EmpId = MPU.EmpId
       AND JP.VisitDate BETWEEN MPU.FromDate AND MPU.ToDate

    LEFT JOIN OneApp_KelloggsMT.dbo.Master_Position MP
        ON MPU.PositionId = MP.PositionId

    WHERE JP.VisitDate BETWEEN '{start_date}'
                          AND '{end_date}'
      AND EM.RightId = 6
      AND EM.EmployeeName NOT LIKE '%test%'
    """

    log.info(
        f"Fetching Attendance data from {start_date} to {end_date}"
    )

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(
        f"Fetched {len(df):,} attendance rows "
        f"for {df['employee_id'].n_unique():,} employees"
    )

    return df

def fetch_login(engine: Engine , mids: list[int]) -> pl.DataFrame:

    sql = """
    WITH login_data AS
    (
        SELECT
            UD.EmpId,
            CAST(UD.LoginDate AS DATE) AS LoginDate,
            CONVERT(VARCHAR(8), UD.InTime, 108) AS LoginTime,

            ROW_NUMBER() OVER
            (
                PARTITION BY
                    UD.EmpId,
                    CAST(UD.LoginDate AS DATE)
                ORDER BY UD.LoginDate
            ) AS rn

        FROM OneApp_KelloggsMT.dbo.T_DeviceLogin UD
        INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
            ON UD.EmpId = EM.EmpId

        WHERE CAST(UD.LoginDate AS DATE) =
              CAST(DATEADD(DAY,-1,GETDATE()) AS DATE)

          AND EM.RightId = 6
          AND EM.EmpName NOT LIKE '%test%'
          AND (
                EM.ResignDate IS NULL
                OR CAST(EM.ResignDate AS DATE) >=
                   CAST(DATEADD(DAY,-1,GETDATE()) AS DATE)
              )
    )

    SELECT
        LD.EmpId AS employee_id,
        LD.LoginDate AS login_date,
        LD.LoginTime AS login_time,

        (
            SELECT MIN(CONVERT(VARCHAR(8), SC.InTime, 108))
            FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
            WHERE SC.IsDel = 0
              AND SC.EmpId = LD.EmpId
              AND CAST(SC.VisitDate AS DATE) = LD.LoginDate
        ) AS first_store_in_time,

        (
            SELECT MAX(CONVERT(VARCHAR(8), SC.OutTime, 108))
            FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
            WHERE SC.IsDel = 0
              AND SC.EmpId = LD.EmpId
              AND CAST(SC.VisitDate AS DATE) = LD.LoginDate
        ) AS last_store_out_time,

        CONCAT('40148','_',CAST(LD.EmpId AS VARCHAR(50))) AS unique_id

    FROM login_data LD
    WHERE LD.rn = 1
    """

 
    log.info("Fetching Login data for yesterday")

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(f"Fetched {len(df):,} Login rows")

    return df



def fetch_Promotion(engine: Engine, mids: list[int]) -> pl.DataFrame:
    
    
    
    if not mids:
        log.warning("No MIDs — nothing to fetch.")
        return pl.DataFrame()

    mid_list = ",".join(str(mid) for mid in mids)

    sql = f"""
        SELECT
            sc.MID                                          AS mid,
            {p}                                   AS project_id,
            sc.StoreId                                     AS store_id,
            Em.EmpId                                       AS employee_id,
            CONVERT(date, sc.VisitDate)                    AS visit_date,
            Em.SupervisorId                                AS supervisor_id,
            sm.ChannelId                                   AS channel_id,
            sm.ChainId                                     AS chain_id,
            sm.StoreTypeId                                 AS storetype_id,
            msd.PromoDefinitionId                          AS promo_definition_id,
            msd.PromoDefinitionName                        AS promo_definition_name,

            -- Resolve which dimension level the promo targets
            -- (Category / SubCategory / Brand / SubBrand)
            CASE
                WHEN ISNULL(ts.PromoTable,'') = 'Master_Category'    THEN 'Category'
                WHEN ISNULL(ts.PromoTable,'') = 'Master_SubCategory' THEN 'SubCategory'
                WHEN ISNULL(ts.PromoTable,'') = 'Master_Brand'       THEN 'Brand'
                WHEN ISNULL(ts.PromoTable,'') = 'Master_SubBrand'    THEN 'SubBrand'
                ELSE ''
            END                                            AS promotion_details,

            ts.PromoValue                                  AS promotion_details_id,

            -- Resolve the actual name of that dimension item
            -- SQL Server evaluates exactly ONE of these subqueries per row
            CASE
                WHEN ISNULL(ts.PromoTable,'') = 'Master_Category'
                    THEN (SELECT a.CategoryName
                          FROM OneApp_KelloggsMT.dbo.Master_Category a
                          WHERE a.CategoryId = ts.PromoValue)
                WHEN ISNULL(ts.PromoTable,'') = 'Master_SubCategory'
                    THEN (SELECT a.SubCategoryName
                          FROM OneApp_KelloggsMT.dbo.Master_SubCategory a
                          WHERE a.SubCategoryId = ts.PromoValue)
                WHEN ISNULL(ts.PromoTable,'') = 'Master_Brand'
                    THEN (SELECT a.BrandName
                          FROM OneApp_KelloggsMT.dbo.Master_Brand a
                          WHERE a.BrandId = ts.PromoValue)
                WHEN ISNULL(ts.PromoTable,'') = 'Master_SubBrand'
                    THEN (SELECT a.SubBrandName
                          FROM OneApp_KelloggsMT.dbo.Master_SubBrand a
                          WHERE a.SubBrandId = ts.PromoValue)
            END                                            AS promo_value_name,

            -- Present flag: 0 → 'N', anything else → 'Y'
            CASE WHEN ts.Present = 0 THEN 'N' ELSE 'Y' END AS is_present,

            -- Reason only populated when promo is ABSENT
            CASE
                WHEN ts.Present = 1 THEN ''
                ELSE ISNULL(mnp.PromoReason, '')
            END                                            AS reason,

            ISNULL(mpq.PromoQuestionName, '')              AS question,
            ISNULL(tpq.PromoAnswerName,   '')              AS answer,

            -- Image URLs — empty string when no image
            CASE
                WHEN ISNULL(SHI.PromoImage1,'') = '' THEN ''
                ELSE 'https://kimt1.parinaam.in/Upload/PromotionImages/'
                     + SHI.PromoImage1
            END                                            AS image1,
            CASE
                WHEN ISNULL(SHI.PromoImage2,'') = '' THEN ''
                ELSE 'https://kimt1.parinaam.in/Upload/PromotionImages/'
                     + SHI.PromoImage2
            END                                            AS image2,

            GETDATE()                                      AS update_date,
            'ETL-SQLAlchemy'                               AS update_by

        FROM OneApp_KelloggsMT.dbo.T_Promotion ts
        INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc
            ON ts.mid = sc.mid
        INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm
            ON sc.StoreId = sm.StoreId
        INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em
            ON sc.EmpId = Em.EmpId
        INNER JOIN OneApp_KelloggsMT.dbo.Master_PromotionDefinition msd
            ON msd.PromoDefinitionId = ts.PromoDefinitionId
        LEFT JOIN OneApp_KelloggsMT.dbo.T_PromotionImages SHI
            ON ts.PId = SHI.PId
        LEFT JOIN OneApp_KelloggsMT.dbo.Master_PromotionReason mnp
            ON ts.PromoReasonId = mnp.PromoReasonId
        LEFT JOIN OneApp_KelloggsMT.dbo.t_promotionquestion tpq
            ON ts.PId = tpq.PId
        LEFT JOIN OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq
            ON tpq.PromoQuestionId = mpq.PromoQuestionId

        WHERE Em.EmpName NOT LIKE 'test%'
          AND sc.MID IN ({mid_list})
    """
    
    log.info(f"Fetching Coverage data for {len(mids):,} MIDs")

    df = pl.read_database(
        query=sql,
        connection=engine
    )

    log.info(f"Fetched {len(df):,} rows from SQL Server")

    return df