diff --git a/kpi/facts.py b/kpi/facts.py deleted file mode 100644 index d079a92..0000000 --- a/kpi/facts.py +++ /dev/null @@ -1,1301 +0,0 @@ -import os -# import pyarrow -import sys -import logging -from datetime import date, timedelta -import polars as pl -from sqlalchemy import create_engine, text -from sqlalchemy.engine import Engine, URL -import clickhouse_connect -from dotenv import load_dotenv - -from log import log -from clickhouse_task.create_table import * -from db_con.connection import * -from mids import * - - - -#PROJECT_ID = 40148 -p=40148 - - - - -def fetch_SOS_OneApp(engine: Engine, mids: list[int]) -> pl.DataFrame: - - - - if not mids: - log.warning("No MIDs — nothing to fetch.") - return pl.DataFrame() - - mid_list = ",".join(str(mid) for mid in mids) - - sql = f""" SELECT - MID, - EmpId AS employee_id, - StoreId AS store_id, - VisitDate AS visit_date, - StoreTypeid AS storetype_id, - ChannelId AS channel_id, - SOSDefinitionName, - SOSHeaderDeatils, - SOSHeaderName, - SOSHeaderID, - SOSChildDeatils, - SOSChildName, - SOSChildID, - SOSHeaderFacing, - ChildTotalFacing, - ChildSelfFacing, - SOSTarget - FROM - ( - SELECT - sm.CountryName, - sc.MID, - sm.RegionName, - sm.StateName, - sm.CityName, - Em.SupervisorName, - Em.EmpId, - Em.EmpName AS EmployeeName, - Em.DesignationName AS Designation, - sm.StoreId, - CAST(sc.VisitDate AS DATE) AS VisitDate, - sm.StoreCode, - sm.StoreName, - sm.Address, - sm.StoreTypeid, - sm.ChannelId, - sm.ChainName, - - MSD.SOSDefinitionName, - - CASE - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand' - END AS SOSHeaderDeatils, - - TS.SOSHeaderName, - TS.SOSHeaderValue AS SOSHeaderID, - - CASE - WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand' - END AS SOSChildDeatils, - - TSC.SOSChildName, - TSC.SOSChildValue AS SOSChildID, - TSC.ChildTotalFacing, - TS.SOSHeaderFacing, - TSC.ChildSelfFacing, - - ( - SELECT TOP 1 SOSTarget - FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a - WHERE a.SOSDefinitionId = MSD.SOSDefinitionId - AND a.StoreId = sm.StoreId - AND a.FromDate <= sc.VisitDate - AND a.ToDate >= sc.VisitDate - ) AS SOSTarget - - FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader TS - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC - ON TS.MID = SC.MID - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM - ON SC.StoreId = SM.StoreId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM - ON SC.EmpId = EM.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild TSC - ON TS.SOSId = TSC.SOSId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition MSD - ON MSD.SOSDefinitionId = TSC.SOSDefinitionId - - WHERE EM.EmpName NOT LIKE 'test%' - AND SC.MID IN ({mid_list}) - ) A - """ - - - log.info(f"Fetching data for {len(mids):,} MIDs") - - df = pl.read_database( - query=sql, - connection=engine - ) - - - log.info(f"Fetched {len(df):,} rows from SQL Server") - - return df - -def fetch_OQaD( - engine: Engine, - run_date: date, -) -> pl.DataFrame: - - sql = f""" - WITH MID_TABLE_COV1 AS - ( - SELECT - EmpId, - CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CAST(CreateDate AS DATE) = '{run_date}' - - UNION - - SELECT - EmpId, - CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CAST(UpdateDate AS DATE) = '{run_date}' - ), - - QUIZ AS - ( - SELECT DISTINCT - E.EmpId, - CAST(DQ.VisitDate AS DATE) AS VisitDate, - DQ.QuestionId, - DQ.AnswerId, - QC.QuestionCategoryId, - QC.QuestionCategory - - FROM OneApp_KelloggsMT.dbo.T_OQAD DQ - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E - ON DQ.EmpId = E.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU - ON DQ.QuestionId = QU.QuestionId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC - ON QU.QuestionCategoryId = QC.QuestionCategoryId - - WHERE E.RightId = 6 - AND E.EmpName NOT LIKE 'test%' - AND E.EmpName NOT LIKE '%TEST%' - AND ( - E.ResignDate IS NULL - OR E.ResignDate >= '{run_date}' - ) - AND EXISTS - ( - SELECT 1 - FROM MID_TABLE_COV1 A - WHERE A.EmpId = DQ.EmpId - AND A.VisitDate = CAST(DQ.VisitDate AS DATE) - ) - ) - - SELECT - 40148 AS project_id, - Q.EmpId AS employee_id, - 0 AS process_id, - Q.VisitDate AS visit_date, - Q.QuestionCategoryId AS question_category_id, - Q.QuestionCategory AS question_category, - QM.QuestionId AS question_id, - QM.Question AS question, - ISNULL(QA.AnswerId, 0) AS answer_id, - ISNULL(QA.Answer, '') AS answer, - - CASE - WHEN QA.AnswerId IS NULL THEN 'Not Answer' - WHEN QA.RightAnswer = 1 THEN 'Y' - WHEN QA.RightAnswer IS NULL THEN 'Not Answer' - ELSE 'N' - END AS correct_answer, - - GETDATE() AS update_date, - 'SP-Pius' AS update_by - - FROM QUIZ Q - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM - ON Q.QuestionId = QM.QuestionId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA - ON Q.AnswerId = QA.AnswerId - """ - - log.info("Fetching OQaD data for run_date=%s", run_date) - - df = pl.read_database( - query=sql, - connection=engine, - ) - - log.info("Fetched %s rows", len(df)) - - return df - - -def fetch_Survey(engine: Engine, mids: list[int]) -> pl.DataFrame: - if not mids: - log.warning("No MIDs — nothing to fetch.") - return pl.DataFrame() - - mid_list = ",".join(map(str, mids)) - - sql = f""" - WITH QUIZ AS - ( - SELECT - SC.MID, - SC.StoreId, - EM.EmpId, - EM.SupervisorId, - CAST(SC.VisitDate AS DATE) AS VisitDate, - - DQ.QuestionId, - QU.Question, - DQ.Answer, - - QC.CategoryId, - QC.Category, - - MSS.SubCategoryId, - MSS.SubCategory - - FROM OneApp_KelloggsMT.dbo.T_SURVEY DQ - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC - ON DQ.MID = SC.MID - - INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyQuestion QU - ON DQ.QuestionId = QU.QuestionId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveySubCategory MSS - ON QU.SubCategoryId = MSS.SubCategoryId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyCategory QC - ON MSS.CategoryId = QC.CategoryId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM - ON SC.EmpId = EM.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_Survey MS - ON DQ.SurveyId = MS.SurveyId - AND QU.SurveyId = MS.SurveyId - - WHERE EM.EmpName NOT LIKE 'test%' - AND SC.MID IN ({mid_list}) - ) - - SELECT - Q.MID, - Q.SupervisorId, - Q.EmpId, - SM.StoreId, - Q.VisitDate, - - SM.StoreTypeId, - SM.ChainId, - - Q.CategoryId, - Q.Category, - - Q.SubCategoryId, - Q.SubCategory, - - Q.QuestionId, - Q.Question, - Q.Answer - - FROM QUIZ Q - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM - ON Q.StoreId = SM.StoreId - """ - - - log.info(f"Fetching Survey data for {len(mids):,} MIDs") - - df = pl.read_database( - query=sql, - connection=engine - ) - - - log.info(f"Fetched {len(df):,} Survey rows") - - return df - - -def fetch_additional_visibility( - engine: Engine, - mids: list[int] -) -> pl.DataFrame: - - if not mids: - log.warning("No MIDs — nothing to fetch.") - return pl.DataFrame() - - mid_list = ",".join(map(str, mids)) - - sql = f""" - SELECT - TS.MID, - - EM.EmpId AS emp_id, - - SM.StoreId AS store_id, - SM.StoreTypeId AS storetype_id, - SM.ChannelId AS channel_id, - SM.ChainId AS chain_id, - - CASE - WHEN SM.CameraAllow = 1 THEN 'Y' - ELSE 'N' - END AS camera_allowed, - - CAST(SC.VisitDate AS DATE) AS visit_date, - - CASE - WHEN TS.Present = 0 THEN 'N' - ELSE 'Y' - END AS is_present, - - ISNULL(MB.BrandId, 0) AS brand_id, - ISNULL(MD.DisplayId, 0) AS display_id, - - ISNULL(TS.Remark, '') AS remarks, - - CASE - WHEN ISNULL(TS.ImageUrl, '') = '' THEN '' - ELSE CONCAT( - 'https://kimt1.parinaam.in/Upload/VisibilityImages/', - TS.ImageUrl - ) - END AS image_url - - FROM OneApp_KelloggsMT.dbo.T_AdditionalVisibility TS - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC - ON TS.MID = SC.MID - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM - ON SC.StoreId = SM.StoreId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM - ON SC.EmpId = EM.EmpId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_Brand MB - ON TS.BrandId = MB.BrandId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_Display MD - ON TS.DisplayId = MD.DisplayId - - WHERE EM.EmpName NOT LIKE 'test%' - AND TS.Present = 1 - AND SC.MID IN ({mid_list}) - """ - - log.info( - f"Fetching Additional Visibility data for {len(mids):,} MIDs" - ) - - df = pl.read_database( - query=sql, - connection=engine - ) - - - log.info( - f"Fetched {len(df):,} Additional Visibility rows" - ) - - return df - - -def fetch_Coverage(engine: Engine, mids: list[int]) -> pl.DataFrame: - - - - if not mids: - log.warning("No MIDs — nothing to fetch.") - return pl.DataFrame() - - mid_list = ",".join(str(mid) for mid in mids) - - sql = f""" - WITH SOS_BASE AS -( - SELECT - sm.CountryName, - sc.MID, - sm.RegionName, - sm.StateName, - sm.CityName, - Em.SupervisorName, - em.EmpId, - Em.EmpName AS EmployeeName, - Em.DesignationName AS Designation, - sm.StoreId, - CONVERT(varchar, sc.VisitDate, 101) AS VisitDate, - sm.StoreCode, - sm.StoreName, - sm.Address, - sm.StoreTypeid, - sm.ChannelId, - sm.ChainName, - - MSD.SOSDefinitionName, - - CASE - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand' - END AS SOSHeaderDeatils, - - TS.SOSHeaderName, - TS.SOSHeaderValue AS SOSHeaderID, - - 'Header_Image' AS HDR1, - - CASE - WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand' - END AS SOSChildDeatils, - - TSC.SOSChildName, - TSC.SOSChildValue AS SOSChildID, - TSC.ChildTotalFacing, - TS.SOSHeaderFacing, - TSC.ChildSelfFacing, - - ( - SELECT TOP 1 SOSTarget - FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a - WHERE a.SOSDefinitionId = MSD.SOSDefinitionId - AND a.StoreId = sm.StoreId - AND a.FromDate <= sc.VisitDate - AND a.ToDate >= sc.VisitDate - ) AS SOSTarget, - - CASE - WHEN ISNULL(SHI.SOSHeaderImage,'') = '' - THEN '' - ELSE - 'https://kimt1.parinaam.in/Upload/SOSImages/' - + SHI.SOSHeaderImage - END AS SOSHeaderImg - - FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc - ON ts.MID = sc.MID - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm - ON sc.StoreId = sm.StoreId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em - ON sc.EmpId = Em.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc - ON ts.SOSId = tsc.SOSId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd - ON msd.SOSDefinitionId = tsc.SOSDefinitionId - - LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI - ON ts.SOSId = SHI.SOSId - AND ts.SOSHeaderValue = SHI.SOSHeaderValue - - LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI - ON tsc.SOSId = SCI.SOSId - AND tsc.SOSChildValue = SCI.SOSChildValue - - WHERE Em.EmpName NOT LIKE 'test%' - AND sc.MID IN ({mid_list}) -), - -SOS_PIVOT AS -( - SELECT * - FROM SOS_BASE - PIVOT - ( - MIN(SOSHeaderImg) - FOR HDR1 IN ([Header_Image]) - ) pvt -) - -SELECT - '40148' AS ProjectId, - MID, - EmpId AS employee_id, - StoreId AS store_id, - VisitDate AS visit_date, - StoreTypeid AS storetype_id, - ChannelId AS channel_id, - SOSDefinitionName, - SOSHeaderDeatils, - SOSHeaderName, - SOSHeaderID, - SOSChildDeatils, - SOSChildName, - SOSChildID, - SOSHeaderFacing, - ChildTotalFacing, - ChildSelfFacing, - SOSTarget, - GETDATE() AS update_date, - 'SP-Pius' AS update_by - -FROM SOS_PIVOT - -GROUP BY - CountryName, - MID, - RegionName, - StateName, - CityName, - SupervisorName, - EmpId, - EmployeeName, - Designation, - StoreId, - VisitDate, - StoreCode, - StoreName, - Address, - StoreTypeid, - ChannelId, - ChainName, - SOSDefinitionName, - SOSHeaderDeatils, - SOSHeaderName, - SOSHeaderID, - SOSChildDeatils, - SOSChildName, - SOSChildID, - ChildTotalFacing, - SOSHeaderFacing, - ChildSelfFacing, - SOSTarget - -ORDER BY - RegionName, - StateName, - CityName, - VisitDate; - """ - - - log.info(f"Fetching coverage data for {len(mids):,} MIDs") - - df = pl.read_database( - query=sql, - connection=engine - ) - - - log.info(f"Fetched {len(df):,} rows from SQL Server") - - return df - - - -def fetch_Login(engine: Engine , mids: list[int]) -> pl.DataFrame: - - sql = """ - WITH login_data AS - ( - SELECT - UD.EmpId, - CAST(UD.LoginDate AS DATE) AS LoginDate, - CONVERT(VARCHAR(8), UD.InTime, 108) AS LoginTime, - - ROW_NUMBER() OVER - ( - PARTITION BY - UD.EmpId, - CAST(UD.LoginDate AS DATE) - ORDER BY UD.LoginDate - ) AS rn - - FROM OneApp_KelloggsMT.dbo.T_DeviceLogin UD - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM - ON UD.EmpId = EM.EmpId - - WHERE CAST(UD.LoginDate AS DATE) = - CAST(DATEADD(DAY,-1,GETDATE()) AS DATE) - - AND EM.RightId = 6 - AND EM.EmpName NOT LIKE '%test%' - AND ( - EM.ResignDate IS NULL - OR CAST(EM.ResignDate AS DATE) >= - CAST(DATEADD(DAY,-1,GETDATE()) AS DATE) - ) - ) - - SELECT - LD.EmpId AS employee_id, - LD.LoginDate AS login_date, - LD.LoginTime AS login_time, - - ( - SELECT MIN(CONVERT(VARCHAR(8), SC.InTime, 108)) - FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC - WHERE SC.IsDel = 0 - AND SC.EmpId = LD.EmpId - AND CAST(SC.VisitDate AS DATE) = LD.LoginDate - ) AS first_store_in_time, - - ( - SELECT MAX(CONVERT(VARCHAR(8), SC.OutTime, 108)) - FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC - WHERE SC.IsDel = 0 - AND SC.EmpId = LD.EmpId - AND CAST(SC.VisitDate AS DATE) = LD.LoginDate - ) AS last_store_out_time, - - CONCAT('40148','_',CAST(LD.EmpId AS VARCHAR(50))) AS unique_id - - FROM login_data LD - WHERE LD.rn = 1 - """ - - - log.info("Fetching Login data for yesterday") - - df = pl.read_database( - query=sql, - connection=engine - ) - - - log.info(f"Fetched {len(df):,} Login rows") - - return df - - - - -def fetch_Stock_Details( - engine: Engine, - mids: list[int] -) -> pl.DataFrame: - - if not mids: - log.warning("No MIDs — nothing to fetch.") - return pl.DataFrame() - - mid_list = ",".join(map(str, mids)) - - sql = f""" - SELECT - SC.MID, - - EM.SupervisorId AS supervisor_id, - EM.EmpId AS employee_id, - - SM.StoreId AS store_id, - - CAST(SC.VisitDate AS DATE) AS visitdate, - - SC.StoreTypeId, - SM.StoreCategoryId, - - VP.ProductId AS product_id, - - TS.MSL, - MP.MBQ, - - ISNULL(TS.OpeningStock, 0) - + ISNULL(TS.MidDayStock, 0) AS stock_qty, - - 0 AS damagedstock, - 0 AS loststock, - 0 AS expirystock, - - CASE - WHEN ISNULL(TS.OpeningStock, 0) - + ISNULL(TS.MidDayStock, 0) >= 1 - THEN 'Y' - ELSE 'N' - END AS skuavailability, - - 'Parinaam' AS stocktype - - FROM OneApp_KelloggsMT.dbo.T_Stock TS - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC - ON TS.MID = SC.MID - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM - ON SC.StoreId = SM.StoreId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM - ON SC.EmpId = EM.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Product VP - ON TS.ProductId = VP.ProductId - - INNER JOIN - ( - SELECT - MBQ, - ProductId, - StateId, - ChainId, - StoreTypeId, - StoreCategoryId, - StoreClassId - FROM OneApp_KelloggsMT.dbo.Mapping_ProductAssortment - WHERE FromDate <= CAST(GETDATE() AS DATE) - AND ToDate >= CAST(GETDATE() AS DATE) - ) MP - ON MP.StateId = SM.StateId - AND MP.ChainId = SM.ChainId - AND MP.StoreTypeId = SM.StoreTypeId - AND MP.StoreCategoryId = SM.StoreCategoryId - AND MP.StoreClassId = SM.StoreClassId - AND MP.ProductId = TS.ProductId - - WHERE EM.EmpName NOT LIKE 'test%' - AND SC.MID IN ({mid_list}) - """ - - log.info( - f"Fetching Stock Details data for {len(mids):,} MIDs" - ) - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Stock Details rows" - ) - - return df - - - - - -def fetch_Attendance( - engine: Engine, - end_date: date | None = None, - days_back: int = 15 -) -> pl.DataFrame: - """ - Fetch attendance source data. - - Default: - end_date = yesterday - start_date = yesterday - 15 days - """ - - if end_date is None: - end_date = date.today() - timedelta(days=1) - - start_date = end_date - timedelta(days=days_back) - - sql = f""" - SELECT - JP.EmpId AS employee_id, - JP.StoreId AS store_id, - CAST(JP.VisitDate AS DATE) AS visit_date, - - EM.ManagerId AS supervisor_id, - EM.JoinDate AS date_of_join, - EM.ResignDate AS date_of_resign, - EM.LegacyCode, - - SC.ReasonId, - SC.InTime, - SC.OutTime, - - MPU.PositionId, - MP.PositionCode - - FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan JP - - INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM - ON JP.EmpId = EM.Id - - LEFT JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC - ON JP.EmpId = SC.EmpId - AND JP.StoreId = SC.StoreId - AND JP.VisitDate = SC.VisitDate - - LEFT JOIN OneApp_KelloggsMT.dbo.Mapping_PositionUser MPU - ON JP.EmpId = MPU.EmpId - AND JP.VisitDate BETWEEN MPU.FromDate AND MPU.ToDate - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_Position MP - ON MPU.PositionId = MP.PositionId - - WHERE JP.VisitDate BETWEEN '{start_date}' - AND '{end_date}' - AND EM.RightId = 6 - AND EM.EmployeeName NOT LIKE '%test%' - """ - - log.info( - f"Fetching Attendance data from {start_date} to {end_date}" - ) - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} attendance rows " - f"for {df['employee_id'].n_unique():,} employees" - ) - - return df - - - - -def fetch_Journey_Plan( - engine: Engine, - report_date: date -) -> pl.DataFrame: - - sql = f""" - SELECT - StoreId AS store_id, - EmpId AS employee_id, - CAST(VisitDate AS DATE) AS visit_date, - Deviation AS process_id - - FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan - - WHERE MONTH(VisitDate) = {report_date.month} - AND YEAR(VisitDate) = {report_date.year} - - AND EmpId NOT IN ( - SELECT ID - FROM OneApp_KelloggsMT.dbo.AspNetUsers - WHERE UserName LIKE 'TEST%' - ) - """ - - log.info( - f"Fetching Journey Plan for {report_date:%Y-%m}" - ) - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Journey Plan records" - ) - return df - - - - - -def fetch_Web_Logins( - engine: Engine, - run_date: date -) -> pl.DataFrame: - """ - Source: - T_User_Activity_Log - - Target: - Web Logins - """ - - sql = f""" - SELECT DISTINCT - 40148 AS project_id, - EM1.Id AS supervisor_id, - EM1.EmployeeName AS supervisor_name, - EM.Id AS emp_id, - EM.EmployeeName AS employee_name, - DM.DesignationName AS designation, - CAST(AL.Date AS DATE) AS date, - CONVERT(VARCHAR(8), AL.Date, 108) AS time, - AL.Thread AS activity_name, - AL.Level AS activity_type, - R.RightName AS right_name, - GETDATE() AS CreateDate, - 'Pius' AS CreateBy - FROM OneApp_KelloggsMT.dbo.T_User_Activity_Log AL - LEFT JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM - ON AL.Logger = EM.USERNAME - INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM1 - ON EM1.Id = EM.ManagerId - INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM2 - ON EM2.Id = EM1.ManagerId - INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation DM - ON EM.DesignationId = DM.DesignationId - INNER JOIN OneApp_KelloggsMT.dbo.Master_City CM - ON EM.CityId = CM.CityId - INNER JOIN OneApp_KelloggsMT.dbo.Master_State ST - ON ST.StateId = CM.StateId - INNER JOIN OneApp_KelloggsMT.dbo.Master_Region RM - ON RM.RegionId = ST.RegionId - INNER JOIN OneApp_KelloggsMT.dbo.Right_Master R - ON EM.RightId = R.RightId - WHERE CAST(AL.Date AS DATE) = '{run_date}' - """ - - - log.info(f"Fetching Web Login data for {run_date}") - - df = pl.read_database( - query=sql, - connection=engine - ) - - - log.info( - f"Fetched {len(df):,} Web Login records" - ) - - return df - - - - -def fetch_Promotion( - engine: Engine, - mids: list[int] -) -> pl.DataFrame: - - if not mids: - log.warning("No MIDs supplied.") - return pl.DataFrame() - - mid_list = ",".join(map(str, mids)) - - sql = f""" - SELECT - SC.MID, - - 40148 AS project_id, - - SC.StoreId AS store_id, - EM.EmpId AS employee_id, - SC.VisitDate AS visit_date, - - EM.SupervisorId AS supervisor_id, - - SM.ChannelId AS channel_id, - SM.ChainId AS chain_id, - SM.StoreTypeId AS storetype_id, - - MSD.PromoDefinitionId AS promo_definition_id, - MSD.PromoDefinitionName AS promo_definition_name, - - CASE - WHEN ISNULL(TS.PromoTable,'')='Master_Category' - THEN 'Category' - WHEN ISNULL(TS.PromoTable,'')='Master_SubCategory' - THEN 'SubCategory' - WHEN ISNULL(TS.PromoTable,'')='Master_Brand' - THEN 'Brand' - WHEN ISNULL(TS.PromoTable,'')='Master_SubBrand' - THEN 'SubBrand' - END AS promotion_details, - - TS.PromoValue AS promotion_details_id, - - CASE - WHEN ISNULL(TS.PromoTable,'')='Master_Category' - THEN ( - SELECT CategoryName - FROM OneApp_KelloggsMT.dbo.Master_Category A - WHERE A.CategoryId = TS.PromoValue - ) - - WHEN ISNULL(TS.PromoTable,'')='Master_SubCategory' - THEN ( - SELECT SubCategoryName - FROM OneApp_KelloggsMT.dbo.Master_SubCategory A - WHERE A.SubCategoryId = TS.PromoValue - ) - - WHEN ISNULL(TS.PromoTable,'')='Master_Brand' - THEN ( - SELECT BrandName - FROM OneApp_KelloggsMT.dbo.Master_Brand A - WHERE A.BrandId = TS.PromoValue - ) - - WHEN ISNULL(TS.PromoTable,'')='Master_SubBrand' - THEN ( - SELECT SubBrandName - FROM OneApp_KelloggsMT.dbo.Master_SubBrand A - WHERE A.SubBrandId = TS.PromoValue - ) - END AS promotion_value_name, - - CASE - WHEN TS.Present = 0 THEN 'N' - ELSE 'Y' - END AS present, - - CASE - WHEN TS.Present = 1 THEN '' - ELSE ISNULL(MNP.PromoReason,'') - END AS reason, - - ISNULL(MPQ.PromoQuestionName,'') AS promo_question, - ISNULL(TPQ.PromoAnswerName,'') AS promo_answer, - - CASE - WHEN ISNULL(SHI.PromoImage1,'') = '' - THEN '' - ELSE - 'https://kimt1.parinaam.in/Upload/PromotionImages/' - + SHI.PromoImage1 - END AS image1, - - CASE - WHEN ISNULL(SHI.PromoImage2,'') = '' - THEN '' - ELSE - 'https://kimt1.parinaam.in/Upload/PromotionImages/' - + SHI.PromoImage2 - END AS image2 - - FROM OneApp_KelloggsMT.dbo.T_Promotion TS - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC - ON TS.MID = SC.MID - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM - ON SC.StoreId = SM.StoreId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM - ON SC.EmpId = EM.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_PromotionDefinition MSD - ON MSD.PromoDefinitionId = TS.PromoDefinitionId - - LEFT JOIN OneApp_KelloggsMT.dbo.T_PromotionImages SHI - ON TS.PId = SHI.PId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_PromotionReason MNP - ON TS.PromoReasonId = MNP.PromoReasonId - - LEFT JOIN OneApp_KelloggsMT.dbo.T_PromotionQuestion TPQ - ON TS.PId = TPQ.PId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_PromotionQuestion MPQ - ON TPQ.PromoQuestionId = MPQ.PromoQuestionId - - WHERE EM.EmpName NOT LIKE 'test%' - AND SC.MID IN ({mid_list}) - """ - - log.info( - f"Fetching Promotion data for {len(mids):,} MIDs" - ) - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Promotion records" - ) - - return df - - - - - -def fetch_PaidVisibility( - engine: Engine, - mids: list[int] -) -> pl.DataFrame: - - if not mids: - log.warning("No MIDs supplied.") - return pl.DataFrame() - - mid_list = ",".join(map(str, mids)) - - sql = f""" - SELECT - SC.MID, - - 40148 AS project_id, - - SC.StoreId AS store_id, - EM.EmpId AS employee_id, - SC.VisitDate AS visit_date, - - EM.SupervisorId AS supervisor_id, - - SM.ChannelId AS channel_id, - SM.ChainId AS chain_id, - SM.StoreTypeId AS storetype_id, - - 0 AS menu_id, - '' AS menu_name, - - TS.VisibilityId AS visibility_id, - MV.VisibilityName AS visibility_name, - - MSD.VisibilityDefinitionId AS visibility_definition_id, - MSD.VisibilityDefinitionName AS visibility_definition_name, - - CASE - WHEN ISNULL(TS.VisibilityTable,'')='Master_Category' - THEN 'Category' - WHEN ISNULL(TS.VisibilityTable,'')='Master_SubCategory' - THEN 'SubCategory' - WHEN ISNULL(TS.VisibilityTable,'')='Master_Brand' - THEN 'Brand' - WHEN ISNULL(TS.VisibilityTable,'')='Master_SubBrand' - THEN 'SubBrand' - END AS visibility_details, - - TS.VisibilityValue AS visibility_details_id, - - CASE - WHEN ISNULL(TS.VisibilityTable,'')='Master_Category' - THEN ( - SELECT CategoryName - FROM OneApp_KelloggsMT.dbo.Master_Category A - WHERE A.CategoryId = TS.VisibilityValue - ) - - WHEN ISNULL(TS.VisibilityTable,'')='Master_SubCategory' - THEN ( - SELECT SubCategoryName - FROM OneApp_KelloggsMT.dbo.Master_SubCategory A - WHERE A.SubCategoryId = TS.VisibilityValue - ) - - WHEN ISNULL(TS.VisibilityTable,'')='Master_Brand' - THEN ( - SELECT BrandName - FROM OneApp_KelloggsMT.dbo.Master_Brand A - WHERE A.BrandId = TS.VisibilityValue - ) - - WHEN ISNULL(TS.VisibilityTable,'')='Master_SubBrand' - THEN ( - SELECT SubBrandName - FROM OneApp_KelloggsMT.dbo.Master_SubBrand A - WHERE A.SubBrandId = TS.VisibilityValue - ) - END AS visibility_value_name, - - CASE - WHEN TS.Present = 0 THEN 'N' - ELSE 'Y' - END AS present, - - CASE - WHEN TS.Present = 0 - THEN TS.VisibilityReasonId - ELSE NULL - END AS reason_id, - - CASE - WHEN TS.Present = 0 - THEN ISNULL(MNP.VisibilityReason,'') - ELSE '' - END AS reason, - - ISNULL(MVQ.VisibilityQuestionName,'') AS visibility_question, - ISNULL(TVQ.VisibilityAnswerName,'') AS visibility_answer, - - CASE - WHEN TS.Present = 0 THEN '' - ELSE - CASE - WHEN ISNULL(SHI.VisibilityImage1,'') = '' - THEN '' - ELSE - 'https://kimt1.parinaam.in/Upload/PaidVisibilityImages/' - + SHI.VisibilityImage1 - END - END AS image1, - - CASE - WHEN ISNULL(SHI.VisibilityImage2,'') = '' - THEN '' - ELSE - 'https://kimt.parinaam.in/Upload/PaidVisibilityImages/' - + SHI.VisibilityImage2 - END AS image2 - - FROM OneApp_KelloggsMT.dbo.T_Visibility TS WITH (NOLOCK) - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC - ON TS.MID = SC.MID - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM - ON SC.StoreId = SM.StoreId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM - ON SC.EmpId = EM.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_VisibilityDefinition MSD - ON MSD.VisibilityDefinitionId = TS.VisibilityDefinitionId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_Visibility MV - ON TS.VisibilityId = MV.VisibilityId - - LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityImages SHI - ON TS.VId = SHI.VId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_VisibilityReason MNP - ON TS.VisibilityReasonId = MNP.VisibilityReasonId - - LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityStock TVS - ON TS.VId = TVS.VId - - LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityQuestion TVQ - ON TS.VId = TVQ.VId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_VisibilityQuestion MVQ - ON TVQ.VisibilityQuestionId = MVQ.VisibilityQuestionId - - WHERE EM.EmpName NOT LIKE 'test%' - AND SC.MID IN ({mid_list}) - """ - - log.info( - f"Fetching Paid Visibility data for {len(mids):,} MIDs" - ) - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Paid Visibility records" - ) - - return df \ No newline at end of file diff --git a/kpi/sql/coverage.sql b/kpi/sql/coverage.sql deleted file mode 100644 index 35b7b75..0000000 --- a/kpi/sql/coverage.sql +++ /dev/null @@ -1,164 +0,0 @@ -WITH SOS_BASE AS -( - SELECT - sm.CountryName, - sc.MID, - sm.RegionName, - sm.StateName, - sm.CityName, - Em.SupervisorName, - em.EmpId, - Em.EmpName AS EmployeeName, - Em.DesignationName AS Designation, - sm.StoreId, - CONVERT(varchar, sc.VisitDate, 101) AS VisitDate, - sm.StoreCode, - sm.StoreName, - sm.Address, - sm.StoreTypeid, - sm.ChannelId, - sm.ChainName, - - MSD.SOSDefinitionName, - - CASE - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand' - END AS SOSHeaderDeatils, - - TS.SOSHeaderName, - TS.SOSHeaderValue AS SOSHeaderID, - - 'Header_Image' AS HDR1, - - CASE - WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand' - END AS SOSChildDeatils, - - TSC.SOSChildName, - TSC.SOSChildValue AS SOSChildID, - TSC.ChildTotalFacing, - TS.SOSHeaderFacing, - TSC.ChildSelfFacing, - - ( - SELECT TOP 1 SOSTarget - FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a - WHERE a.SOSDefinitionId = MSD.SOSDefinitionId - AND a.StoreId = sm.StoreId - AND a.FromDate <= sc.VisitDate - AND a.ToDate >= sc.VisitDate - ) AS SOSTarget, - - CASE - WHEN ISNULL(SHI.SOSHeaderImage,'') = '' - THEN '' - ELSE - 'https://kimt1.parinaam.in/Upload/SOSImages/' - + SHI.SOSHeaderImage - END AS SOSHeaderImg - - FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc - ON ts.MID = sc.MID - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm - ON sc.StoreId = sm.StoreId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em - ON sc.EmpId = Em.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc - ON ts.SOSId = tsc.SOSId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd - ON msd.SOSDefinitionId = tsc.SOSDefinitionId - - LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI - ON ts.SOSId = SHI.SOSId - AND ts.SOSHeaderValue = SHI.SOSHeaderValue - - LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI - ON tsc.SOSId = SCI.SOSId - AND tsc.SOSChildValue = SCI.SOSChildValue - - WHERE Em.EmpName NOT LIKE 'test%' - AND sc.MID IN ({mid_list}) -), - -SOS_PIVOT AS -( - SELECT * - FROM SOS_BASE - PIVOT - ( - MIN(SOSHeaderImg) - FOR HDR1 IN ([Header_Image]) - ) pvt -) - -SELECT - '40148' AS ProjectId, - MID, - EmpId AS employee_id, - StoreId AS store_id, - VisitDate AS visit_date, - StoreTypeid AS storetype_id, - ChannelId AS channel_id, - SOSDefinitionName, - SOSHeaderDeatils, - SOSHeaderName, - SOSHeaderID, - SOSChildDeatils, - SOSChildName, - SOSChildID, - SOSHeaderFacing, - ChildTotalFacing, - ChildSelfFacing, - SOSTarget, - GETDATE() AS update_date, - 'SP-Pius' AS update_by - -FROM SOS_PIVOT - -GROUP BY - CountryName, - MID, - RegionName, - StateName, - CityName, - SupervisorName, - EmpId, - EmployeeName, - Designation, - StoreId, - VisitDate, - StoreCode, - StoreName, - Address, - StoreTypeid, - ChannelId, - ChainName, - SOSDefinitionName, - SOSHeaderDeatils, - SOSHeaderName, - SOSHeaderID, - SOSChildDeatils, - SOSChildName, - SOSChildID, - ChildTotalFacing, - SOSHeaderFacing, - ChildSelfFacing, - SOSTarget - -ORDER BY - RegionName, - StateName, - CityName, - VisitDate; \ No newline at end of file diff --git a/logs/etl_20260610.log b/logs/etl_20260610.log deleted file mode 100644 index e69de29..0000000 diff --git a/logs/etl_20260612.log b/logs/etl_20260612.log deleted file mode 100644 index 1c58d25..0000000 --- a/logs/etl_20260612.log +++ /dev/null @@ -1,1024 +0,0 @@ -2026-06-12 13:10:05 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:10:05 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:10:44 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:10:44 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:10:45 | INFO | -2026-06-12 13:10:50 | INFO | -2026-06-12 13:12:36 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:12:36 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:12:37 | INFO | -2026-06-12 13:12:39 | INFO | -2026-06-12 13:14:24 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:14:24 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:14:26 | INFO | -2026-06-12 13:14:27 | INFO | -2026-06-12 13:14:28 | INFO | Both databases connected successfully -2026-06-12 13:14:28 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:14:28 | INFO | Found 818 MIDs -2026-06-12 13:15:39 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:15:39 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:15:40 | INFO | -2026-06-12 13:15:41 | INFO | -2026-06-12 13:15:42 | INFO | Both databases connected successfully -2026-06-12 13:15:42 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:15:42 | INFO | Found 818 MIDs -2026-06-12 13:23:57 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:23:57 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:23:58 | INFO | -2026-06-12 13:24:01 | INFO | -2026-06-12 13:24:01 | INFO | Both databases connected successfully -2026-06-12 13:24:01 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:24:02 | INFO | Found 818 MIDs -2026-06-12 13:24:02 | INFO | Fetching data for 818 MIDs -2026-06-12 13:24:06 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:24:06 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:24:09 | INFO | Fetched 464 rows -2026-06-12 13:24:09 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:24:10 | INFO | Fetched 142 Survey rows -2026-06-12 13:24:10 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:24:11 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 13:24:11 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 13:24:11 | INFO | Fetched 761 rows from SQL Server -2026-06-12 13:24:11 | INFO | Fetching Login data for yesterday -2026-06-12 13:24:12 | INFO | Fetched 479 Login rows -2026-06-12 13:24:12 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 13:24:17 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 13:24:17 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 13:24:18 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 13:24:18 | INFO | Fetching Store Master data -2026-06-12 13:24:20 | INFO | Fetched 5,984 stores -2026-06-12 13:24:20 | INFO | Fetching SKU Master data -2026-06-12 13:24:20 | INFO | Fetched 160 SKU Master rows -2026-06-12 13:24:20 | INFO | Fetching Display Master data -2026-06-12 13:24:20 | INFO | Fetched 135 Display Master records -2026-06-12 13:24:20 | INFO | Fetching Employee Master data -2026-06-12 13:24:21 | INFO | Fetched 2,269 Employee Master records -2026-06-12 13:24:21 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 13:24:21 | INFO | Fetched 20,969 Journey Plan records -2026-06-12 13:24:21 | INFO | Fetching Coverage Remarks -2026-06-12 13:24:21 | INFO | Fetched 29 Coverage Remark records -2026-06-12 13:24:21 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 13:24:22 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 13:24:22 | INFO | Fetching Master Visibility Reason data -2026-06-12 13:24:22 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 13:24:22 | INFO | Fetching Master Visibility Definition data -2026-06-12 13:24:22 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 13:24:22 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 13:24:22 | INFO | Fetched 223 Web Login records -2026-06-12 13:24:22 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 13:24:25 | INFO | Fetched 8,410 Promotion records -2026-06-12 13:26:34 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:26:34 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:26:34 | INFO | -2026-06-12 13:26:36 | INFO | -2026-06-12 13:26:36 | INFO | Both databases connected successfully -2026-06-12 13:26:36 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:26:37 | INFO | Found 818 MIDs -2026-06-12 13:26:37 | INFO | Fetching data for 818 MIDs -2026-06-12 13:26:37 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:26:37 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:26:40 | INFO | Fetched 464 rows -2026-06-12 13:26:40 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:26:40 | INFO | Fetched 142 Survey rows -2026-06-12 13:26:40 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:26:41 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 13:26:41 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 13:26:41 | INFO | Fetched 761 rows from SQL Server -2026-06-12 13:26:41 | INFO | Fetching Login data for yesterday -2026-06-12 13:26:41 | INFO | Fetched 479 Login rows -2026-06-12 13:26:41 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 13:26:45 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 13:26:45 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 13:26:46 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 13:26:46 | INFO | Fetching Store Master data -2026-06-12 13:26:48 | INFO | Fetched 5,984 stores -2026-06-12 13:26:48 | INFO | Fetching SKU Master data -2026-06-12 13:26:48 | INFO | Fetched 160 SKU Master rows -2026-06-12 13:26:48 | INFO | Fetching Display Master data -2026-06-12 13:26:48 | INFO | Fetched 135 Display Master records -2026-06-12 13:26:48 | INFO | Fetching Employee Master data -2026-06-12 13:26:49 | INFO | Fetched 2,269 Employee Master records -2026-06-12 13:26:49 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 13:26:50 | INFO | Fetched 20,969 Journey Plan records -2026-06-12 13:26:50 | INFO | Fetching Coverage Remarks -2026-06-12 13:26:50 | INFO | Fetched 29 Coverage Remark records -2026-06-12 13:26:50 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 13:26:50 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 13:26:50 | INFO | Fetching Master Visibility Reason data -2026-06-12 13:26:50 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 13:26:50 | INFO | Fetching Master Visibility Definition data -2026-06-12 13:26:50 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 13:26:50 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 13:26:50 | INFO | Fetched 223 Web Login records -2026-06-12 13:26:50 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 13:26:53 | INFO | Fetched 8,410 Promotion records -2026-06-12 13:29:18 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:29:18 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:29:19 | INFO | -2026-06-12 13:29:21 | INFO | -2026-06-12 13:29:21 | INFO | Both databases connected successfully -2026-06-12 13:29:21 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:29:21 | INFO | Found 818 MIDs -2026-06-12 13:29:21 | INFO | Fetching data for 818 MIDs -2026-06-12 13:29:22 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:29:22 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:29:24 | INFO | Fetched 464 rows -2026-06-12 13:29:24 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:29:24 | INFO | Fetched 142 Survey rows -2026-06-12 13:29:24 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:29:27 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 13:29:27 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 13:29:28 | INFO | Fetched 761 rows from SQL Server -2026-06-12 13:29:28 | INFO | Fetching Login data for yesterday -2026-06-12 13:29:28 | INFO | Fetched 479 Login rows -2026-06-12 13:29:28 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 13:29:38 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 13:29:38 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 13:29:41 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 13:29:41 | INFO | Fetching Store Master data -2026-06-12 13:29:46 | INFO | Fetched 5,984 stores -2026-06-12 13:29:46 | INFO | Fetching SKU Master data -2026-06-12 13:29:46 | INFO | Fetched 160 SKU Master rows -2026-06-12 13:29:46 | INFO | Fetching Display Master data -2026-06-12 13:29:46 | INFO | Fetched 135 Display Master records -2026-06-12 13:29:46 | INFO | Fetching Employee Master data -2026-06-12 13:29:47 | INFO | Fetched 2,269 Employee Master records -2026-06-12 13:29:47 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 13:29:48 | INFO | Fetched 20,969 Journey Plan records -2026-06-12 13:29:48 | INFO | Fetching Coverage Remarks -2026-06-12 13:29:48 | INFO | Fetched 29 Coverage Remark records -2026-06-12 13:29:48 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 13:29:48 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 13:29:48 | INFO | Fetching Master Visibility Reason data -2026-06-12 13:29:48 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 13:29:48 | INFO | Fetching Master Visibility Definition data -2026-06-12 13:29:48 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 13:29:48 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 13:29:49 | INFO | Fetched 223 Web Login records -2026-06-12 13:29:49 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 13:29:55 | INFO | Fetched 8,410 Promotion records -2026-06-12 13:29:55 | INFO | Fetching Paid Visibility data for 818 MIDs -2026-06-12 13:30:05 | INFO | Fetched 1,098 Paid Visibility records -2026-06-12 13:30:05 | INFO | Fetching Master Sales Territory Layer data -2026-06-12 13:30:05 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-12 13:31:55 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:31:55 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:31:56 | INFO | -2026-06-12 13:31:57 | INFO | -2026-06-12 13:31:58 | INFO | Both databases connected successfully -2026-06-12 13:31:58 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:31:58 | INFO | Found 818 MIDs -2026-06-12 13:31:58 | INFO | Fetching data for 818 MIDs -2026-06-12 13:32:00 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:32:00 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:32:02 | INFO | Fetched 464 rows -2026-06-12 13:32:02 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:32:02 | INFO | Fetched 142 Survey rows -2026-06-12 13:32:02 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:32:04 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 13:32:04 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 13:32:04 | INFO | Fetched 761 rows from SQL Server -2026-06-12 13:32:04 | INFO | Fetching Login data for yesterday -2026-06-12 13:32:04 | INFO | Fetched 479 Login rows -2026-06-12 13:32:04 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 13:32:08 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 13:32:08 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 13:32:09 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 13:32:09 | INFO | Fetching Store Master data -2026-06-12 13:32:12 | INFO | Fetched 5,984 stores -2026-06-12 13:32:12 | INFO | Fetching SKU Master data -2026-06-12 13:32:12 | INFO | Fetched 160 SKU Master rows -2026-06-12 13:32:12 | INFO | Fetching Display Master data -2026-06-12 13:32:12 | INFO | Fetched 135 Display Master records -2026-06-12 13:32:12 | INFO | Fetching Employee Master data -2026-06-12 13:32:13 | INFO | Fetched 2,269 Employee Master records -2026-06-12 13:32:13 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 13:32:13 | INFO | Fetched 20,969 Journey Plan records -2026-06-12 13:32:13 | INFO | Fetching Coverage Remarks -2026-06-12 13:32:13 | INFO | Fetched 29 Coverage Remark records -2026-06-12 13:32:13 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 13:32:14 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 13:32:14 | INFO | Fetching Master Visibility Reason data -2026-06-12 13:32:14 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 13:32:14 | INFO | Fetching Master Visibility Definition data -2026-06-12 13:32:14 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 13:32:14 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 13:32:14 | INFO | Fetched 223 Web Login records -2026-06-12 13:32:14 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 13:32:21 | INFO | Fetched 8,410 Promotion records -2026-06-12 13:32:21 | INFO | Fetching Paid Visibility data for 818 MIDs -2026-06-12 13:32:28 | INFO | Fetched 1,098 Paid Visibility records -2026-06-12 13:32:28 | INFO | Fetching Master Sales Territory Layer data -2026-06-12 13:32:29 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-12 13:33:34 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:33:34 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:33:34 | INFO | -2026-06-12 13:33:36 | INFO | -2026-06-12 13:33:36 | INFO | Both databases connected successfully -2026-06-12 13:33:36 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:33:36 | INFO | Found 818 MIDs -2026-06-12 13:33:37 | INFO | Fetching data for 818 MIDs -2026-06-12 13:33:37 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:33:37 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:33:39 | INFO | Fetched 464 rows -2026-06-12 13:33:39 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:33:40 | INFO | Fetched 142 Survey rows -2026-06-12 13:33:40 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:35:57 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:35:57 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:35:57 | INFO | -2026-06-12 13:35:59 | INFO | -2026-06-12 13:35:59 | INFO | Both databases connected successfully -2026-06-12 13:35:59 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:36:00 | INFO | Found 818 MIDs -2026-06-12 13:36:00 | INFO | ================================================================================ -2026-06-12 13:36:00 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:00 | INFO | Fetching data for 818 MIDs -2026-06-12 13:36:00 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:36:00 | INFO | ================================================================================ -2026-06-12 13:36:00 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:00 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:36:03 | INFO | Fetched 464 rows -2026-06-12 13:36:03 | INFO | ================================================================================ -2026-06-12 13:36:03 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:03 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:36:03 | INFO | Fetched 142 Survey rows -2026-06-12 13:36:03 | INFO | ================================================================================ -2026-06-12 13:36:03 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:03 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:36:04 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 13:36:04 | INFO | ================================================================================ -2026-06-12 13:36:04 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:04 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 13:36:04 | INFO | Fetched 761 rows from SQL Server -2026-06-12 13:36:04 | INFO | ================================================================================ -2026-06-12 13:36:04 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:04 | INFO | Fetching Login data for yesterday -2026-06-12 13:36:04 | INFO | Fetched 479 Login rows -2026-06-12 13:36:04 | INFO | ================================================================================ -2026-06-12 13:36:04 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:04 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 13:36:08 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 13:36:08 | INFO | ================================================================================ -2026-06-12 13:36:08 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 13:36:08 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 13:36:13 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 13:36:13 | INFO | ================================================================================ -2026-06-12 13:36:13 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:36:13 | INFO | Fetching Store Master data -2026-06-12 13:36:17 | INFO | Fetched 5,984 stores -2026-06-12 13:36:17 | INFO | ================================================================================ -2026-06-12 13:36:17 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:36:17 | INFO | Fetching SKU Master data -2026-06-12 13:36:17 | INFO | Fetched 160 SKU Master rows -2026-06-12 13:36:17 | INFO | ================================================================================ -2026-06-12 13:36:17 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:36:17 | INFO | Fetching Display Master data -2026-06-12 13:36:17 | INFO | Fetched 135 Display Master records -2026-06-12 13:36:17 | INFO | ================================================================================ -2026-06-12 13:36:17 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:36:17 | INFO | Fetching Employee Master data -2026-06-12 13:36:18 | INFO | Fetched 2,269 Employee Master records -2026-06-12 13:36:18 | INFO | ================================================================================ -2026-06-12 13:36:18 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:18 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 13:36:18 | INFO | Fetched 20,970 Journey Plan records -2026-06-12 13:36:18 | INFO | ================================================================================ -2026-06-12 13:36:18 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:18 | INFO | Fetching Coverage Remarks -2026-06-12 13:36:18 | INFO | Fetched 29 Coverage Remark records -2026-06-12 13:36:18 | INFO | ================================================================================ -2026-06-12 13:36:18 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT -2026-06-12 13:36:18 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 13:36:18 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 13:36:18 | INFO | ================================================================================ -2026-06-12 13:36:18 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:36:18 | INFO | Fetching Master Visibility Reason data -2026-06-12 13:36:19 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 13:36:19 | INFO | ================================================================================ -2026-06-12 13:36:19 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:36:19 | INFO | Fetching Master Visibility Definition data -2026-06-12 13:36:19 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 13:36:19 | INFO | ================================================================================ -2026-06-12 13:36:19 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:19 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 13:36:19 | INFO | Fetched 223 Web Login records -2026-06-12 13:36:19 | INFO | ================================================================================ -2026-06-12 13:36:19 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:19 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 13:36:22 | INFO | Fetched 8,410 Promotion records -2026-06-12 13:36:22 | INFO | ================================================================================ -2026-06-12 13:36:22 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:36:22 | INFO | Fetching Paid Visibility data for 818 MIDs -2026-06-12 13:36:30 | INFO | Fetched 1,098 Paid Visibility records -2026-06-12 13:36:30 | INFO | ================================================================================ -2026-06-12 13:36:30 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:36:30 | INFO | Fetching Master Sales Territory Layer data -2026-06-12 13:36:31 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-12 13:40:32 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:40:32 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:40:32 | INFO | -2026-06-12 13:40:34 | INFO | -2026-06-12 13:40:35 | INFO | Both databases connected successfully -2026-06-12 13:40:35 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:40:35 | INFO | Found 818 MIDs -2026-06-12 13:40:35 | INFO | ================================================================================ -2026-06-12 13:40:35 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:35 | INFO | Fetching data for 818 MIDs -2026-06-12 13:40:36 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:40:36 | INFO | ================================================================================ -2026-06-12 13:40:36 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:36 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:40:38 | INFO | Fetched 464 rows -2026-06-12 13:40:38 | INFO | ================================================================================ -2026-06-12 13:40:38 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:38 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:40:38 | INFO | Fetched 142 Survey rows -2026-06-12 13:40:38 | INFO | ================================================================================ -2026-06-12 13:40:38 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:38 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:40:39 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 13:40:39 | INFO | ================================================================================ -2026-06-12 13:40:39 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:39 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 13:40:39 | INFO | Fetched 761 rows from SQL Server -2026-06-12 13:40:39 | INFO | ================================================================================ -2026-06-12 13:40:39 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:39 | INFO | Fetching Login data for yesterday -2026-06-12 13:40:39 | INFO | Fetched 479 Login rows -2026-06-12 13:40:39 | INFO | ================================================================================ -2026-06-12 13:40:39 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:39 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 13:40:41 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 13:40:41 | INFO | ================================================================================ -2026-06-12 13:40:41 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 13:40:41 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 13:40:42 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 13:40:42 | INFO | ================================================================================ -2026-06-12 13:40:42 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:40:42 | INFO | Fetching Store Master data -2026-06-12 13:40:44 | INFO | Fetched 5,984 stores -2026-06-12 13:40:44 | INFO | ================================================================================ -2026-06-12 13:40:44 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:40:44 | INFO | Fetching SKU Master data -2026-06-12 13:40:44 | INFO | Fetched 160 SKU Master rows -2026-06-12 13:40:44 | INFO | ================================================================================ -2026-06-12 13:40:44 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:40:44 | INFO | Fetching Display Master data -2026-06-12 13:40:44 | INFO | Fetched 135 Display Master records -2026-06-12 13:40:44 | INFO | ================================================================================ -2026-06-12 13:40:44 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:40:44 | INFO | Fetching Employee Master data -2026-06-12 13:40:44 | INFO | Fetched 2,269 Employee Master records -2026-06-12 13:40:44 | INFO | ================================================================================ -2026-06-12 13:40:44 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:44 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 13:40:45 | INFO | Fetched 20,972 Journey Plan records -2026-06-12 13:40:45 | INFO | ================================================================================ -2026-06-12 13:40:45 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:45 | INFO | Fetching Coverage Remarks -2026-06-12 13:40:45 | INFO | Fetched 29 Coverage Remark records -2026-06-12 13:40:45 | INFO | ================================================================================ -2026-06-12 13:40:45 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT -2026-06-12 13:40:45 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 13:40:45 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 13:40:45 | INFO | ================================================================================ -2026-06-12 13:40:45 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:40:45 | INFO | Fetching Master Visibility Reason data -2026-06-12 13:40:45 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 13:40:45 | INFO | ================================================================================ -2026-06-12 13:40:45 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:40:45 | INFO | Fetching Master Visibility Definition data -2026-06-12 13:40:45 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 13:40:45 | INFO | ================================================================================ -2026-06-12 13:40:45 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:45 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 13:40:45 | INFO | Fetched 223 Web Login records -2026-06-12 13:40:45 | INFO | ================================================================================ -2026-06-12 13:40:45 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:45 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 13:40:48 | INFO | Fetched 8,410 Promotion records -2026-06-12 13:40:48 | INFO | ================================================================================ -2026-06-12 13:40:48 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:40:48 | INFO | Fetching Paid Visibility data for 818 MIDs -2026-06-12 13:40:55 | INFO | Fetched 1,098 Paid Visibility records -2026-06-12 13:40:55 | INFO | ================================================================================ -2026-06-12 13:40:55 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:40:55 | INFO | Fetching Master Sales Territory Layer data -2026-06-12 13:40:55 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-12 13:45:23 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:45:23 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:45:24 | INFO | -2026-06-12 13:45:26 | INFO | -2026-06-12 13:45:26 | INFO | Both databases connected successfully -2026-06-12 13:45:26 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:45:26 | INFO | Found 818 MIDs -2026-06-12 13:45:26 | INFO | ================================================================================ -2026-06-12 13:45:26 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:26 | INFO | Fetching data for 818 MIDs -2026-06-12 13:45:27 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:45:27 | INFO | ================================================================================ -2026-06-12 13:45:27 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:27 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:45:29 | INFO | Fetched 464 rows -2026-06-12 13:45:29 | INFO | ================================================================================ -2026-06-12 13:45:29 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:29 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:45:30 | INFO | Fetched 142 Survey rows -2026-06-12 13:45:30 | INFO | ================================================================================ -2026-06-12 13:45:30 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:30 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:45:31 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 13:45:31 | INFO | ================================================================================ -2026-06-12 13:45:31 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:31 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 13:45:31 | INFO | Fetched 761 rows from SQL Server -2026-06-12 13:45:31 | INFO | ================================================================================ -2026-06-12 13:45:31 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:31 | INFO | Fetching Login data for yesterday -2026-06-12 13:45:31 | INFO | Fetched 479 Login rows -2026-06-12 13:45:31 | INFO | ================================================================================ -2026-06-12 13:45:31 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:31 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 13:45:34 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 13:45:34 | INFO | ================================================================================ -2026-06-12 13:45:34 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 13:45:34 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 13:45:35 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 13:45:35 | INFO | ================================================================================ -2026-06-12 13:45:35 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:45:35 | INFO | ================================================================================ -2026-06-12 13:45:35 | INFO | Fetching Store Master data -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | Fetched 5,984 stores -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | Fetching SKU Master data -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | Fetched 160 SKU Master rows -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | Fetching Display Master data -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | Fetched 135 Display Master records -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:45:37 | INFO | ================================================================================ -2026-06-12 13:45:37 | INFO | Fetching Employee Master data -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | Fetched 2,269 Employee Master records -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:38 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 13:45:38 | INFO | Fetched 20,972 Journey Plan records -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:38 | INFO | Fetching Coverage Remarks -2026-06-12 13:45:38 | INFO | Fetched 29 Coverage Remark records -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT -2026-06-12 13:45:38 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 13:45:38 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | Fetching Master Visibility Reason data -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | Fetching Master Visibility Definition data -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 13:45:38 | INFO | ================================================================================ -2026-06-12 13:45:38 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:38 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 13:45:39 | INFO | Fetched 223 Web Login records -2026-06-12 13:45:39 | INFO | ================================================================================ -2026-06-12 13:45:39 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:39 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 13:45:42 | INFO | Fetched 8,410 Promotion records -2026-06-12 13:45:42 | INFO | ================================================================================ -2026-06-12 13:45:42 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:45:42 | INFO | Fetching Paid Visibility data for 818 MIDs -2026-06-12 13:45:49 | INFO | Fetched 1,098 Paid Visibility records -2026-06-12 13:45:49 | INFO | ================================================================================ -2026-06-12 13:45:50 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:45:50 | INFO | ================================================================================ -2026-06-12 13:45:50 | INFO | Fetching Master Sales Territory Layer data -2026-06-12 13:45:50 | INFO | ================================================================================ -2026-06-12 13:45:50 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-12 13:48:24 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 13:48:24 | INFO | ================================================================================ -2026-06-12 13:48:24 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 13:48:24 | INFO | ================================================================================ -2026-06-12 13:48:24 | INFO | -2026-06-12 13:48:26 | INFO | ================================================================================ -2026-06-12 13:48:26 | INFO | -2026-06-12 13:48:26 | INFO | ================================================================================ -2026-06-12 13:48:26 | INFO | Both databases connected successfully -2026-06-12 13:48:26 | INFO | ================================================================================ -2026-06-12 13:48:26 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 13:48:27 | INFO | ================================================================================ -2026-06-12 13:48:27 | INFO | Found 818 MIDs -2026-06-12 13:48:27 | INFO | ================================================================================ -2026-06-12 13:48:27 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:27 | INFO | ================================================================================ -2026-06-12 13:48:27 | INFO | Fetching data for 818 MIDs -2026-06-12 13:48:27 | INFO | ================================================================================ -2026-06-12 13:48:27 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 13:48:27 | INFO | ================================================================================ -2026-06-12 13:48:27 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:27 | INFO | ================================================================================ -2026-06-12 13:48:27 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 13:48:29 | INFO | ================================================================================ -2026-06-12 13:48:29 | INFO | Fetched 464 rows -2026-06-12 13:48:29 | INFO | ================================================================================ -2026-06-12 13:48:29 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:29 | INFO | ================================================================================ -2026-06-12 13:48:29 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | Fetched 142 Survey rows -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | Fetched 761 rows from SQL Server -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:30 | INFO | ================================================================================ -2026-06-12 13:48:30 | INFO | Fetching Login data for yesterday -2026-06-12 13:48:31 | INFO | ================================================================================ -2026-06-12 13:48:31 | INFO | Fetched 479 Login rows -2026-06-12 13:48:31 | INFO | ================================================================================ -2026-06-12 13:48:31 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:31 | INFO | ================================================================================ -2026-06-12 13:48:31 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 13:48:34 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 13:48:34 | INFO | ================================================================================ -2026-06-12 13:48:34 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 13:48:34 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 13:48:35 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 13:48:35 | INFO | ================================================================================ -2026-06-12 13:48:35 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:48:35 | INFO | Fetching Store Master data -2026-06-12 13:48:37 | INFO | Fetched 5,984 stores -2026-06-12 13:48:37 | INFO | ================================================================================ -2026-06-12 13:48:37 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:48:37 | INFO | Fetching SKU Master data -2026-06-12 13:48:37 | INFO | Fetched 160 SKU Master rows -2026-06-12 13:48:37 | INFO | ================================================================================ -2026-06-12 13:48:37 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:48:37 | INFO | Fetching Display Master data -2026-06-12 13:48:37 | INFO | Fetched 135 Display Master records -2026-06-12 13:48:37 | INFO | ================================================================================ -2026-06-12 13:48:37 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 13:48:37 | INFO | Fetching Employee Master data -2026-06-12 13:48:37 | INFO | Fetched 2,269 Employee Master records -2026-06-12 13:48:37 | INFO | ================================================================================ -2026-06-12 13:48:37 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:37 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 13:48:38 | INFO | Fetched 20,973 Journey Plan records -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:38 | INFO | Fetching Coverage Remarks -2026-06-12 13:48:38 | INFO | Fetched 29 Coverage Remark records -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:48:38 | INFO | Fetching Master Visibility Reason data -2026-06-12 13:48:38 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:48:38 | INFO | Fetching Master Visibility Definition data -2026-06-12 13:48:38 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | Fetched 223 Web Login records -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:38 | INFO | ================================================================================ -2026-06-12 13:48:38 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 13:48:41 | INFO | ================================================================================ -2026-06-12 13:48:41 | INFO | Fetched 8,410 Promotion records -2026-06-12 13:48:41 | INFO | ================================================================================ -2026-06-12 13:48:41 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 13:48:41 | INFO | ================================================================================ -2026-06-12 13:48:41 | INFO | Fetching Paid Visibility data for 818 MIDs -2026-06-12 13:48:48 | INFO | ================================================================================ -2026-06-12 13:48:48 | INFO | Fetched 1,098 Paid Visibility records -2026-06-12 13:48:48 | INFO | ================================================================================ -2026-06-12 13:48:48 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 13:48:48 | INFO | Fetching Master Sales Territory Layer data -2026-06-12 13:48:48 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-12 14:57:00 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 14:57:00 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 14:57:01 | INFO | ================================================================================ -2026-06-12 14:57:01 | INFO | -2026-06-12 14:57:03 | INFO | -2026-06-12 14:57:04 | INFO | Both databases connected successfully -2026-06-12 14:57:04 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 14:57:04 | INFO | Found 818 MIDs -2026-06-12 14:59:11 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 14:59:11 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 14:59:12 | INFO | ================================================================================ -2026-06-12 14:59:12 | INFO | -2026-06-12 14:59:14 | INFO | -2026-06-12 14:59:14 | INFO | Both databases connected successfully -2026-06-12 14:59:14 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 14:59:14 | INFO | Found 818 MIDs -2026-06-12 14:59:14 | INFO | ================================================================================ -2026-06-12 14:59:14 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:14 | INFO | ================================================================================ -2026-06-12 14:59:14 | INFO | Fetching data for 818 MIDs -2026-06-12 14:59:15 | INFO | ================================================================================ -2026-06-12 14:59:15 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 14:59:15 | INFO | ================================================================================ -2026-06-12 14:59:15 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:15 | INFO | ================================================================================ -2026-06-12 14:59:15 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 14:59:18 | INFO | ================================================================================ -2026-06-12 14:59:18 | INFO | Fetched 464 rows -2026-06-12 14:59:18 | INFO | ================================================================================ -2026-06-12 14:59:18 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:18 | INFO | ================================================================================ -2026-06-12 14:59:18 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 14:59:19 | INFO | ================================================================================ -2026-06-12 14:59:19 | INFO | Fetched 142 Survey rows -2026-06-12 14:59:19 | INFO | ================================================================================ -2026-06-12 14:59:19 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:19 | INFO | ================================================================================ -2026-06-12 14:59:19 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 14:59:19 | INFO | ================================================================================ -2026-06-12 14:59:19 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 14:59:19 | INFO | ================================================================================ -2026-06-12 14:59:19 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:19 | INFO | ================================================================================ -2026-06-12 14:59:19 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 14:59:20 | INFO | ================================================================================ -2026-06-12 14:59:20 | INFO | Fetched 761 rows from SQL Server -2026-06-12 14:59:20 | INFO | ================================================================================ -2026-06-12 14:59:20 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:20 | INFO | ================================================================================ -2026-06-12 14:59:20 | INFO | Fetching Login data for yesterday -2026-06-12 14:59:21 | INFO | ================================================================================ -2026-06-12 14:59:21 | INFO | Fetched 479 Login rows -2026-06-12 14:59:21 | INFO | ================================================================================ -2026-06-12 14:59:21 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:21 | INFO | ================================================================================ -2026-06-12 14:59:21 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 14:59:26 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 14:59:26 | INFO | ================================================================================ -2026-06-12 14:59:26 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 14:59:26 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 14:59:27 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 14:59:27 | INFO | ================================================================================ -2026-06-12 14:59:27 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 14:59:27 | INFO | Fetching Store Master data -2026-06-12 14:59:28 | INFO | Fetched 5,984 stores -2026-06-12 14:59:29 | INFO | ================================================================================ -2026-06-12 14:59:29 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 14:59:29 | INFO | Fetching SKU Master data -2026-06-12 14:59:29 | INFO | Fetched 160 SKU Master rows -2026-06-12 14:59:29 | INFO | ================================================================================ -2026-06-12 14:59:29 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 14:59:29 | INFO | Fetching Display Master data -2026-06-12 14:59:29 | INFO | Fetched 135 Display Master records -2026-06-12 14:59:29 | INFO | ================================================================================ -2026-06-12 14:59:29 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 14:59:29 | INFO | Fetching Employee Master data -2026-06-12 14:59:29 | INFO | Fetched 2,269 Employee Master records -2026-06-12 14:59:29 | INFO | ================================================================================ -2026-06-12 14:59:29 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:29 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 14:59:30 | INFO | Fetched 20,976 Journey Plan records -2026-06-12 14:59:30 | INFO | ================================================================================ -2026-06-12 14:59:30 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:30 | INFO | Fetching Coverage Remarks -2026-06-12 14:59:30 | INFO | Fetched 29 Coverage Remark records -2026-06-12 14:59:30 | INFO | ================================================================================ -2026-06-12 14:59:30 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT -2026-06-12 14:59:30 | INFO | ================================================================================ -2026-06-12 14:59:30 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 14:59:30 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 14:59:30 | INFO | ================================================================================ -2026-06-12 14:59:30 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 14:59:30 | INFO | Fetching Master Visibility Reason data -2026-06-12 14:59:30 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 14:59:30 | INFO | ================================================================================ -2026-06-12 14:59:30 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 14:59:30 | INFO | Fetching Master Visibility Definition data -2026-06-12 14:59:30 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 14:59:30 | INFO | ================================================================================ -2026-06-12 14:59:30 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:30 | INFO | ================================================================================ -2026-06-12 14:59:30 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 14:59:31 | INFO | Fetched 223 Web Login records -2026-06-12 14:59:31 | INFO | ================================================================================ -2026-06-12 14:59:31 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:31 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 14:59:33 | INFO | Fetched 8,410 Promotion records -2026-06-12 14:59:33 | INFO | ================================================================================ -2026-06-12 14:59:33 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 14:59:33 | INFO | Fetching Paid Visibility data for 818 MIDs -2026-06-12 14:59:41 | INFO | Fetched 1,098 Paid Visibility records -2026-06-12 14:59:41 | INFO | ================================================================================ -2026-06-12 14:59:41 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 14:59:41 | INFO | Fetching Master Sales Territory Layer data -2026-06-12 14:59:41 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-12 15:20:11 | INFO | ================================================================================ -2026-06-12 15:20:11 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 15:20:11 | INFO | Data-pipeline running Date is -:2026-06-11 -2026-06-12 15:20:11 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 15:20:12 | INFO | -2026-06-12 15:20:13 | INFO | -2026-06-12 15:20:14 | INFO | Both databases connected successfully -2026-06-12 15:20:14 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 15:20:14 | INFO | Found 818 MIDs -2026-06-12 15:20:14 | INFO | ================================================================================ -2026-06-12 15:20:14 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:14 | INFO | Fetching data for 818 MIDs -2026-06-12 15:20:16 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 15:20:16 | INFO | ================================================================================ -2026-06-12 15:20:16 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:16 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 15:20:20 | INFO | Fetched 464 rows -2026-06-12 15:20:20 | INFO | ================================================================================ -2026-06-12 15:20:20 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:20 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 15:20:21 | INFO | Fetched 142 Survey rows -2026-06-12 15:20:21 | INFO | ================================================================================ -2026-06-12 15:20:21 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:21 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 15:20:22 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 15:20:22 | INFO | ================================================================================ -2026-06-12 15:20:22 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:22 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 15:20:24 | INFO | Fetched 761 rows from SQL Server -2026-06-12 15:20:24 | INFO | ================================================================================ -2026-06-12 15:20:24 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:24 | INFO | Fetching Login data for yesterday -2026-06-12 15:20:25 | INFO | Fetched 479 Login rows -2026-06-12 15:20:25 | INFO | ================================================================================ -2026-06-12 15:20:25 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:25 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 15:20:30 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 15:20:30 | INFO | ================================================================================ -2026-06-12 15:20:30 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 15:20:30 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 15:20:31 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 15:20:31 | INFO | ================================================================================ -2026-06-12 15:20:31 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 15:20:31 | INFO | Fetching Store Master data -2026-06-12 15:20:33 | INFO | Fetched 5,984 stores -2026-06-12 15:20:33 | INFO | ================================================================================ -2026-06-12 15:20:33 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 15:20:33 | INFO | Fetching SKU Master data -2026-06-12 15:20:33 | INFO | Fetched 160 SKU Master rows -2026-06-12 15:20:33 | INFO | ================================================================================ -2026-06-12 15:20:33 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 15:20:33 | INFO | Fetching Display Master data -2026-06-12 15:20:33 | INFO | Fetched 135 Display Master records -2026-06-12 15:20:33 | INFO | ================================================================================ -2026-06-12 15:20:33 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 15:20:33 | INFO | Fetching Employee Master data -2026-06-12 15:20:34 | INFO | Fetched 2,269 Employee Master records -2026-06-12 15:20:34 | INFO | ================================================================================ -2026-06-12 15:20:34 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:34 | INFO | Fetching Journey Plan for 2026-06 -2026-06-12 15:20:35 | INFO | Fetched 20,979 Journey Plan records -2026-06-12 15:20:35 | INFO | ================================================================================ -2026-06-12 15:20:35 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:35 | INFO | Fetching Coverage Remarks -2026-06-12 15:20:35 | INFO | Fetched 29 Coverage Remark records -2026-06-12 15:20:35 | INFO | ================================================================================ -2026-06-12 15:20:35 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT -2026-06-12 15:20:35 | INFO | Fetching Mapping Store Visibility for 2026-06-11 -2026-06-12 15:20:35 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-12 15:20:35 | INFO | ================================================================================ -2026-06-12 15:20:35 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 15:20:35 | INFO | Fetching Master Visibility Reason data -2026-06-12 15:20:35 | INFO | Fetched 17 Master Visibility Reason records -2026-06-12 15:20:35 | INFO | ================================================================================ -2026-06-12 15:20:35 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 15:20:35 | INFO | Fetching Master Visibility Definition data -2026-06-12 15:20:35 | INFO | Fetched 861 Master Visibility Definition records -2026-06-12 15:20:35 | INFO | ================================================================================ -2026-06-12 15:20:35 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:35 | INFO | Fetching Web Login data for 2026-06-11 -2026-06-12 15:20:36 | INFO | Fetched 223 Web Login records -2026-06-12 15:20:36 | INFO | ================================================================================ -2026-06-12 15:20:36 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:36 | INFO | Fetching Promotion data for 818 MIDs -2026-06-12 15:20:39 | INFO | Fetched 8,410 Promotion records -2026-06-12 15:20:39 | INFO | ================================================================================ -2026-06-12 15:20:39 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:20:39 | INFO | Fetching Paid Visibility data for 818 MIDs -2026-06-12 15:20:47 | INFO | Fetched 1,098 Paid Visibility records -2026-06-12 15:20:47 | INFO | ================================================================================ -2026-06-12 15:20:47 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT -2026-06-12 15:20:47 | INFO | Fetching Master Sales Territory Layer data -2026-06-12 15:20:47 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-12 15:41:09 | INFO | ================================================================================ -2026-06-12 15:41:09 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 15:41:09 | INFO | Data-pipeline running Date is -:2026-06-11 -2026-06-12 15:41:09 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 15:41:10 | INFO | -2026-06-12 15:41:12 | INFO | -2026-06-12 15:41:12 | INFO | Both databases connected successfully -2026-06-12 15:41:12 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 15:41:13 | INFO | Found 818 MIDs -2026-06-12 15:41:13 | INFO | ================================================================================ -2026-06-12 15:41:13 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:41:13 | INFO | Fetching data for 818 MIDs -2026-06-12 15:41:13 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 15:41:13 | INFO | ================================================================================ -2026-06-12 15:41:13 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:41:13 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 15:41:16 | INFO | Fetched 464 rows -2026-06-12 15:41:16 | INFO | ================================================================================ -2026-06-12 15:41:16 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:41:16 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 15:41:17 | INFO | Fetched 142 Survey rows -2026-06-12 15:41:17 | INFO | ================================================================================ -2026-06-12 15:41:17 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:41:17 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 15:41:18 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 15:41:18 | INFO | ================================================================================ -2026-06-12 15:41:18 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:41:18 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 15:41:19 | INFO | Fetched 761 rows from SQL Server -2026-06-12 15:41:19 | INFO | ================================================================================ -2026-06-12 15:41:19 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:41:19 | INFO | Fetching Login data for yesterday -2026-06-12 15:41:19 | INFO | Fetched 479 Login rows -2026-06-12 15:41:19 | INFO | ================================================================================ -2026-06-12 15:41:19 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:41:19 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 15:41:25 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 15:41:25 | INFO | ================================================================================ -2026-06-12 15:41:25 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 15:41:25 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 15:41:25 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 15:44:38 | INFO | ================================================================================ -2026-06-12 15:44:38 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 15:44:38 | INFO | Data-pipeline running Date is -:2026-06-11 -2026-06-12 15:44:38 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 15:44:40 | INFO | -2026-06-12 15:44:42 | INFO | -2026-06-12 15:44:42 | INFO | Both databases connected successfully -2026-06-12 15:44:42 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 15:44:42 | INFO | Found 818 MIDs -2026-06-12 15:44:42 | INFO | ================================================================================ -2026-06-12 15:44:42 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:44:42 | INFO | Fetching data for 818 MIDs -2026-06-12 15:44:43 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 15:44:43 | INFO | ================================================================================ -2026-06-12 15:44:43 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:44:43 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 15:44:46 | INFO | Fetched 464 rows -2026-06-12 15:44:46 | INFO | ================================================================================ -2026-06-12 15:44:46 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:44:46 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 15:44:46 | INFO | Fetched 142 Survey rows -2026-06-12 15:44:46 | INFO | ================================================================================ -2026-06-12 15:44:46 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:44:46 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 15:44:47 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 15:44:47 | INFO | ================================================================================ -2026-06-12 15:44:47 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:44:47 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 15:44:47 | INFO | Fetched 761 rows from SQL Server -2026-06-12 15:44:47 | INFO | ================================================================================ -2026-06-12 15:44:47 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:44:47 | INFO | Fetching Login data for yesterday -2026-06-12 15:44:48 | INFO | Fetched 479 Login rows -2026-06-12 15:44:48 | INFO | ================================================================================ -2026-06-12 15:44:48 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 15:44:48 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 15:44:50 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 15:44:50 | INFO | ================================================================================ -2026-06-12 15:44:50 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 15:44:50 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 15:44:51 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 19:03:59 | INFO | ================================================================================ -2026-06-12 19:03:59 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 19:03:59 | INFO | Data-pipeline running Date is -:2026-06-11 -2026-06-12 19:03:59 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 19:04:01 | INFO | -2026-06-12 19:04:03 | INFO | -2026-06-12 19:04:03 | INFO | Both databases connected successfully -2026-06-12 19:04:03 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 19:04:05 | INFO | Found 818 MIDs -2026-06-12 19:04:05 | INFO | ================================================================================ -2026-06-12 19:04:05 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:04:05 | INFO | Fetching data for 818 MIDs -2026-06-12 19:04:12 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 19:04:12 | INFO | Table ready: SOS_OneApp -2026-06-12 19:04:13 | INFO | SOS_OneApp: inserted 3,677 rows into ClickHouse -2026-06-12 19:04:13 | INFO | ================================================================================ -2026-06-12 19:04:13 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:04:13 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 19:04:26 | INFO | Fetched 464 rows -2026-06-12 19:04:27 | INFO | Table ready: OQaD -2026-06-12 19:04:27 | INFO | OQaD: inserted 464 rows into ClickHouse -2026-06-12 19:04:27 | INFO | ================================================================================ -2026-06-12 19:04:27 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:04:27 | INFO | Fetching Survey data for 818 MIDs -2026-06-12 19:04:28 | INFO | Fetched 142 Survey rows -2026-06-12 19:04:29 | INFO | Table ready: Survey -2026-06-12 19:04:29 | INFO | Survey: inserted 142 rows into ClickHouse -2026-06-12 19:04:29 | INFO | ================================================================================ -2026-06-12 19:04:29 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:04:29 | INFO | Fetching Additional Visibility data for 818 MIDs -2026-06-12 19:04:30 | INFO | Fetched 1,922 Additional Visibility rows -2026-06-12 19:04:30 | INFO | Table ready: additional_visibility -2026-06-12 19:04:31 | INFO | additional_visibility: inserted 1,922 rows into ClickHouse -2026-06-12 19:04:31 | INFO | ================================================================================ -2026-06-12 19:04:31 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:04:31 | INFO | Fetching coverage data for 818 MIDs -2026-06-12 19:04:34 | INFO | Fetched 761 rows from SQL Server -2026-06-12 19:04:35 | INFO | Table ready: Coverage -2026-06-12 19:04:35 | INFO | Coverage: inserted 761 rows into ClickHouse -2026-06-12 19:04:35 | INFO | ================================================================================ -2026-06-12 19:04:35 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:04:35 | INFO | Fetching Login data for yesterday -2026-06-12 19:04:36 | INFO | Fetched 479 Login rows -2026-06-12 19:04:36 | INFO | Table ready: Login -2026-06-12 19:04:36 | INFO | Login: inserted 479 rows into ClickHouse -2026-06-12 19:04:36 | INFO | ================================================================================ -2026-06-12 19:04:36 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:04:36 | INFO | Fetching Stock Details data for 818 MIDs -2026-06-12 19:04:49 | INFO | Fetched 41,628 Stock Details rows -2026-06-12 19:04:49 | INFO | Table ready: Stock_Details -2026-06-12 19:04:49 | INFO | Stock_Details: inserted 41,628 rows into ClickHouse -2026-06-12 19:04:49 | INFO | ================================================================================ -2026-06-12 19:04:49 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT -2026-06-12 19:04:49 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 -2026-06-12 19:04:50 | INFO | Fetched 11,918 attendance rows for 592 employees -2026-06-12 19:04:51 | INFO | Table ready: Attendance -2026-06-12 19:04:51 | INFO | Attendance: inserted 11,918 rows into ClickHouse -2026-06-12 19:04:51 | INFO | ================================================================================ -2026-06-12 19:04:51 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT -2026-06-12 19:04:51 | INFO | Fetching Store Master data -2026-06-12 19:04:53 | INFO | Fetched 5,984 stores -2026-06-12 19:04:53 | INFO | Table Store_Master truncated successfully. -2026-06-12 19:04:53 | INFO | Truncate a ClickHouse table - Store_Master -2026-06-12 19:07:43 | INFO | ================================================================================ -2026-06-12 19:07:43 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 19:07:43 | INFO | Data-pipeline running Date is -:2026-06-11 -2026-06-12 19:07:43 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 19:07:44 | INFO | -2026-06-12 19:07:45 | INFO | -2026-06-12 19:07:46 | INFO | Both databases connected successfully -2026-06-12 19:07:46 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 19:07:46 | INFO | Found 818 MIDs -2026-06-12 19:07:46 | INFO | ================================================================================ -2026-06-12 19:07:46 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:07:46 | INFO | Fetching data for 818 MIDs -2026-06-12 19:07:47 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 19:07:47 | INFO | Delete command submitted successfully. -2026-06-12 19:07:47 | INFO | SOS_OneApp: inserted 3,677 rows into ClickHouse -2026-06-12 19:07:47 | INFO | ================================================================================ -2026-06-12 19:07:47 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:07:47 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 19:07:50 | INFO | Fetched 464 rows -2026-06-12 19:09:36 | INFO | ================================================================================ -2026-06-12 19:09:36 | INFO | Hello from data-move Python data pipeline ! -2026-06-12 19:09:36 | INFO | Data-pipeline running Date is -:2026-06-11 -2026-06-12 19:09:36 | INFO | connecting with both db servers sql-serveras well as clickhouse DB -2026-06-12 19:09:37 | INFO | -2026-06-12 19:09:38 | INFO | -2026-06-12 19:09:39 | INFO | Both databases connected successfully -2026-06-12 19:09:39 | INFO | Collecting MIDs for: 2026-06-11 -2026-06-12 19:09:39 | INFO | Found 818 MIDs -2026-06-12 19:09:39 | INFO | ================================================================================ -2026-06-12 19:09:39 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:09:39 | INFO | Fetching data for 818 MIDs -2026-06-12 19:09:40 | INFO | Fetched 3,677 rows from SQL Server -2026-06-12 19:09:40 | INFO | Deleting rows from SOS_OneApp -2026-06-12 19:09:40 | INFO | Delete command submitted successfully. -2026-06-12 19:09:41 | INFO | SOS_OneApp: inserted 3,677 rows into ClickHouse -2026-06-12 19:09:41 | INFO | ================================================================================ -2026-06-12 19:09:41 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT -2026-06-12 19:09:41 | INFO | Fetching OQaD data for 818 MIDs -2026-06-12 19:09:44 | INFO | Fetched 464 rows -2026-06-12 19:09:44 | INFO | Deleting rows from OQaD diff --git a/logs/etl_20260615.log b/logs/etl_20260615.log deleted file mode 100644 index d3342e2..0000000 --- a/logs/etl_20260615.log +++ /dev/null @@ -1,3583 +0,0 @@ -2026-06-15 11:21:29 | INFO | ================================================================================ -2026-06-15 11:21:29 | INFO | Hello from data-move Python data pipeline ! -2026-06-15 11:21:29 | INFO | Pipeline Run Date : 2026-06-14 -2026-06-15 11:21:29 | INFO | Connecting to SQL Server and ClickHouse -2026-06-15 11:21:31 | INFO | -2026-06-15 11:21:34 | INFO | -2026-06-15 11:21:34 | INFO | Database connections established -2026-06-15 11:21:34 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:21:35 | INFO | Found 571 MIDs -2026-06-15 11:21:35 | INFO | Deleting data from additional_visibility -2026-06-15 11:27:46 | INFO | ================================================================================ -2026-06-15 11:27:46 | INFO | Hello from data-move Python data pipeline ! -2026-06-15 11:27:46 | INFO | Pipeline Run Date : 2026-06-14 -2026-06-15 11:27:46 | INFO | Connecting to SQL Server and ClickHouse -2026-06-15 11:27:46 | INFO | -2026-06-15 11:27:48 | INFO | -2026-06-15 11:27:48 | INFO | Database connections established -2026-06-15 11:27:48 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:27:49 | INFO | Found 571 MIDs -2026-06-15 11:27:49 | INFO | Deleting data from additional_visibility -2026-06-15 11:27:49 | INFO | Deleting data from Coverage -2026-06-15 11:27:49 | INFO | Deleting data from Survey -2026-06-15 11:27:49 | INFO | Deleting data from Promotion -2026-06-15 11:41:04 | INFO | ================================================================================ -2026-06-15 11:41:04 | INFO | Hello from data-move Python data pipeline! -2026-06-15 11:41:04 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 11:41:04 | INFO | Connecting to databases... -2026-06-15 11:41:05 | INFO | -2026-06-15 11:41:06 | INFO | -2026-06-15 11:41:07 | INFO | Database connections established -2026-06-15 11:41:07 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:41:07 | INFO | Found 571 MIDs -2026-06-15 11:41:07 | INFO | ================================================================================ -2026-06-15 11:41:07 | INFO | Processing Table: SOS_OneApp -2026-06-15 11:41:07 | INFO | Fetching data for 571 MIDs -2026-06-15 11:41:10 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 11:41:10 | INFO | Fetched 1995 rows -2026-06-15 11:41:10 | INFO | Deleting from SOS_OneApp -2026-06-15 11:41:10 | ERROR | Failed processing table SOS_OneApp -Traceback (most recent call last): - File "D:\data_move\test2.py", line 232, in main - load_to_clickhouse( - ~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - table_name=table_name, - ^^^^^^^^^^^^^^^^^^^^^^ - df=df, - ^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\load_table.py", line 22, in load_to_clickhouse - arrow_table = df.to_arrow() - File "d:\data_move\.venv\Lib\site-packages\polars\_utils\deprecation.py", line 128, in wrapper - return function(*args, **kwargs) - File "d:\data_move\.venv\Lib\site-packages\polars\dataframe\frame.py", line 1797, in to_arrow - record_batches = self._df.to_arrow(compat_level_py) -ModuleNotFoundError: No module named 'pyarrow' -2026-06-15 11:42:30 | INFO | ================================================================================ -2026-06-15 11:42:30 | INFO | Hello from data-move Python data pipeline! -2026-06-15 11:42:30 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 11:42:30 | INFO | Connecting to databases... -2026-06-15 11:42:31 | INFO | -2026-06-15 11:42:33 | INFO | -2026-06-15 11:42:33 | INFO | Database connections established -2026-06-15 11:42:33 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:42:34 | INFO | Found 571 MIDs -2026-06-15 11:42:34 | INFO | ================================================================================ -2026-06-15 11:42:34 | INFO | Processing Table: SOS_OneApp -2026-06-15 11:42:34 | INFO | Fetching data for 571 MIDs -2026-06-15 11:42:34 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 11:42:34 | INFO | Fetched 1995 rows -2026-06-15 11:42:35 | INFO | Deleting from SOS_OneApp -2026-06-15 11:42:35 | ERROR | Failed processing table SOS_OneApp -Traceback (most recent call last): - File "D:\data_move\test2.py", line 233, in main - load_to_clickhouse( - ~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - table_name=table_name, - ^^^^^^^^^^^^^^^^^^^^^^ - df=df, - ^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\load_table.py", line 22, in load_to_clickhouse - arrow_table = df.to_arrow() - File "d:\data_move\.venv\Lib\site-packages\polars\_utils\deprecation.py", line 128, in wrapper - return function(*args, **kwargs) - File "d:\data_move\.venv\Lib\site-packages\polars\dataframe\frame.py", line 1797, in to_arrow - record_batches = self._df.to_arrow(compat_level_py) -ModuleNotFoundError: No module named 'pyarrow' -2026-06-15 11:45:18 | INFO | ================================================================================ -2026-06-15 11:45:18 | INFO | Hello from data-move Python data pipeline! -2026-06-15 11:45:18 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 11:45:18 | INFO | Connecting to databases... -2026-06-15 11:45:20 | INFO | -2026-06-15 11:45:24 | INFO | -2026-06-15 11:45:24 | INFO | Database connections established -2026-06-15 11:45:24 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:45:25 | INFO | Found 571 MIDs -2026-06-15 11:45:25 | INFO | ================================================================================ -2026-06-15 11:45:25 | INFO | Processing Table: SOS_OneApp -2026-06-15 11:45:25 | INFO | Fetching data for 571 MIDs -2026-06-15 11:45:26 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 11:45:26 | INFO | Fetched 1995 rows -2026-06-15 11:45:26 | INFO | Deleting from SOS_OneApp -2026-06-15 11:45:26 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse -2026-06-15 11:45:26 | INFO | SOS_OneApp loaded successfully (1995 rows) -2026-06-15 11:45:26 | INFO | ================================================================================ -2026-06-15 11:45:26 | INFO | Processing Table: OQaD -2026-06-15 11:45:26 | INFO | Fetching OQaD data for run_date=[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650] -2026-06-15 11:45:27 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.DataError: ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\test2.py", line 177, in main - df = get_dataframe( - fn_name=fn_name, - ...<3 lines>... - run_date=run_date, - ) - File "D:\data_move\test2.py", line 78, in get_dataframe - return fn(sql_engine, mids) - File "D:\data_move\kpi\facts.py", line 236, in fetch_OQaD - df = pl.read_database( - query=sql, - connection=engine, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.DataError: (pyodbc.DataError) ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') -[SQL: - WITH MID_TABLE_COV1 AS - ( - SELECT - EmpId, - CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CAST(CreateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - - UNION - - SELECT - EmpId, - CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CAST(UpdateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - ), - - QUIZ AS - ( - SELECT DISTINCT - E.EmpId, - CAST(DQ.VisitDate AS DATE) AS VisitDate, - DQ.QuestionId, - DQ.AnswerId, - QC.QuestionCategoryId, - QC.QuestionCategory - - FROM OneApp_KelloggsMT.dbo.T_OQAD DQ - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E - ON DQ.EmpId = E.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU - ON DQ.QuestionId = QU.QuestionId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC - ON QU.QuestionCategoryId = QC.QuestionCategoryId - - WHERE E.RightId = 6 - AND E.EmpName NOT LIKE 'test%' - AND E.EmpName NOT LIKE '%TEST%' - AND ( - E.ResignDate IS NULL - OR E.ResignDate >= '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - ) - AND EXISTS - ( - SELECT 1 - FROM MID_TABLE_COV1 A - WHERE A.EmpId = DQ.EmpId - AND A.VisitDate = CAST(DQ.VisitDate AS DATE) - ) - ) - - SELECT - 40148 AS project_id, - Q.EmpId AS employee_id, - 0 AS process_id, - Q.VisitDate AS visit_date, - Q.QuestionCategoryId AS question_category_id, - Q.QuestionCategory AS question_category, - QM.QuestionId AS question_id, - QM.Question AS question, - ISNULL(QA.AnswerId, 0) AS answer_id, - ISNULL(QA.Answer, '') AS answer, - - CASE - WHEN QA.AnswerId IS NULL THEN 'Not Answer' - WHEN QA.RightAnswer = 1 THEN 'Y' - WHEN QA.RightAnswer IS NULL THEN 'Not Answer' - ELSE 'N' - END AS correct_answer, - - GETDATE() AS update_date, - 'SP-Pius' AS update_by - - FROM QUIZ Q - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM - ON Q.QuestionId = QM.QuestionId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA - ON Q.AnswerId = QA.AnswerId - ] -(Background on this error at: https://sqlalche.me/e/20/9h9h) -2026-06-15 11:47:45 | INFO | ================================================================================ -2026-06-15 11:47:45 | INFO | Hello from data-move Python data pipeline! -2026-06-15 11:47:45 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 11:47:45 | INFO | Connecting to databases... -2026-06-15 11:47:46 | INFO | -2026-06-15 11:47:48 | INFO | -2026-06-15 11:47:48 | INFO | Database connections established -2026-06-15 11:47:48 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:47:49 | INFO | Found 571 MIDs -2026-06-15 11:47:49 | INFO | ================================================================================ -2026-06-15 11:47:49 | INFO | Processing Table: SOS_OneApp -2026-06-15 11:47:49 | INFO | Fetching data for 571 MIDs -2026-06-15 11:47:49 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 11:47:49 | INFO | Fetched 1995 rows -2026-06-15 11:47:50 | INFO | Deleting from SOS_OneApp -2026-06-15 11:47:50 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse -2026-06-15 11:47:50 | INFO | SOS_OneApp loaded successfully (1995 rows) -2026-06-15 11:47:50 | INFO | ================================================================================ -2026-06-15 11:47:50 | INFO | Processing Table: OQaD -2026-06-15 11:47:50 | INFO | Fetching OQaD data for run_date=[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650] -2026-06-15 11:47:50 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.DataError: ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\test2.py", line 177, in main - df = get_dataframe( - fn_name=fn_name, - ...<3 lines>... - run_date=run_date, - ) - File "D:\data_move\test2.py", line 78, in get_dataframe - return fn(sql_engine, mids) - File "D:\data_move\kpi\facts.py", line 236, in fetch_OQaD - df = pl.read_database( - query=sql, - connection=engine, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.DataError: (pyodbc.DataError) ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') -[SQL: - WITH MID_TABLE_COV1 AS - ( - SELECT - EmpId, - CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CAST(CreateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - - UNION - - SELECT - EmpId, - CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CAST(UpdateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - ), - - QUIZ AS - ( - SELECT DISTINCT - E.EmpId, - CAST(DQ.VisitDate AS DATE) AS VisitDate, - DQ.QuestionId, - DQ.AnswerId, - QC.QuestionCategoryId, - QC.QuestionCategory - - FROM OneApp_KelloggsMT.dbo.T_OQAD DQ - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E - ON DQ.EmpId = E.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU - ON DQ.QuestionId = QU.QuestionId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC - ON QU.QuestionCategoryId = QC.QuestionCategoryId - - WHERE E.RightId = 6 - AND E.EmpName NOT LIKE 'test%' - AND E.EmpName NOT LIKE '%TEST%' - AND ( - E.ResignDate IS NULL - OR E.ResignDate >= '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - ) - AND EXISTS - ( - SELECT 1 - FROM MID_TABLE_COV1 A - WHERE A.EmpId = DQ.EmpId - AND A.VisitDate = CAST(DQ.VisitDate AS DATE) - ) - ) - - SELECT - 40148 AS project_id, - Q.EmpId AS employee_id, - 0 AS process_id, - Q.VisitDate AS visit_date, - Q.QuestionCategoryId AS question_category_id, - Q.QuestionCategory AS question_category, - QM.QuestionId AS question_id, - QM.Question AS question, - ISNULL(QA.AnswerId, 0) AS answer_id, - ISNULL(QA.Answer, '') AS answer, - - CASE - WHEN QA.AnswerId IS NULL THEN 'Not Answer' - WHEN QA.RightAnswer = 1 THEN 'Y' - WHEN QA.RightAnswer IS NULL THEN 'Not Answer' - ELSE 'N' - END AS correct_answer, - - GETDATE() AS update_date, - 'SP-Pius' AS update_by - - FROM QUIZ Q - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM - ON Q.QuestionId = QM.QuestionId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA - ON Q.AnswerId = QA.AnswerId - ] -(Background on this error at: https://sqlalche.me/e/20/9h9h) -2026-06-15 11:49:49 | INFO | ================================================================================ -2026-06-15 11:49:49 | INFO | Hello from data-move Python data pipeline! -2026-06-15 11:49:49 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 11:49:49 | INFO | Connecting to databases... -2026-06-15 11:49:50 | INFO | -2026-06-15 11:49:51 | INFO | -2026-06-15 11:49:52 | INFO | Database connections established -2026-06-15 11:49:52 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:49:52 | INFO | Found 571 MIDs -2026-06-15 11:49:52 | INFO | ================================================================================ -2026-06-15 11:49:52 | INFO | Processing Table: SOS_OneApp -2026-06-15 11:49:52 | INFO | Fetching data for 571 MIDs -2026-06-15 11:49:52 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 11:49:52 | INFO | Fetched 1995 rows -2026-06-15 11:49:52 | INFO | Deleting from SOS_OneApp -2026-06-15 11:49:53 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse -2026-06-15 11:49:53 | INFO | SOS_OneApp loaded successfully (1995 rows) -2026-06-15 11:49:53 | INFO | ================================================================================ -2026-06-15 11:49:53 | INFO | Processing Table: OQaD -2026-06-15 11:49:53 | INFO | Fetching OQaD data for run_date=[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650] -2026-06-15 11:49:53 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.DataError: ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\test2.py", line 177, in main - df = get_dataframe( - fn_name=fn_name, - ...<3 lines>... - run_date=run_date, - ) - File "D:\data_move\test2.py", line 78, in get_dataframe - return fn(sql_engine, mids) - File "D:\data_move\kpi\facts.py", line 236, in fetch_OQaD - df = pl.read_database( - query=sql, - connection=engine, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.DataError: (pyodbc.DataError) ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') -[SQL: - WITH MID_TABLE_COV1 AS - ( - SELECT - EmpId, - CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CAST(CreateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - - UNION - - SELECT - EmpId, - CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CAST(UpdateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - ), - - QUIZ AS - ( - SELECT DISTINCT - E.EmpId, - CAST(DQ.VisitDate AS DATE) AS VisitDate, - DQ.QuestionId, - DQ.AnswerId, - QC.QuestionCategoryId, - QC.QuestionCategory - - FROM OneApp_KelloggsMT.dbo.T_OQAD DQ - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E - ON DQ.EmpId = E.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU - ON DQ.QuestionId = QU.QuestionId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC - ON QU.QuestionCategoryId = QC.QuestionCategoryId - - WHERE E.RightId = 6 - AND E.EmpName NOT LIKE 'test%' - AND E.EmpName NOT LIKE '%TEST%' - AND ( - E.ResignDate IS NULL - OR E.ResignDate >= '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' - ) - AND EXISTS - ( - SELECT 1 - FROM MID_TABLE_COV1 A - WHERE A.EmpId = DQ.EmpId - AND A.VisitDate = CAST(DQ.VisitDate AS DATE) - ) - ) - - SELECT - 40148 AS project_id, - Q.EmpId AS employee_id, - 0 AS process_id, - Q.VisitDate AS visit_date, - Q.QuestionCategoryId AS question_category_id, - Q.QuestionCategory AS question_category, - QM.QuestionId AS question_id, - QM.Question AS question, - ISNULL(QA.AnswerId, 0) AS answer_id, - ISNULL(QA.Answer, '') AS answer, - - CASE - WHEN QA.AnswerId IS NULL THEN 'Not Answer' - WHEN QA.RightAnswer = 1 THEN 'Y' - WHEN QA.RightAnswer IS NULL THEN 'Not Answer' - ELSE 'N' - END AS correct_answer, - - GETDATE() AS update_date, - 'SP-Pius' AS update_by - - FROM QUIZ Q - - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM - ON Q.QuestionId = QM.QuestionId - - LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA - ON Q.AnswerId = QA.AnswerId - ] -(Background on this error at: https://sqlalche.me/e/20/9h9h) -2026-06-15 11:51:58 | INFO | ================================================================================ -2026-06-15 11:51:58 | INFO | Hello from data-move Python data pipeline! -2026-06-15 11:51:58 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 11:51:58 | INFO | Connecting to databases... -2026-06-15 11:51:59 | INFO | -2026-06-15 11:52:01 | INFO | -2026-06-15 11:52:01 | INFO | Database connections established -2026-06-15 11:52:01 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:52:01 | INFO | Found 571 MIDs -2026-06-15 11:52:02 | INFO | ================================================================================ -2026-06-15 11:52:02 | INFO | Processing Table: SOS_OneApp -2026-06-15 11:52:02 | INFO | Fetching data for 571 MIDs -2026-06-15 11:52:02 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 11:52:02 | INFO | Fetched 1995 rows -2026-06-15 11:52:02 | INFO | Deleting from SOS_OneApp -2026-06-15 11:52:02 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse -2026-06-15 11:52:02 | INFO | SOS_OneApp loaded successfully (1995 rows) -2026-06-15 11:52:02 | INFO | ================================================================================ -2026-06-15 11:52:02 | INFO | Processing Table: OQaD -2026-06-15 11:52:02 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 11:52:06 | INFO | Fetched 443 rows -2026-06-15 11:52:06 | INFO | Fetched 443 rows -2026-06-15 11:52:07 | INFO | Deleting from OQaD -2026-06-15 11:52:07 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\test2.py", line 248, in main - load_to_clickhouse( - ~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - table_name=table_name, - ^^^^^^^^^^^^^^^^^^^^^^ - df=df, - ^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse - client.insert_arrow( - ~~~~~~~~~~~~~~~~~~~^ - table=table_name, - ^^^^^^^^^^^^^^^^^ - arrow_table=arrow_table, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow - return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) - ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert - response = self._raw_request(insert_block, params, headers, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs.OQaD (baea0f0f-5216-49dd-b156-fa06fe3e62f1). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) -2026-06-15 11:55:57 | INFO | ================================================================================ -2026-06-15 11:55:57 | INFO | Hello from data-move Python data pipeline! -2026-06-15 11:55:57 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 11:55:57 | INFO | Connecting to databases... -2026-06-15 11:55:58 | INFO | -2026-06-15 11:56:00 | INFO | -2026-06-15 11:56:00 | INFO | Database connections established -2026-06-15 11:56:00 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:56:00 | INFO | Found 571 MIDs -2026-06-15 11:56:01 | INFO | ================================================================================ -2026-06-15 11:56:01 | INFO | Processing Table: SOS_OneApp -2026-06-15 11:56:01 | INFO | Fetching data for 571 MIDs -2026-06-15 11:56:01 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 11:56:01 | INFO | Fetched 1995 rows -2026-06-15 11:56:01 | INFO | Creating table SOS_OneApp -2026-06-15 11:56:01 | INFO | Table ready: SOS_OneApp -2026-06-15 11:56:02 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse -2026-06-15 11:56:02 | INFO | SOS_OneApp loaded successfully (1995 rows) -2026-06-15 11:56:02 | INFO | ================================================================================ -2026-06-15 11:56:02 | INFO | Processing Table: OQaD -2026-06-15 11:56:02 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 11:56:04 | INFO | Fetched 443 rows -2026-06-15 11:56:04 | INFO | Fetched 443 rows -2026-06-15 11:56:04 | INFO | Creating table OQaD -2026-06-15 11:56:04 | INFO | Table ready: OQaD -2026-06-15 11:56:04 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 11:56:04 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 11:56:04 | INFO | ================================================================================ -2026-06-15 11:56:04 | INFO | Processing Table: Survey -2026-06-15 11:56:04 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 11:56:07 | INFO | Fetched 173 Survey rows -2026-06-15 11:56:07 | INFO | Fetched 173 rows -2026-06-15 11:56:07 | INFO | Creating table Survey -2026-06-15 11:56:07 | INFO | Table ready: Survey -2026-06-15 11:56:07 | INFO | Survey: inserted 173 rows into ClickHouse -2026-06-15 11:56:07 | INFO | Survey loaded successfully (173 rows) -2026-06-15 11:56:07 | INFO | ================================================================================ -2026-06-15 11:56:07 | INFO | Processing Table: additional_visibility -2026-06-15 11:56:07 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 11:56:17 | INFO | Fetched 1,648 Additional Visibility rows -2026-06-15 11:56:17 | INFO | Fetched 1648 rows -2026-06-15 11:56:17 | INFO | Creating table additional_visibility -2026-06-15 11:56:18 | INFO | Table ready: additional_visibility -2026-06-15 11:56:18 | INFO | additional_visibility: inserted 1,648 rows into ClickHouse -2026-06-15 11:56:18 | INFO | additional_visibility loaded successfully (1648 rows) -2026-06-15 11:56:18 | INFO | ================================================================================ -2026-06-15 11:56:18 | INFO | Processing Table: Coverage -2026-06-15 11:56:18 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 11:56:20 | INFO | Fetched 514 rows from SQL Server -2026-06-15 11:56:20 | INFO | Fetched 514 rows -2026-06-15 11:56:20 | INFO | Creating table Coverage -2026-06-15 11:56:20 | INFO | Table ready: Coverage -2026-06-15 11:56:20 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 11:56:20 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 11:56:20 | INFO | ================================================================================ -2026-06-15 11:56:20 | INFO | Processing Table: Login -2026-06-15 11:56:20 | INFO | Fetching Login data for yesterday -2026-06-15 11:56:22 | INFO | Fetched 454 Login rows -2026-06-15 11:56:22 | INFO | Fetched 454 rows -2026-06-15 11:56:22 | INFO | Creating table Login -2026-06-15 11:56:22 | INFO | Table ready: Login -2026-06-15 11:56:23 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 11:56:23 | INFO | Login loaded successfully (454 rows) -2026-06-15 11:56:23 | INFO | ================================================================================ -2026-06-15 11:56:23 | INFO | Processing Table: Stock_Details -2026-06-15 11:56:23 | INFO | Fetching Stock Details data for 571 MIDs -2026-06-15 11:56:30 | INFO | Fetched 28,493 Stock Details rows -2026-06-15 11:56:30 | INFO | Fetched 28493 rows -2026-06-15 11:56:31 | INFO | Creating table Stock_Details -2026-06-15 11:56:31 | INFO | Table ready: Stock_Details -2026-06-15 11:56:31 | INFO | Stock_Details: inserted 28,493 rows into ClickHouse -2026-06-15 11:56:31 | INFO | Stock_Details loaded successfully (28493 rows) -2026-06-15 11:56:31 | INFO | ================================================================================ -2026-06-15 11:56:31 | INFO | Processing Table: Attendance -2026-06-15 11:56:31 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 -2026-06-15 11:56:46 | INFO | Fetched 11,543 attendance rows for 594 employees -2026-06-15 11:56:46 | INFO | Fetched 11543 rows -2026-06-15 11:56:47 | INFO | Creating table Attendance -2026-06-15 11:56:47 | INFO | Table ready: Attendance -2026-06-15 11:56:47 | INFO | Attendance: inserted 11,543 rows into ClickHouse -2026-06-15 11:56:47 | INFO | Attendance loaded successfully (11543 rows) -2026-06-15 11:56:47 | INFO | ================================================================================ -2026-06-15 11:56:47 | INFO | Processing Table: Store_Master -2026-06-15 11:56:47 | INFO | Fetching Store Master data -2026-06-15 11:57:01 | INFO | Fetched 5,984 stores -2026-06-15 11:57:01 | INFO | Fetched 5984 rows -2026-06-15 11:57:01 | INFO | Creating table Store_Master -2026-06-15 11:57:01 | INFO | Table ready: Store_Master -2026-06-15 11:57:02 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 11:57:02 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 11:57:02 | INFO | ================================================================================ -2026-06-15 11:57:02 | INFO | Processing Table: SKU_Master -2026-06-15 11:57:02 | INFO | Fetching SKU Master data -2026-06-15 11:57:02 | INFO | Fetched 160 SKU Master rows -2026-06-15 11:57:02 | INFO | Fetched 160 rows -2026-06-15 11:57:02 | INFO | Creating table SKU_Master -2026-06-15 11:57:02 | INFO | Table ready: SKU_Master -2026-06-15 11:57:02 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 11:57:02 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 11:57:02 | INFO | ================================================================================ -2026-06-15 11:57:02 | INFO | Processing Table: display_master -2026-06-15 11:57:02 | INFO | Fetching Display Master data -2026-06-15 11:57:03 | INFO | Fetched 135 Display Master records -2026-06-15 11:57:03 | INFO | Fetched 135 rows -2026-06-15 11:57:03 | INFO | Creating table display_master -2026-06-15 11:57:03 | INFO | Table ready: display_master -2026-06-15 11:57:03 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 11:57:03 | INFO | display_master loaded successfully (135 rows) -2026-06-15 11:57:03 | INFO | ================================================================================ -2026-06-15 11:57:03 | INFO | Processing Table: Employee_Master -2026-06-15 11:57:03 | INFO | Fetching Employee Master data -2026-06-15 11:57:04 | INFO | Fetched 2,273 Employee Master records -2026-06-15 11:57:04 | INFO | Fetched 2273 rows -2026-06-15 11:57:04 | INFO | Creating table Employee_Master -2026-06-15 11:57:04 | INFO | Table ready: Employee_Master -2026-06-15 11:57:04 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 11:57:04 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 11:57:04 | INFO | ================================================================================ -2026-06-15 11:57:04 | INFO | Processing Table: Journey_Plan -2026-06-15 11:57:04 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 11:57:10 | INFO | Fetched 21,133 Journey Plan records -2026-06-15 11:57:10 | INFO | Fetched 21133 rows -2026-06-15 11:57:10 | INFO | Creating table Journey_Plan -2026-06-15 11:57:10 | INFO | Table ready: Journey_Plan -2026-06-15 11:57:10 | INFO | Journey_Plan: inserted 21,133 rows into ClickHouse -2026-06-15 11:57:10 | INFO | Journey_Plan loaded successfully (21133 rows) -2026-06-15 11:57:10 | INFO | ================================================================================ -2026-06-15 11:57:10 | INFO | Processing Table: coverage_remarks -2026-06-15 11:57:10 | INFO | Fetching Coverage Remarks -2026-06-15 11:57:10 | INFO | Fetched 29 Coverage Remark records -2026-06-15 11:57:10 | INFO | Fetched 29 rows -2026-06-15 11:57:10 | INFO | Creating table coverage_remarks -2026-06-15 11:57:10 | INFO | Table ready: coverage_remarks -2026-06-15 11:57:11 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 11:57:11 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 11:57:11 | INFO | ================================================================================ -2026-06-15 11:57:11 | INFO | Processing Table: mapping_storevisibility -2026-06-15 11:57:11 | INFO | Fetching Mapping Store Visibility for 2026-06-14 -2026-06-15 11:57:11 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 11:57:11 | WARNING | mapping_storevisibility returned no rows -2026-06-15 11:57:11 | INFO | ================================================================================ -2026-06-15 11:57:11 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 11:57:11 | INFO | Fetching Master Visibility Reason data -2026-06-15 11:57:11 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 11:57:11 | INFO | Fetched 17 rows -2026-06-15 11:57:11 | INFO | Creating table Master_VisibilityReason -2026-06-15 11:57:11 | INFO | Table ready: Master_VisibilityReason -2026-06-15 11:57:12 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 11:57:12 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 11:57:12 | INFO | ================================================================================ -2026-06-15 11:57:12 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 11:57:12 | INFO | Fetching Master Visibility Definition data -2026-06-15 11:57:12 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 11:57:12 | INFO | Fetched 861 rows -2026-06-15 11:57:12 | INFO | Creating table Master_VisibilityDefinition -2026-06-15 11:57:12 | INFO | Table ready: Master_VisibilityDefinition -2026-06-15 11:57:12 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 11:57:12 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 11:57:12 | INFO | ================================================================================ -2026-06-15 11:57:12 | INFO | Processing Table: Web_Logins -2026-06-15 11:57:12 | INFO | Fetching Web Login data for 2026-06-14 -2026-06-15 11:57:13 | INFO | Fetched 25 Web Login records -2026-06-15 11:57:13 | INFO | Fetched 25 rows -2026-06-15 11:57:13 | INFO | Creating table Web_Logins -2026-06-15 11:57:13 | INFO | Table ready: Web_Logins -2026-06-15 11:57:13 | INFO | Web_Logins: inserted 25 rows into ClickHouse -2026-06-15 11:57:13 | INFO | Web_Logins loaded successfully (25 rows) -2026-06-15 11:57:13 | INFO | ================================================================================ -2026-06-15 11:57:13 | INFO | Processing Table: Promotion -2026-06-15 11:57:13 | INFO | Fetching Promotion data for 571 MIDs -2026-06-15 11:57:34 | INFO | Fetched 4,426 Promotion records -2026-06-15 11:57:34 | INFO | Fetched 4426 rows -2026-06-15 11:57:34 | INFO | Creating table Promotion -2026-06-15 11:57:34 | INFO | Table ready: Promotion -2026-06-15 11:57:34 | INFO | Promotion: inserted 4,426 rows into ClickHouse -2026-06-15 11:57:34 | INFO | Promotion loaded successfully (4426 rows) -2026-06-15 11:57:34 | INFO | ================================================================================ -2026-06-15 11:57:34 | INFO | Processing Table: PaidVisibility -2026-06-15 11:57:34 | INFO | Fetching Paid Visibility data for 571 MIDs -2026-06-15 11:57:37 | INFO | Fetched 1,296 Paid Visibility records -2026-06-15 11:57:37 | INFO | Fetched 1296 rows -2026-06-15 11:57:37 | INFO | Creating table PaidVisibility -2026-06-15 11:57:37 | INFO | Table ready: PaidVisibility -2026-06-15 11:57:37 | INFO | PaidVisibility: inserted 1,296 rows into ClickHouse -2026-06-15 11:57:37 | INFO | PaidVisibility loaded successfully (1296 rows) -2026-06-15 11:57:37 | INFO | ================================================================================ -2026-06-15 11:57:37 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 11:57:37 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 11:57:38 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 11:57:38 | INFO | Fetched 33 rows -2026-06-15 11:57:38 | INFO | Creating table Master_Salesterritorylayer -2026-06-15 11:57:38 | INFO | Table ready: Master_Salesterritorylayer -2026-06-15 11:57:38 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 11:57:38 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 11:57:38 | INFO | ================================================================================ -2026-06-15 11:57:38 | INFO | Pipeline Completed Successfully -2026-06-15 11:57:38 | INFO | ================================================================================ -2026-06-15 11:58:05 | INFO | ================================================================================ -2026-06-15 11:58:05 | INFO | Hello from data-move Python data pipeline! -2026-06-15 11:58:05 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 11:58:05 | INFO | Connecting to databases... -2026-06-15 11:58:06 | INFO | -2026-06-15 11:58:07 | INFO | -2026-06-15 11:58:08 | INFO | Database connections established -2026-06-15 11:58:08 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 11:58:08 | INFO | Found 571 MIDs -2026-06-15 11:58:09 | INFO | ================================================================================ -2026-06-15 11:58:09 | INFO | Processing Table: SOS_OneApp -2026-06-15 11:58:09 | INFO | Fetching data for 571 MIDs -2026-06-15 11:58:09 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 11:58:09 | INFO | Fetched 1995 rows -2026-06-15 11:58:09 | INFO | Deleting from SOS_OneApp -2026-06-15 11:58:09 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse -2026-06-15 11:58:09 | INFO | SOS_OneApp loaded successfully (1995 rows) -2026-06-15 11:58:09 | INFO | ================================================================================ -2026-06-15 11:58:09 | INFO | Processing Table: OQaD -2026-06-15 11:58:09 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 11:58:12 | INFO | Fetched 443 rows -2026-06-15 11:58:12 | INFO | Fetched 443 rows -2026-06-15 11:58:12 | INFO | Deleting from OQaD -2026-06-15 11:58:12 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 11:58:12 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 11:58:12 | INFO | ================================================================================ -2026-06-15 11:58:12 | INFO | Processing Table: Survey -2026-06-15 11:58:12 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 11:58:12 | INFO | Fetched 173 Survey rows -2026-06-15 11:58:12 | INFO | Fetched 173 rows -2026-06-15 11:58:13 | INFO | Deleting from Survey -2026-06-15 11:58:13 | INFO | Survey: inserted 173 rows into ClickHouse -2026-06-15 11:58:13 | INFO | Survey loaded successfully (173 rows) -2026-06-15 11:58:13 | INFO | ================================================================================ -2026-06-15 11:58:13 | INFO | Processing Table: additional_visibility -2026-06-15 11:58:13 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 11:58:13 | INFO | Fetched 1,648 Additional Visibility rows -2026-06-15 11:58:13 | INFO | Fetched 1648 rows -2026-06-15 11:58:13 | INFO | Deleting from additional_visibility -2026-06-15 11:58:14 | INFO | additional_visibility: inserted 1,648 rows into ClickHouse -2026-06-15 11:58:14 | INFO | additional_visibility loaded successfully (1648 rows) -2026-06-15 11:58:14 | INFO | ================================================================================ -2026-06-15 11:58:14 | INFO | Processing Table: Coverage -2026-06-15 11:58:14 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 11:58:14 | INFO | Fetched 514 rows from SQL Server -2026-06-15 11:58:14 | INFO | Fetched 514 rows -2026-06-15 11:58:14 | INFO | Deleting from Coverage -2026-06-15 11:58:14 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 11:58:14 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 11:58:14 | INFO | ================================================================================ -2026-06-15 11:58:14 | INFO | Processing Table: Login -2026-06-15 11:58:14 | INFO | Fetching Login data for yesterday -2026-06-15 11:58:15 | INFO | Fetched 454 Login rows -2026-06-15 11:58:15 | INFO | Fetched 454 rows -2026-06-15 11:58:15 | INFO | Deleting from Login -2026-06-15 11:58:15 | ERROR | Failed processing table Login -Traceback (most recent call last): - File "D:\data_move\test2.py", line 236, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 83, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - table_name, - ^^^^^^^^^^^ - f"MID IN ({mids_str})", - ^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650)), _database, _part_data_version, _part_granule_offset, _part_offset, _disk_name, _sample_factor, _block_offset, _table, login_date, _part_uuid, _part_starting_offset, first_store_in_time, _part_index, _part, unique_id, _block_number, login_time, _partition_id, last_store_out_time, _distance, employee_id', required columns: '_part_data_version' '_database' '_part_offset' '_part_granule_offset' 'MID' '_disk_name' '_sample_factor' 'login_date' '_block_offset' '_table' '_part_uuid' '_part' '_part_index' '_part_starting_offset' 'first_store_in_time' 'login_time' '_block_number' 'unique_id' '_partition_id' 'employee_id' '_distance' 'last_store_out_time', maybe you meant: 'login_date', 'first_store_in_time', 'login_time', 'unique_id', 'employee_id' or 'last_store_out_time'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-15 12:03:38 | INFO | ================================================================================ -2026-06-15 12:03:38 | INFO | Hello from data-move Python data pipeline! -2026-06-15 12:03:38 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 12:03:38 | INFO | Connecting to databases... -2026-06-15 12:03:39 | INFO | -2026-06-15 12:03:40 | INFO | -2026-06-15 12:03:41 | INFO | Database connections established -2026-06-15 12:03:41 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 12:03:41 | INFO | Found 571 MIDs -2026-06-15 12:03:41 | INFO | ================================================================================ -2026-06-15 12:03:41 | INFO | Processing Table: SOS_OneApp -2026-06-15 12:03:41 | INFO | Fetching data for 571 MIDs -2026-06-15 12:03:42 | INFO | Fetched 1,995 rows from SQL Server -2026-06-15 12:03:42 | INFO | Fetched 1995 rows -2026-06-15 12:03:42 | INFO | Deleting from SOS_OneApp -2026-06-15 12:03:42 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse -2026-06-15 12:03:42 | INFO | SOS_OneApp loaded successfully (1995 rows) -2026-06-15 12:03:42 | INFO | ================================================================================ -2026-06-15 12:03:42 | INFO | Processing Table: OQaD -2026-06-15 12:03:42 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 12:03:45 | INFO | Fetched 443 rows -2026-06-15 12:03:45 | INFO | Fetched 443 rows -2026-06-15 12:03:45 | INFO | Deleting from OQaD -2026-06-15 12:03:45 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 12:03:45 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 12:03:45 | INFO | ================================================================================ -2026-06-15 12:03:45 | INFO | Processing Table: Survey -2026-06-15 12:03:45 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 12:03:45 | INFO | Fetched 173 Survey rows -2026-06-15 12:03:45 | INFO | Fetched 173 rows -2026-06-15 12:03:45 | INFO | Deleting from Survey -2026-06-15 12:03:45 | INFO | Survey: inserted 173 rows into ClickHouse -2026-06-15 12:03:45 | INFO | Survey loaded successfully (173 rows) -2026-06-15 12:03:45 | INFO | ================================================================================ -2026-06-15 12:03:45 | INFO | Processing Table: additional_visibility -2026-06-15 12:03:45 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 12:03:46 | INFO | Fetched 1,648 Additional Visibility rows -2026-06-15 12:03:46 | INFO | Fetched 1648 rows -2026-06-15 12:03:46 | INFO | Deleting from additional_visibility -2026-06-15 12:03:46 | INFO | additional_visibility: inserted 1,648 rows into ClickHouse -2026-06-15 12:03:46 | INFO | additional_visibility loaded successfully (1648 rows) -2026-06-15 12:03:46 | INFO | ================================================================================ -2026-06-15 12:03:46 | INFO | Processing Table: Coverage -2026-06-15 12:03:46 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 12:03:46 | INFO | Fetched 514 rows from SQL Server -2026-06-15 12:03:46 | INFO | Fetched 514 rows -2026-06-15 12:03:46 | INFO | Deleting from Coverage -2026-06-15 12:03:46 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 12:03:46 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 12:03:46 | INFO | ================================================================================ -2026-06-15 12:03:46 | INFO | Processing Table: Login -2026-06-15 12:03:46 | INFO | Fetching Login data for yesterday -2026-06-15 12:03:47 | INFO | Fetched 454 Login rows -2026-06-15 12:03:47 | INFO | Fetched 454 rows -2026-06-15 12:03:47 | INFO | Deleting from Login -2026-06-15 12:03:47 | ERROR | Failed processing table Login -Traceback (most recent call last): - File "D:\data_move\test2.py", line 236, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 83, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - table_name, - ^^^^^^^^^^^ - f"MID IN ({mids_str})", - ^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650)), _database, _part_data_version, _part_granule_offset, _part_offset, _disk_name, _sample_factor, _block_offset, _table, login_date, _part_uuid, _part_starting_offset, first_store_in_time, _part_index, _part, unique_id, _block_number, login_time, _partition_id, last_store_out_time, _distance, employee_id', required columns: '_part_data_version' '_database' '_part_offset' '_part_granule_offset' 'MID' '_disk_name' '_sample_factor' 'login_date' '_block_offset' '_table' '_part_uuid' '_part' '_part_index' '_part_starting_offset' 'first_store_in_time' 'login_time' '_block_number' 'unique_id' '_partition_id' 'employee_id' '_distance' 'last_store_out_time', maybe you meant: 'login_date', 'first_store_in_time', 'login_time', 'unique_id', 'employee_id' or 'last_store_out_time'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-15 12:53:14 | INFO | ================================================================================ -2026-06-15 12:53:14 | INFO | Hello from data-move Python data pipeline! -2026-06-15 12:53:14 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 12:53:14 | INFO | Connecting to databases... -2026-06-15 12:53:17 | INFO | -2026-06-15 12:53:18 | INFO | -2026-06-15 12:53:19 | INFO | Database connections established -2026-06-15 12:53:19 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 12:53:21 | INFO | Found 571 MIDs -2026-06-15 12:53:21 | INFO | ================================================================================ -2026-06-15 12:53:21 | INFO | Processing Table: SOS_OneApp -2026-06-15 12:53:21 | INFO | Fetching data for 571 MIDs -2026-06-15 12:53:22 | INFO | Fetched 1,997 rows from SQL Server -2026-06-15 12:53:22 | INFO | Fetched 1997 rows -2026-06-15 12:53:22 | INFO | Deleting from SOS_OneApp -2026-06-15 12:53:23 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse -2026-06-15 12:53:23 | INFO | SOS_OneApp loaded successfully (1997 rows) -2026-06-15 12:53:23 | INFO | ================================================================================ -2026-06-15 12:53:23 | INFO | Processing Table: OQaD -2026-06-15 12:53:23 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 12:53:26 | INFO | Fetched 443 rows -2026-06-15 12:53:26 | INFO | Fetched 443 rows -2026-06-15 12:53:26 | INFO | Deleting from OQaD -2026-06-15 12:53:26 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 12:53:26 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 12:53:26 | INFO | ================================================================================ -2026-06-15 12:53:26 | INFO | Processing Table: Survey -2026-06-15 12:53:26 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 12:53:27 | INFO | Fetched 174 Survey rows -2026-06-15 12:53:27 | INFO | Fetched 174 rows -2026-06-15 12:53:27 | INFO | Deleting from Survey -2026-06-15 12:53:27 | INFO | Survey: inserted 174 rows into ClickHouse -2026-06-15 12:53:27 | INFO | Survey loaded successfully (174 rows) -2026-06-15 12:53:27 | INFO | ================================================================================ -2026-06-15 12:53:27 | INFO | Processing Table: additional_visibility -2026-06-15 12:53:27 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 12:53:28 | INFO | Fetched 1,651 Additional Visibility rows -2026-06-15 12:53:28 | INFO | Fetched 1651 rows -2026-06-15 12:53:28 | INFO | Deleting from additional_visibility -2026-06-15 12:53:28 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse -2026-06-15 12:53:28 | INFO | additional_visibility loaded successfully (1651 rows) -2026-06-15 12:53:28 | INFO | ================================================================================ -2026-06-15 12:53:28 | INFO | Processing Table: Coverage -2026-06-15 12:53:28 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 12:53:28 | INFO | Fetched 514 rows from SQL Server -2026-06-15 12:53:28 | INFO | Fetched 514 rows -2026-06-15 12:53:29 | INFO | Deleting from Coverage -2026-06-15 12:53:29 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 12:53:29 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 12:53:29 | INFO | ================================================================================ -2026-06-15 12:53:29 | INFO | Processing Table: Login -2026-06-15 12:53:29 | INFO | Fetching Login data for yesterday -2026-06-15 12:53:31 | INFO | Fetched 454 Login rows -2026-06-15 12:53:31 | INFO | Fetched 454 rows -2026-06-15 12:53:31 | INFO | Deleting from Login -2026-06-15 12:53:31 | ERROR | Failed processing table Login -Traceback (most recent call last): - File "D:\data_move\test2.py", line 236, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 112, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<4 lines>... - """ - ^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'project_id' while processing: 'isZeroOrNull((project_id = 40148) AND (toDate(login_date) = toDate('2026-06-14'))), _database, _part_data_version, _part_granule_offset, _part_offset, _disk_name, _sample_factor, _block_offset, _table, login_date, _part_uuid, _part_starting_offset, first_store_in_time, _part_index, _part, unique_id, _block_number, login_time, _partition_id, last_store_out_time, _distance, employee_id', required columns: '_part_offset' '_sample_factor' '_disk_name' '_part_data_version' 'project_id' '_database' '_part_granule_offset' '_part_uuid' '_block_offset' '_table' 'login_date' '_part' '_part_index' '_part_starting_offset' 'first_store_in_time' 'login_time' '_block_number' 'unique_id' '_partition_id' 'employee_id' '_distance' 'last_store_out_time', maybe you meant: 'login_date', 'first_store_in_time', 'login_time', 'unique_id', 'employee_id' or 'last_store_out_time'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-15 13:19:07 | INFO | ================================================================================ -2026-06-15 13:19:07 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:19:07 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 13:19:07 | INFO | Connecting to databases... -2026-06-15 13:19:07 | INFO | -2026-06-15 13:19:08 | INFO | -2026-06-15 13:19:09 | INFO | Database connections established -2026-06-15 13:19:09 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 13:19:09 | INFO | Found 571 MIDs -2026-06-15 13:19:09 | INFO | ================================================================================ -2026-06-15 13:19:09 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:19:09 | INFO | Fetching data for 571 MIDs -2026-06-15 13:19:10 | INFO | Fetched 1,997 rows from SQL Server -2026-06-15 13:19:10 | INFO | Fetched 1997 rows -2026-06-15 13:19:10 | INFO | Deleting from SOS_OneApp -2026-06-15 13:19:10 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse -2026-06-15 13:19:10 | INFO | SOS_OneApp loaded successfully (1997 rows) -2026-06-15 13:19:10 | INFO | ================================================================================ -2026-06-15 13:19:10 | INFO | Processing Table: OQaD -2026-06-15 13:19:10 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 13:19:13 | INFO | Fetched 443 rows -2026-06-15 13:19:13 | INFO | Fetched 443 rows -2026-06-15 13:19:13 | INFO | Deleting from OQaD -2026-06-15 13:19:14 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 13:19:14 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 13:19:14 | INFO | ================================================================================ -2026-06-15 13:19:14 | INFO | Processing Table: Survey -2026-06-15 13:19:14 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 13:19:14 | INFO | Fetched 174 Survey rows -2026-06-15 13:19:14 | INFO | Fetched 174 rows -2026-06-15 13:19:14 | INFO | Deleting from Survey -2026-06-15 13:19:14 | INFO | Survey: inserted 174 rows into ClickHouse -2026-06-15 13:19:14 | INFO | Survey loaded successfully (174 rows) -2026-06-15 13:19:14 | INFO | ================================================================================ -2026-06-15 13:19:14 | INFO | Processing Table: additional_visibility -2026-06-15 13:19:14 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 13:19:32 | INFO | Fetched 1,651 Additional Visibility rows -2026-06-15 13:19:32 | INFO | Fetched 1651 rows -2026-06-15 13:19:32 | INFO | Deleting from additional_visibility -2026-06-15 13:19:33 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse -2026-06-15 13:19:33 | INFO | additional_visibility loaded successfully (1651 rows) -2026-06-15 13:19:33 | INFO | ================================================================================ -2026-06-15 13:19:33 | INFO | Processing Table: Coverage -2026-06-15 13:19:33 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 13:19:34 | INFO | Fetched 514 rows from SQL Server -2026-06-15 13:19:34 | INFO | Fetched 514 rows -2026-06-15 13:19:34 | INFO | Deleting from Coverage -2026-06-15 13:19:34 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 13:19:34 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 13:19:34 | INFO | ================================================================================ -2026-06-15 13:19:34 | INFO | Processing Table: Login -2026-06-15 13:19:34 | INFO | Fetching Login data for yesterday -2026-06-15 13:19:36 | INFO | Fetched 454 Login rows -2026-06-15 13:19:36 | INFO | Fetched 454 rows -2026-06-15 13:19:36 | INFO | Deleting from Login -2026-06-15 13:19:36 | ERROR | Failed processing table Login -Traceback (most recent call last): - File "D:\data_move\test2.py", line 236, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 110, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<3 lines>... - """ - ^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 62, server response: Code: 62. DB::Exception: Syntax error: failed at position 58 (toDate) (line 5, col 13): toDate(login_date) = toDate('2026-06-14') - - . Expected one of: token sequence, Dot, token, OR, AND, IS NOT DISTINCT FROM, IS DISTINCT FROM, IS NULL, IS NOT NULL, BETWEEN, NOT BETWEEN, LIKE, ILIKE, NOT LIKE, NOT ILIKE, REGEXP, IN, NOT IN, GLOBAL IN, GLOBAL NOT IN, MOD, DIV, alias, AS, Comma, INTO OUTFILE, FORMAT, SETTINGS, ParallelWithClause, PARALLEL WITH, end of query. (SYNTAX_ERROR) (for url http://172.188.12.194:8123) -2026-06-15 13:21:54 | INFO | ================================================================================ -2026-06-15 13:21:54 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:21:54 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 13:21:54 | INFO | Connecting to databases... -2026-06-15 13:21:55 | INFO | -2026-06-15 13:21:56 | INFO | -2026-06-15 13:21:56 | INFO | Database connections established -2026-06-15 13:21:56 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 13:21:57 | INFO | Found 571 MIDs -2026-06-15 13:21:57 | INFO | ================================================================================ -2026-06-15 13:21:57 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:21:57 | INFO | Fetching data for 571 MIDs -2026-06-15 13:21:58 | INFO | Fetched 1,997 rows from SQL Server -2026-06-15 13:21:58 | INFO | Fetched 1997 rows -2026-06-15 13:21:58 | INFO | Deleting from SOS_OneApp -2026-06-15 13:21:58 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse -2026-06-15 13:21:58 | INFO | SOS_OneApp loaded successfully (1997 rows) -2026-06-15 13:21:58 | INFO | ================================================================================ -2026-06-15 13:21:58 | INFO | Processing Table: OQaD -2026-06-15 13:21:58 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 13:22:00 | INFO | Fetched 443 rows -2026-06-15 13:22:00 | INFO | Fetched 443 rows -2026-06-15 13:22:00 | INFO | Deleting from OQaD -2026-06-15 13:22:00 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 13:22:00 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 13:22:01 | INFO | ================================================================================ -2026-06-15 13:22:01 | INFO | Processing Table: Survey -2026-06-15 13:22:01 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 13:22:01 | INFO | Fetched 174 Survey rows -2026-06-15 13:22:01 | INFO | Fetched 174 rows -2026-06-15 13:22:01 | INFO | Deleting from Survey -2026-06-15 13:22:01 | INFO | Survey: inserted 174 rows into ClickHouse -2026-06-15 13:22:01 | INFO | Survey loaded successfully (174 rows) -2026-06-15 13:22:01 | INFO | ================================================================================ -2026-06-15 13:22:01 | INFO | Processing Table: additional_visibility -2026-06-15 13:22:01 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 13:22:02 | INFO | Fetched 1,651 Additional Visibility rows -2026-06-15 13:22:02 | INFO | Fetched 1651 rows -2026-06-15 13:22:02 | INFO | Deleting from additional_visibility -2026-06-15 13:22:02 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse -2026-06-15 13:22:02 | INFO | additional_visibility loaded successfully (1651 rows) -2026-06-15 13:22:02 | INFO | ================================================================================ -2026-06-15 13:22:02 | INFO | Processing Table: Coverage -2026-06-15 13:22:02 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 13:22:02 | INFO | Fetched 514 rows from SQL Server -2026-06-15 13:22:02 | INFO | Fetched 514 rows -2026-06-15 13:22:02 | INFO | Deleting from Coverage -2026-06-15 13:22:02 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 13:22:02 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 13:22:02 | INFO | ================================================================================ -2026-06-15 13:22:02 | INFO | Processing Table: Login -2026-06-15 13:22:02 | INFO | Fetching Login data for yesterday -2026-06-15 13:22:03 | INFO | Fetched 454 Login rows -2026-06-15 13:22:03 | INFO | Fetched 454 rows -2026-06-15 13:22:03 | INFO | Deleting from Login -2026-06-15 13:22:03 | INFO | No delete logic required for Login -2026-06-15 13:22:03 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:22:03 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:22:03 | INFO | ================================================================================ -2026-06-15 13:22:03 | INFO | Processing Table: Stock_Details -2026-06-15 13:22:03 | INFO | Fetching Stock Details data for 571 MIDs -2026-06-15 13:22:12 | INFO | Fetched 28,574 Stock Details rows -2026-06-15 13:22:12 | INFO | Fetched 28574 rows -2026-06-15 13:22:12 | INFO | Deleting from Stock_Details -2026-06-15 13:22:13 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse -2026-06-15 13:22:13 | INFO | Stock_Details loaded successfully (28574 rows) -2026-06-15 13:22:13 | INFO | ================================================================================ -2026-06-15 13:22:13 | INFO | Processing Table: Attendance -2026-06-15 13:22:13 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 -2026-06-15 13:22:14 | INFO | Fetched 11,543 attendance rows for 594 employees -2026-06-15 13:22:14 | INFO | Fetched 11543 rows -2026-06-15 13:22:14 | INFO | Truncated table Attendance -2026-06-15 13:22:15 | INFO | Attendance: inserted 11,543 rows into ClickHouse -2026-06-15 13:22:15 | INFO | Attendance loaded successfully (11543 rows) -2026-06-15 13:22:15 | INFO | ================================================================================ -2026-06-15 13:22:15 | INFO | Processing Table: Store_Master -2026-06-15 13:22:15 | INFO | Fetching Store Master data -2026-06-15 13:22:17 | INFO | Fetched 5,984 stores -2026-06-15 13:22:17 | INFO | Fetched 5984 rows -2026-06-15 13:22:17 | INFO | Truncated table Store_Master -2026-06-15 13:22:18 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:22:18 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:22:18 | INFO | ================================================================================ -2026-06-15 13:22:18 | INFO | Processing Table: SKU_Master -2026-06-15 13:22:18 | INFO | Fetching SKU Master data -2026-06-15 13:22:18 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:22:18 | INFO | Fetched 160 rows -2026-06-15 13:22:18 | INFO | Truncated table SKU_Master -2026-06-15 13:22:18 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:22:18 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:22:18 | INFO | ================================================================================ -2026-06-15 13:22:18 | INFO | Processing Table: display_master -2026-06-15 13:22:18 | INFO | Fetching Display Master data -2026-06-15 13:22:18 | INFO | Fetched 135 Display Master records -2026-06-15 13:22:18 | INFO | Fetched 135 rows -2026-06-15 13:22:19 | INFO | Truncated table display_master -2026-06-15 13:22:19 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:22:19 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:22:19 | INFO | ================================================================================ -2026-06-15 13:22:19 | INFO | Processing Table: Employee_Master -2026-06-15 13:22:19 | INFO | Fetching Employee Master data -2026-06-15 13:22:19 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:22:19 | INFO | Fetched 2273 rows -2026-06-15 13:22:19 | INFO | Truncated table Employee_Master -2026-06-15 13:22:20 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:22:20 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:22:20 | INFO | ================================================================================ -2026-06-15 13:22:20 | INFO | Processing Table: Journey_Plan -2026-06-15 13:22:20 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:22:20 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:22:20 | INFO | Fetched 21141 rows -2026-06-15 13:22:20 | INFO | Deleting from Journey_Plan -2026-06-15 13:22:21 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:22:21 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:22:21 | INFO | ================================================================================ -2026-06-15 13:22:21 | INFO | Processing Table: coverage_remarks -2026-06-15 13:22:21 | INFO | Fetching Coverage Remarks -2026-06-15 13:22:21 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:22:21 | INFO | Fetched 29 rows -2026-06-15 13:22:21 | INFO | Truncated table coverage_remarks -2026-06-15 13:22:21 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:22:21 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:22:21 | INFO | ================================================================================ -2026-06-15 13:22:21 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:22:21 | INFO | Fetching Mapping Store Visibility for 2026-06-14 -2026-06-15 13:22:21 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:22:21 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:22:21 | INFO | ================================================================================ -2026-06-15 13:22:21 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:22:21 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:22:21 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:22:21 | INFO | Fetched 17 rows -2026-06-15 13:22:21 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:22:21 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:22:21 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:22:21 | INFO | ================================================================================ -2026-06-15 13:22:22 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:22:22 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:22:22 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:22:22 | INFO | Fetched 861 rows -2026-06-15 13:22:22 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:22:22 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:22:22 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:22:22 | INFO | ================================================================================ -2026-06-15 13:22:22 | INFO | Processing Table: Web_Logins -2026-06-15 13:22:22 | INFO | Fetching Web Login data for 2026-06-14 -2026-06-15 13:22:22 | INFO | Fetched 25 Web Login records -2026-06-15 13:22:22 | INFO | Fetched 25 rows -2026-06-15 13:22:22 | INFO | Deleting from Web_Logins -2026-06-15 13:22:22 | ERROR | Failed processing table Web_Logins -Traceback (most recent call last): - File "D:\data_move\test2.py", line 236, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 129, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<4 lines>... - """, - ^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'date' while processing: 'isZeroOrNull(toDate(date) = toDate('2026-06-14')), _block_offset, _block_number, _table, _database, _distance, _part_starting_offset, _part_data_version, _part_granule_offset, _disk_name, _sample_factor, _partition_id, right_name, _part_index, _part, create_by, employee_name, activity_type, _part_uuid, login_time, create_date, project_id, activity_name, login_date, _part_offset, emp_id, designation, supervisor_name, supervisor_id', required columns: 'date' '_block_number' '_table' '_block_offset' '_part_uuid' 'activity_name' 'project_id' '_distance' '_database' '_part_starting_offset' '_part_data_version' '_part_granule_offset' '_disk_name' '_sample_factor' '_partition_id' 'create_date' 'login_time' 'designation' '_part_index' 'right_name' 'activity_type' 'employee_name' 'login_date' 'create_by' '_part_offset' 'emp_id' '_part' 'supervisor_name' 'supervisor_id', maybe you meant: 'activity_name', 'project_id', 'create_date', 'login_time', 'designation', 'right_name', 'activity_type', 'employee_name', 'login_date', 'create_by', 'emp_id', 'supervisor_name' or 'supervisor_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-15 13:26:05 | INFO | ================================================================================ -2026-06-15 13:26:05 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:26:05 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 13:26:05 | INFO | Connecting to databases... -2026-06-15 13:26:06 | INFO | -2026-06-15 13:26:08 | INFO | -2026-06-15 13:26:08 | INFO | Database connections established -2026-06-15 13:26:08 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 13:26:08 | INFO | Found 571 MIDs -2026-06-15 13:26:09 | INFO | ================================================================================ -2026-06-15 13:26:09 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:26:09 | INFO | Fetching data for 571 MIDs -2026-06-15 13:26:09 | INFO | Fetched 1,997 rows from SQL Server -2026-06-15 13:26:09 | INFO | Fetched 1997 rows -2026-06-15 13:26:09 | INFO | Deleting from SOS_OneApp -2026-06-15 13:26:09 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse -2026-06-15 13:26:09 | INFO | SOS_OneApp loaded successfully (1997 rows) -2026-06-15 13:26:09 | INFO | ================================================================================ -2026-06-15 13:26:09 | INFO | Processing Table: OQaD -2026-06-15 13:26:09 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 13:26:11 | INFO | Fetched 443 rows -2026-06-15 13:26:11 | INFO | Fetched 443 rows -2026-06-15 13:26:12 | INFO | Deleting from OQaD -2026-06-15 13:26:12 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 13:26:12 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 13:26:12 | INFO | ================================================================================ -2026-06-15 13:26:12 | INFO | Processing Table: Survey -2026-06-15 13:26:12 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 13:26:12 | INFO | Fetched 174 Survey rows -2026-06-15 13:26:12 | INFO | Fetched 174 rows -2026-06-15 13:26:12 | INFO | Deleting from Survey -2026-06-15 13:26:12 | INFO | Survey: inserted 174 rows into ClickHouse -2026-06-15 13:26:12 | INFO | Survey loaded successfully (174 rows) -2026-06-15 13:26:12 | INFO | ================================================================================ -2026-06-15 13:26:12 | INFO | Processing Table: additional_visibility -2026-06-15 13:26:12 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 13:26:13 | INFO | Fetched 1,651 Additional Visibility rows -2026-06-15 13:26:13 | INFO | Fetched 1651 rows -2026-06-15 13:26:13 | INFO | Deleting from additional_visibility -2026-06-15 13:26:13 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse -2026-06-15 13:26:13 | INFO | additional_visibility loaded successfully (1651 rows) -2026-06-15 13:26:13 | INFO | ================================================================================ -2026-06-15 13:26:13 | INFO | Processing Table: Coverage -2026-06-15 13:26:13 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 13:26:13 | INFO | Fetched 514 rows from SQL Server -2026-06-15 13:26:13 | INFO | Fetched 514 rows -2026-06-15 13:26:13 | INFO | Deleting from Coverage -2026-06-15 13:26:13 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 13:26:13 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 13:26:13 | INFO | ================================================================================ -2026-06-15 13:26:13 | INFO | Processing Table: Login -2026-06-15 13:26:13 | INFO | Fetching Login data for yesterday -2026-06-15 13:26:14 | INFO | Fetched 454 Login rows -2026-06-15 13:26:14 | INFO | Fetched 454 rows -2026-06-15 13:26:14 | INFO | Deleting from Login -2026-06-15 13:26:14 | INFO | No delete logic required for Login -2026-06-15 13:26:14 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:26:14 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:26:14 | INFO | ================================================================================ -2026-06-15 13:26:14 | INFO | Processing Table: Stock_Details -2026-06-15 13:26:14 | INFO | Fetching Stock Details data for 571 MIDs -2026-06-15 13:26:17 | INFO | Fetched 28,574 Stock Details rows -2026-06-15 13:26:17 | INFO | Fetched 28574 rows -2026-06-15 13:26:17 | INFO | Deleting from Stock_Details -2026-06-15 13:26:18 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse -2026-06-15 13:26:18 | INFO | Stock_Details loaded successfully (28574 rows) -2026-06-15 13:26:18 | INFO | ================================================================================ -2026-06-15 13:26:18 | INFO | Processing Table: Attendance -2026-06-15 13:26:18 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 -2026-06-15 13:26:18 | INFO | Fetched 11,543 attendance rows for 594 employees -2026-06-15 13:26:18 | INFO | Fetched 11543 rows -2026-06-15 13:26:19 | INFO | Truncated table Attendance -2026-06-15 13:26:19 | INFO | Attendance: inserted 11,543 rows into ClickHouse -2026-06-15 13:26:19 | INFO | Attendance loaded successfully (11543 rows) -2026-06-15 13:26:19 | INFO | ================================================================================ -2026-06-15 13:26:19 | INFO | Processing Table: Store_Master -2026-06-15 13:26:19 | INFO | Fetching Store Master data -2026-06-15 13:26:21 | INFO | Fetched 5,984 stores -2026-06-15 13:26:21 | INFO | Fetched 5984 rows -2026-06-15 13:26:22 | INFO | Truncated table Store_Master -2026-06-15 13:26:22 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:26:22 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:26:22 | INFO | ================================================================================ -2026-06-15 13:26:22 | INFO | Processing Table: SKU_Master -2026-06-15 13:26:22 | INFO | Fetching SKU Master data -2026-06-15 13:26:22 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:26:22 | INFO | Fetched 160 rows -2026-06-15 13:26:22 | INFO | Truncated table SKU_Master -2026-06-15 13:26:22 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:26:22 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:26:22 | INFO | ================================================================================ -2026-06-15 13:26:22 | INFO | Processing Table: display_master -2026-06-15 13:26:22 | INFO | Fetching Display Master data -2026-06-15 13:26:23 | INFO | Fetched 135 Display Master records -2026-06-15 13:26:23 | INFO | Fetched 135 rows -2026-06-15 13:26:23 | INFO | Truncated table display_master -2026-06-15 13:26:23 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:26:23 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:26:23 | INFO | ================================================================================ -2026-06-15 13:26:23 | INFO | Processing Table: Employee_Master -2026-06-15 13:26:23 | INFO | Fetching Employee Master data -2026-06-15 13:26:24 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:26:24 | INFO | Fetched 2273 rows -2026-06-15 13:26:24 | INFO | Truncated table Employee_Master -2026-06-15 13:26:24 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:26:24 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:26:24 | INFO | ================================================================================ -2026-06-15 13:26:24 | INFO | Processing Table: Journey_Plan -2026-06-15 13:26:24 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:26:24 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:26:24 | INFO | Fetched 21141 rows -2026-06-15 13:26:24 | INFO | Deleting from Journey_Plan -2026-06-15 13:26:25 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:26:25 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:26:25 | INFO | ================================================================================ -2026-06-15 13:26:25 | INFO | Processing Table: coverage_remarks -2026-06-15 13:26:25 | INFO | Fetching Coverage Remarks -2026-06-15 13:26:25 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:26:25 | INFO | Fetched 29 rows -2026-06-15 13:26:25 | INFO | Truncated table coverage_remarks -2026-06-15 13:26:25 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:26:25 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:26:25 | INFO | ================================================================================ -2026-06-15 13:26:25 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:26:25 | INFO | Fetching Mapping Store Visibility for 2026-06-14 -2026-06-15 13:26:25 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:26:25 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:26:25 | INFO | ================================================================================ -2026-06-15 13:26:25 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:26:25 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:26:25 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:26:25 | INFO | Fetched 17 rows -2026-06-15 13:26:25 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:26:25 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:26:25 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:26:25 | INFO | ================================================================================ -2026-06-15 13:26:25 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:26:25 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:26:25 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:26:25 | INFO | Fetched 861 rows -2026-06-15 13:26:26 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:26:26 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:26:26 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:26:26 | INFO | ================================================================================ -2026-06-15 13:26:26 | INFO | Processing Table: Web_Logins -2026-06-15 13:26:26 | INFO | Fetching Web Login data for 2026-06-14 -2026-06-15 13:26:26 | INFO | Fetched 25 Web Login records -2026-06-15 13:26:26 | INFO | Fetched 25 rows -2026-06-15 13:26:26 | INFO | Deleting from Web_Logins -2026-06-15 13:26:26 | ERROR | Failed processing table Web_Logins -Traceback (most recent call last): - File "D:\data_move\test2.py", line 236, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 129, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<4 lines>... - """, - ^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'date' while processing: 'isZeroOrNull(toDate(date) = toDate('2026-06-14')), _block_offset, _block_number, _table, _database, _distance, _part_starting_offset, _part_data_version, _part_granule_offset, _disk_name, _sample_factor, _partition_id, right_name, _part_index, _part, create_by, employee_name, activity_type, _part_uuid, login_time, create_date, project_id, activity_name, login_date, _part_offset, emp_id, designation, supervisor_name, supervisor_id', required columns: 'date' '_block_number' '_table' '_block_offset' '_part_uuid' 'activity_name' 'project_id' '_distance' '_database' '_part_starting_offset' '_part_data_version' '_part_granule_offset' '_disk_name' '_sample_factor' '_partition_id' 'create_date' 'login_time' 'designation' '_part_index' 'right_name' 'activity_type' 'employee_name' 'login_date' 'create_by' '_part_offset' 'emp_id' '_part' 'supervisor_name' 'supervisor_id', maybe you meant: 'activity_name', 'project_id', 'create_date', 'login_time', 'designation', 'right_name', 'activity_type', 'employee_name', 'login_date', 'create_by', 'emp_id', 'supervisor_name' or 'supervisor_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-15 13:27:51 | INFO | ================================================================================ -2026-06-15 13:27:51 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:27:51 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 13:27:51 | INFO | Connecting to databases... -2026-06-15 13:27:52 | INFO | -2026-06-15 13:27:53 | INFO | -2026-06-15 13:27:53 | INFO | Database connections established -2026-06-15 13:27:53 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 13:27:54 | INFO | Found 571 MIDs -2026-06-15 13:27:54 | INFO | ================================================================================ -2026-06-15 13:27:54 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:27:54 | INFO | Fetching data for 571 MIDs -2026-06-15 13:27:54 | INFO | Fetched 1,997 rows from SQL Server -2026-06-15 13:27:54 | INFO | Fetched 1997 rows -2026-06-15 13:27:54 | INFO | Deleting from SOS_OneApp -2026-06-15 13:27:55 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse -2026-06-15 13:27:55 | INFO | SOS_OneApp loaded successfully (1997 rows) -2026-06-15 13:27:55 | INFO | ================================================================================ -2026-06-15 13:27:55 | INFO | Processing Table: OQaD -2026-06-15 13:27:55 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 13:27:57 | INFO | Fetched 443 rows -2026-06-15 13:27:57 | INFO | Fetched 443 rows -2026-06-15 13:27:57 | INFO | Deleting from OQaD -2026-06-15 13:27:57 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 13:27:57 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 13:27:57 | INFO | ================================================================================ -2026-06-15 13:27:57 | INFO | Processing Table: Survey -2026-06-15 13:27:57 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 13:27:57 | INFO | Fetched 174 Survey rows -2026-06-15 13:27:57 | INFO | Fetched 174 rows -2026-06-15 13:27:57 | INFO | Deleting from Survey -2026-06-15 13:27:58 | INFO | Survey: inserted 174 rows into ClickHouse -2026-06-15 13:27:58 | INFO | Survey loaded successfully (174 rows) -2026-06-15 13:27:58 | INFO | ================================================================================ -2026-06-15 13:27:58 | INFO | Processing Table: additional_visibility -2026-06-15 13:27:58 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 13:27:58 | INFO | Fetched 1,651 Additional Visibility rows -2026-06-15 13:27:58 | INFO | Fetched 1651 rows -2026-06-15 13:27:58 | INFO | Deleting from additional_visibility -2026-06-15 13:27:58 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse -2026-06-15 13:27:58 | INFO | additional_visibility loaded successfully (1651 rows) -2026-06-15 13:27:58 | INFO | ================================================================================ -2026-06-15 13:27:58 | INFO | Processing Table: Coverage -2026-06-15 13:27:58 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 13:27:58 | INFO | Fetched 514 rows from SQL Server -2026-06-15 13:27:58 | INFO | Fetched 514 rows -2026-06-15 13:27:59 | INFO | Deleting from Coverage -2026-06-15 13:27:59 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 13:27:59 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 13:27:59 | INFO | ================================================================================ -2026-06-15 13:27:59 | INFO | Processing Table: Login -2026-06-15 13:27:59 | INFO | Fetching Login data for yesterday -2026-06-15 13:27:59 | INFO | Fetched 454 Login rows -2026-06-15 13:27:59 | INFO | Fetched 454 rows -2026-06-15 13:27:59 | INFO | Deleting from Login -2026-06-15 13:28:00 | INFO | No delete logic required for Login -2026-06-15 13:28:00 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:28:00 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:28:00 | INFO | ================================================================================ -2026-06-15 13:28:00 | INFO | Processing Table: Stock_Details -2026-06-15 13:28:00 | INFO | Fetching Stock Details data for 571 MIDs -2026-06-15 13:28:03 | INFO | Fetched 28,574 Stock Details rows -2026-06-15 13:28:03 | INFO | Fetched 28574 rows -2026-06-15 13:28:03 | INFO | Deleting from Stock_Details -2026-06-15 13:28:03 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse -2026-06-15 13:28:03 | INFO | Stock_Details loaded successfully (28574 rows) -2026-06-15 13:28:03 | INFO | ================================================================================ -2026-06-15 13:28:03 | INFO | Processing Table: Attendance -2026-06-15 13:28:03 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 -2026-06-15 13:28:04 | INFO | Fetched 11,543 attendance rows for 594 employees -2026-06-15 13:28:04 | INFO | Fetched 11543 rows -2026-06-15 13:28:04 | INFO | Truncated table Attendance -2026-06-15 13:28:05 | INFO | Attendance: inserted 11,543 rows into ClickHouse -2026-06-15 13:28:05 | INFO | Attendance loaded successfully (11543 rows) -2026-06-15 13:28:05 | INFO | ================================================================================ -2026-06-15 13:28:05 | INFO | Processing Table: Store_Master -2026-06-15 13:28:05 | INFO | Fetching Store Master data -2026-06-15 13:28:06 | INFO | Fetched 5,984 stores -2026-06-15 13:28:06 | INFO | Fetched 5984 rows -2026-06-15 13:28:06 | INFO | Truncated table Store_Master -2026-06-15 13:28:07 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:28:07 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:28:07 | INFO | ================================================================================ -2026-06-15 13:28:07 | INFO | Processing Table: SKU_Master -2026-06-15 13:28:07 | INFO | Fetching SKU Master data -2026-06-15 13:28:07 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:28:07 | INFO | Fetched 160 rows -2026-06-15 13:28:07 | INFO | Truncated table SKU_Master -2026-06-15 13:28:07 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:28:07 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:28:07 | INFO | ================================================================================ -2026-06-15 13:28:07 | INFO | Processing Table: display_master -2026-06-15 13:28:07 | INFO | Fetching Display Master data -2026-06-15 13:28:08 | INFO | Fetched 135 Display Master records -2026-06-15 13:28:08 | INFO | Fetched 135 rows -2026-06-15 13:28:08 | INFO | Truncated table display_master -2026-06-15 13:28:08 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:28:08 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:28:08 | INFO | ================================================================================ -2026-06-15 13:28:08 | INFO | Processing Table: Employee_Master -2026-06-15 13:28:08 | INFO | Fetching Employee Master data -2026-06-15 13:28:08 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:28:08 | INFO | Fetched 2273 rows -2026-06-15 13:28:08 | INFO | Truncated table Employee_Master -2026-06-15 13:28:09 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:28:09 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:28:09 | INFO | ================================================================================ -2026-06-15 13:28:09 | INFO | Processing Table: Journey_Plan -2026-06-15 13:28:09 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:28:09 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:28:09 | INFO | Fetched 21141 rows -2026-06-15 13:28:09 | INFO | Deleting from Journey_Plan -2026-06-15 13:28:09 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:28:09 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:28:09 | INFO | ================================================================================ -2026-06-15 13:28:09 | INFO | Processing Table: coverage_remarks -2026-06-15 13:28:09 | INFO | Fetching Coverage Remarks -2026-06-15 13:28:09 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:28:09 | INFO | Fetched 29 rows -2026-06-15 13:28:10 | INFO | Truncated table coverage_remarks -2026-06-15 13:28:10 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:28:10 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:28:10 | INFO | ================================================================================ -2026-06-15 13:28:10 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:28:10 | INFO | Fetching Mapping Store Visibility for 2026-06-14 -2026-06-15 13:28:10 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:28:10 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:28:10 | INFO | ================================================================================ -2026-06-15 13:28:10 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:28:10 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:28:10 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:28:10 | INFO | Fetched 17 rows -2026-06-15 13:28:10 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:28:10 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:28:10 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:28:10 | INFO | ================================================================================ -2026-06-15 13:28:10 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:28:10 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:28:10 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:28:10 | INFO | Fetched 861 rows -2026-06-15 13:28:10 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:28:11 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:28:11 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:28:11 | INFO | ================================================================================ -2026-06-15 13:28:11 | INFO | Processing Table: Web_Logins -2026-06-15 13:28:11 | INFO | Fetching Web Login data for 2026-06-14 -2026-06-15 13:28:11 | INFO | Fetched 25 Web Login records -2026-06-15 13:28:11 | INFO | Fetched 25 rows -2026-06-15 13:28:11 | INFO | Deleting from Web_Logins -2026-06-15 13:28:11 | ERROR | Failed processing table Web_Logins -Traceback (most recent call last): - File "D:\data_move\test2.py", line 248, in main - load_to_clickhouse( - ~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - table_name=table_name, - ^^^^^^^^^^^^^^^^^^^^^^ - df=df, - ^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse - client.insert_arrow( - ~~~~~~~~~~~~~~~~~~~^ - table=table_name, - ^^^^^^^^^^^^^^^^^ - arrow_table=arrow_table, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow - return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) - ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert - response = self._raw_request(insert_block, params, headers, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column date in table kelloggs_1.Web_Logins (c845112d-d41d-478b-ba61-6c113b115c9d). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) -2026-06-15 13:34:41 | INFO | ================================================================================ -2026-06-15 13:34:41 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:34:41 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 13:34:41 | INFO | Connecting to databases... -2026-06-15 13:34:41 | INFO | -2026-06-15 13:34:42 | INFO | -2026-06-15 13:34:43 | INFO | Database connections established -2026-06-15 13:34:43 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 13:34:43 | INFO | Found 571 MIDs -2026-06-15 13:34:43 | INFO | ================================================================================ -2026-06-15 13:34:43 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:34:43 | INFO | Fetching data for 571 MIDs -2026-06-15 13:34:44 | INFO | Fetched 1,997 rows from SQL Server -2026-06-15 13:34:44 | INFO | Fetched 1997 rows -2026-06-15 13:34:44 | INFO | Creating table SOS_OneApp -2026-06-15 13:34:44 | INFO | Table ready: SOS_OneApp -2026-06-15 13:34:44 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse -2026-06-15 13:34:44 | INFO | SOS_OneApp loaded successfully (1997 rows) -2026-06-15 13:34:44 | INFO | ================================================================================ -2026-06-15 13:34:44 | INFO | Processing Table: OQaD -2026-06-15 13:34:44 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 13:34:47 | INFO | Fetched 443 rows -2026-06-15 13:34:47 | INFO | Fetched 443 rows -2026-06-15 13:34:47 | INFO | Creating table OQaD -2026-06-15 13:34:47 | INFO | Table ready: OQaD -2026-06-15 13:34:47 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 13:34:47 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 13:34:47 | INFO | ================================================================================ -2026-06-15 13:34:47 | INFO | Processing Table: Survey -2026-06-15 13:34:47 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 13:34:47 | INFO | Fetched 174 Survey rows -2026-06-15 13:34:47 | INFO | Fetched 174 rows -2026-06-15 13:34:47 | INFO | Creating table Survey -2026-06-15 13:34:47 | INFO | Table ready: Survey -2026-06-15 13:34:48 | INFO | Survey: inserted 174 rows into ClickHouse -2026-06-15 13:34:48 | INFO | Survey loaded successfully (174 rows) -2026-06-15 13:34:48 | INFO | ================================================================================ -2026-06-15 13:34:48 | INFO | Processing Table: additional_visibility -2026-06-15 13:34:48 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 13:34:48 | INFO | Fetched 1,651 Additional Visibility rows -2026-06-15 13:34:48 | INFO | Fetched 1651 rows -2026-06-15 13:34:48 | INFO | Creating table additional_visibility -2026-06-15 13:34:48 | INFO | Table ready: additional_visibility -2026-06-15 13:34:48 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse -2026-06-15 13:34:48 | INFO | additional_visibility loaded successfully (1651 rows) -2026-06-15 13:34:48 | INFO | ================================================================================ -2026-06-15 13:34:48 | INFO | Processing Table: Coverage -2026-06-15 13:34:48 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 13:34:48 | INFO | Fetched 514 rows from SQL Server -2026-06-15 13:34:48 | INFO | Fetched 514 rows -2026-06-15 13:34:48 | INFO | Creating table Coverage -2026-06-15 13:34:49 | INFO | Table ready: Coverage -2026-06-15 13:34:49 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 13:34:49 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 13:34:49 | INFO | ================================================================================ -2026-06-15 13:34:49 | INFO | Processing Table: Login -2026-06-15 13:34:49 | INFO | Fetching Login data for yesterday -2026-06-15 13:34:49 | INFO | Fetched 454 Login rows -2026-06-15 13:34:49 | INFO | Fetched 454 rows -2026-06-15 13:34:49 | INFO | Creating table Login -2026-06-15 13:34:49 | INFO | Table ready: Login -2026-06-15 13:34:50 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:34:50 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:34:50 | INFO | ================================================================================ -2026-06-15 13:34:50 | INFO | Processing Table: Stock_Details -2026-06-15 13:34:50 | INFO | Fetching Stock Details data for 571 MIDs -2026-06-15 13:34:52 | INFO | Fetched 28,574 Stock Details rows -2026-06-15 13:34:52 | INFO | Fetched 28574 rows -2026-06-15 13:34:52 | INFO | Creating table Stock_Details -2026-06-15 13:34:52 | INFO | Table ready: Stock_Details -2026-06-15 13:34:52 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse -2026-06-15 13:34:52 | INFO | Stock_Details loaded successfully (28574 rows) -2026-06-15 13:34:52 | INFO | ================================================================================ -2026-06-15 13:34:52 | INFO | Processing Table: Attendance -2026-06-15 13:34:52 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 -2026-06-15 13:34:53 | INFO | Fetched 11,543 attendance rows for 594 employees -2026-06-15 13:34:53 | INFO | Fetched 11543 rows -2026-06-15 13:34:53 | INFO | Creating table Attendance -2026-06-15 13:34:54 | INFO | Table ready: Attendance -2026-06-15 13:34:54 | INFO | Attendance: inserted 11,543 rows into ClickHouse -2026-06-15 13:34:54 | INFO | Attendance loaded successfully (11543 rows) -2026-06-15 13:34:54 | INFO | ================================================================================ -2026-06-15 13:34:54 | INFO | Processing Table: Store_Master -2026-06-15 13:34:54 | INFO | Fetching Store Master data -2026-06-15 13:34:56 | INFO | Fetched 5,984 stores -2026-06-15 13:34:56 | INFO | Fetched 5984 rows -2026-06-15 13:34:56 | INFO | Creating table Store_Master -2026-06-15 13:34:56 | INFO | Table ready: Store_Master -2026-06-15 13:34:57 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:34:57 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:34:57 | INFO | ================================================================================ -2026-06-15 13:34:57 | INFO | Processing Table: SKU_Master -2026-06-15 13:34:57 | INFO | Fetching SKU Master data -2026-06-15 13:34:57 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:34:57 | INFO | Fetched 160 rows -2026-06-15 13:34:57 | INFO | Creating table SKU_Master -2026-06-15 13:34:57 | INFO | Table ready: SKU_Master -2026-06-15 13:34:57 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:34:57 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:34:57 | INFO | ================================================================================ -2026-06-15 13:34:57 | INFO | Processing Table: display_master -2026-06-15 13:34:57 | INFO | Fetching Display Master data -2026-06-15 13:34:57 | INFO | Fetched 135 Display Master records -2026-06-15 13:34:57 | INFO | Fetched 135 rows -2026-06-15 13:34:57 | INFO | Creating table display_master -2026-06-15 13:34:58 | INFO | Table ready: display_master -2026-06-15 13:34:58 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:34:58 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:34:58 | INFO | ================================================================================ -2026-06-15 13:34:58 | INFO | Processing Table: Employee_Master -2026-06-15 13:34:58 | INFO | Fetching Employee Master data -2026-06-15 13:34:58 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:34:58 | INFO | Fetched 2273 rows -2026-06-15 13:34:58 | INFO | Creating table Employee_Master -2026-06-15 13:34:58 | INFO | Table ready: Employee_Master -2026-06-15 13:34:59 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:34:59 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:34:59 | INFO | ================================================================================ -2026-06-15 13:34:59 | INFO | Processing Table: Journey_Plan -2026-06-15 13:34:59 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:35:00 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:35:00 | INFO | Fetched 21141 rows -2026-06-15 13:35:00 | INFO | Creating table Journey_Plan -2026-06-15 13:35:00 | INFO | Table ready: Journey_Plan -2026-06-15 13:35:00 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:35:00 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:35:00 | INFO | ================================================================================ -2026-06-15 13:35:00 | INFO | Processing Table: coverage_remarks -2026-06-15 13:35:00 | INFO | Fetching Coverage Remarks -2026-06-15 13:35:00 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:35:00 | INFO | Fetched 29 rows -2026-06-15 13:35:00 | INFO | Creating table coverage_remarks -2026-06-15 13:35:00 | INFO | Table ready: coverage_remarks -2026-06-15 13:35:01 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:35:01 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:35:01 | INFO | ================================================================================ -2026-06-15 13:35:01 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:35:01 | INFO | Fetching Mapping Store Visibility for 2026-06-14 -2026-06-15 13:35:01 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:35:01 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:35:01 | INFO | ================================================================================ -2026-06-15 13:35:01 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:35:01 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:35:01 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:35:01 | INFO | Fetched 17 rows -2026-06-15 13:35:01 | INFO | Creating table Master_VisibilityReason -2026-06-15 13:35:01 | INFO | Table ready: Master_VisibilityReason -2026-06-15 13:35:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:35:01 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:35:01 | INFO | ================================================================================ -2026-06-15 13:35:01 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:35:01 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:35:01 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:35:01 | INFO | Fetched 861 rows -2026-06-15 13:35:01 | INFO | Creating table Master_VisibilityDefinition -2026-06-15 13:35:02 | INFO | Table ready: Master_VisibilityDefinition -2026-06-15 13:35:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:35:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:35:02 | INFO | ================================================================================ -2026-06-15 13:35:02 | INFO | Processing Table: Web_Logins -2026-06-15 13:35:02 | INFO | Fetching Web Login data for 2026-06-14 -2026-06-15 13:35:02 | INFO | Fetched 25 Web Login records -2026-06-15 13:35:02 | INFO | Fetched 25 rows -2026-06-15 13:35:02 | INFO | Creating table Web_Logins -2026-06-15 13:35:02 | INFO | Table ready: Web_Logins -2026-06-15 13:35:02 | INFO | Web_Logins: inserted 25 rows into ClickHouse -2026-06-15 13:35:02 | INFO | Web_Logins loaded successfully (25 rows) -2026-06-15 13:35:02 | INFO | ================================================================================ -2026-06-15 13:35:02 | INFO | Processing Table: Promotion -2026-06-15 13:35:02 | INFO | Fetching Promotion data for 571 MIDs -2026-06-15 13:35:05 | INFO | Fetched 4,426 Promotion records -2026-06-15 13:35:05 | INFO | Fetched 4426 rows -2026-06-15 13:35:05 | INFO | Creating table Promotion -2026-06-15 13:35:06 | INFO | Table ready: Promotion -2026-06-15 13:35:06 | INFO | Promotion: inserted 4,426 rows into ClickHouse -2026-06-15 13:35:06 | INFO | Promotion loaded successfully (4426 rows) -2026-06-15 13:35:06 | INFO | ================================================================================ -2026-06-15 13:35:06 | INFO | Processing Table: PaidVisibility -2026-06-15 13:35:06 | INFO | Fetching Paid Visibility data for 571 MIDs -2026-06-15 13:35:07 | INFO | Fetched 1,296 Paid Visibility records -2026-06-15 13:35:07 | INFO | Fetched 1296 rows -2026-06-15 13:35:07 | INFO | Creating table PaidVisibility -2026-06-15 13:35:07 | INFO | Table ready: PaidVisibility -2026-06-15 13:35:07 | INFO | PaidVisibility: inserted 1,296 rows into ClickHouse -2026-06-15 13:35:07 | INFO | PaidVisibility loaded successfully (1296 rows) -2026-06-15 13:35:07 | INFO | ================================================================================ -2026-06-15 13:35:07 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:35:07 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:35:08 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:35:08 | INFO | Fetched 33 rows -2026-06-15 13:35:08 | INFO | Creating table Master_Salesterritorylayer -2026-06-15 13:35:08 | INFO | Table ready: Master_Salesterritorylayer -2026-06-15 13:35:08 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:35:08 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:35:08 | INFO | ================================================================================ -2026-06-15 13:35:08 | INFO | Pipeline Completed Successfully -2026-06-15 13:35:08 | INFO | ================================================================================ -2026-06-15 13:35:25 | INFO | ================================================================================ -2026-06-15 13:35:25 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:35:25 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 13:35:25 | INFO | Connecting to databases... -2026-06-15 13:35:26 | INFO | -2026-06-15 13:35:27 | INFO | -2026-06-15 13:35:27 | INFO | Database connections established -2026-06-15 13:35:27 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 13:35:27 | INFO | Found 571 MIDs -2026-06-15 13:35:28 | INFO | ================================================================================ -2026-06-15 13:35:28 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:35:28 | INFO | Fetching data for 571 MIDs -2026-06-15 13:35:28 | INFO | Fetched 1,997 rows from SQL Server -2026-06-15 13:35:28 | INFO | Fetched 1997 rows -2026-06-15 13:35:28 | INFO | Deleting from SOS_OneApp -2026-06-15 13:35:29 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse -2026-06-15 13:35:29 | INFO | SOS_OneApp loaded successfully (1997 rows) -2026-06-15 13:35:29 | INFO | ================================================================================ -2026-06-15 13:35:29 | INFO | Processing Table: OQaD -2026-06-15 13:35:29 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 13:35:31 | INFO | Fetched 443 rows -2026-06-15 13:35:31 | INFO | Fetched 443 rows -2026-06-15 13:35:31 | INFO | Deleting from OQaD -2026-06-15 13:35:31 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 13:35:31 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 13:35:31 | INFO | ================================================================================ -2026-06-15 13:35:31 | INFO | Processing Table: Survey -2026-06-15 13:35:31 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 13:35:31 | INFO | Fetched 174 Survey rows -2026-06-15 13:35:31 | INFO | Fetched 174 rows -2026-06-15 13:35:32 | INFO | Deleting from Survey -2026-06-15 13:35:32 | INFO | Survey: inserted 174 rows into ClickHouse -2026-06-15 13:35:32 | INFO | Survey loaded successfully (174 rows) -2026-06-15 13:35:32 | INFO | ================================================================================ -2026-06-15 13:35:32 | INFO | Processing Table: additional_visibility -2026-06-15 13:35:32 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 13:35:32 | INFO | Fetched 1,651 Additional Visibility rows -2026-06-15 13:35:32 | INFO | Fetched 1651 rows -2026-06-15 13:35:32 | INFO | Deleting from additional_visibility -2026-06-15 13:35:32 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse -2026-06-15 13:35:32 | INFO | additional_visibility loaded successfully (1651 rows) -2026-06-15 13:35:32 | INFO | ================================================================================ -2026-06-15 13:35:32 | INFO | Processing Table: Coverage -2026-06-15 13:35:32 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 13:35:33 | INFO | Fetched 514 rows from SQL Server -2026-06-15 13:35:33 | INFO | Fetched 514 rows -2026-06-15 13:35:33 | INFO | Deleting from Coverage -2026-06-15 13:35:33 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 13:35:33 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 13:35:33 | INFO | ================================================================================ -2026-06-15 13:35:33 | INFO | Processing Table: Login -2026-06-15 13:35:33 | INFO | Fetching Login data for yesterday -2026-06-15 13:35:33 | INFO | Fetched 454 Login rows -2026-06-15 13:35:33 | INFO | Fetched 454 rows -2026-06-15 13:35:34 | INFO | Deleting from Login -2026-06-15 13:35:34 | INFO | No delete logic required for Login -2026-06-15 13:35:34 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:35:34 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:35:34 | INFO | ================================================================================ -2026-06-15 13:35:34 | INFO | Processing Table: Stock_Details -2026-06-15 13:35:34 | INFO | Fetching Stock Details data for 571 MIDs -2026-06-15 13:35:36 | INFO | Fetched 28,574 Stock Details rows -2026-06-15 13:35:36 | INFO | Fetched 28574 rows -2026-06-15 13:35:36 | INFO | Deleting from Stock_Details -2026-06-15 13:35:36 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse -2026-06-15 13:35:36 | INFO | Stock_Details loaded successfully (28574 rows) -2026-06-15 13:35:36 | INFO | ================================================================================ -2026-06-15 13:35:36 | INFO | Processing Table: Attendance -2026-06-15 13:35:36 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 -2026-06-15 13:35:37 | INFO | Fetched 11,543 attendance rows for 594 employees -2026-06-15 13:35:37 | INFO | Fetched 11543 rows -2026-06-15 13:35:37 | INFO | Truncated table Attendance -2026-06-15 13:35:37 | INFO | Attendance: inserted 11,543 rows into ClickHouse -2026-06-15 13:35:37 | INFO | Attendance loaded successfully (11543 rows) -2026-06-15 13:35:37 | INFO | ================================================================================ -2026-06-15 13:35:37 | INFO | Processing Table: Store_Master -2026-06-15 13:35:37 | INFO | Fetching Store Master data -2026-06-15 13:35:39 | INFO | Fetched 5,984 stores -2026-06-15 13:35:39 | INFO | Fetched 5984 rows -2026-06-15 13:35:39 | INFO | Truncated table Store_Master -2026-06-15 13:35:40 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:35:40 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:35:40 | INFO | ================================================================================ -2026-06-15 13:35:40 | INFO | Processing Table: SKU_Master -2026-06-15 13:35:40 | INFO | Fetching SKU Master data -2026-06-15 13:35:40 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:35:40 | INFO | Fetched 160 rows -2026-06-15 13:35:40 | INFO | Truncated table SKU_Master -2026-06-15 13:35:40 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:35:40 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:35:40 | INFO | ================================================================================ -2026-06-15 13:35:40 | INFO | Processing Table: display_master -2026-06-15 13:35:40 | INFO | Fetching Display Master data -2026-06-15 13:35:40 | INFO | Fetched 135 Display Master records -2026-06-15 13:35:40 | INFO | Fetched 135 rows -2026-06-15 13:35:41 | INFO | Truncated table display_master -2026-06-15 13:35:41 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:35:41 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:35:41 | INFO | ================================================================================ -2026-06-15 13:35:41 | INFO | Processing Table: Employee_Master -2026-06-15 13:35:41 | INFO | Fetching Employee Master data -2026-06-15 13:35:41 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:35:41 | INFO | Fetched 2273 rows -2026-06-15 13:35:41 | INFO | Truncated table Employee_Master -2026-06-15 13:35:42 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:35:42 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:35:42 | INFO | ================================================================================ -2026-06-15 13:35:42 | INFO | Processing Table: Journey_Plan -2026-06-15 13:35:42 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:35:42 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:35:42 | INFO | Fetched 21141 rows -2026-06-15 13:35:42 | INFO | Deleting from Journey_Plan -2026-06-15 13:35:42 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:35:42 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:35:42 | INFO | ================================================================================ -2026-06-15 13:35:42 | INFO | Processing Table: coverage_remarks -2026-06-15 13:35:42 | INFO | Fetching Coverage Remarks -2026-06-15 13:35:43 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:35:43 | INFO | Fetched 29 rows -2026-06-15 13:35:43 | INFO | Truncated table coverage_remarks -2026-06-15 13:35:43 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:35:43 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:35:43 | INFO | ================================================================================ -2026-06-15 13:35:43 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:35:43 | INFO | Fetching Mapping Store Visibility for 2026-06-14 -2026-06-15 13:35:43 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:35:43 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:35:43 | INFO | ================================================================================ -2026-06-15 13:35:43 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:35:43 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:35:43 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:35:43 | INFO | Fetched 17 rows -2026-06-15 13:35:43 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:35:43 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:35:43 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:35:43 | INFO | ================================================================================ -2026-06-15 13:35:43 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:35:43 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:35:44 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:35:44 | INFO | Fetched 861 rows -2026-06-15 13:35:44 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:35:44 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:35:44 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:35:44 | INFO | ================================================================================ -2026-06-15 13:35:44 | INFO | Processing Table: Web_Logins -2026-06-15 13:35:44 | INFO | Fetching Web Login data for 2026-06-14 -2026-06-15 13:35:44 | INFO | Fetched 25 Web Login records -2026-06-15 13:35:44 | INFO | Fetched 25 rows -2026-06-15 13:35:44 | INFO | Deleting from Web_Logins -2026-06-15 13:35:45 | INFO | Web_Logins: inserted 25 rows into ClickHouse -2026-06-15 13:35:45 | INFO | Web_Logins loaded successfully (25 rows) -2026-06-15 13:35:45 | INFO | ================================================================================ -2026-06-15 13:35:45 | INFO | Processing Table: Promotion -2026-06-15 13:35:45 | INFO | Fetching Promotion data for 571 MIDs -2026-06-15 13:35:46 | INFO | Fetched 4,426 Promotion records -2026-06-15 13:35:46 | INFO | Fetched 4426 rows -2026-06-15 13:35:46 | INFO | Deleting from Promotion -2026-06-15 13:35:47 | INFO | Promotion: inserted 4,426 rows into ClickHouse -2026-06-15 13:35:47 | INFO | Promotion loaded successfully (4426 rows) -2026-06-15 13:35:47 | INFO | ================================================================================ -2026-06-15 13:35:47 | INFO | Processing Table: PaidVisibility -2026-06-15 13:35:47 | INFO | Fetching Paid Visibility data for 571 MIDs -2026-06-15 13:35:48 | INFO | Fetched 1,296 Paid Visibility records -2026-06-15 13:35:48 | INFO | Fetched 1296 rows -2026-06-15 13:35:48 | INFO | Deleting from PaidVisibility -2026-06-15 13:35:48 | INFO | PaidVisibility: inserted 1,296 rows into ClickHouse -2026-06-15 13:35:48 | INFO | PaidVisibility loaded successfully (1296 rows) -2026-06-15 13:35:48 | INFO | ================================================================================ -2026-06-15 13:35:48 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:35:48 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:35:48 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:35:48 | INFO | Fetched 33 rows -2026-06-15 13:35:48 | INFO | Truncated table Master_Salesterritorylayer -2026-06-15 13:35:48 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:35:48 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:35:48 | INFO | ================================================================================ -2026-06-15 13:35:48 | INFO | Pipeline Completed Successfully -2026-06-15 13:35:48 | INFO | ================================================================================ -2026-06-15 13:35:54 | INFO | ================================================================================ -2026-06-15 13:35:54 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:35:54 | INFO | Pipeline Run Date: 2026-06-14 -2026-06-15 13:35:54 | INFO | Connecting to databases... -2026-06-15 13:35:55 | INFO | -2026-06-15 13:35:56 | INFO | -2026-06-15 13:35:56 | INFO | Database connections established -2026-06-15 13:35:56 | INFO | Collecting MIDs for: 2026-06-14 -2026-06-15 13:35:56 | INFO | Found 571 MIDs -2026-06-15 13:35:57 | INFO | ================================================================================ -2026-06-15 13:35:57 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:35:57 | INFO | Fetching data for 571 MIDs -2026-06-15 13:35:57 | INFO | Fetched 1,997 rows from SQL Server -2026-06-15 13:35:57 | INFO | Fetched 1997 rows -2026-06-15 13:35:57 | INFO | Deleting from SOS_OneApp -2026-06-15 13:35:58 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse -2026-06-15 13:35:58 | INFO | SOS_OneApp loaded successfully (1997 rows) -2026-06-15 13:35:58 | INFO | ================================================================================ -2026-06-15 13:35:58 | INFO | Processing Table: OQaD -2026-06-15 13:35:58 | INFO | Fetching OQaD data for run_date=2026-06-14 -2026-06-15 13:36:01 | INFO | Fetched 443 rows -2026-06-15 13:36:01 | INFO | Fetched 443 rows -2026-06-15 13:36:01 | INFO | Deleting from OQaD -2026-06-15 13:36:01 | INFO | OQaD: inserted 443 rows into ClickHouse -2026-06-15 13:36:01 | INFO | OQaD loaded successfully (443 rows) -2026-06-15 13:36:01 | INFO | ================================================================================ -2026-06-15 13:36:01 | INFO | Processing Table: Survey -2026-06-15 13:36:01 | INFO | Fetching Survey data for 571 MIDs -2026-06-15 13:36:01 | INFO | Fetched 174 Survey rows -2026-06-15 13:36:01 | INFO | Fetched 174 rows -2026-06-15 13:36:01 | INFO | Deleting from Survey -2026-06-15 13:36:01 | INFO | Survey: inserted 174 rows into ClickHouse -2026-06-15 13:36:01 | INFO | Survey loaded successfully (174 rows) -2026-06-15 13:36:01 | INFO | ================================================================================ -2026-06-15 13:36:01 | INFO | Processing Table: additional_visibility -2026-06-15 13:36:01 | INFO | Fetching Additional Visibility data for 571 MIDs -2026-06-15 13:36:02 | INFO | Fetched 1,651 Additional Visibility rows -2026-06-15 13:36:02 | INFO | Fetched 1651 rows -2026-06-15 13:36:02 | INFO | Deleting from additional_visibility -2026-06-15 13:36:02 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse -2026-06-15 13:36:02 | INFO | additional_visibility loaded successfully (1651 rows) -2026-06-15 13:36:02 | INFO | ================================================================================ -2026-06-15 13:36:02 | INFO | Processing Table: Coverage -2026-06-15 13:36:02 | INFO | Fetching coverage data for 571 MIDs -2026-06-15 13:36:02 | INFO | Fetched 514 rows from SQL Server -2026-06-15 13:36:02 | INFO | Fetched 514 rows -2026-06-15 13:36:02 | INFO | Deleting from Coverage -2026-06-15 13:36:02 | INFO | Coverage: inserted 514 rows into ClickHouse -2026-06-15 13:36:02 | INFO | Coverage loaded successfully (514 rows) -2026-06-15 13:36:02 | INFO | ================================================================================ -2026-06-15 13:36:02 | INFO | Processing Table: Login -2026-06-15 13:36:02 | INFO | Fetching Login data for yesterday -2026-06-15 13:36:03 | INFO | Fetched 454 Login rows -2026-06-15 13:36:03 | INFO | Fetched 454 rows -2026-06-15 13:36:03 | INFO | Deleting from Login -2026-06-15 13:36:03 | INFO | No delete logic required for Login -2026-06-15 13:36:03 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:36:03 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:36:03 | INFO | ================================================================================ -2026-06-15 13:36:03 | INFO | Processing Table: Stock_Details -2026-06-15 13:36:03 | INFO | Fetching Stock Details data for 571 MIDs -2026-06-15 13:36:09 | INFO | Fetched 28,574 Stock Details rows -2026-06-15 13:36:09 | INFO | Fetched 28574 rows -2026-06-15 13:36:09 | INFO | Deleting from Stock_Details -2026-06-15 13:36:09 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse -2026-06-15 13:36:09 | INFO | Stock_Details loaded successfully (28574 rows) -2026-06-15 13:36:09 | INFO | ================================================================================ -2026-06-15 13:36:09 | INFO | Processing Table: Attendance -2026-06-15 13:36:09 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 -2026-06-15 13:36:10 | INFO | Fetched 11,543 attendance rows for 594 employees -2026-06-15 13:36:10 | INFO | Fetched 11543 rows -2026-06-15 13:36:10 | INFO | Truncated table Attendance -2026-06-15 13:36:10 | INFO | Attendance: inserted 11,543 rows into ClickHouse -2026-06-15 13:36:10 | INFO | Attendance loaded successfully (11543 rows) -2026-06-15 13:36:10 | INFO | ================================================================================ -2026-06-15 13:36:10 | INFO | Processing Table: Store_Master -2026-06-15 13:36:10 | INFO | Fetching Store Master data -2026-06-15 13:36:12 | INFO | Fetched 5,984 stores -2026-06-15 13:36:12 | INFO | Fetched 5984 rows -2026-06-15 13:36:12 | INFO | Truncated table Store_Master -2026-06-15 13:36:13 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:36:13 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:36:13 | INFO | ================================================================================ -2026-06-15 13:36:13 | INFO | Processing Table: SKU_Master -2026-06-15 13:36:13 | INFO | Fetching SKU Master data -2026-06-15 13:36:13 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:36:13 | INFO | Fetched 160 rows -2026-06-15 13:36:13 | INFO | Truncated table SKU_Master -2026-06-15 13:36:13 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:36:13 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:36:13 | INFO | ================================================================================ -2026-06-15 13:36:13 | INFO | Processing Table: display_master -2026-06-15 13:36:13 | INFO | Fetching Display Master data -2026-06-15 13:36:13 | INFO | Fetched 135 Display Master records -2026-06-15 13:36:13 | INFO | Fetched 135 rows -2026-06-15 13:36:13 | INFO | Truncated table display_master -2026-06-15 13:36:13 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:36:13 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:36:13 | INFO | ================================================================================ -2026-06-15 13:36:13 | INFO | Processing Table: Employee_Master -2026-06-15 13:36:13 | INFO | Fetching Employee Master data -2026-06-15 13:36:14 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:36:14 | INFO | Fetched 2273 rows -2026-06-15 13:36:14 | INFO | Truncated table Employee_Master -2026-06-15 13:36:14 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:36:14 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:36:14 | INFO | ================================================================================ -2026-06-15 13:36:14 | INFO | Processing Table: Journey_Plan -2026-06-15 13:36:14 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:36:15 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:36:15 | INFO | Fetched 21141 rows -2026-06-15 13:36:15 | INFO | Deleting from Journey_Plan -2026-06-15 13:36:15 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:36:15 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:36:15 | INFO | ================================================================================ -2026-06-15 13:36:15 | INFO | Processing Table: coverage_remarks -2026-06-15 13:36:15 | INFO | Fetching Coverage Remarks -2026-06-15 13:36:15 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:36:15 | INFO | Fetched 29 rows -2026-06-15 13:36:15 | INFO | Truncated table coverage_remarks -2026-06-15 13:36:15 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:36:15 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:36:15 | INFO | ================================================================================ -2026-06-15 13:36:15 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:36:15 | INFO | Fetching Mapping Store Visibility for 2026-06-14 -2026-06-15 13:36:15 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:36:15 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:36:15 | INFO | ================================================================================ -2026-06-15 13:36:15 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:36:15 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:36:15 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:36:15 | INFO | Fetched 17 rows -2026-06-15 13:36:16 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:36:16 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:36:16 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:36:16 | INFO | ================================================================================ -2026-06-15 13:36:16 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:36:16 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:36:16 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:36:16 | INFO | Fetched 861 rows -2026-06-15 13:36:16 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:36:16 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:36:16 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:36:16 | INFO | ================================================================================ -2026-06-15 13:36:16 | INFO | Processing Table: Web_Logins -2026-06-15 13:36:16 | INFO | Fetching Web Login data for 2026-06-14 -2026-06-15 13:36:16 | INFO | Fetched 25 Web Login records -2026-06-15 13:36:16 | INFO | Fetched 25 rows -2026-06-15 13:36:16 | INFO | Deleting from Web_Logins -2026-06-15 13:36:16 | INFO | Web_Logins: inserted 25 rows into ClickHouse -2026-06-15 13:36:16 | INFO | Web_Logins loaded successfully (25 rows) -2026-06-15 13:36:16 | INFO | ================================================================================ -2026-06-15 13:36:16 | INFO | Processing Table: Promotion -2026-06-15 13:36:16 | INFO | Fetching Promotion data for 571 MIDs -2026-06-15 13:36:18 | INFO | Fetched 4,426 Promotion records -2026-06-15 13:36:18 | INFO | Fetched 4426 rows -2026-06-15 13:36:18 | INFO | Deleting from Promotion -2026-06-15 13:36:18 | INFO | Promotion: inserted 4,426 rows into ClickHouse -2026-06-15 13:36:18 | INFO | Promotion loaded successfully (4426 rows) -2026-06-15 13:36:18 | INFO | ================================================================================ -2026-06-15 13:36:18 | INFO | Processing Table: PaidVisibility -2026-06-15 13:36:18 | INFO | Fetching Paid Visibility data for 571 MIDs -2026-06-15 13:36:19 | INFO | Fetched 1,296 Paid Visibility records -2026-06-15 13:36:19 | INFO | Fetched 1296 rows -2026-06-15 13:36:19 | INFO | Deleting from PaidVisibility -2026-06-15 13:36:20 | INFO | PaidVisibility: inserted 1,296 rows into ClickHouse -2026-06-15 13:36:20 | INFO | PaidVisibility loaded successfully (1296 rows) -2026-06-15 13:36:20 | INFO | ================================================================================ -2026-06-15 13:36:20 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:36:20 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:36:20 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:36:20 | INFO | Fetched 33 rows -2026-06-15 13:36:20 | INFO | Truncated table Master_Salesterritorylayer -2026-06-15 13:36:20 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:36:20 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:36:20 | INFO | ================================================================================ -2026-06-15 13:36:20 | INFO | Pipeline Completed Successfully -2026-06-15 13:36:20 | INFO | ================================================================================ -2026-06-15 13:37:03 | INFO | ================================================================================ -2026-06-15 13:37:03 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:37:03 | INFO | Pipeline Run Date: 2026-06-03 -2026-06-15 13:37:03 | INFO | Connecting to databases... -2026-06-15 13:37:04 | INFO | -2026-06-15 13:37:05 | INFO | -2026-06-15 13:37:06 | INFO | Database connections established -2026-06-15 13:37:06 | INFO | Collecting MIDs for: 2026-06-03 -2026-06-15 13:37:06 | INFO | Found 808 MIDs -2026-06-15 13:37:06 | INFO | ================================================================================ -2026-06-15 13:37:06 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:37:06 | INFO | Fetching data for 808 MIDs -2026-06-15 13:37:07 | INFO | Fetched 3,565 rows from SQL Server -2026-06-15 13:37:07 | INFO | Fetched 3565 rows -2026-06-15 13:37:08 | INFO | Deleting from SOS_OneApp -2026-06-15 13:37:08 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse -2026-06-15 13:37:08 | INFO | SOS_OneApp loaded successfully (3565 rows) -2026-06-15 13:37:08 | INFO | ================================================================================ -2026-06-15 13:37:08 | INFO | Processing Table: OQaD -2026-06-15 13:37:08 | INFO | Fetching OQaD data for run_date=2026-06-03 -2026-06-15 13:37:12 | INFO | Fetched 455 rows -2026-06-15 13:37:12 | INFO | Fetched 455 rows -2026-06-15 13:37:12 | INFO | Deleting from OQaD -2026-06-15 13:37:12 | INFO | OQaD: inserted 455 rows into ClickHouse -2026-06-15 13:37:12 | INFO | OQaD loaded successfully (455 rows) -2026-06-15 13:37:12 | INFO | ================================================================================ -2026-06-15 13:37:12 | INFO | Processing Table: Survey -2026-06-15 13:37:12 | INFO | Fetching Survey data for 808 MIDs -2026-06-15 13:37:13 | INFO | Fetched 139 Survey rows -2026-06-15 13:37:13 | INFO | Fetched 139 rows -2026-06-15 13:37:13 | INFO | Deleting from Survey -2026-06-15 13:37:13 | INFO | Survey: inserted 139 rows into ClickHouse -2026-06-15 13:37:13 | INFO | Survey loaded successfully (139 rows) -2026-06-15 13:37:13 | INFO | ================================================================================ -2026-06-15 13:37:13 | INFO | Processing Table: additional_visibility -2026-06-15 13:37:13 | INFO | Fetching Additional Visibility data for 808 MIDs -2026-06-15 13:37:14 | INFO | Fetched 1,884 Additional Visibility rows -2026-06-15 13:37:14 | INFO | Fetched 1884 rows -2026-06-15 13:37:14 | INFO | Deleting from additional_visibility -2026-06-15 13:37:14 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse -2026-06-15 13:37:14 | INFO | additional_visibility loaded successfully (1884 rows) -2026-06-15 13:37:14 | INFO | ================================================================================ -2026-06-15 13:37:14 | INFO | Processing Table: Coverage -2026-06-15 13:37:14 | INFO | Fetching coverage data for 808 MIDs -2026-06-15 13:37:15 | INFO | Fetched 751 rows from SQL Server -2026-06-15 13:37:15 | INFO | Fetched 751 rows -2026-06-15 13:37:15 | INFO | Deleting from Coverage -2026-06-15 13:37:15 | INFO | Coverage: inserted 751 rows into ClickHouse -2026-06-15 13:37:15 | INFO | Coverage loaded successfully (751 rows) -2026-06-15 13:37:15 | INFO | ================================================================================ -2026-06-15 13:37:15 | INFO | Processing Table: Login -2026-06-15 13:37:15 | INFO | Fetching Login data for yesterday -2026-06-15 13:37:16 | INFO | Fetched 454 Login rows -2026-06-15 13:37:16 | INFO | Fetched 454 rows -2026-06-15 13:37:16 | INFO | Deleting from Login -2026-06-15 13:37:16 | INFO | No delete logic required for Login -2026-06-15 13:37:16 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:37:16 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:37:16 | INFO | ================================================================================ -2026-06-15 13:37:16 | INFO | Processing Table: Stock_Details -2026-06-15 13:37:16 | INFO | Fetching Stock Details data for 808 MIDs -2026-06-15 13:37:19 | INFO | Fetched 40,840 Stock Details rows -2026-06-15 13:37:19 | INFO | Fetched 40840 rows -2026-06-15 13:37:19 | INFO | Deleting from Stock_Details -2026-06-15 13:37:20 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse -2026-06-15 13:37:20 | INFO | Stock_Details loaded successfully (40840 rows) -2026-06-15 13:37:20 | INFO | ================================================================================ -2026-06-15 13:37:20 | INFO | Processing Table: Attendance -2026-06-15 13:37:20 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 -2026-06-15 13:37:21 | INFO | Fetched 12,432 attendance rows for 579 employees -2026-06-15 13:37:21 | INFO | Fetched 12432 rows -2026-06-15 13:37:21 | INFO | Truncated table Attendance -2026-06-15 13:37:21 | INFO | Attendance: inserted 12,432 rows into ClickHouse -2026-06-15 13:37:21 | INFO | Attendance loaded successfully (12432 rows) -2026-06-15 13:37:21 | INFO | ================================================================================ -2026-06-15 13:37:21 | INFO | Processing Table: Store_Master -2026-06-15 13:37:21 | INFO | Fetching Store Master data -2026-06-15 13:37:22 | INFO | Fetched 5,984 stores -2026-06-15 13:37:22 | INFO | Fetched 5984 rows -2026-06-15 13:37:22 | INFO | Truncated table Store_Master -2026-06-15 13:37:22 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:37:22 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:37:22 | INFO | ================================================================================ -2026-06-15 13:37:22 | INFO | Processing Table: SKU_Master -2026-06-15 13:37:22 | INFO | Fetching SKU Master data -2026-06-15 13:37:22 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:37:22 | INFO | Fetched 160 rows -2026-06-15 13:37:23 | INFO | Truncated table SKU_Master -2026-06-15 13:37:23 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:37:23 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:37:23 | INFO | ================================================================================ -2026-06-15 13:37:23 | INFO | Processing Table: display_master -2026-06-15 13:37:23 | INFO | Fetching Display Master data -2026-06-15 13:37:23 | INFO | Fetched 135 Display Master records -2026-06-15 13:37:23 | INFO | Fetched 135 rows -2026-06-15 13:37:23 | INFO | Truncated table display_master -2026-06-15 13:37:23 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:37:23 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:37:23 | INFO | ================================================================================ -2026-06-15 13:37:23 | INFO | Processing Table: Employee_Master -2026-06-15 13:37:23 | INFO | Fetching Employee Master data -2026-06-15 13:37:24 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:37:24 | INFO | Fetched 2273 rows -2026-06-15 13:37:24 | INFO | Truncated table Employee_Master -2026-06-15 13:37:25 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:37:25 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:37:25 | INFO | ================================================================================ -2026-06-15 13:37:25 | INFO | Processing Table: Journey_Plan -2026-06-15 13:37:25 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:37:25 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:37:25 | INFO | Fetched 21141 rows -2026-06-15 13:37:25 | INFO | Deleting from Journey_Plan -2026-06-15 13:37:25 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:37:25 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:37:25 | INFO | ================================================================================ -2026-06-15 13:37:25 | INFO | Processing Table: coverage_remarks -2026-06-15 13:37:25 | INFO | Fetching Coverage Remarks -2026-06-15 13:37:25 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:37:25 | INFO | Fetched 29 rows -2026-06-15 13:37:25 | INFO | Truncated table coverage_remarks -2026-06-15 13:37:26 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:37:26 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:37:26 | INFO | ================================================================================ -2026-06-15 13:37:26 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:37:26 | INFO | Fetching Mapping Store Visibility for 2026-06-03 -2026-06-15 13:37:26 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:37:26 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:37:26 | INFO | ================================================================================ -2026-06-15 13:37:26 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:37:26 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:37:26 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:37:26 | INFO | Fetched 17 rows -2026-06-15 13:37:26 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:37:26 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:37:26 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:37:26 | INFO | ================================================================================ -2026-06-15 13:37:26 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:37:26 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:37:26 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:37:26 | INFO | Fetched 861 rows -2026-06-15 13:37:26 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:37:26 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:37:26 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:37:26 | INFO | ================================================================================ -2026-06-15 13:37:26 | INFO | Processing Table: Web_Logins -2026-06-15 13:37:26 | INFO | Fetching Web Login data for 2026-06-03 -2026-06-15 13:37:27 | INFO | Fetched 200 Web Login records -2026-06-15 13:37:27 | INFO | Fetched 200 rows -2026-06-15 13:37:27 | INFO | Deleting from Web_Logins -2026-06-15 13:37:27 | INFO | Web_Logins: inserted 200 rows into ClickHouse -2026-06-15 13:37:27 | INFO | Web_Logins loaded successfully (200 rows) -2026-06-15 13:37:27 | INFO | ================================================================================ -2026-06-15 13:37:27 | INFO | Processing Table: Promotion -2026-06-15 13:37:27 | INFO | Fetching Promotion data for 808 MIDs -2026-06-15 13:37:32 | INFO | Fetched 8,162 Promotion records -2026-06-15 13:37:32 | INFO | Fetched 8162 rows -2026-06-15 13:37:32 | INFO | Deleting from Promotion -2026-06-15 13:37:32 | INFO | Promotion: inserted 8,162 rows into ClickHouse -2026-06-15 13:37:32 | INFO | Promotion loaded successfully (8162 rows) -2026-06-15 13:37:32 | INFO | ================================================================================ -2026-06-15 13:37:32 | INFO | Processing Table: PaidVisibility -2026-06-15 13:37:32 | INFO | Fetching Paid Visibility data for 808 MIDs -2026-06-15 13:37:34 | INFO | Fetched 1,064 Paid Visibility records -2026-06-15 13:37:34 | INFO | Fetched 1064 rows -2026-06-15 13:37:34 | INFO | Deleting from PaidVisibility -2026-06-15 13:37:34 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse -2026-06-15 13:37:34 | INFO | PaidVisibility loaded successfully (1064 rows) -2026-06-15 13:37:34 | INFO | ================================================================================ -2026-06-15 13:37:34 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:37:34 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:37:34 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:37:34 | INFO | Fetched 33 rows -2026-06-15 13:37:34 | INFO | Truncated table Master_Salesterritorylayer -2026-06-15 13:37:34 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:37:34 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:37:34 | INFO | ================================================================================ -2026-06-15 13:37:34 | INFO | Pipeline Completed Successfully -2026-06-15 13:37:34 | INFO | ================================================================================ -2026-06-15 13:37:43 | INFO | ================================================================================ -2026-06-15 13:37:43 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:37:43 | INFO | Pipeline Run Date: 2026-06-03 -2026-06-15 13:37:43 | INFO | Connecting to databases... -2026-06-15 13:37:44 | INFO | -2026-06-15 13:37:44 | INFO | -2026-06-15 13:37:45 | INFO | Database connections established -2026-06-15 13:37:45 | INFO | Collecting MIDs for: 2026-06-03 -2026-06-15 13:37:45 | INFO | Found 808 MIDs -2026-06-15 13:37:45 | INFO | ================================================================================ -2026-06-15 13:37:45 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:37:45 | INFO | Fetching data for 808 MIDs -2026-06-15 13:37:46 | INFO | Fetched 3,565 rows from SQL Server -2026-06-15 13:37:46 | INFO | Fetched 3565 rows -2026-06-15 13:37:46 | INFO | Deleting from SOS_OneApp -2026-06-15 13:37:46 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse -2026-06-15 13:37:46 | INFO | SOS_OneApp loaded successfully (3565 rows) -2026-06-15 13:37:46 | INFO | ================================================================================ -2026-06-15 13:37:46 | INFO | Processing Table: OQaD -2026-06-15 13:37:46 | INFO | Fetching OQaD data for run_date=2026-06-03 -2026-06-15 13:37:49 | INFO | Fetched 455 rows -2026-06-15 13:37:49 | INFO | Fetched 455 rows -2026-06-15 13:37:49 | INFO | Deleting from OQaD -2026-06-15 13:37:49 | INFO | OQaD: inserted 455 rows into ClickHouse -2026-06-15 13:37:49 | INFO | OQaD loaded successfully (455 rows) -2026-06-15 13:37:49 | INFO | ================================================================================ -2026-06-15 13:37:49 | INFO | Processing Table: Survey -2026-06-15 13:37:49 | INFO | Fetching Survey data for 808 MIDs -2026-06-15 13:37:49 | INFO | Fetched 139 Survey rows -2026-06-15 13:37:49 | INFO | Fetched 139 rows -2026-06-15 13:37:49 | INFO | Deleting from Survey -2026-06-15 13:37:49 | INFO | Survey: inserted 139 rows into ClickHouse -2026-06-15 13:37:49 | INFO | Survey loaded successfully (139 rows) -2026-06-15 13:37:49 | INFO | ================================================================================ -2026-06-15 13:37:49 | INFO | Processing Table: additional_visibility -2026-06-15 13:37:49 | INFO | Fetching Additional Visibility data for 808 MIDs -2026-06-15 13:37:50 | INFO | Fetched 1,884 Additional Visibility rows -2026-06-15 13:37:50 | INFO | Fetched 1884 rows -2026-06-15 13:37:50 | INFO | Deleting from additional_visibility -2026-06-15 13:37:50 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse -2026-06-15 13:37:50 | INFO | additional_visibility loaded successfully (1884 rows) -2026-06-15 13:37:50 | INFO | ================================================================================ -2026-06-15 13:37:50 | INFO | Processing Table: Coverage -2026-06-15 13:37:50 | INFO | Fetching coverage data for 808 MIDs -2026-06-15 13:37:50 | INFO | Fetched 751 rows from SQL Server -2026-06-15 13:37:50 | INFO | Fetched 751 rows -2026-06-15 13:37:50 | INFO | Deleting from Coverage -2026-06-15 13:37:50 | INFO | Coverage: inserted 751 rows into ClickHouse -2026-06-15 13:37:50 | INFO | Coverage loaded successfully (751 rows) -2026-06-15 13:37:50 | INFO | ================================================================================ -2026-06-15 13:37:50 | INFO | Processing Table: Login -2026-06-15 13:37:50 | INFO | Fetching Login data for yesterday -2026-06-15 13:37:51 | INFO | Fetched 454 Login rows -2026-06-15 13:37:51 | INFO | Fetched 454 rows -2026-06-15 13:37:51 | INFO | Deleting from Login -2026-06-15 13:37:51 | INFO | No delete logic required for Login -2026-06-15 13:37:51 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:37:51 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:37:51 | INFO | ================================================================================ -2026-06-15 13:37:51 | INFO | Processing Table: Stock_Details -2026-06-15 13:37:51 | INFO | Fetching Stock Details data for 808 MIDs -2026-06-15 13:37:55 | INFO | Fetched 40,840 Stock Details rows -2026-06-15 13:37:55 | INFO | Fetched 40840 rows -2026-06-15 13:37:55 | INFO | Deleting from Stock_Details -2026-06-15 13:37:55 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse -2026-06-15 13:37:55 | INFO | Stock_Details loaded successfully (40840 rows) -2026-06-15 13:37:55 | INFO | ================================================================================ -2026-06-15 13:37:55 | INFO | Processing Table: Attendance -2026-06-15 13:37:55 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 -2026-06-15 13:37:56 | INFO | Fetched 12,432 attendance rows for 579 employees -2026-06-15 13:37:56 | INFO | Fetched 12432 rows -2026-06-15 13:37:57 | INFO | Truncated table Attendance -2026-06-15 13:37:57 | INFO | Attendance: inserted 12,432 rows into ClickHouse -2026-06-15 13:37:57 | INFO | Attendance loaded successfully (12432 rows) -2026-06-15 13:37:57 | INFO | ================================================================================ -2026-06-15 13:37:57 | INFO | Processing Table: Store_Master -2026-06-15 13:37:57 | INFO | Fetching Store Master data -2026-06-15 13:37:58 | INFO | Fetched 5,984 stores -2026-06-15 13:37:58 | INFO | Fetched 5984 rows -2026-06-15 13:37:58 | INFO | Truncated table Store_Master -2026-06-15 13:37:59 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:37:59 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:37:59 | INFO | ================================================================================ -2026-06-15 13:37:59 | INFO | Processing Table: SKU_Master -2026-06-15 13:37:59 | INFO | Fetching SKU Master data -2026-06-15 13:37:59 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:37:59 | INFO | Fetched 160 rows -2026-06-15 13:37:59 | INFO | Truncated table SKU_Master -2026-06-15 13:37:59 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:37:59 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:37:59 | INFO | ================================================================================ -2026-06-15 13:37:59 | INFO | Processing Table: display_master -2026-06-15 13:37:59 | INFO | Fetching Display Master data -2026-06-15 13:37:59 | INFO | Fetched 135 Display Master records -2026-06-15 13:37:59 | INFO | Fetched 135 rows -2026-06-15 13:37:59 | INFO | Truncated table display_master -2026-06-15 13:37:59 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:37:59 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:37:59 | INFO | ================================================================================ -2026-06-15 13:37:59 | INFO | Processing Table: Employee_Master -2026-06-15 13:37:59 | INFO | Fetching Employee Master data -2026-06-15 13:38:00 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:38:00 | INFO | Fetched 2273 rows -2026-06-15 13:38:00 | INFO | Truncated table Employee_Master -2026-06-15 13:38:00 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:38:00 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:38:00 | INFO | ================================================================================ -2026-06-15 13:38:00 | INFO | Processing Table: Journey_Plan -2026-06-15 13:38:00 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:38:00 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:38:00 | INFO | Fetched 21141 rows -2026-06-15 13:38:00 | INFO | Deleting from Journey_Plan -2026-06-15 13:38:01 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:38:01 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:38:01 | INFO | ================================================================================ -2026-06-15 13:38:01 | INFO | Processing Table: coverage_remarks -2026-06-15 13:38:01 | INFO | Fetching Coverage Remarks -2026-06-15 13:38:01 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:38:01 | INFO | Fetched 29 rows -2026-06-15 13:38:01 | INFO | Truncated table coverage_remarks -2026-06-15 13:38:01 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:38:01 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:38:01 | INFO | ================================================================================ -2026-06-15 13:38:01 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:38:01 | INFO | Fetching Mapping Store Visibility for 2026-06-03 -2026-06-15 13:38:01 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:38:01 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:38:01 | INFO | ================================================================================ -2026-06-15 13:38:01 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:38:01 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:38:01 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:38:01 | INFO | Fetched 17 rows -2026-06-15 13:38:01 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:38:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:38:01 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:38:01 | INFO | ================================================================================ -2026-06-15 13:38:01 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:38:01 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:38:02 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:38:02 | INFO | Fetched 861 rows -2026-06-15 13:38:02 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:38:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:38:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:38:02 | INFO | ================================================================================ -2026-06-15 13:38:02 | INFO | Processing Table: Web_Logins -2026-06-15 13:38:02 | INFO | Fetching Web Login data for 2026-06-03 -2026-06-15 13:38:02 | INFO | Fetched 200 Web Login records -2026-06-15 13:38:02 | INFO | Fetched 200 rows -2026-06-15 13:38:02 | INFO | Deleting from Web_Logins -2026-06-15 13:38:02 | INFO | Web_Logins: inserted 200 rows into ClickHouse -2026-06-15 13:38:02 | INFO | Web_Logins loaded successfully (200 rows) -2026-06-15 13:38:02 | INFO | ================================================================================ -2026-06-15 13:38:02 | INFO | Processing Table: Promotion -2026-06-15 13:38:02 | INFO | Fetching Promotion data for 808 MIDs -2026-06-15 13:38:04 | INFO | Fetched 8,162 Promotion records -2026-06-15 13:38:04 | INFO | Fetched 8162 rows -2026-06-15 13:38:04 | INFO | Deleting from Promotion -2026-06-15 13:38:04 | INFO | Promotion: inserted 8,162 rows into ClickHouse -2026-06-15 13:38:04 | INFO | Promotion loaded successfully (8162 rows) -2026-06-15 13:38:04 | INFO | ================================================================================ -2026-06-15 13:38:04 | INFO | Processing Table: PaidVisibility -2026-06-15 13:38:04 | INFO | Fetching Paid Visibility data for 808 MIDs -2026-06-15 13:38:05 | INFO | Fetched 1,064 Paid Visibility records -2026-06-15 13:38:05 | INFO | Fetched 1064 rows -2026-06-15 13:38:06 | INFO | Deleting from PaidVisibility -2026-06-15 13:38:06 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse -2026-06-15 13:38:06 | INFO | PaidVisibility loaded successfully (1064 rows) -2026-06-15 13:38:06 | INFO | ================================================================================ -2026-06-15 13:38:06 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:38:06 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:38:06 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:38:06 | INFO | Fetched 33 rows -2026-06-15 13:38:06 | INFO | Truncated table Master_Salesterritorylayer -2026-06-15 13:38:07 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:38:07 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:38:07 | INFO | ================================================================================ -2026-06-15 13:38:07 | INFO | Pipeline Completed Successfully -2026-06-15 13:38:07 | INFO | ================================================================================ -2026-06-15 13:40:53 | INFO | ================================================================================ -2026-06-15 13:40:53 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:40:53 | INFO | Pipeline Run Date: 2026-06-03 -2026-06-15 13:40:53 | INFO | Connecting to databases... -2026-06-15 13:40:53 | INFO | -2026-06-15 13:40:55 | INFO | -2026-06-15 13:40:56 | INFO | Database connections established -2026-06-15 13:40:56 | INFO | Collecting MIDs for: 2026-06-03 -2026-06-15 13:40:56 | INFO | Found 808 MIDs -2026-06-15 13:40:56 | INFO | ================================================================================ -2026-06-15 13:40:56 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:40:56 | INFO | Fetching data for 808 MIDs -2026-06-15 13:40:57 | INFO | Fetched 3,565 rows from SQL Server -2026-06-15 13:40:57 | INFO | Fetched 3565 rows -2026-06-15 13:40:57 | INFO | Deleting from SOS_OneApp -2026-06-15 13:40:57 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse -2026-06-15 13:40:57 | INFO | SOS_OneApp loaded successfully (3565 rows) -2026-06-15 13:40:57 | INFO | ================================================================================ -2026-06-15 13:40:57 | INFO | Processing Table: OQaD -2026-06-15 13:40:57 | INFO | Fetching OQaD data for run_date=2026-06-03 -2026-06-15 13:41:00 | INFO | Fetched 455 rows -2026-06-15 13:41:00 | INFO | Fetched 455 rows -2026-06-15 13:41:00 | INFO | Deleting from OQaD -2026-06-15 13:41:00 | INFO | OQaD: inserted 455 rows into ClickHouse -2026-06-15 13:41:00 | INFO | OQaD loaded successfully (455 rows) -2026-06-15 13:41:00 | INFO | ================================================================================ -2026-06-15 13:41:00 | INFO | Processing Table: Survey -2026-06-15 13:41:00 | INFO | Fetching Survey data for 808 MIDs -2026-06-15 13:41:00 | INFO | Fetched 139 Survey rows -2026-06-15 13:41:00 | INFO | Fetched 139 rows -2026-06-15 13:41:00 | INFO | Deleting from Survey -2026-06-15 13:41:00 | INFO | Survey: inserted 139 rows into ClickHouse -2026-06-15 13:41:00 | INFO | Survey loaded successfully (139 rows) -2026-06-15 13:41:00 | INFO | ================================================================================ -2026-06-15 13:41:00 | INFO | Processing Table: additional_visibility -2026-06-15 13:41:00 | INFO | Fetching Additional Visibility data for 808 MIDs -2026-06-15 13:41:01 | INFO | Fetched 1,884 Additional Visibility rows -2026-06-15 13:41:01 | INFO | Fetched 1884 rows -2026-06-15 13:41:01 | INFO | Deleting from additional_visibility -2026-06-15 13:41:01 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse -2026-06-15 13:41:01 | INFO | additional_visibility loaded successfully (1884 rows) -2026-06-15 13:41:01 | INFO | ================================================================================ -2026-06-15 13:41:01 | INFO | Processing Table: Coverage -2026-06-15 13:41:01 | INFO | Fetching coverage data for 808 MIDs -2026-06-15 13:41:01 | INFO | Fetched 751 rows from SQL Server -2026-06-15 13:41:01 | INFO | Fetched 751 rows -2026-06-15 13:41:01 | INFO | Deleting from Coverage -2026-06-15 13:41:02 | INFO | Coverage: inserted 751 rows into ClickHouse -2026-06-15 13:41:02 | INFO | Coverage loaded successfully (751 rows) -2026-06-15 13:41:02 | INFO | ================================================================================ -2026-06-15 13:41:02 | INFO | Processing Table: Login -2026-06-15 13:41:02 | INFO | Fetching Login data for yesterday -2026-06-15 13:41:02 | INFO | Fetched 454 Login rows -2026-06-15 13:41:02 | INFO | Fetched 454 rows -2026-06-15 13:41:02 | INFO | Deleting from Login -2026-06-15 13:41:02 | INFO | No delete logic required for Login -2026-06-15 13:41:02 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:41:02 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:41:02 | INFO | ================================================================================ -2026-06-15 13:41:02 | INFO | Processing Table: Stock_Details -2026-06-15 13:41:02 | INFO | Fetching Stock Details data for 808 MIDs -2026-06-15 13:41:07 | INFO | Fetched 40,840 Stock Details rows -2026-06-15 13:41:07 | INFO | Fetched 40840 rows -2026-06-15 13:41:08 | INFO | Deleting from Stock_Details -2026-06-15 13:41:08 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse -2026-06-15 13:41:08 | INFO | Stock_Details loaded successfully (40840 rows) -2026-06-15 13:41:08 | INFO | ================================================================================ -2026-06-15 13:41:08 | INFO | Processing Table: Attendance -2026-06-15 13:41:08 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 -2026-06-15 13:41:10 | INFO | Fetched 12,432 attendance rows for 579 employees -2026-06-15 13:41:10 | INFO | Fetched 12432 rows -2026-06-15 13:41:10 | INFO | Truncated table Attendance -2026-06-15 13:41:10 | INFO | Attendance: inserted 12,432 rows into ClickHouse -2026-06-15 13:41:10 | INFO | Attendance loaded successfully (12432 rows) -2026-06-15 13:41:10 | INFO | ================================================================================ -2026-06-15 13:41:10 | INFO | Processing Table: Store_Master -2026-06-15 13:41:10 | INFO | Fetching Store Master data -2026-06-15 13:41:18 | INFO | Fetched 5,984 stores -2026-06-15 13:41:18 | INFO | Fetched 5984 rows -2026-06-15 13:41:18 | INFO | Truncated table Store_Master -2026-06-15 13:41:18 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:41:18 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:41:18 | INFO | ================================================================================ -2026-06-15 13:41:18 | INFO | Processing Table: SKU_Master -2026-06-15 13:41:18 | INFO | Fetching SKU Master data -2026-06-15 13:41:18 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:41:18 | INFO | Fetched 160 rows -2026-06-15 13:41:18 | INFO | Truncated table SKU_Master -2026-06-15 13:41:19 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:41:19 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:41:19 | INFO | ================================================================================ -2026-06-15 13:41:19 | INFO | Processing Table: display_master -2026-06-15 13:41:19 | INFO | Fetching Display Master data -2026-06-15 13:41:19 | INFO | Fetched 135 Display Master records -2026-06-15 13:41:19 | INFO | Fetched 135 rows -2026-06-15 13:41:19 | INFO | Truncated table display_master -2026-06-15 13:41:19 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:41:19 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:41:19 | INFO | ================================================================================ -2026-06-15 13:41:19 | INFO | Processing Table: Employee_Master -2026-06-15 13:41:19 | INFO | Fetching Employee Master data -2026-06-15 13:41:20 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:41:20 | INFO | Fetched 2273 rows -2026-06-15 13:41:20 | INFO | Truncated table Employee_Master -2026-06-15 13:41:20 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:41:20 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:41:20 | INFO | ================================================================================ -2026-06-15 13:41:20 | INFO | Processing Table: Journey_Plan -2026-06-15 13:41:20 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:41:21 | INFO | Fetched 21,141 Journey Plan records -2026-06-15 13:41:21 | INFO | Fetched 21141 rows -2026-06-15 13:41:21 | INFO | Deleting from Journey_Plan -2026-06-15 13:41:21 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse -2026-06-15 13:41:21 | INFO | Journey_Plan loaded successfully (21141 rows) -2026-06-15 13:41:21 | INFO | ================================================================================ -2026-06-15 13:41:21 | INFO | Processing Table: coverage_remarks -2026-06-15 13:41:21 | INFO | Fetching Coverage Remarks -2026-06-15 13:41:21 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:41:21 | INFO | Fetched 29 rows -2026-06-15 13:41:21 | INFO | Truncated table coverage_remarks -2026-06-15 13:41:22 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:41:22 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:41:22 | INFO | ================================================================================ -2026-06-15 13:41:22 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:41:22 | INFO | Fetching Mapping Store Visibility for 2026-06-03 -2026-06-15 13:41:22 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:41:22 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:41:22 | INFO | ================================================================================ -2026-06-15 13:41:22 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:41:22 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:41:22 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:41:22 | INFO | Fetched 17 rows -2026-06-15 13:41:22 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:41:22 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:41:22 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:41:22 | INFO | ================================================================================ -2026-06-15 13:41:22 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:41:22 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:41:22 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:41:22 | INFO | Fetched 861 rows -2026-06-15 13:41:22 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:41:22 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:41:22 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:41:22 | INFO | ================================================================================ -2026-06-15 13:41:22 | INFO | Processing Table: Web_Logins -2026-06-15 13:41:22 | INFO | Fetching Web Login data for 2026-06-03 -2026-06-15 13:41:23 | INFO | Fetched 200 Web Login records -2026-06-15 13:41:23 | INFO | Fetched 200 rows -2026-06-15 13:41:23 | INFO | Deleting from Web_Logins -2026-06-15 13:41:23 | INFO | Web_Logins: inserted 200 rows into ClickHouse -2026-06-15 13:41:23 | INFO | Web_Logins loaded successfully (200 rows) -2026-06-15 13:41:23 | INFO | ================================================================================ -2026-06-15 13:41:23 | INFO | Processing Table: Promotion -2026-06-15 13:41:23 | INFO | Fetching Promotion data for 808 MIDs -2026-06-15 13:41:28 | INFO | Fetched 8,162 Promotion records -2026-06-15 13:41:28 | INFO | Fetched 8162 rows -2026-06-15 13:41:28 | INFO | Deleting from Promotion -2026-06-15 13:41:28 | INFO | Promotion: inserted 8,162 rows into ClickHouse -2026-06-15 13:41:28 | INFO | Promotion loaded successfully (8162 rows) -2026-06-15 13:41:28 | INFO | ================================================================================ -2026-06-15 13:41:28 | INFO | Processing Table: PaidVisibility -2026-06-15 13:41:28 | INFO | Fetching Paid Visibility data for 808 MIDs -2026-06-15 13:41:29 | INFO | Fetched 1,064 Paid Visibility records -2026-06-15 13:41:29 | INFO | Fetched 1064 rows -2026-06-15 13:41:29 | INFO | Deleting from PaidVisibility -2026-06-15 13:41:30 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse -2026-06-15 13:41:30 | INFO | PaidVisibility loaded successfully (1064 rows) -2026-06-15 13:41:30 | INFO | ================================================================================ -2026-06-15 13:41:30 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:41:30 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:41:30 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:41:30 | INFO | Fetched 33 rows -2026-06-15 13:41:30 | INFO | Truncated table Master_Salesterritorylayer -2026-06-15 13:41:30 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:41:30 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:41:30 | INFO | ================================================================================ -2026-06-15 13:41:30 | INFO | Pipeline Completed Successfully -2026-06-15 13:41:30 | INFO | ================================================================================ -2026-06-15 13:46:09 | INFO | ================================================================================ -2026-06-15 13:46:09 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:46:09 | INFO | Pipeline Run Date: 2026-06-03 -2026-06-15 13:46:09 | INFO | Connecting to databases... -2026-06-15 13:46:10 | INFO | -2026-06-15 13:46:11 | INFO | -2026-06-15 13:46:11 | INFO | Database connections established -2026-06-15 13:46:11 | INFO | Collecting MIDs for: 2026-06-03 -2026-06-15 13:46:11 | INFO | Found 808 MIDs -2026-06-15 13:46:11 | INFO | ================================================================================ -2026-06-15 13:46:11 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:46:11 | INFO | Fetching data for 808 MIDs -2026-06-15 13:46:12 | INFO | Fetched 3,565 rows from SQL Server -2026-06-15 13:46:12 | INFO | Fetched 3565 rows -2026-06-15 13:46:12 | INFO | Deleting from SOS_OneApp -2026-06-15 13:46:13 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse -2026-06-15 13:46:13 | INFO | SOS_OneApp loaded successfully (3565 rows) -2026-06-15 13:46:13 | INFO | ================================================================================ -2026-06-15 13:46:13 | INFO | Processing Table: OQaD -2026-06-15 13:46:13 | INFO | Fetching OQaD data for run_date=2026-06-03 -2026-06-15 13:46:15 | INFO | Fetched 455 rows -2026-06-15 13:46:15 | INFO | Fetched 455 rows -2026-06-15 13:46:15 | INFO | Deleting from OQaD -2026-06-15 13:46:15 | INFO | OQaD: inserted 455 rows into ClickHouse -2026-06-15 13:46:15 | INFO | OQaD loaded successfully (455 rows) -2026-06-15 13:46:15 | INFO | ================================================================================ -2026-06-15 13:46:15 | INFO | Processing Table: Survey -2026-06-15 13:46:15 | INFO | Fetching Survey data for 808 MIDs -2026-06-15 13:46:15 | INFO | Fetched 139 Survey rows -2026-06-15 13:46:15 | INFO | Fetched 139 rows -2026-06-15 13:46:15 | INFO | Deleting from Survey -2026-06-15 13:46:16 | INFO | Survey: inserted 139 rows into ClickHouse -2026-06-15 13:46:16 | INFO | Survey loaded successfully (139 rows) -2026-06-15 13:46:16 | INFO | ================================================================================ -2026-06-15 13:46:16 | INFO | Processing Table: additional_visibility -2026-06-15 13:46:16 | INFO | Fetching Additional Visibility data for 808 MIDs -2026-06-15 13:46:16 | INFO | Fetched 1,884 Additional Visibility rows -2026-06-15 13:46:16 | INFO | Fetched 1884 rows -2026-06-15 13:46:16 | INFO | Deleting from additional_visibility -2026-06-15 13:46:16 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse -2026-06-15 13:46:16 | INFO | additional_visibility loaded successfully (1884 rows) -2026-06-15 13:46:16 | INFO | ================================================================================ -2026-06-15 13:46:16 | INFO | Processing Table: Coverage -2026-06-15 13:46:16 | INFO | Fetching coverage data for 808 MIDs -2026-06-15 13:46:16 | INFO | Fetched 751 rows from SQL Server -2026-06-15 13:46:16 | INFO | Fetched 751 rows -2026-06-15 13:46:16 | INFO | Deleting from Coverage -2026-06-15 13:46:17 | INFO | Coverage: inserted 751 rows into ClickHouse -2026-06-15 13:46:17 | INFO | Coverage loaded successfully (751 rows) -2026-06-15 13:46:17 | INFO | ================================================================================ -2026-06-15 13:46:17 | INFO | Processing Table: Login -2026-06-15 13:46:17 | INFO | Fetching Login data for yesterday -2026-06-15 13:46:17 | INFO | Fetched 454 Login rows -2026-06-15 13:46:17 | INFO | Fetched 454 rows -2026-06-15 13:46:17 | INFO | Deleting from Login -2026-06-15 13:46:17 | INFO | No delete logic required for Login -2026-06-15 13:46:18 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:46:18 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:46:18 | INFO | ================================================================================ -2026-06-15 13:46:18 | INFO | Processing Table: Stock_Details -2026-06-15 13:46:18 | INFO | Fetching Stock Details data for 808 MIDs -2026-06-15 13:46:22 | INFO | Fetched 40,840 Stock Details rows -2026-06-15 13:46:22 | INFO | Fetched 40840 rows -2026-06-15 13:46:22 | INFO | Deleting from Stock_Details -2026-06-15 13:46:23 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse -2026-06-15 13:46:23 | INFO | Stock_Details loaded successfully (40840 rows) -2026-06-15 13:46:23 | INFO | ================================================================================ -2026-06-15 13:46:23 | INFO | Processing Table: Attendance -2026-06-15 13:46:23 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 -2026-06-15 13:46:25 | INFO | Fetched 12,432 attendance rows for 579 employees -2026-06-15 13:46:25 | INFO | Fetched 12432 rows -2026-06-15 13:46:25 | INFO | Truncated table Attendance -2026-06-15 13:46:25 | INFO | Attendance: inserted 12,432 rows into ClickHouse -2026-06-15 13:46:25 | INFO | Attendance loaded successfully (12432 rows) -2026-06-15 13:46:25 | INFO | ================================================================================ -2026-06-15 13:46:25 | INFO | Processing Table: Store_Master -2026-06-15 13:46:25 | INFO | Fetching Store Master data -2026-06-15 13:46:29 | INFO | Fetched 5,984 stores -2026-06-15 13:46:29 | INFO | Fetched 5984 rows -2026-06-15 13:46:29 | INFO | Truncated table Store_Master -2026-06-15 13:46:29 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:46:29 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:46:29 | INFO | ================================================================================ -2026-06-15 13:46:29 | INFO | Processing Table: SKU_Master -2026-06-15 13:46:29 | INFO | Fetching SKU Master data -2026-06-15 13:46:29 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:46:29 | INFO | Fetched 160 rows -2026-06-15 13:46:30 | INFO | Truncated table SKU_Master -2026-06-15 13:46:30 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:46:30 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:46:30 | INFO | ================================================================================ -2026-06-15 13:46:30 | INFO | Processing Table: display_master -2026-06-15 13:46:30 | INFO | Fetching Display Master data -2026-06-15 13:46:30 | INFO | Fetched 135 Display Master records -2026-06-15 13:46:30 | INFO | Fetched 135 rows -2026-06-15 13:46:30 | INFO | Truncated table display_master -2026-06-15 13:46:30 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:46:30 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:46:30 | INFO | ================================================================================ -2026-06-15 13:46:30 | INFO | Processing Table: Employee_Master -2026-06-15 13:46:30 | INFO | Fetching Employee Master data -2026-06-15 13:46:31 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:46:31 | INFO | Fetched 2273 rows -2026-06-15 13:46:31 | INFO | Truncated table Employee_Master -2026-06-15 13:46:31 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:46:31 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:46:31 | INFO | ================================================================================ -2026-06-15 13:46:31 | INFO | Processing Table: Journey_Plan -2026-06-15 13:46:31 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:46:31 | INFO | Fetched 21,142 Journey Plan records -2026-06-15 13:46:31 | INFO | Fetched 21142 rows -2026-06-15 13:46:32 | INFO | Deleting from Journey_Plan -2026-06-15 13:46:32 | INFO | Journey_Plan: inserted 21,142 rows into ClickHouse -2026-06-15 13:46:32 | INFO | Journey_Plan loaded successfully (21142 rows) -2026-06-15 13:46:32 | INFO | ================================================================================ -2026-06-15 13:46:32 | INFO | Processing Table: coverage_remarks -2026-06-15 13:46:32 | INFO | Fetching Coverage Remarks -2026-06-15 13:46:32 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:46:32 | INFO | Fetched 29 rows -2026-06-15 13:46:32 | INFO | Truncated table coverage_remarks -2026-06-15 13:46:33 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:46:33 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:46:33 | INFO | ================================================================================ -2026-06-15 13:46:33 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:46:33 | INFO | Fetching Mapping Store Visibility for 2026-06-03 -2026-06-15 13:46:33 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:46:33 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:46:33 | INFO | ================================================================================ -2026-06-15 13:46:33 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:46:33 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:46:33 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:46:33 | INFO | Fetched 17 rows -2026-06-15 13:46:33 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:46:33 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:46:33 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:46:33 | INFO | ================================================================================ -2026-06-15 13:46:33 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:46:33 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:46:33 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:46:33 | INFO | Fetched 861 rows -2026-06-15 13:46:34 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:46:34 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:46:34 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:46:34 | INFO | ================================================================================ -2026-06-15 13:46:34 | INFO | Processing Table: Web_Logins -2026-06-15 13:46:34 | INFO | Fetching Web Login data for 2026-06-03 -2026-06-15 13:46:34 | INFO | Fetched 200 Web Login records -2026-06-15 13:46:34 | INFO | Fetched 200 rows -2026-06-15 13:46:34 | INFO | Deleting from Web_Logins -2026-06-15 13:46:34 | INFO | Web_Logins: inserted 200 rows into ClickHouse -2026-06-15 13:46:34 | INFO | Web_Logins loaded successfully (200 rows) -2026-06-15 13:46:34 | INFO | ================================================================================ -2026-06-15 13:46:34 | INFO | Processing Table: Promotion -2026-06-15 13:46:34 | INFO | Fetching Promotion data for 808 MIDs -2026-06-15 13:46:37 | INFO | Fetched 8,162 Promotion records -2026-06-15 13:46:37 | INFO | Fetched 8162 rows -2026-06-15 13:46:37 | INFO | Deleting from Promotion -2026-06-15 13:46:37 | INFO | Promotion: inserted 8,162 rows into ClickHouse -2026-06-15 13:46:37 | INFO | Promotion loaded successfully (8162 rows) -2026-06-15 13:46:37 | INFO | ================================================================================ -2026-06-15 13:46:37 | INFO | Processing Table: PaidVisibility -2026-06-15 13:46:37 | INFO | Fetching Paid Visibility data for 808 MIDs -2026-06-15 13:46:38 | INFO | Fetched 1,064 Paid Visibility records -2026-06-15 13:46:38 | INFO | Fetched 1064 rows -2026-06-15 13:46:38 | INFO | Deleting from PaidVisibility -2026-06-15 13:46:38 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse -2026-06-15 13:46:38 | INFO | PaidVisibility loaded successfully (1064 rows) -2026-06-15 13:46:38 | INFO | ================================================================================ -2026-06-15 13:46:38 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:46:38 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:46:38 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:46:38 | INFO | Fetched 33 rows -2026-06-15 13:46:38 | INFO | Truncated table Master_Salesterritorylayer -2026-06-15 13:46:38 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:46:38 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:46:38 | INFO | ================================================================================ -2026-06-15 13:46:38 | INFO | Pipeline Completed Successfully -2026-06-15 13:46:38 | INFO | ================================================================================ -2026-06-15 13:46:43 | INFO | ================================================================================ -2026-06-15 13:46:43 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:46:43 | INFO | Pipeline Run Date: 2026-06-03 -2026-06-15 13:46:43 | INFO | Connecting to databases... -2026-06-15 13:46:43 | INFO | -2026-06-15 13:46:44 | INFO | -2026-06-15 13:46:45 | INFO | Database connections established -2026-06-15 13:46:45 | INFO | Collecting MIDs for: 2026-06-03 -2026-06-15 13:46:45 | INFO | Found 808 MIDs -2026-06-15 13:46:45 | INFO | ================================================================================ -2026-06-15 13:46:45 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:46:45 | INFO | Fetching data for 808 MIDs -2026-06-15 13:46:45 | INFO | Fetched 3,565 rows from SQL Server -2026-06-15 13:46:45 | INFO | Fetched 3565 rows -2026-06-15 13:46:46 | INFO | Deleting from SOS_OneApp -2026-06-15 13:46:46 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse -2026-06-15 13:46:46 | INFO | SOS_OneApp loaded successfully (3565 rows) -2026-06-15 13:46:46 | INFO | ================================================================================ -2026-06-15 13:46:46 | INFO | Processing Table: OQaD -2026-06-15 13:46:46 | INFO | Fetching OQaD data for run_date=2026-06-03 -2026-06-15 13:46:48 | INFO | Fetched 455 rows -2026-06-15 13:46:48 | INFO | Fetched 455 rows -2026-06-15 13:46:48 | INFO | Deleting from OQaD -2026-06-15 13:46:48 | INFO | OQaD: inserted 455 rows into ClickHouse -2026-06-15 13:46:48 | INFO | OQaD loaded successfully (455 rows) -2026-06-15 13:46:48 | INFO | ================================================================================ -2026-06-15 13:46:48 | INFO | Processing Table: Survey -2026-06-15 13:46:48 | INFO | Fetching Survey data for 808 MIDs -2026-06-15 13:46:48 | INFO | Fetched 139 Survey rows -2026-06-15 13:46:48 | INFO | Fetched 139 rows -2026-06-15 13:46:49 | INFO | Deleting from Survey -2026-06-15 13:46:49 | INFO | Survey: inserted 139 rows into ClickHouse -2026-06-15 13:46:49 | INFO | Survey loaded successfully (139 rows) -2026-06-15 13:46:49 | INFO | ================================================================================ -2026-06-15 13:46:49 | INFO | Processing Table: additional_visibility -2026-06-15 13:46:49 | INFO | Fetching Additional Visibility data for 808 MIDs -2026-06-15 13:46:50 | INFO | Fetched 1,884 Additional Visibility rows -2026-06-15 13:46:50 | INFO | Fetched 1884 rows -2026-06-15 13:46:50 | INFO | Deleting from additional_visibility -2026-06-15 13:46:50 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse -2026-06-15 13:46:50 | INFO | additional_visibility loaded successfully (1884 rows) -2026-06-15 13:46:50 | INFO | ================================================================================ -2026-06-15 13:46:50 | INFO | Processing Table: Coverage -2026-06-15 13:46:50 | INFO | Fetching coverage data for 808 MIDs -2026-06-15 13:46:50 | INFO | Fetched 751 rows from SQL Server -2026-06-15 13:46:50 | INFO | Fetched 751 rows -2026-06-15 13:46:50 | INFO | Deleting from Coverage -2026-06-15 13:46:50 | INFO | Coverage: inserted 751 rows into ClickHouse -2026-06-15 13:46:50 | INFO | Coverage loaded successfully (751 rows) -2026-06-15 13:46:50 | INFO | ================================================================================ -2026-06-15 13:46:50 | INFO | Processing Table: Login -2026-06-15 13:46:50 | INFO | Fetching Login data for yesterday -2026-06-15 13:46:51 | INFO | Fetched 454 Login rows -2026-06-15 13:46:51 | INFO | Fetched 454 rows -2026-06-15 13:46:51 | INFO | Deleting from Login -2026-06-15 13:46:51 | INFO | No delete logic required for Login -2026-06-15 13:46:51 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:46:51 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:46:51 | INFO | ================================================================================ -2026-06-15 13:46:51 | INFO | Processing Table: Stock_Details -2026-06-15 13:46:51 | INFO | Fetching Stock Details data for 808 MIDs -2026-06-15 13:46:53 | INFO | Fetched 40,840 Stock Details rows -2026-06-15 13:46:53 | INFO | Fetched 40840 rows -2026-06-15 13:46:53 | INFO | Deleting from Stock_Details -2026-06-15 13:46:54 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse -2026-06-15 13:46:54 | INFO | Stock_Details loaded successfully (40840 rows) -2026-06-15 13:46:54 | INFO | ================================================================================ -2026-06-15 13:46:54 | INFO | Processing Table: Attendance -2026-06-15 13:46:54 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 -2026-06-15 13:46:54 | INFO | Fetched 12,432 attendance rows for 579 employees -2026-06-15 13:46:54 | INFO | Fetched 12432 rows -2026-06-15 13:46:55 | INFO | Truncated table Attendance -2026-06-15 13:46:55 | INFO | Attendance: inserted 12,432 rows into ClickHouse -2026-06-15 13:46:55 | INFO | Attendance loaded successfully (12432 rows) -2026-06-15 13:46:55 | INFO | ================================================================================ -2026-06-15 13:46:55 | INFO | Processing Table: Store_Master -2026-06-15 13:46:55 | INFO | Fetching Store Master data -2026-06-15 13:46:56 | INFO | Fetched 5,984 stores -2026-06-15 13:46:56 | INFO | Fetched 5984 rows -2026-06-15 13:46:56 | INFO | Truncated table Store_Master -2026-06-15 13:46:56 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:46:56 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:46:56 | INFO | ================================================================================ -2026-06-15 13:46:56 | INFO | Processing Table: SKU_Master -2026-06-15 13:46:56 | INFO | Fetching SKU Master data -2026-06-15 13:46:57 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:46:57 | INFO | Fetched 160 rows -2026-06-15 13:46:57 | INFO | Truncated table SKU_Master -2026-06-15 13:46:57 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:46:57 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:46:57 | INFO | ================================================================================ -2026-06-15 13:46:57 | INFO | Processing Table: display_master -2026-06-15 13:46:57 | INFO | Fetching Display Master data -2026-06-15 13:46:57 | INFO | Fetched 135 Display Master records -2026-06-15 13:46:57 | INFO | Fetched 135 rows -2026-06-15 13:46:57 | INFO | Truncated table display_master -2026-06-15 13:46:57 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:46:57 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:46:57 | INFO | ================================================================================ -2026-06-15 13:46:57 | INFO | Processing Table: Employee_Master -2026-06-15 13:46:57 | INFO | Fetching Employee Master data -2026-06-15 13:46:58 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:46:58 | INFO | Fetched 2273 rows -2026-06-15 13:46:58 | INFO | Truncated table Employee_Master -2026-06-15 13:46:58 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:46:58 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:46:58 | INFO | ================================================================================ -2026-06-15 13:46:58 | INFO | Processing Table: Journey_Plan -2026-06-15 13:46:58 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:46:58 | INFO | Fetched 21,142 Journey Plan records -2026-06-15 13:46:58 | INFO | Fetched 21142 rows -2026-06-15 13:46:58 | INFO | Deleting from Journey_Plan -2026-06-15 13:46:58 | INFO | Journey_Plan: inserted 21,142 rows into ClickHouse -2026-06-15 13:46:58 | INFO | Journey_Plan loaded successfully (21142 rows) -2026-06-15 13:46:58 | INFO | ================================================================================ -2026-06-15 13:46:58 | INFO | Processing Table: coverage_remarks -2026-06-15 13:46:58 | INFO | Fetching Coverage Remarks -2026-06-15 13:46:58 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:46:59 | INFO | Fetched 29 rows -2026-06-15 13:46:59 | INFO | Truncated table coverage_remarks -2026-06-15 13:46:59 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:46:59 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:46:59 | INFO | ================================================================================ -2026-06-15 13:46:59 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:46:59 | INFO | Fetching Mapping Store Visibility for 2026-06-03 -2026-06-15 13:46:59 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:46:59 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:46:59 | INFO | ================================================================================ -2026-06-15 13:46:59 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:46:59 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:46:59 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:46:59 | INFO | Fetched 17 rows -2026-06-15 13:46:59 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:46:59 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:46:59 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:46:59 | INFO | ================================================================================ -2026-06-15 13:46:59 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:46:59 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:46:59 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:46:59 | INFO | Fetched 861 rows -2026-06-15 13:46:59 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:47:00 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:47:00 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:47:00 | INFO | ================================================================================ -2026-06-15 13:47:00 | INFO | Processing Table: Web_Logins -2026-06-15 13:47:00 | INFO | Fetching Web Login data for 2026-06-03 -2026-06-15 13:47:00 | INFO | Fetched 200 Web Login records -2026-06-15 13:47:00 | INFO | Fetched 200 rows -2026-06-15 13:47:00 | INFO | Deleting from Web_Logins -2026-06-15 13:47:00 | INFO | Web_Logins: inserted 200 rows into ClickHouse -2026-06-15 13:47:00 | INFO | Web_Logins loaded successfully (200 rows) -2026-06-15 13:47:00 | INFO | ================================================================================ -2026-06-15 13:47:00 | INFO | Processing Table: Promotion -2026-06-15 13:47:00 | INFO | Fetching Promotion data for 808 MIDs -2026-06-15 13:47:02 | INFO | Fetched 8,162 Promotion records -2026-06-15 13:47:02 | INFO | Fetched 8162 rows -2026-06-15 13:47:02 | INFO | Deleting from Promotion -2026-06-15 13:47:02 | INFO | Promotion: inserted 8,162 rows into ClickHouse -2026-06-15 13:47:02 | INFO | Promotion loaded successfully (8162 rows) -2026-06-15 13:47:02 | INFO | ================================================================================ -2026-06-15 13:47:02 | INFO | Processing Table: PaidVisibility -2026-06-15 13:47:02 | INFO | Fetching Paid Visibility data for 808 MIDs -2026-06-15 13:47:03 | INFO | Fetched 1,064 Paid Visibility records -2026-06-15 13:47:03 | INFO | Fetched 1064 rows -2026-06-15 13:47:03 | INFO | Deleting from PaidVisibility -2026-06-15 13:47:03 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse -2026-06-15 13:47:03 | INFO | PaidVisibility loaded successfully (1064 rows) -2026-06-15 13:47:03 | INFO | ================================================================================ -2026-06-15 13:47:03 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:47:03 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:47:03 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:47:03 | INFO | Fetched 33 rows -2026-06-15 13:47:03 | INFO | Truncated table Master_Salesterritorylayer -2026-06-15 13:47:03 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:47:03 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:47:03 | INFO | ================================================================================ -2026-06-15 13:47:03 | INFO | Pipeline Completed Successfully -2026-06-15 13:47:03 | INFO | ================================================================================ -2026-06-15 13:50:19 | INFO | ================================================================================ -2026-06-15 13:50:19 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:50:19 | INFO | Pipeline Run Date: 2026-06-03 -2026-06-15 13:50:19 | INFO | Connecting to databases... -2026-06-15 13:50:20 | INFO | -2026-06-15 13:50:21 | INFO | -2026-06-15 13:50:21 | INFO | Database connections established -2026-06-15 13:50:21 | INFO | Collecting MIDs for: 2026-06-03 -2026-06-15 13:50:21 | INFO | Found 808 MIDs -2026-06-15 13:50:22 | INFO | ================================================================================ -2026-06-15 13:50:22 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:50:22 | INFO | Fetching data for 808 MIDs -2026-06-15 13:50:22 | INFO | Fetched 3,565 rows from SQL Server -2026-06-15 13:50:22 | INFO | Fetched 3565 rows -2026-06-15 13:50:22 | INFO | Deleting from SOS_OneApp -2026-06-15 13:50:23 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse -2026-06-15 13:50:23 | INFO | SOS_OneApp loaded successfully (3565 rows) -2026-06-15 13:50:23 | INFO | ================================================================================ -2026-06-15 13:50:23 | INFO | Processing Table: OQaD -2026-06-15 13:50:23 | INFO | Fetching OQaD data for run_date=2026-06-03 -2026-06-15 13:50:27 | INFO | Fetched 455 rows -2026-06-15 13:50:27 | INFO | Fetched 455 rows -2026-06-15 13:50:27 | INFO | Deleting from OQaD -2026-06-15 13:50:27 | INFO | OQaD: inserted 455 rows into ClickHouse -2026-06-15 13:50:27 | INFO | OQaD loaded successfully (455 rows) -2026-06-15 13:50:27 | INFO | ================================================================================ -2026-06-15 13:50:27 | INFO | Processing Table: Survey -2026-06-15 13:50:27 | INFO | Fetching Survey data for 808 MIDs -2026-06-15 13:50:27 | INFO | Fetched 139 Survey rows -2026-06-15 13:50:27 | INFO | Fetched 139 rows -2026-06-15 13:50:27 | INFO | Deleting from Survey -2026-06-15 13:50:27 | INFO | Survey: inserted 139 rows into ClickHouse -2026-06-15 13:50:27 | INFO | Survey loaded successfully (139 rows) -2026-06-15 13:50:27 | INFO | ================================================================================ -2026-06-15 13:50:27 | INFO | Processing Table: additional_visibility -2026-06-15 13:50:27 | INFO | Fetching Additional Visibility data for 808 MIDs -2026-06-15 13:50:28 | INFO | Fetched 1,884 Additional Visibility rows -2026-06-15 13:50:28 | INFO | Fetched 1884 rows -2026-06-15 13:50:28 | INFO | Deleting from additional_visibility -2026-06-15 13:50:28 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse -2026-06-15 13:50:28 | INFO | additional_visibility loaded successfully (1884 rows) -2026-06-15 13:50:28 | INFO | ================================================================================ -2026-06-15 13:50:28 | INFO | Processing Table: Coverage -2026-06-15 13:50:28 | INFO | Fetching coverage data for 808 MIDs -2026-06-15 13:50:28 | INFO | Fetched 751 rows from SQL Server -2026-06-15 13:50:28 | INFO | Fetched 751 rows -2026-06-15 13:50:28 | INFO | Deleting from Coverage -2026-06-15 13:50:29 | INFO | Coverage: inserted 751 rows into ClickHouse -2026-06-15 13:50:29 | INFO | Coverage loaded successfully (751 rows) -2026-06-15 13:50:29 | INFO | ================================================================================ -2026-06-15 13:50:29 | INFO | Processing Table: Login -2026-06-15 13:50:29 | INFO | Fetching Login data for yesterday -2026-06-15 13:50:29 | INFO | Fetched 454 Login rows -2026-06-15 13:50:29 | INFO | Fetched 454 rows -2026-06-15 13:50:29 | INFO | Deleting from Login -2026-06-15 13:50:29 | INFO | No delete logic required for Login -2026-06-15 13:50:29 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:50:29 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:50:29 | INFO | ================================================================================ -2026-06-15 13:50:29 | INFO | Processing Table: Stock_Details -2026-06-15 13:50:29 | INFO | Fetching Stock Details data for 808 MIDs -2026-06-15 13:50:33 | INFO | Fetched 40,840 Stock Details rows -2026-06-15 13:50:33 | INFO | Fetched 40840 rows -2026-06-15 13:50:33 | INFO | Deleting from Stock_Details -2026-06-15 13:50:34 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse -2026-06-15 13:50:34 | INFO | Stock_Details loaded successfully (40840 rows) -2026-06-15 13:50:34 | INFO | ================================================================================ -2026-06-15 13:50:34 | INFO | Processing Table: Attendance -2026-06-15 13:50:34 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 -2026-06-15 13:50:35 | INFO | Fetched 12,432 attendance rows for 579 employees -2026-06-15 13:50:35 | INFO | Fetched 12432 rows -2026-06-15 13:50:35 | INFO | Truncated table Attendance -2026-06-15 13:50:35 | INFO | Attendance: inserted 12,432 rows into ClickHouse -2026-06-15 13:50:35 | INFO | Attendance loaded successfully (12432 rows) -2026-06-15 13:50:35 | INFO | ================================================================================ -2026-06-15 13:50:35 | INFO | Processing Table: Store_Master -2026-06-15 13:50:35 | INFO | Fetching Store Master data -2026-06-15 13:50:37 | INFO | Fetched 5,984 stores -2026-06-15 13:50:37 | INFO | Fetched 5984 rows -2026-06-15 13:50:37 | INFO | Truncated table Store_Master -2026-06-15 13:50:38 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:50:38 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:50:38 | INFO | ================================================================================ -2026-06-15 13:50:38 | INFO | Processing Table: SKU_Master -2026-06-15 13:50:38 | INFO | Fetching SKU Master data -2026-06-15 13:50:38 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:50:38 | INFO | Fetched 160 rows -2026-06-15 13:50:38 | INFO | Truncated table SKU_Master -2026-06-15 13:50:38 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:50:38 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:50:38 | INFO | ================================================================================ -2026-06-15 13:50:38 | INFO | Processing Table: display_master -2026-06-15 13:50:38 | INFO | Fetching Display Master data -2026-06-15 13:50:38 | INFO | Fetched 135 Display Master records -2026-06-15 13:50:38 | INFO | Fetched 135 rows -2026-06-15 13:50:38 | INFO | Truncated table display_master -2026-06-15 13:50:39 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:50:39 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:50:39 | INFO | ================================================================================ -2026-06-15 13:50:39 | INFO | Processing Table: Employee_Master -2026-06-15 13:50:39 | INFO | Fetching Employee Master data -2026-06-15 13:50:39 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:50:39 | INFO | Fetched 2273 rows -2026-06-15 13:50:39 | INFO | Truncated table Employee_Master -2026-06-15 13:50:39 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:50:39 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:50:39 | INFO | ================================================================================ -2026-06-15 13:50:39 | INFO | Processing Table: Journey_Plan -2026-06-15 13:50:39 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:50:40 | INFO | Fetched 21,142 Journey Plan records -2026-06-15 13:50:40 | INFO | Fetched 21142 rows -2026-06-15 13:50:40 | INFO | Deleting from Journey_Plan -2026-06-15 13:50:40 | INFO | Journey_Plan: inserted 21,142 rows into ClickHouse -2026-06-15 13:50:40 | INFO | Journey_Plan loaded successfully (21142 rows) -2026-06-15 13:50:40 | INFO | ================================================================================ -2026-06-15 13:50:40 | INFO | Processing Table: coverage_remarks -2026-06-15 13:50:40 | INFO | Fetching Coverage Remarks -2026-06-15 13:50:40 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:50:40 | INFO | Fetched 29 rows -2026-06-15 13:50:40 | INFO | Truncated table coverage_remarks -2026-06-15 13:50:40 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:50:40 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:50:40 | INFO | ================================================================================ -2026-06-15 13:50:40 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:50:40 | ERROR | Failed processing table mapping_storevisibility -Traceback (most recent call last): - File "D:\data_move\main.py", line 177, in main - df = get_dataframe( - fn_name=fn_name, - ...<3 lines>... - run_date=run_date, - ) - File "D:\data_move\main.py", line 83, in get_dataframe - return fn(sql_engine) -TypeError: fetch_mapping_storevisibility() missing 1 required positional argument: 'run_date' -2026-06-15 13:51:03 | INFO | ================================================================================ -2026-06-15 13:51:03 | INFO | Hello from data-move Python data pipeline! -2026-06-15 13:51:03 | INFO | Pipeline Run Date: 2026-06-03 -2026-06-15 13:51:03 | INFO | Connecting to databases... -2026-06-15 13:51:03 | INFO | -2026-06-15 13:51:04 | INFO | -2026-06-15 13:51:05 | INFO | Database connections established -2026-06-15 13:51:05 | INFO | Collecting MIDs for: 2026-06-03 -2026-06-15 13:51:05 | INFO | Found 808 MIDs -2026-06-15 13:51:05 | INFO | ================================================================================ -2026-06-15 13:51:05 | INFO | Processing Table: SOS_OneApp -2026-06-15 13:51:05 | INFO | Fetching data for 808 MIDs -2026-06-15 13:51:06 | INFO | Fetched 3,565 rows from SQL Server -2026-06-15 13:51:06 | INFO | Fetched 3565 rows -2026-06-15 13:51:06 | INFO | Deleting from SOS_OneApp -2026-06-15 13:51:06 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse -2026-06-15 13:51:06 | INFO | SOS_OneApp loaded successfully (3565 rows) -2026-06-15 13:51:06 | INFO | ================================================================================ -2026-06-15 13:51:06 | INFO | Processing Table: OQaD -2026-06-15 13:51:06 | INFO | Fetching OQaD data for run_date=2026-06-03 -2026-06-15 13:51:09 | INFO | Fetched 455 rows -2026-06-15 13:51:09 | INFO | Fetched 455 rows -2026-06-15 13:51:09 | INFO | Deleting from OQaD -2026-06-15 13:51:09 | INFO | OQaD: inserted 455 rows into ClickHouse -2026-06-15 13:51:09 | INFO | OQaD loaded successfully (455 rows) -2026-06-15 13:51:09 | INFO | ================================================================================ -2026-06-15 13:51:09 | INFO | Processing Table: Survey -2026-06-15 13:51:09 | INFO | Fetching Survey data for 808 MIDs -2026-06-15 13:51:09 | INFO | Fetched 139 Survey rows -2026-06-15 13:51:09 | INFO | Fetched 139 rows -2026-06-15 13:51:09 | INFO | Deleting from Survey -2026-06-15 13:51:09 | INFO | Survey: inserted 139 rows into ClickHouse -2026-06-15 13:51:09 | INFO | Survey loaded successfully (139 rows) -2026-06-15 13:51:09 | INFO | ================================================================================ -2026-06-15 13:51:09 | INFO | Processing Table: additional_visibility -2026-06-15 13:51:09 | INFO | Fetching Additional Visibility data for 808 MIDs -2026-06-15 13:51:10 | INFO | Fetched 1,884 Additional Visibility rows -2026-06-15 13:51:10 | INFO | Fetched 1884 rows -2026-06-15 13:51:10 | INFO | Deleting from additional_visibility -2026-06-15 13:51:10 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse -2026-06-15 13:51:10 | INFO | additional_visibility loaded successfully (1884 rows) -2026-06-15 13:51:10 | INFO | ================================================================================ -2026-06-15 13:51:10 | INFO | Processing Table: Coverage -2026-06-15 13:51:10 | INFO | Fetching coverage data for 808 MIDs -2026-06-15 13:51:10 | INFO | Fetched 751 rows from SQL Server -2026-06-15 13:51:10 | INFO | Fetched 751 rows -2026-06-15 13:51:10 | INFO | Deleting from Coverage -2026-06-15 13:51:11 | INFO | Coverage: inserted 751 rows into ClickHouse -2026-06-15 13:51:11 | INFO | Coverage loaded successfully (751 rows) -2026-06-15 13:51:11 | INFO | ================================================================================ -2026-06-15 13:51:11 | INFO | Processing Table: Login -2026-06-15 13:51:11 | INFO | Fetching Login data for yesterday -2026-06-15 13:51:11 | INFO | Fetched 454 Login rows -2026-06-15 13:51:11 | INFO | Fetched 454 rows -2026-06-15 13:51:11 | INFO | Deleting from Login -2026-06-15 13:51:11 | INFO | No delete logic required for Login -2026-06-15 13:51:11 | INFO | Login: inserted 454 rows into ClickHouse -2026-06-15 13:51:11 | INFO | Login loaded successfully (454 rows) -2026-06-15 13:51:11 | INFO | ================================================================================ -2026-06-15 13:51:11 | INFO | Processing Table: Stock_Details -2026-06-15 13:51:11 | INFO | Fetching Stock Details data for 808 MIDs -2026-06-15 13:51:15 | INFO | Fetched 40,840 Stock Details rows -2026-06-15 13:51:15 | INFO | Fetched 40840 rows -2026-06-15 13:51:15 | INFO | Deleting from Stock_Details -2026-06-15 13:51:15 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse -2026-06-15 13:51:15 | INFO | Stock_Details loaded successfully (40840 rows) -2026-06-15 13:51:15 | INFO | ================================================================================ -2026-06-15 13:51:15 | INFO | Processing Table: Attendance -2026-06-15 13:51:15 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 -2026-06-15 13:51:16 | INFO | Fetched 12,432 attendance rows for 579 employees -2026-06-15 13:51:16 | INFO | Fetched 12432 rows -2026-06-15 13:51:17 | INFO | Truncated table Attendance -2026-06-15 13:51:17 | INFO | Attendance: inserted 12,432 rows into ClickHouse -2026-06-15 13:51:17 | INFO | Attendance loaded successfully (12432 rows) -2026-06-15 13:51:17 | INFO | ================================================================================ -2026-06-15 13:51:17 | INFO | Processing Table: Store_Master -2026-06-15 13:51:17 | INFO | Fetching Store Master data -2026-06-15 13:51:20 | INFO | Fetched 5,984 stores -2026-06-15 13:51:20 | INFO | Fetched 5984 rows -2026-06-15 13:51:20 | INFO | Truncated table Store_Master -2026-06-15 13:51:20 | INFO | Store_Master: inserted 5,984 rows into ClickHouse -2026-06-15 13:51:20 | INFO | Store_Master loaded successfully (5984 rows) -2026-06-15 13:51:20 | INFO | ================================================================================ -2026-06-15 13:51:20 | INFO | Processing Table: SKU_Master -2026-06-15 13:51:20 | INFO | Fetching SKU Master data -2026-06-15 13:51:20 | INFO | Fetched 160 SKU Master rows -2026-06-15 13:51:20 | INFO | Fetched 160 rows -2026-06-15 13:51:21 | INFO | Truncated table SKU_Master -2026-06-15 13:51:21 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-15 13:51:21 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-15 13:51:21 | INFO | ================================================================================ -2026-06-15 13:51:21 | INFO | Processing Table: display_master -2026-06-15 13:51:21 | INFO | Fetching Display Master data -2026-06-15 13:51:21 | INFO | Fetched 135 Display Master records -2026-06-15 13:51:21 | INFO | Fetched 135 rows -2026-06-15 13:51:21 | INFO | Truncated table display_master -2026-06-15 13:51:21 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-15 13:51:21 | INFO | display_master loaded successfully (135 rows) -2026-06-15 13:51:21 | INFO | ================================================================================ -2026-06-15 13:51:21 | INFO | Processing Table: Employee_Master -2026-06-15 13:51:21 | INFO | Fetching Employee Master data -2026-06-15 13:51:21 | INFO | Fetched 2,273 Employee Master records -2026-06-15 13:51:21 | INFO | Fetched 2273 rows -2026-06-15 13:51:22 | INFO | Truncated table Employee_Master -2026-06-15 13:51:22 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse -2026-06-15 13:51:22 | INFO | Employee_Master loaded successfully (2273 rows) -2026-06-15 13:51:22 | INFO | ================================================================================ -2026-06-15 13:51:22 | INFO | Processing Table: Journey_Plan -2026-06-15 13:51:22 | INFO | Fetching Journey Plan for 2026-06 -2026-06-15 13:51:22 | INFO | Fetched 21,142 Journey Plan records -2026-06-15 13:51:22 | INFO | Fetched 21142 rows -2026-06-15 13:51:22 | INFO | Deleting from Journey_Plan -2026-06-15 13:51:22 | INFO | Journey_Plan: inserted 21,142 rows into ClickHouse -2026-06-15 13:51:22 | INFO | Journey_Plan loaded successfully (21142 rows) -2026-06-15 13:51:22 | INFO | ================================================================================ -2026-06-15 13:51:22 | INFO | Processing Table: coverage_remarks -2026-06-15 13:51:22 | INFO | Fetching Coverage Remarks -2026-06-15 13:51:22 | INFO | Fetched 29 Coverage Remark records -2026-06-15 13:51:22 | INFO | Fetched 29 rows -2026-06-15 13:51:23 | INFO | Truncated table coverage_remarks -2026-06-15 13:51:23 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-15 13:51:23 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-15 13:51:23 | INFO | ================================================================================ -2026-06-15 13:51:23 | INFO | Processing Table: mapping_storevisibility -2026-06-15 13:51:23 | INFO | Fetching Mapping Store Visibility for 2026-06-03 -2026-06-15 13:51:23 | INFO | Fetched 0 Mapping Store Visibility records -2026-06-15 13:51:23 | WARNING | mapping_storevisibility returned no rows -2026-06-15 13:51:23 | INFO | ================================================================================ -2026-06-15 13:51:23 | INFO | Processing Table: Master_VisibilityReason -2026-06-15 13:51:23 | INFO | Fetching Master Visibility Reason data -2026-06-15 13:51:23 | INFO | Fetched 17 Master Visibility Reason records -2026-06-15 13:51:23 | INFO | Fetched 17 rows -2026-06-15 13:51:23 | INFO | Truncated table Master_VisibilityReason -2026-06-15 13:51:23 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-15 13:51:23 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-15 13:51:23 | INFO | ================================================================================ -2026-06-15 13:51:23 | INFO | Processing Table: Master_VisibilityDefinition -2026-06-15 13:51:23 | INFO | Fetching Master Visibility Definition data -2026-06-15 13:51:23 | INFO | Fetched 861 Master Visibility Definition records -2026-06-15 13:51:23 | INFO | Fetched 861 rows -2026-06-15 13:51:23 | INFO | Truncated table Master_VisibilityDefinition -2026-06-15 13:51:24 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-15 13:51:24 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-15 13:51:24 | INFO | ================================================================================ -2026-06-15 13:51:24 | INFO | Processing Table: Web_Logins -2026-06-15 13:51:24 | INFO | Fetching Web Login data for 2026-06-03 -2026-06-15 13:51:24 | INFO | Fetched 200 Web Login records -2026-06-15 13:51:24 | INFO | Fetched 200 rows -2026-06-15 13:51:24 | INFO | Deleting from Web_Logins -2026-06-15 13:51:24 | INFO | Web_Logins: inserted 200 rows into ClickHouse -2026-06-15 13:51:24 | INFO | Web_Logins loaded successfully (200 rows) -2026-06-15 13:51:24 | INFO | ================================================================================ -2026-06-15 13:51:24 | INFO | Processing Table: Promotion -2026-06-15 13:51:24 | INFO | Fetching Promotion data for 808 MIDs -2026-06-15 13:51:28 | INFO | Fetched 8,162 Promotion records -2026-06-15 13:51:28 | INFO | Fetched 8162 rows -2026-06-15 13:51:28 | INFO | Deleting from Promotion -2026-06-15 13:51:29 | INFO | Promotion: inserted 8,162 rows into ClickHouse -2026-06-15 13:51:29 | INFO | Promotion loaded successfully (8162 rows) -2026-06-15 13:51:29 | INFO | ================================================================================ -2026-06-15 13:51:29 | INFO | Processing Table: PaidVisibility -2026-06-15 13:51:29 | INFO | Fetching Paid Visibility data for 808 MIDs -2026-06-15 13:51:30 | INFO | Fetched 1,064 Paid Visibility records -2026-06-15 13:51:30 | INFO | Fetched 1064 rows -2026-06-15 13:51:30 | INFO | Deleting from PaidVisibility -2026-06-15 13:51:30 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse -2026-06-15 13:51:30 | INFO | PaidVisibility loaded successfully (1064 rows) -2026-06-15 13:51:30 | INFO | ================================================================================ -2026-06-15 13:51:30 | INFO | Processing Table: Master_Salesterritorylayer -2026-06-15 13:51:30 | INFO | Fetching Master Sales Territory Layer data -2026-06-15 13:51:30 | INFO | Fetched 33 Master Sales Territory Layer records -2026-06-15 13:51:30 | INFO | Fetched 33 rows -2026-06-15 13:51:30 | INFO | Truncated table Master_Salesterritorylayer -2026-06-15 13:51:30 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-15 13:51:30 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-15 13:51:30 | INFO | ================================================================================ -2026-06-15 13:51:30 | INFO | Pipeline Completed Successfully -2026-06-15 13:51:30 | INFO | ================================================================================ diff --git a/logs/etl_20260616.log b/logs/etl_20260616.log deleted file mode 100644 index 9c7b84d..0000000 --- a/logs/etl_20260616.log +++ /dev/null @@ -1,71 +0,0 @@ -2026-06-16 17:56:17 | INFO | ================================================================================ -2026-06-16 17:56:17 | INFO | Hello from data-move Python data pipeline! -2026-06-16 17:56:17 | INFO | Pipeline Run Date: 2026-06-15 -2026-06-16 17:56:17 | INFO | Connecting to databases... -2026-06-16 17:56:19 | INFO | -2026-06-16 17:56:23 | INFO | -2026-06-16 17:56:24 | INFO | Database connections established -2026-06-16 17:56:24 | INFO | Collecting MIDs for: 2026-06-15 -2026-06-16 17:56:24 | INFO | Found 849 MIDs -2026-06-16 17:57:33 | INFO | ================================================================================ -2026-06-16 17:57:33 | INFO | Hello from data-move Python data pipeline! -2026-06-16 17:57:33 | INFO | Pipeline Run Date: 2026-06-15 -2026-06-16 17:57:33 | INFO | Connecting to databases... -2026-06-16 17:57:34 | INFO | -2026-06-16 17:57:36 | INFO | -2026-06-16 17:57:36 | INFO | Database connections established -2026-06-16 17:57:36 | INFO | Collecting MIDs for: 2026-06-15 -2026-06-16 17:57:37 | INFO | Found 849 MIDs -2026-06-16 17:58:52 | INFO | ================================================================================ -2026-06-16 17:58:52 | INFO | Hello from data-move Python data pipeline! -2026-06-16 17:58:52 | INFO | Pipeline Run Date: 2026-06-15 -2026-06-16 17:58:52 | INFO | Connecting to databases... -2026-06-16 17:58:53 | INFO | -2026-06-16 17:58:54 | INFO | -2026-06-16 17:58:55 | INFO | Database connections established -2026-06-16 17:58:55 | INFO | Collecting MIDs for: 2026-06-15 -2026-06-16 17:58:55 | INFO | Found 849 MIDs -2026-06-16 18:00:31 | INFO | ================================================================================ -2026-06-16 18:00:31 | INFO | Hello from data-move Python data pipeline! -2026-06-16 18:00:31 | INFO | Pipeline Run Date: 2026-06-15 -2026-06-16 18:00:31 | INFO | Connecting to databases... -2026-06-16 18:00:38 | INFO | -2026-06-16 18:00:40 | INFO | -2026-06-16 18:00:40 | INFO | Database connections established -2026-06-16 18:00:40 | INFO | Collecting MIDs for: 2026-06-15 -2026-06-16 18:00:42 | INFO | Found 849 MIDs -2026-06-16 18:00:44 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-16 18:02:58 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server -2026-06-16 18:07:47 | INFO | ================================================================================ -2026-06-16 18:07:47 | INFO | Hello from data-move Python data pipeline! -2026-06-16 18:07:47 | INFO | Pipeline Run Date: 2026-06-15 -2026-06-16 18:07:47 | INFO | Connecting to databases... -2026-06-16 18:07:48 | INFO | -2026-06-16 18:07:50 | INFO | -2026-06-16 18:07:50 | INFO | Database connections established -2026-06-16 18:07:50 | INFO | Collecting MIDs for: 2026-06-15 -2026-06-16 18:07:50 | INFO | Found 849 MIDs -2026-06-16 18:07:51 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-16 18:08:09 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server -2026-06-16 18:08:58 | INFO | ================================================================================ -2026-06-16 18:08:58 | INFO | Hello from data-move Python data pipeline! -2026-06-16 18:08:58 | INFO | Pipeline Run Date: 2026-06-15 -2026-06-16 18:08:58 | INFO | Connecting to databases... -2026-06-16 18:08:59 | INFO | -2026-06-16 18:09:01 | INFO | -2026-06-16 18:09:01 | INFO | Database connections established -2026-06-16 18:09:01 | INFO | Collecting MIDs for: 2026-06-15 -2026-06-16 18:09:02 | INFO | Found 849 MIDs -2026-06-16 18:09:02 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-16 18:09:19 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server -2026-06-16 18:11:21 | INFO | ================================================================================ -2026-06-16 18:11:21 | INFO | Hello from data-move Python data pipeline! -2026-06-16 18:11:21 | INFO | Pipeline Run Date: 2026-06-15 -2026-06-16 18:11:21 | INFO | Connecting to databases... -2026-06-16 18:11:23 | INFO | -2026-06-16 18:11:24 | INFO | -2026-06-16 18:11:24 | INFO | Database connections established -2026-06-16 18:11:24 | INFO | Collecting MIDs for: 2026-06-15 -2026-06-16 18:11:25 | INFO | Found 849 MIDs -2026-06-16 18:11:25 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-16 18:13:05 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server diff --git a/logs/etl_20260617.log b/logs/etl_20260617.log deleted file mode 100644 index 0d18acb..0000000 --- a/logs/etl_20260617.log +++ /dev/null @@ -1,688 +0,0 @@ -2026-06-17 10:17:16 | INFO | ================================================================================ -2026-06-17 10:17:16 | INFO | Hello from data-move Python data pipeline! -2026-06-17 10:17:16 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 10:17:16 | INFO | Connecting to databases... -2026-06-17 10:17:18 | INFO | -2026-06-17 10:17:21 | INFO | -2026-06-17 10:17:22 | INFO | Database connections established -2026-06-17 10:17:22 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 10:17:22 | INFO | Found 863 MIDs -2026-06-17 10:17:23 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 10:17:44 | INFO | Fetched 290,984 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 10:43:05 | INFO | ================================================================================ -2026-06-17 10:43:05 | INFO | Hello from data-move Python data pipeline! -2026-06-17 10:43:05 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 10:43:05 | INFO | Connecting to databases... -2026-06-17 10:43:07 | INFO | -2026-06-17 10:43:08 | INFO | -2026-06-17 10:43:09 | INFO | Database connections established -2026-06-17 10:43:09 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 10:43:10 | INFO | Found 863 MIDs -2026-06-17 10:43:11 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 10:43:55 | INFO | Fetched 291,001 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 10:45:51 | INFO | ================================================================================ -2026-06-17 10:45:51 | INFO | Hello from data-move Python data pipeline! -2026-06-17 10:45:51 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 10:45:51 | INFO | Connecting to databases... -2026-06-17 10:45:52 | INFO | -2026-06-17 10:45:53 | INFO | -2026-06-17 10:45:54 | INFO | Database connections established -2026-06-17 10:45:54 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 10:45:55 | INFO | Found 863 MIDs -2026-06-17 10:45:55 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 10:46:11 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 10:46:11 | INFO | ================================================================================ -2026-06-17 10:46:11 | INFO | Processing Table: OQaD -2026-06-17 10:46:11 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\main2.py", line 212, in main - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - ...<2 lines>... - run_date=run_date - ) - File "D:\data_move\src\fact.py", line 58, in fetch_OQaD - with open(sql_file, "r", encoding="utf-8") as f: - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql' -2026-06-17 10:48:44 | INFO | ================================================================================ -2026-06-17 10:48:44 | INFO | Hello from data-move Python data pipeline! -2026-06-17 10:48:44 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 10:48:44 | INFO | Connecting to databases... -2026-06-17 10:48:45 | INFO | -2026-06-17 10:48:46 | INFO | -2026-06-17 10:48:47 | INFO | Database connections established -2026-06-17 10:48:47 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 10:48:48 | INFO | Found 863 MIDs -2026-06-17 10:48:48 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 10:49:00 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 10:49:01 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 10:49:01 | INFO | ================================================================================ -2026-06-17 10:49:01 | INFO | Processing Table: OQaD -2026-06-17 10:49:01 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\main2.py", line 215, in main - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - ...<2 lines>... - run_date=run_date - ) - File "D:\data_move\src\fact.py", line 58, in fetch_OQaD - with open(sql_file, "r", encoding="utf-8") as f: - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql' -2026-06-17 10:53:50 | INFO | ================================================================================ -2026-06-17 10:53:50 | INFO | Hello from data-move Python data pipeline! -2026-06-17 10:53:50 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 10:53:50 | INFO | Connecting to databases... -2026-06-17 10:53:51 | INFO | -2026-06-17 10:53:52 | INFO | -2026-06-17 10:53:53 | INFO | Database connections established -2026-06-17 10:53:53 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 10:53:54 | INFO | Found 863 MIDs -2026-06-17 10:53:54 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 10:54:08 | INFO | Fetched 291,006 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 10:54:09 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 10:54:09 | INFO | ================================================================================ -2026-06-17 10:54:09 | INFO | Processing Table: OQaD -2026-06-17 10:54:09 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\main2.py", line 217, in main - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - ...<2 lines>... - run_date=run_date - ) - File "D:\data_move\src\fact.py", line 58, in fetch_OQaD - with open(sql_file, "r", encoding="utf-8") as f: - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql' -2026-06-17 10:57:28 | INFO | ================================================================================ -2026-06-17 10:57:28 | INFO | Hello from data-move Python data pipeline! -2026-06-17 10:57:28 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 10:57:28 | INFO | Connecting to databases... -2026-06-17 10:57:30 | INFO | -2026-06-17 10:57:31 | INFO | -2026-06-17 10:57:32 | INFO | Database connections established -2026-06-17 10:57:32 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 10:57:32 | INFO | Found 863 MIDs -2026-06-17 10:57:33 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 10:57:45 | INFO | Fetched 291,010 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 10:57:46 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 10:57:46 | INFO | ================================================================================ -2026-06-17 10:57:46 | INFO | Processing Table: OQaD -2026-06-17 10:57:46 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\main2.py", line 217, in main - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - ...<2 lines>... - run_date=run_date - ) - File "D:\data_move\src\fact.py", line 58, in fetch_OQaD - with open(sql_file, "r", encoding="utf-8") as f: - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql' -2026-06-17 10:59:57 | INFO | ================================================================================ -2026-06-17 10:59:57 | INFO | Hello from data-move Python data pipeline! -2026-06-17 10:59:57 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 10:59:57 | INFO | Connecting to databases... -2026-06-17 10:59:58 | INFO | -2026-06-17 11:00:00 | INFO | -2026-06-17 11:00:00 | INFO | Database connections established -2026-06-17 11:00:00 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 11:00:01 | INFO | Found 863 MIDs -2026-06-17 11:00:02 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 11:00:24 | INFO | Fetched 291,011 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 11:00:25 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 11:00:25 | INFO | ================================================================================ -2026-06-17 11:00:25 | INFO | Processing Table: OQaD -2026-06-17 11:00:25 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\main2.py", line 217, in main - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - ...<2 lines>... - run_date=run_date - ) - File "D:\data_move\src\fact.py", line 58, in fetch_OQaD - with open(sql_file, "r", encoding="utf-8") as f: - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql' -2026-06-17 12:09:34 | INFO | ================================================================================ -2026-06-17 12:09:34 | INFO | Hello from data-move Python data pipeline! -2026-06-17 12:09:34 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 12:09:34 | INFO | Connecting to databases... -2026-06-17 12:09:37 | INFO | -2026-06-17 12:09:39 | INFO | -2026-06-17 12:09:39 | INFO | Database connections established -2026-06-17 12:09:39 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 12:09:41 | INFO | Found 863 MIDs -2026-06-17 12:09:43 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 12:10:48 | INFO | Fetched 291,093 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 12:10:48 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 12:10:48 | INFO | ================================================================================ -2026-06-17 12:10:48 | INFO | Processing Table: OQaD -2026-06-17 12:10:48 | INFO | Exists: False -2026-06-17 12:10:48 | INFO | Path: D:\data_move\sql\fact\OQaD.sql -2026-06-17 12:10:48 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\main2.py", line 217, in main - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - ...<2 lines>... - run_date=run_date - ) - File "D:\data_move\src\fact.py", line 62, in fetch_OQaD - with open(sql_file, "r", encoding="utf-8") as f: - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql' -2026-06-17 12:11:47 | INFO | ================================================================================ -2026-06-17 12:11:47 | INFO | Hello from data-move Python data pipeline! -2026-06-17 12:11:47 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 12:11:47 | INFO | Connecting to databases... -2026-06-17 12:11:48 | INFO | -2026-06-17 12:11:50 | INFO | -2026-06-17 12:11:51 | INFO | Database connections established -2026-06-17 12:11:51 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 12:11:52 | INFO | Found 863 MIDs -2026-06-17 12:11:52 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 12:12:14 | INFO | Fetched 291,097 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 12:12:14 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 12:12:14 | INFO | ================================================================================ -2026-06-17 12:12:14 | INFO | Processing Table: OQaD -2026-06-17 12:12:14 | INFO | Exists: True -2026-06-17 12:12:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql -2026-06-17 12:12:14 | INFO | Fetching data for 893 EMPIDs -2026-06-17 12:12:14 | INFO | Fetching OQaD data for run_date=2026-06-16 -2026-06-17 12:12:14 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)") - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\main2.py", line 217, in main - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - ...<2 lines>... - run_date=run_date - ) - File "D:\data_move\src\fact.py", line 74, in fetch_OQaD - df = pl.read_database( - query=sql, - connection=engine, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)") -[SQL: WITH MID_TABLE_COV1 AS -( - SELECT EmpId, VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CreateDate >= 2026-06-16 - AND CreateDate < DATEADD(DAY,1,'2026-06-16') - - UNION ALL - - SELECT EmpId, VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE UpdateDate >= 2026-06-16 - AND UpdateDate < DATEADD(DAY,1, '2026-06-16') -), - -OQaD - - -QUIZ AS -( - SELECT DISTINCT - E.EmpId, - E.EmpName, - E.SupervisorId, - E.SupervisorName, - E.DesignationName, - E.CityName, - E.StateName, - E.RegionName, - CAST(DQ.VisitDate AS DATE) AS VisitDate, - DQ.QuestionId, - DQ.AnswerId, - QC.QuestionCategoryId, - QC.QuestionCategory - FROM OneApp_KelloggsMT.dbo.T_OQAD DQ - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E - ON DQ.EmpId = E.EmpId - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU - ON DQ.QuestionId = QU.QuestionId - INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC - ON QU.QuestionCategoryId = QC.QuestionCategoryId - WHERE E.EmpName NOT LIKE '%TEST%' - AND E.RightId = 6 - AND ( - E.ResignDate IS NULL - OR CAST(E.ResignDate AS DATE) >= '2026-06-16' - ) - AND EXISTS - ( - SELECT 1 - FROM MID_TABLE_COV1 A - WHERE A.EmpId = DQ.EmpId - AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE) - ) -) - -SELECT - 40148 AS project_id, - Q.EmpId AS employee_id, - 0 AS process_id, - Q.VisitDate AS visit_date, - Q.QuestionCategoryId AS question_category_id, - Q.QuestionCategory AS question_category, - QM.QuestionId AS question_id, - QM.Question AS question, - ISNULL(QA.AnswerId,0) AS answer_id, - ISNULL(QA.Answer,'') AS answer, - CASE - WHEN QA.AnswerId IS NULL THEN 'Not Answer' - WHEN QA.RightAnswer = 1 THEN 'Y' - WHEN QA.RightAnswer IS NULL THEN 'Not Answer' - ELSE 'N' - END AS correct_answer, - GETDATE() AS update_date, - 'SP-Pius' AS update_by -FROM QUIZ Q -INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM - ON Q.QuestionId = QM.QuestionId -LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA - ON Q.AnswerId = QA.AnswerId; -where Q.EmpId not in (3167,4724,4019,4902,4375,3352,3081,5244,3181,4810,5166,4895,4817,5173,5237,3608,4646,4126,3243,3948,3950,3599,3777,5235,5346,3606,3079,5175,5242,5064,4366,5071,5339,4812,4722,3169,5270,3326,3689,4062,3131,5054,5178,3371,4829,5185,4698,4836,3760,4665,5218,4451,4280,3269,3098,5327,4978,5261,3335,5090,5320,4214,4919,4570,3150,4221,3328,3112,3290,4950,4430,5104,3383,5213,4864,5199,5206,4439,3207,5033,4684,3979,5211,4774,3117,5102,5280,3309,4931,4753,5123,3155,5085,4914,5092,4387,5325,5154,4100,4805,3402,5187,3580,3409,3060,3962,4696,4831,3715,4710,4242,5282,5121,4933,5289,5282,3444,5104,3095,4933,4142,3437,3530,3088,3715,4840,5002,4831,5187,3361,3962,5113,3086,5106,5256,5085,4914,5092,3470,4387,5277,3309,5239,4712,4945,5246,5123,4774,3952,3603,4999,3727,3378,3207,5341,4992,5199,5334,4672,5206,3342,3520,3454,3171,4916,3105,4396,3112,3290,5265,3119,5227,3143,3150,5056,4529,3328,5313,5220,4693,5320,4615,3755,4444,3406,4978,4451,3746,4280,3399,5322,4617,4795,5218,3330,3159,4698,5144,5054,3473,3131,4726,4688,4743,3689,3812,3107,3169,5332,3347,5023,5022,4812,3380,3067,3950,4954,5132,3081,5296,5244,3140,4019,3492,4375,5210,3126,5289,3482,4847,5296,4954,4436,3380,5127,5305,3306,5291,5023,5120,4531,4346,5229,3107,3812,4353,3145,3152,3330,5322,4439,5144,4795,3363,4054,4232,3399,4239,3755,5279,4930,5213,3406,4864,5286,5220,4693,5101,5227,3985,3297,3636,4529,3420,5265,4916,3591,3335,4460,5343,4638,3342,5327,3520,3171,4467,3700,5336,4987,3515,4992,3760,5267,4562,3247,5082,4555,4733,5061,5239,3090,4712,5246,4904,5270,3059,4866,5277,4750,5106,3361,4968,5175,5310,3496,3608,4840,5248,4543,3615,4142,3437,3088,3444,3078,4902,3095,3567,4543,4450,3140,5319,4614,3496,4970,4443,5132,3916,3149,5310,3327,5312,3156,4968,4894,3394,5065,4538,4688,4866,3251,3090,3268,4671,4849,4904,4726,5082,4555,5191,4733,4562,3515,5300,4951,5115,3173,5293,4944,4460,4987,4638,4467,3591,3064,4526,3821,3232,3420,5056,5046,5008,3985,3636,5350,3109,5272,5101,5279,3358,4396,5172,5110,4054,4232,5281,3363,4999,3638,3603,3104,5027,5205,4856,4151,3083,4346,3764,3819,3415,4353,3470,3299,5167,3346,5113,3306,5331,5291,5160,5120,5127,4913,5091,4564,5262,5196,4847,5077,4666,4139,3612,3978,3078,5340,4139,3612,4991,5248,3170,5155,5333,4628,5077,4217,5084,4032,4913,4564,5257,5324,4381,5331,5250,5160,4134,4312,4141,3059,3764,3415,4305,5027,3220,3104,5321,4972,5307,5267,5136,5314,4787,3315,3322,4540,3835,3700,3486,4547,5172,5001,5008,3427,3379,5046,3372,3821,4526,3650,5124,5302,5286,3130,5115,4937,3132,5293,5117,3118,4239,5300,4951,4343,3897,5219,5191,4671,4849,5212,5229,5003,5181,4531,5065,5345,3538,4538,4189,4229,3531,5276,5305,4578,3106,3149,5312,5269,3113,4436,3099,4443,4970,3916,3567,3975,4153,4844,5210,5200,4844,3533,4666,5022,3184,5200,4402,4153,3704,3975,4922,3099,5262,4751,5091,3106,5269,5093,5276,4578,3346,5202,4675,4229,3531,3182,5195,5167,3203,5003,5205,5150,5328,3220,4979,4856,5157,4452,5335,3514,5219,3638,4343,4383,3289,4390,3118,3296,5281,5110,3438,3267,5074,5117,5252,3780,3431,5005,3372,3379,5176,3232,5316,4611,4967,3068,5309,3139,3495,3673,4540,3835,4891,3486,3488,3315,3322,5307,5136,3070,4877,4787,5314,5055,3939,3234,4072,5181,4305,3251,3374,5119,4134,3080,5297,4065,4894,3127,5250,4763,4941,5257,3298,4381,3327,3156,4687,4338,5221,4450,5155,3400,5333,5207,4628,5162,3742,4098,3742,4943,5121,3129,5207,5114,5292,4687,3122,4338,4696,3464,3113,4333,3298,3127,5031,4941,4763,5209,5119,3388,3566,3538,4065,3744,4072,3374,4753,3234,3939,5102,5088,5266,3070,5055,4877,3096,3979,3132,3488,3139,4841,5019,5302,3495,5124,5309,5012,4611,3369,3509,5176,5145,5323,5005,3153,4656,4371,3431,5254,5252,4547,3091,5074,4198,3782,3433,4383,4651,4829,4302,3953,5349,5321,3514,4972,5178,5150,5328,4979,5342,4452,5157,5304,3203,5195,3490,4312,3141,5126,4141,4675,4366,5057,5235,5064,3243,5093,4217,4922,3407,4751,4810,3170,4283,3756,3704,4974,3533,4991,3184,4974,3492,3143,4098,4283,3400,3756,3407,4710,5221,3067,3580,5057,3409,5196,3060,4805,4100,3402,5325,5297,3490,3141,5154,5126,3155,5304,4931,5109,5287,4117,5280,3473,4651,4302,3953,5033,5211,3782,4684,3433,3084,4891,3746,3383,4371,4022,5254,3673,5313,4207,3153,5316,3509,5145,5323,4430,4919,4570,3369,4221,3105,5090,4563,5012,3269,4841,3098,5019,3454,5261,3438,4672,3267,4836,3096,4665,5259,5088,3727,5266,3378,5185,4390,4397,3371,4062,4945,3528,3326,4722,3388,5256,3566,3744,5202,5242,3086,5071,5209,3777,4333,3606,3079,5002,3948,4126,3599,3464,5344,4817,5173,5351,4646,3122,3352,3530,5337,3181,3359,5166,5292,5114,4943) - -- select d.employee_id from OQaD d where project_id=40148 and d.employee_id=Q.EmpId and CONVERT(date,d.visit_date,101)= - -- CONVERT(date,q.VisitDate,101)) - -] -(Background on this error at: https://sqlalche.me/e/20/f405) -2026-06-17 12:19:35 | INFO | ================================================================================ -2026-06-17 12:19:35 | INFO | Hello from data-move Python data pipeline! -2026-06-17 12:19:35 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 12:19:35 | INFO | Connecting to databases... -2026-06-17 12:19:37 | INFO | -2026-06-17 12:19:38 | INFO | -2026-06-17 12:19:39 | INFO | Database connections established -2026-06-17 12:19:39 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 12:19:40 | INFO | Found 863 MIDs -2026-06-17 12:19:40 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 12:20:14 | INFO | Fetched 291,100 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 12:20:14 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 12:20:14 | INFO | ================================================================================ -2026-06-17 12:20:14 | INFO | Processing Table: OQaD -2026-06-17 12:20:14 | INFO | Exists: True -2026-06-17 12:20:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql -2026-06-17 12:20:14 | INFO | Fetching data for 893 EMPIDs -2026-06-17 12:20:14 | INFO | Fetching OQaD data for run_date=2026-06-16 -2026-06-17 12:20:30 | INFO | fn name is fetch_OQad ------Fetched 4998 rows -2026-06-17 12:20:30 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\main2.py", line 226, in main - df = get_dataframe( - fn_name=fn_name, - ...<3 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 81, in get_dataframe - return fn(sql_engine, run_date) -TypeError: fetch_OQaD() missing 3 required positional arguments: 'table_type', 'empids', and 'run_date' -2026-06-17 12:38:43 | INFO | ================================================================================ -2026-06-17 12:38:43 | INFO | Hello from data-move Python data pipeline! -2026-06-17 12:38:43 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 12:38:43 | INFO | Connecting to databases... -2026-06-17 12:38:45 | INFO | -2026-06-17 12:38:46 | INFO | -2026-06-17 12:38:46 | INFO | Database connections established -2026-06-17 12:38:46 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 12:38:47 | INFO | Found 863 MIDs -2026-06-17 12:38:47 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 12:38:58 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 12:38:58 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 12:38:58 | INFO | ================================================================================ -2026-06-17 12:38:58 | INFO | Processing Table: OQaD -2026-06-17 12:38:58 | INFO | Exists: True -2026-06-17 12:38:58 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql -2026-06-17 12:38:58 | INFO | Fetching data for 893 EMPIDs -2026-06-17 12:38:58 | INFO | Fetching OQaD data for run_date=2026-06-16 -2026-06-17 12:39:05 | INFO | fn name is fetch_OQad ------Fetched 4998 rows -2026-06-17 12:39:05 | INFO | ================================================================================ -2026-06-17 12:39:05 | INFO | Pipeline Completed Successfully -2026-06-17 12:39:05 | INFO | ================================================================================ -2026-06-17 12:39:30 | INFO | ================================================================================ -2026-06-17 12:39:30 | INFO | Hello from data-move Python data pipeline! -2026-06-17 12:39:30 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 12:39:30 | INFO | Connecting to databases... -2026-06-17 12:39:31 | INFO | -2026-06-17 12:39:32 | INFO | -2026-06-17 12:39:33 | INFO | Database connections established -2026-06-17 12:39:33 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 12:39:33 | INFO | Found 863 MIDs -2026-06-17 12:39:33 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 12:39:43 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 12:39:43 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 12:39:43 | INFO | ================================================================================ -2026-06-17 12:39:43 | INFO | Processing Table: OQaD -2026-06-17 12:39:43 | INFO | Exists: True -2026-06-17 12:39:43 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql -2026-06-17 12:39:43 | INFO | Fetching data for 893 EMPIDs -2026-06-17 12:39:43 | INFO | Fetching OQaD data for run_date=2026-06-16 -2026-06-17 12:39:46 | INFO | fn name is fetch_OQad ------Fetched 4998 rows -2026-06-17 12:39:46 | INFO | ================================================================================ -2026-06-17 12:39:46 | INFO | Pipeline Completed Successfully -2026-06-17 12:39:46 | INFO | ================================================================================ -2026-06-17 12:44:50 | INFO | ================================================================================ -2026-06-17 12:44:50 | INFO | Hello from data-move Python data pipeline! -2026-06-17 12:44:50 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 12:44:50 | INFO | Connecting to databases... -2026-06-17 12:44:51 | INFO | -2026-06-17 12:44:52 | INFO | -2026-06-17 12:44:53 | INFO | Database connections established -2026-06-17 12:44:53 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 12:44:53 | INFO | Found 863 MIDs -2026-06-17 12:44:53 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 12:45:07 | INFO | Fetched 291,115 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 12:45:08 | INFO | Fetched 893 matched empids fetched for OQAD -2026-06-17 12:45:08 | INFO | ================================================================================ -2026-06-17 12:45:08 | INFO | Processing Table: OQaD -2026-06-17 12:45:08 | INFO | Exists: True -2026-06-17 12:45:08 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql -2026-06-17 12:45:08 | INFO | Fetching data for 893 EMPIDs -2026-06-17 12:45:08 | INFO | Fetching OQaD data for run_date=2026-06-16 -2026-06-17 12:45:10 | INFO | fn name is fetch_OQad ------Fetched 4998 rows -2026-06-17 12:45:10 | INFO | Fetched 4998 rows -2026-06-17 12:45:10 | INFO | Deleting from OQaD -2026-06-17 12:45:11 | INFO | OQaD: inserted 4,998 rows into ClickHouse -2026-06-17 12:45:11 | INFO | OQaD loaded successfully (4998 rows) -2026-06-17 12:45:11 | INFO | ================================================================================ -2026-06-17 12:45:11 | INFO | Pipeline Completed Successfully -2026-06-17 12:45:11 | INFO | ================================================================================ -2026-06-17 13:23:53 | INFO | ================================================================================ -2026-06-17 13:23:53 | INFO | Hello from data-move Python data pipeline! -2026-06-17 13:23:53 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 13:23:53 | INFO | Connecting to databases... -2026-06-17 13:23:54 | INFO | -2026-06-17 13:23:56 | INFO | -2026-06-17 13:23:56 | INFO | Database connections established -2026-06-17 13:23:56 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 13:23:57 | INFO | Found 863 MIDs -2026-06-17 13:23:58 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 13:24:15 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 13:24:15 | INFO | Fetched 4,819 matched empids fetched for OQAD -2026-06-17 13:24:15 | INFO | ================================================================================ -2026-06-17 13:24:15 | INFO | Processing Table: OQaD -2026-06-17 13:24:15 | INFO | Exists: True -2026-06-17 13:24:15 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql -2026-06-17 13:24:15 | INFO | Fetching data for 4,819 EMPIDs -2026-06-17 13:24:15 | INFO | Fetching OQaD data for run_date=2026-06-16 -2026-06-17 13:24:22 | INFO | fn name is fetch_OQad ------Fetched 1072 rows -2026-06-17 13:24:22 | INFO | Fetched 1072 rows -2026-06-17 13:24:22 | INFO | Deleting from OQaD -2026-06-17 13:24:23 | INFO | OQaD: inserted 1,072 rows into ClickHouse -2026-06-17 13:24:23 | INFO | OQaD loaded successfully (1072 rows) -2026-06-17 13:24:23 | INFO | ================================================================================ -2026-06-17 13:24:23 | INFO | Processing Table: Survey -2026-06-17 13:24:23 | INFO | Fetching data for 863 MIDs -2026-06-17 13:24:23 | INFO | Fetched 142 rows from SQL Server -2026-06-17 13:24:23 | INFO | Fetched 142 rows -2026-06-17 13:24:23 | INFO | Deleting from Survey -2026-06-17 13:24:24 | ERROR | Failed processing table Survey -Traceback (most recent call last): - File "D:\data_move\main2.py", line 299, in main - load_to_clickhouse( - ~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - table_name=table_name, - ^^^^^^^^^^^^^^^^^^^^^^ - df=df, - ^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse - client.insert_arrow( - ~~~~~~~~~~~~~~~~~~~^ - table=table_name, - ^^^^^^^^^^^^^^^^^ - arrow_table=arrow_table, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow - return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) - ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert - response = self._raw_request(insert_block, params, headers, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column Project_Id in table kelloggs_1.Survey (d8c44908-2ed7-4d29-bd10-8e70fa84dfba). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) -2026-06-17 13:27:22 | INFO | ================================================================================ -2026-06-17 13:27:22 | INFO | Hello from data-move Python data pipeline! -2026-06-17 13:27:22 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 13:27:22 | INFO | Connecting to databases... -2026-06-17 13:27:23 | INFO | -2026-06-17 13:27:25 | INFO | -2026-06-17 13:27:25 | INFO | Database connections established -2026-06-17 13:27:25 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 13:27:26 | INFO | Found 863 MIDs -2026-06-17 13:27:26 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 13:27:37 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 13:27:38 | INFO | Fetched 4,814 matched empids fetched for OQAD -2026-06-17 13:27:38 | INFO | ================================================================================ -2026-06-17 13:27:38 | INFO | Processing Table: OQaD -2026-06-17 13:27:38 | INFO | Exists: True -2026-06-17 13:27:38 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql -2026-06-17 13:27:38 | INFO | Fetching data for 4,814 EMPIDs -2026-06-17 13:27:38 | INFO | Fetching OQaD data for run_date=2026-06-16 -2026-06-17 13:27:44 | INFO | fn name is fetch_OQad ------Fetched 1072 rows -2026-06-17 13:27:44 | INFO | Fetched 1072 rows -2026-06-17 13:27:44 | INFO | Deleting from OQaD -2026-06-17 13:27:45 | INFO | OQaD: inserted 1,072 rows into ClickHouse -2026-06-17 13:27:45 | INFO | OQaD loaded successfully (1072 rows) -2026-06-17 13:27:45 | INFO | ================================================================================ -2026-06-17 13:27:45 | INFO | Processing Table: Survey -2026-06-17 13:27:45 | INFO | Fetching data for 863 MIDs -2026-06-17 13:27:45 | INFO | Fetched 142 rows from SQL Server -2026-06-17 13:27:45 | INFO | Fetched 142 rows -2026-06-17 13:27:45 | INFO | Creating table Survey -2026-06-17 13:27:45 | INFO | Table ready: Survey -2026-06-17 13:27:45 | INFO | Survey: inserted 142 rows into ClickHouse -2026-06-17 13:27:45 | INFO | Survey loaded successfully (142 rows) -2026-06-17 13:27:45 | INFO | ================================================================================ -2026-06-17 13:27:45 | INFO | Pipeline Completed Successfully -2026-06-17 13:27:45 | INFO | ================================================================================ -2026-06-17 15:05:42 | INFO | ================================================================================ -2026-06-17 15:05:42 | INFO | Hello from data-move Python data pipeline! -2026-06-17 15:05:42 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 15:05:42 | INFO | Connecting to databases... -2026-06-17 15:05:43 | INFO | -2026-06-17 15:05:45 | INFO | -2026-06-17 15:05:45 | INFO | Database connections established -2026-06-17 15:05:45 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 15:05:45 | INFO | Found 863 MIDs -2026-06-17 15:05:46 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 15:05:57 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 15:05:57 | INFO | Fetched 4,814 matched empids fetched for OQAD -2026-06-17 15:05:57 | INFO | ================================================================================ -2026-06-17 15:05:57 | INFO | Processing Table: additional_visibility -2026-06-17 15:05:57 | INFO | Fetching data for 863 MIDs -2026-06-17 15:05:58 | ERROR | Failed processing table additional_visibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 228, in main - df = fetch_data( - engine=sql_engine, - ...<3 lines>... - run_date=run_date, - ) - File "D:\data_move\src\fact.py", line 34, in fetch_data - df = pl.read_database( - query=sql, - connection=engine - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 287, in read_database - ).to_polars( - ~~~~~~~~~^ - batch_size=batch_size, - ^^^^^^^^^^^^^^^^^^^^^^ - ...<2 lines>... - infer_schema_length=infer_schema_length, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 588, in to_polars - frame = frame_init( - batch_size=batch_size, - ...<2 lines>... - infer_schema_length=infer_schema_length, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 285, in _from_rows - (d[0], d[1:]) for d in self.result.cursor.description - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -AttributeError: 'NoneType' object has no attribute 'description' -2026-06-17 15:07:15 | INFO | ================================================================================ -2026-06-17 15:07:15 | INFO | Hello from data-move Python data pipeline! -2026-06-17 15:07:15 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 15:07:15 | INFO | Connecting to databases... -2026-06-17 15:07:16 | INFO | -2026-06-17 15:07:17 | INFO | -2026-06-17 15:07:18 | INFO | Database connections established -2026-06-17 15:07:18 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 15:07:18 | INFO | Found 863 MIDs -2026-06-17 15:07:18 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 15:07:31 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 15:07:31 | INFO | Fetched 4,814 matched empids fetched for OQAD -2026-06-17 15:07:31 | INFO | ================================================================================ -2026-06-17 15:07:31 | INFO | Processing Table: additional_visibility -2026-06-17 15:07:31 | INFO | Fetching data for 863 MIDs -2026-06-17 15:07:32 | INFO | Fetched 1,773 rows from SQL Server -2026-06-17 15:07:32 | INFO | Fetched 1773 rows -2026-06-17 15:07:32 | INFO | Deleting from additional_visibility -2026-06-17 15:07:32 | ERROR | Failed processing table additional_visibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 299, in main - load_to_clickhouse( - ~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - table_name=table_name, - ^^^^^^^^^^^^^^^^^^^^^^ - df=df, - ^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse - client.insert_arrow( - ~~~~~~~~~~~~~~~~~~~^ - table=table_name, - ^^^^^^^^^^^^^^^^^ - arrow_table=arrow_table, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow - return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) - ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert - response = self._raw_request(insert_block, params, headers, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.additional_visibility (9e03e625-028a-47db-8f60-e8ab93e907ea). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) -2026-06-17 15:08:55 | INFO | ================================================================================ -2026-06-17 15:08:55 | INFO | Hello from data-move Python data pipeline! -2026-06-17 15:08:55 | INFO | Pipeline Run Date: 2026-06-16 -2026-06-17 15:08:55 | INFO | Connecting to databases... -2026-06-17 15:08:56 | INFO | -2026-06-17 15:08:58 | INFO | -2026-06-17 15:08:58 | INFO | Database connections established -2026-06-17 15:08:58 | INFO | Collecting MIDs for: 2026-06-16 -2026-06-17 15:08:59 | INFO | Found 863 MIDs -2026-06-17 15:08:59 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-17 15:09:12 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server -2026-06-17 15:09:12 | INFO | Fetched 4,814 matched empids fetched for OQAD -2026-06-17 15:09:12 | INFO | ================================================================================ -2026-06-17 15:09:12 | INFO | Processing Table: additional_visibility -2026-06-17 15:09:12 | INFO | Fetching data for 863 MIDs -2026-06-17 15:09:14 | INFO | Fetched 1,773 rows from SQL Server -2026-06-17 15:09:14 | INFO | Fetched 1773 rows -2026-06-17 15:09:14 | INFO | Creating table additional_visibility -2026-06-17 15:09:14 | INFO | Table ready: additional_visibility -2026-06-17 15:09:15 | INFO | additional_visibility: inserted 1,773 rows into ClickHouse -2026-06-17 15:09:15 | INFO | additional_visibility loaded successfully (1773 rows) -2026-06-17 15:09:15 | INFO | ================================================================================ -2026-06-17 15:09:15 | INFO | Pipeline Completed Successfully -2026-06-17 15:09:15 | INFO | ================================================================================ diff --git a/logs/etl_20260618.log b/logs/etl_20260618.log deleted file mode 100644 index 6165602..0000000 --- a/logs/etl_20260618.log +++ /dev/null @@ -1,2134 +0,0 @@ -2026-06-18 11:27:30 | INFO | ================================================================================ -2026-06-18 11:27:30 | INFO | Hello from data-move Python data pipeline! -2026-06-18 11:27:30 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 11:27:30 | INFO | Connecting to databases... -2026-06-18 11:27:31 | INFO | -2026-06-18 11:27:33 | INFO | -2026-06-18 11:27:33 | INFO | Database connections established -2026-06-18 11:27:33 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 11:27:34 | INFO | Found 836 MIDs -2026-06-18 11:27:35 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-18 11:28:13 | INFO | Fetched 289,830 total empid and visitdate fetched for OQAD from SQL Server -2026-06-18 11:28:13 | INFO | Fetched 4,800 matched empids fetched for OQAD -2026-06-18 11:28:13 | INFO | ================================================================================ -2026-06-18 11:28:13 | INFO | Processing Table: Login -2026-06-18 11:28:13 | INFO | Fetching data for 836 MIDs -2026-06-18 11:28:15 | INFO | Fetched 475 rows from SQL Server -2026-06-18 11:28:15 | INFO | Fetched 475 rows -2026-06-18 11:28:16 | INFO | Deleting from Login -2026-06-18 11:28:16 | INFO | No delete logic required for Login -2026-06-18 11:28:16 | ERROR | Failed processing table Login -Traceback (most recent call last): - File "D:\data_move\main2.py", line 299, in main - load_to_clickhouse( - ~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - table_name=table_name, - ^^^^^^^^^^^^^^^^^^^^^^ - df=df, - ^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse - client.insert_arrow( - ~~~~~~~~~~~~~~~~~~~^ - table=table_name, - ^^^^^^^^^^^^^^^^^ - arrow_table=arrow_table, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow - return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) - ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert - response = self._raw_request(insert_block, params, headers, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.Login (3444490f-e300-409d-87dc-fc98fff9e514). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) -2026-06-18 11:30:44 | INFO | ================================================================================ -2026-06-18 11:30:44 | INFO | Hello from data-move Python data pipeline! -2026-06-18 11:30:44 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 11:30:44 | INFO | Connecting to databases... -2026-06-18 11:30:45 | INFO | -2026-06-18 11:30:46 | INFO | -2026-06-18 11:30:47 | INFO | Database connections established -2026-06-18 11:30:47 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 11:30:48 | INFO | Found 836 MIDs -2026-06-18 11:30:51 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-18 11:32:29 | INFO | Fetched 289,833 total empid and visitdate fetched for OQAD from SQL Server -2026-06-18 11:32:30 | INFO | Fetched 4,800 matched empids fetched for OQAD -2026-06-18 11:32:30 | INFO | ================================================================================ -2026-06-18 11:32:30 | INFO | Processing Table: Login -2026-06-18 11:32:30 | INFO | Fetching data for 836 MIDs -2026-06-18 11:32:37 | INFO | Fetched 475 rows from SQL Server -2026-06-18 11:32:37 | INFO | Fetched 475 rows -2026-06-18 11:32:37 | INFO | Creating table Login -2026-06-18 11:32:37 | INFO | Table ready: Login -2026-06-18 11:32:38 | INFO | Login: inserted 475 rows into ClickHouse -2026-06-18 11:32:38 | INFO | Login loaded successfully (475 rows) -2026-06-18 11:32:38 | INFO | ================================================================================ -2026-06-18 11:32:38 | INFO | Pipeline Completed Successfully -2026-06-18 11:32:38 | INFO | ================================================================================ -2026-06-18 11:49:35 | INFO | ================================================================================ -2026-06-18 11:49:35 | INFO | Hello from data-move Python data pipeline! -2026-06-18 11:49:35 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 11:49:35 | INFO | Connecting to databases... -2026-06-18 11:49:37 | INFO | -2026-06-18 11:49:38 | INFO | -2026-06-18 11:49:39 | INFO | Database connections established -2026-06-18 11:49:39 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 11:49:40 | INFO | Found 836 MIDs -2026-06-18 11:49:40 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-18 11:49:59 | INFO | Fetched 289,843 total empid and visitdate fetched for OQAD from SQL Server -2026-06-18 11:50:00 | INFO | Fetched 4,800 matched empids fetched for OQAD -2026-06-18 11:50:00 | INFO | ================================================================================ -2026-06-18 11:50:00 | INFO | Processing Table: Login -2026-06-18 11:50:00 | INFO | Fetching data for 836 MIDs -2026-06-18 11:50:18 | INFO | Fetched 475 rows from SQL Server -2026-06-18 11:50:18 | INFO | Fetched 475 rows -2026-06-18 11:50:18 | INFO | Deleting from Login -2026-06-18 11:50:19 | INFO | No delete logic required for Login -2026-06-18 11:50:19 | INFO | Login: inserted 475 rows into ClickHouse -2026-06-18 11:50:19 | INFO | Login loaded successfully (475 rows) -2026-06-18 11:50:19 | INFO | ================================================================================ -2026-06-18 11:50:19 | INFO | Pipeline Completed Successfully -2026-06-18 11:50:19 | INFO | ================================================================================ -2026-06-18 11:51:32 | INFO | ================================================================================ -2026-06-18 11:51:32 | INFO | Hello from data-move Python data pipeline! -2026-06-18 11:51:32 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 11:51:32 | INFO | Connecting to databases... -2026-06-18 11:51:34 | INFO | -2026-06-18 11:51:35 | INFO | -2026-06-18 11:51:36 | INFO | Database connections established -2026-06-18 11:51:36 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 11:51:36 | INFO | Found 836 MIDs -2026-06-18 11:51:37 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-18 11:52:03 | INFO | Fetched 289,846 total empid and visitdate fetched for OQAD from SQL Server -2026-06-18 11:52:03 | INFO | Fetched 4,800 matched empids fetched for OQAD -2026-06-18 11:52:03 | INFO | ================================================================================ -2026-06-18 11:52:04 | INFO | Processing Table: Stock_Details -2026-06-18 11:52:04 | INFO | Fetching data for 836 MIDs -2026-06-18 11:52:23 | INFO | Fetched 39,313 rows from SQL Server -2026-06-18 11:52:23 | INFO | Fetched 39313 rows -2026-06-18 11:52:24 | INFO | Deleting from Stock_Details -2026-06-18 11:52:24 | ERROR | Failed processing table Stock_Details -Traceback (most recent call last): - File "D:\data_move\main2.py", line 299, in main - load_to_clickhouse( - ~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - table_name=table_name, - ^^^^^^^^^^^^^^^^^^^^^^ - df=df, - ^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse - client.insert_arrow( - ~~~~~~~~~~~~~~~~~~~^ - table=table_name, - ^^^^^^^^^^^^^^^^^ - arrow_table=arrow_table, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow - return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) - ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert - response = self._raw_request(insert_block, params, headers, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.Stock_Details (cf4b6e46-3078-4bfb-9aa3-39357da21740). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) -2026-06-18 11:55:44 | INFO | ================================================================================ -2026-06-18 11:55:44 | INFO | Hello from data-move Python data pipeline! -2026-06-18 11:55:44 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 11:55:44 | INFO | Connecting to databases... -2026-06-18 11:55:45 | INFO | -2026-06-18 11:55:46 | INFO | -2026-06-18 11:55:46 | INFO | Database connections established -2026-06-18 11:55:46 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 11:55:47 | INFO | Found 836 MIDs -2026-06-18 11:55:47 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-18 11:56:11 | INFO | Fetched 289,856 total empid and visitdate fetched for OQAD from SQL Server -2026-06-18 11:56:11 | INFO | Fetched 4,800 matched empids fetched for OQAD -2026-06-18 11:56:11 | INFO | ================================================================================ -2026-06-18 11:56:11 | INFO | Processing Table: Stock_Details -2026-06-18 11:56:11 | INFO | Fetching data for 836 MIDs -2026-06-18 11:56:26 | INFO | Fetched 39,313 rows from SQL Server -2026-06-18 11:56:26 | INFO | Fetched 39313 rows -2026-06-18 11:56:26 | INFO | Creating table Stock_Details -2026-06-18 11:56:26 | INFO | Table ready: Stock_Details -2026-06-18 11:56:26 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 11:56:27 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse -2026-06-18 11:56:27 | INFO | Stock_Details loaded successfully (39313 rows) -2026-06-18 11:56:27 | INFO | ================================================================================ -2026-06-18 11:56:27 | INFO | Pipeline Completed Successfully -2026-06-18 11:56:27 | INFO | ================================================================================ -2026-06-18 12:41:09 | INFO | ================================================================================ -2026-06-18 12:41:09 | INFO | Hello from data-move Python data pipeline! -2026-06-18 12:41:09 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 12:41:09 | INFO | Connecting to databases... -2026-06-18 12:41:11 | INFO | -2026-06-18 12:41:12 | INFO | -2026-06-18 12:41:13 | INFO | Database connections established -2026-06-18 12:41:13 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 12:41:14 | INFO | Found 836 MIDs -2026-06-18 12:41:15 | INFO | ================================================================================ -2026-06-18 12:41:15 | INFO | Processing Table: SOS_OneApp -2026-06-18 12:41:15 | INFO | Fetching Data from sql server for table-:SOS_OneApp--------------------------- -2026-06-18 12:41:15 | ERROR | Failed processing table SOS_OneApp -Traceback (most recent call last): - File "D:\data_move\main2.py", line 184, in main - df = get_dataframe( - engine=sql_engine, - ...<5 lines>... - run_date=run_date, - ) -TypeError: get_dataframe() got an unexpected keyword argument 'engine' -2026-06-18 12:43:11 | INFO | ================================================================================ -2026-06-18 12:43:11 | INFO | Hello from data-move Python data pipeline! -2026-06-18 12:43:11 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 12:43:11 | INFO | Connecting to databases... -2026-06-18 12:43:13 | INFO | -2026-06-18 12:43:14 | INFO | -2026-06-18 12:43:15 | INFO | Database connections established -2026-06-18 12:43:15 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 12:43:15 | INFO | Found 836 MIDs -2026-06-18 12:43:16 | INFO | ================================================================================ -2026-06-18 12:43:16 | INFO | Processing Table: SOS_OneApp -2026-06-18 12:43:16 | INFO | Fetching Data from sql server for table-:SOS_OneApp--------------------------- -2026-06-18 12:43:16 | INFO | Fetching data for 836 MIDs -2026-06-18 12:43:16 | ERROR | Failed processing table SOS_OneApp -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.Error: ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\main2.py", line 184, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 92, in fetch_SOS_OneApp - df = pl.read_database( - query=sql, - connection=engine - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.DBAPIError: (pyodbc.Error) ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') -(Background on this error at: https://sqlalche.me/e/20/dbapi) -2026-06-18 12:47:32 | INFO | ================================================================================ -2026-06-18 12:47:32 | INFO | Hello from data-move Python data pipeline! -2026-06-18 12:47:32 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 12:47:32 | INFO | Connecting to databases... -2026-06-18 12:47:33 | INFO | -2026-06-18 12:47:35 | INFO | -2026-06-18 12:47:37 | INFO | Database connections established -2026-06-18 12:47:37 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 12:47:37 | INFO | Found 836 MIDs -2026-06-18 12:47:38 | INFO | ================================================================================ -2026-06-18 12:47:38 | INFO | Processing Table: SOS_OneApp -2026-06-18 12:47:38 | INFO | Fetching Data from sql server for table-:SOS_OneApp--------------------------- -2026-06-18 12:47:38 | INFO | Fetching data for 836 MIDs -2026-06-18 12:47:38 | ERROR | Failed processing table SOS_OneApp -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.Error: ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\main2.py", line 184, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 92, in fetch_SOS_OneApp - df = pl.read_database( - query=sql, - connection=engine - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.DBAPIError: (pyodbc.Error) ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') -(Background on this error at: https://sqlalche.me/e/20/dbapi) -2026-06-18 12:54:11 | INFO | ================================================================================ -2026-06-18 12:54:11 | INFO | Hello from data-move Python data pipeline! -2026-06-18 12:54:11 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 12:54:11 | INFO | Connecting to databases... -2026-06-18 12:54:12 | INFO | -2026-06-18 12:54:14 | INFO | -2026-06-18 12:54:15 | INFO | Database connections established -2026-06-18 12:54:15 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 12:54:15 | INFO | Found 836 MIDs -2026-06-18 12:54:16 | INFO | ================================================================================ -2026-06-18 12:54:16 | INFO | Processing Table: SOS_OneApp -2026-06-18 12:54:16 | INFO | Fetching Data from sql server for table-:SOS_OneApp--------------------------- -2026-06-18 12:54:16 | INFO | Fetching data for 836 MIDs -2026-06-18 12:54:16 | ERROR | Failed processing table SOS_OneApp -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.Error: ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\main2.py", line 184, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 92, in fetch_SOS_OneApp - df = pl.read_database( - query=sql, - connection=engine - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.DBAPIError: (pyodbc.Error) ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') -(Background on this error at: https://sqlalche.me/e/20/dbapi) -2026-06-18 12:57:59 | INFO | ================================================================================ -2026-06-18 12:57:59 | INFO | Hello from data-move Python data pipeline! -2026-06-18 12:57:59 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 12:57:59 | INFO | Connecting to databases... -2026-06-18 12:58:00 | INFO | -2026-06-18 12:58:01 | INFO | -2026-06-18 12:58:02 | INFO | Database connections established -2026-06-18 12:58:02 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 12:58:02 | INFO | Found 836 MIDs -2026-06-18 12:58:03 | INFO | ================================================================================ -2026-06-18 12:58:03 | INFO | Processing Table: OQaD -2026-06-18 12:58:03 | INFO | Fetching Data from sql server for table-:OQaD--------------------------- -2026-06-18 12:58:03 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-18 12:58:22 | INFO | Fetched 289,901 total empid and visitdate fetched for OQAD from SQL Server -2026-06-18 12:58:23 | INFO | Fetched 4,800 matched empids fetched for OQAD -2026-06-18 12:58:23 | ERROR | Failed processing table OQaD -Traceback (most recent call last): - File "D:\data_move\main2.py", line 184, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 284, in fetch_OQaD - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - ...<2 lines>... - run_date=run_date - ) -TypeError: fetch_OQaD() got an unexpected keyword argument 'engine' -2026-06-18 12:59:46 | INFO | ================================================================================ -2026-06-18 12:59:46 | INFO | Hello from data-move Python data pipeline! -2026-06-18 12:59:46 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 12:59:46 | INFO | Connecting to databases... -2026-06-18 12:59:46 | INFO | -2026-06-18 12:59:48 | INFO | -2026-06-18 12:59:49 | INFO | Database connections established -2026-06-18 12:59:49 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 12:59:49 | INFO | Found 836 MIDs -2026-06-18 12:59:50 | INFO | ================================================================================ -2026-06-18 12:59:50 | INFO | Processing Table: OQaD -2026-06-18 12:59:50 | INFO | Fetching Data from sql server for table-:OQaD--------------------------- -2026-06-18 12:59:50 | INFO | Fetching quiz_empids data for EMPID and Visitid -2026-06-18 13:00:25 | INFO | Fetched 289,901 total empid and visitdate fetched for OQAD from SQL Server -2026-06-18 13:00:25 | INFO | Fetched 4,800 matched empids fetched for OQAD -2026-06-18 13:00:25 | INFO | Exists: True -2026-06-18 13:00:25 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql -2026-06-18 13:00:25 | INFO | Fetching data for 4,800 EMPIDs -2026-06-18 13:00:25 | INFO | Fetching OQaD data for run_date=2026-06-17 -2026-06-18 13:00:40 | INFO | fn name is fetch_OQad ------Fetched 2341 rows -2026-06-18 13:00:40 | INFO | Fetched 2,341 rows from SQL Server -2026-06-18 13:00:40 | INFO | Fetched total row -: 2341 from sql server for table-:OQaD ...........!!! -2026-06-18 13:00:40 | INFO | Fetched 2341 rows -2026-06-18 13:00:41 | INFO | Deleting from OQaD -2026-06-18 13:00:41 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:00:41 | INFO | OQaD: inserted 2,341 rows into ClickHouse -2026-06-18 13:00:41 | INFO | OQaD loaded successfully (2341 rows) -2026-06-18 13:00:41 | INFO | ================================================================================ -2026-06-18 13:00:41 | INFO | Processing Table: additional_visibility -2026-06-18 13:00:41 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- -2026-06-18 13:00:41 | INFO | Fetching data for 836 MIDs -2026-06-18 13:00:43 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:00:43 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:00:43 | INFO | Fetched 1885 rows -2026-06-18 13:00:43 | INFO | Deleting from additional_visibility -2026-06-18 13:00:44 | ERROR | Failed processing table additional_visibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 246, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 81, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - table_name, - ^^^^^^^^^^^ - f"MID IN ({mids_str})", - ^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144)), _block_offset, _block_number, _table, _database, _distance, _disk_name, _part_granule_offset, _partition_id, _sample_factor, brand_id, _part_uuid, _part_index, _part, created_by, storetype_id, created_date, camera_allowed, image_url, display_id, Remarks, is_present, _part_data_version, _part_starting_offset, visit_date, Mid, channel_id, store_id, chain_id, _part_offset, emp_id, project_id', required columns: 'MID' '_part_granule_offset' '_block_number' '_table' '_block_offset' '_database' '_distance' 'Remarks' 'is_present' '_disk_name' '_partition_id' '_sample_factor' 'display_id' 'brand_id' '_part_uuid' '_part_index' '_part' 'created_by' 'created_date' 'storetype_id' 'camera_allowed' 'image_url' '_part_data_version' '_part_starting_offset' 'visit_date' 'Mid' 'channel_id' 'store_id' 'chain_id' '_part_offset' 'emp_id' 'project_id', maybe you meant: 'Mid', 'Remarks', 'is_present', 'display_id', 'brand_id', 'created_by', 'created_date', 'storetype_id', 'camera_allowed', 'image_url', 'visit_date', 'channel_id', 'store_id', 'chain_id', 'emp_id' or 'project_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-18 13:05:16 | INFO | ================================================================================ -2026-06-18 13:05:16 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:05:16 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:05:16 | INFO | Connecting to databases... -2026-06-18 13:05:17 | INFO | -2026-06-18 13:05:19 | INFO | -2026-06-18 13:05:20 | INFO | Database connections established -2026-06-18 13:05:20 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:05:21 | INFO | Found 836 MIDs -2026-06-18 13:05:22 | INFO | ================================================================================ -2026-06-18 13:05:22 | INFO | Processing Table: additional_visibility Table type is -: FACT Based on mids and operation is used INSERT -2026-06-18 13:05:22 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- -2026-06-18 13:05:22 | INFO | Fetching data for 836 MIDs -2026-06-18 13:05:25 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:05:25 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:05:25 | INFO | Fetched 1885 rows -2026-06-18 13:05:25 | INFO | Deleting from additional_visibility -2026-06-18 13:05:25 | ERROR | Failed processing table additional_visibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 81, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - table_name, - ^^^^^^^^^^^ - f"MID IN ({mids_str})", - ^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144)), _block_offset, _block_number, _table, _database, _distance, _disk_name, _part_granule_offset, _partition_id, _sample_factor, brand_id, _part_uuid, _part_index, _part, created_by, storetype_id, created_date, camera_allowed, image_url, display_id, Remarks, is_present, _part_data_version, _part_starting_offset, visit_date, Mid, channel_id, store_id, chain_id, _part_offset, emp_id, project_id', required columns: 'MID' '_part_granule_offset' '_block_number' '_table' '_block_offset' '_database' '_distance' 'Remarks' 'is_present' '_disk_name' '_partition_id' '_sample_factor' 'display_id' 'brand_id' '_part_uuid' '_part_index' '_part' 'created_by' 'created_date' 'storetype_id' 'camera_allowed' 'image_url' '_part_data_version' '_part_starting_offset' 'visit_date' 'Mid' 'channel_id' 'store_id' 'chain_id' '_part_offset' 'emp_id' 'project_id', maybe you meant: 'Mid', 'Remarks', 'is_present', 'display_id', 'brand_id', 'created_by', 'created_date', 'storetype_id', 'camera_allowed', 'image_url', 'visit_date', 'channel_id', 'store_id', 'chain_id', 'emp_id' or 'project_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-18 13:11:42 | INFO | ================================================================================ -2026-06-18 13:11:42 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:11:42 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:11:42 | INFO | Connecting to databases... -2026-06-18 13:11:43 | INFO | -2026-06-18 13:11:45 | INFO | -2026-06-18 13:11:45 | INFO | Database connections established -2026-06-18 13:11:45 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:11:46 | INFO | Found 836 MIDs -2026-06-18 13:11:47 | INFO | ================================================================================ -2026-06-18 13:11:47 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:11:47 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- -2026-06-18 13:11:47 | INFO | Fetching data for 836 MIDs -2026-06-18 13:11:51 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:11:51 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:11:51 | INFO | Fetched 1885 rows -2026-06-18 13:11:51 | INFO | Deleting from additional_visibility -2026-06-18 13:11:51 | ERROR | Failed processing table additional_visibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 81, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - table_name, - ^^^^^^^^^^^ - f"MID IN ({mids_str})", - ^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147)), _block_offset, _block_number, _table, _database, _distance, _disk_name, _part_granule_offset, _partition_id, _sample_factor, brand_id, _part_uuid, _part_index, _part, created_by, storetype_id, created_date, camera_allowed, image_url, display_id, Remarks, is_present, _part_data_version, _part_starting_offset, visit_date, Mid, channel_id, store_id, chain_id, _part_offset, emp_id, project_id', required columns: 'MID' '_part_granule_offset' '_block_number' '_table' '_block_offset' '_database' '_distance' 'Remarks' 'is_present' '_disk_name' '_partition_id' '_sample_factor' 'display_id' 'brand_id' '_part_uuid' '_part_index' '_part' 'created_by' 'created_date' 'storetype_id' 'camera_allowed' 'image_url' '_part_data_version' '_part_starting_offset' 'visit_date' 'Mid' 'channel_id' 'store_id' 'chain_id' '_part_offset' 'emp_id' 'project_id', maybe you meant: 'Mid', 'Remarks', 'is_present', 'display_id', 'brand_id', 'created_by', 'created_date', 'storetype_id', 'camera_allowed', 'image_url', 'visit_date', 'channel_id', 'store_id', 'chain_id', 'emp_id' or 'project_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-18 13:15:48 | INFO | ================================================================================ -2026-06-18 13:15:48 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:15:48 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:15:48 | INFO | Connecting to databases... -2026-06-18 13:15:49 | INFO | -2026-06-18 13:15:51 | INFO | -2026-06-18 13:15:51 | INFO | Database connections established -2026-06-18 13:15:51 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:15:52 | INFO | Found 836 MIDs -2026-06-18 13:15:52 | INFO | ================================================================================ -2026-06-18 13:15:52 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:15:52 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- -2026-06-18 13:15:52 | INFO | Fetching data for 836 MIDs -2026-06-18 13:15:54 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:15:54 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:15:54 | INFO | Fetched 1885 rows -2026-06-18 13:15:54 | INFO | Deleting from additional_visibility -2026-06-18 13:15:54 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:15:54 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:15:54 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:15:54 | INFO | ================================================================================ -2026-06-18 13:15:54 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:15:54 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- -2026-06-18 13:15:54 | INFO | Fetching data for 836 MIDs -2026-06-18 13:15:54 | ERROR | Failed processing table Coverage -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 363, in fetch_Coverage - df = pl.read_database( - query=sql, - connection=engine - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") -[SQL: -Executed as -(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' -ELSE 'N' END AS COVERED, -CASE WHEN isnull(C.MID,'')<>'' then 'Y' -ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON - -from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join -(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) -inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B - ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) - AND JP.ReasonId in (0,1,3,9,10,19,20))b -on JP.MID=b.MID - -left join -(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) -inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B - ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) - AND JP.ReasonId in (0,19))c -on JP.MID=c.MID -left join -OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId -Where 1=1 AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146) - -) , - Coverage - (project_id,MID, -store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, -is_covered,is_executed,reason_remarks,storetype_id, -supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, -update_by,Unique_Id) - -AS ( - -Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, - Convert(VARCHAR,Jp.VisitDate) AS VisitDate -, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, - Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, - Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId - And VisitDate = Jp.VisitDate),'') As [In Time] - , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, - Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, - Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage - Where EmpId = Jp.EmpId And StoreId = Jp.StoreId - And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] - , - Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 - Else Case When outtime0 Then 'Y' -Else 'N' End As Covered -, isnull(Exe.Executed,'') as Executed - -,Isnull((Select Top 1 Case -When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + -Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') -Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason -From OneApp_KelloggsMT.dbo.T_StoreCoverage D -Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId -Where D.Isdel = 0 -And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], -sm.StoreTypeId,Em.SupervisorId, -Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' -When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' -When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] -,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 - Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 -Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 - End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId -And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] -,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, - -GETDATE(),'SP-Pius' , - - CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) - + '_' + CAST(EM.EMPID AS VARCHAR) - - - -FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join -Executed Exe on Jp.MID= Exe.MID Inner Join -OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join -OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId -left join -OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) -on jp.mid=pv.MID -Where 1=1 and em.UserName not like 'test%' AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146 ) - -) -select * from Coverage] -(Background on this error at: https://sqlalche.me/e/20/f405) -2026-06-18 13:19:16 | INFO | ================================================================================ -2026-06-18 13:19:16 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:19:16 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:19:16 | INFO | Connecting to databases... -2026-06-18 13:19:17 | INFO | -2026-06-18 13:19:19 | INFO | -2026-06-18 13:19:20 | INFO | Database connections established -2026-06-18 13:19:20 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:19:21 | INFO | Found 836 MIDs -2026-06-18 13:19:21 | INFO | ================================================================================ -2026-06-18 13:19:21 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:19:21 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- -2026-06-18 13:19:21 | INFO | Fetching data for 836 MIDs -2026-06-18 13:19:26 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:19:26 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:19:26 | INFO | Fetched 1885 rows -2026-06-18 13:19:26 | INFO | Deleting from additional_visibility -2026-06-18 13:19:26 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:19:26 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:19:26 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:19:26 | INFO | ================================================================================ -2026-06-18 13:19:26 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:19:26 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- -2026-06-18 13:19:26 | INFO | Fetching data for 836 MIDs -2026-06-18 13:19:26 | ERROR | Failed processing table Coverage -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 363, in fetch_Coverage - df = pl.read_database( - query=sql, - connection=engine - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") -[SQL: -Executed as -(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' -ELSE 'N' END AS COVERED, -CASE WHEN isnull(C.MID,'')<>'' then 'Y' -ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON - -from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join -(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) -inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B - ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) - AND JP.ReasonId in (0,1,3,9,10,19,20))b -on JP.MID=b.MID - -left join -(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) -inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B - ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) - AND JP.ReasonId in (0,19))c -on JP.MID=c.MID -left join -OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId -Where 1=1 AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146) - -) , - Coverage - (project_id,MID, -store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, -is_covered,is_executed,reason_remarks,storetype_id, -supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, -update_by,Unique_Id) - -AS ( - -Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, - Convert(VARCHAR,Jp.VisitDate) AS VisitDate -, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, - Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, - Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId - And VisitDate = Jp.VisitDate),'') As [In Time] - , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, - Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, - Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage - Where EmpId = Jp.EmpId And StoreId = Jp.StoreId - And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] - , - Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 - Else Case When outtime0 Then 'Y' -Else 'N' End As Covered -, isnull(Exe.Executed,'') as Executed - -,Isnull((Select Top 1 Case -When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + -Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') -Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason -From OneApp_KelloggsMT.dbo.T_StoreCoverage D -Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId -Where D.Isdel = 0 -And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], -sm.StoreTypeId,Em.SupervisorId, -Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' -When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' -When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] -,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 - Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 -Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 - End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId -And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] -,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, - -GETDATE(),'SP-Pius' , - - CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) - + '_' + CAST(EM.EMPID AS VARCHAR) - - - -FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join -Executed Exe on Jp.MID= Exe.MID Inner Join -OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join -OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId -left join -OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) -on jp.mid=pv.MID -Where 1=1 and em.UserName not like 'test%' AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146 ) - -) -select * from Coverage] -(Background on this error at: https://sqlalche.me/e/20/f405) -2026-06-18 13:21:52 | INFO | ================================================================================ -2026-06-18 13:21:52 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:21:52 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:21:52 | INFO | Connecting to databases... -2026-06-18 13:21:53 | INFO | -2026-06-18 13:21:55 | INFO | -2026-06-18 13:21:56 | INFO | Database connections established -2026-06-18 13:21:56 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:21:56 | INFO | Found 836 MIDs -2026-06-18 13:21:57 | INFO | ================================================================================ -2026-06-18 13:21:57 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:21:57 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- -2026-06-18 13:21:57 | INFO | Fetching data for 836 MIDs -2026-06-18 13:21:58 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:21:58 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:21:58 | INFO | Fetched 1885 rows -2026-06-18 13:21:58 | INFO | Creating table additional_visibility -2026-06-18 13:21:58 | INFO | Table ready: additional_visibility -2026-06-18 13:21:58 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:21:59 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:21:59 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:21:59 | INFO | ================================================================================ -2026-06-18 13:21:59 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:21:59 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- -2026-06-18 13:21:59 | INFO | Fetching data for 836 MIDs -2026-06-18 13:21:59 | ERROR | Failed processing table Coverage -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 363, in fetch_Coverage - df = pl.read_database( - query=sql, - connection=engine - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") -[SQL: -Executed as -(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' -ELSE 'N' END AS COVERED, -CASE WHEN isnull(C.MID,'')<>'' then 'Y' -ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON - -from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join -(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) -inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B - ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) - AND JP.ReasonId in (0,1,3,9,10,19,20))b -on JP.MID=b.MID - -left join -(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) -inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B - ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) - AND JP.ReasonId in (0,19))c -on JP.MID=c.MID -left join -OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId -Where 1=1 AND JP.MID in (1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146) - -) , - Coverage - (project_id,MID, -store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, -is_covered,is_executed,reason_remarks,storetype_id, -supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, -update_by,Unique_Id) - -AS ( - -Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, - Convert(VARCHAR,Jp.VisitDate) AS VisitDate -, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, - Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, - Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId - And VisitDate = Jp.VisitDate),'') As [In Time] - , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, - Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, - Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage - Where EmpId = Jp.EmpId And StoreId = Jp.StoreId - And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] - , - Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 - Else Case When outtime0 Then 'Y' -Else 'N' End As Covered -, isnull(Exe.Executed,'') as Executed - -,Isnull((Select Top 1 Case -When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + -Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') -Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason -From OneApp_KelloggsMT.dbo.T_StoreCoverage D -Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId -Where D.Isdel = 0 -And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], -sm.StoreTypeId,Em.SupervisorId, -Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' -When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' -When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] -,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 - Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 -Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 - End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId -And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] -,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, - -GETDATE(),'SP-Pius' , - - CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) - + '_' + CAST(EM.EMPID AS VARCHAR) - - - -FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join -Executed Exe on Jp.MID= Exe.MID Inner Join -OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join -OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId -left join -OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) -on jp.mid=pv.MID -Where 1=1 and em.UserName not like 'test%' AND JP.MID in (1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146 ) - -) -select * from Coverage] -(Background on this error at: https://sqlalche.me/e/20/f405) -2026-06-18 13:26:03 | INFO | ================================================================================ -2026-06-18 13:26:03 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:26:03 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:26:03 | INFO | Connecting to databases... -2026-06-18 13:26:05 | INFO | -2026-06-18 13:26:07 | INFO | -2026-06-18 13:26:07 | INFO | Database connections established -2026-06-18 13:26:07 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:26:08 | INFO | Found 836 MIDs -2026-06-18 13:26:09 | INFO | ================================================================================ -2026-06-18 13:26:09 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:26:09 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- -2026-06-18 13:26:09 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:26:09 | INFO | Fetching data for 836 MIDs -2026-06-18 13:26:12 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:26:12 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:26:12 | INFO | Fetched 1885 rows -2026-06-18 13:26:13 | INFO | Deleting from additional_visibility -2026-06-18 13:26:13 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:26:13 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:26:13 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:26:13 | INFO | ================================================================================ -2026-06-18 13:26:13 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:26:13 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- -2026-06-18 13:26:13 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:26:13 | INFO | Fetching data for 836 MIDs -2026-06-18 13:26:14 | ERROR | Failed processing table Coverage -Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 364, in fetch_Coverage - df = pl.read_database( - query=sql, - connection=engine - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database - return cx.execute( - ~~~~~~~~~~^ - query=query, - ^^^^^^^^^^^^ - options=execute_options, - ^^^^^^^^^^^^^^^^^^^^^^^^ - ).to_polars( - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute - result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute - return meth( - self, - distilled_parameters, - execution_options or NO_OPTIONS, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection - return connection._execute_clauseelement( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - self, distilled_params, execution_options - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement - ret = self._execute_context( - dialect, - ...<8 lines>... - cache_hit=cache_hit, - ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context - return self._exec_single_context( - ~~~~~~~~~~~~~~~~~~~~~~~~~^ - dialect, context, statement, parameters - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context - self._handle_dbapi_exception( - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ - e, str_statement, effective_parameters, cursor, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception - raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context - self.dialect.do_execute( - ~~~~~~~~~~~~~~~~~~~~~~~^ - cursor, str_statement, effective_parameters, context - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute - cursor.execute(statement, parameters) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") -[SQL: -Executed as -(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' -ELSE 'N' END AS COVERED, -CASE WHEN isnull(C.MID,'')<>'' then 'Y' -ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON - -from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join -(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) -inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B - ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) - AND JP.ReasonId in (0,1,3,9,10,19,20))b -on JP.MID=b.MID - -left join -(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) -inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B - ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) - AND JP.ReasonId in (0,19))c -on JP.MID=c.MID -left join -OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId -Where 1=1 AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135) - -) , - Coverage - (project_id,MID, -store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, -is_covered,is_executed,reason_remarks,storetype_id, -supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, -update_by,Unique_Id) - -AS ( - -Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, - Convert(VARCHAR,Jp.VisitDate) AS VisitDate -, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, - Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, - Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId - And VisitDate = Jp.VisitDate),'') As [In Time] - , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, - Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, - Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage - Where EmpId = Jp.EmpId And StoreId = Jp.StoreId - And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] - , - Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 - Else Case When outtime0 Then 'Y' -Else 'N' End As Covered -, isnull(Exe.Executed,'') as Executed - -,Isnull((Select Top 1 Case -When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + -Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') -Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason -From OneApp_KelloggsMT.dbo.T_StoreCoverage D -Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId -Where D.Isdel = 0 -And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], -sm.StoreTypeId,Em.SupervisorId, -Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' -When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' -When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] -,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 - Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 -Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 - End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId -And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] -,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, - -GETDATE(),'SP-Pius' , - - CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) - + '_' + CAST(EM.EMPID AS VARCHAR) - - - -FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join -Executed Exe on Jp.MID= Exe.MID Inner Join -OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join -OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId -left join -OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) -on jp.mid=pv.MID -Where 1=1 and em.UserName not like 'test%' AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135 ) - -) -select * from Coverage] -(Background on this error at: https://sqlalche.me/e/20/f405) -2026-06-18 13:35:43 | INFO | ================================================================================ -2026-06-18 13:35:43 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:35:43 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:35:43 | INFO | Connecting to databases... -2026-06-18 13:35:44 | INFO | -2026-06-18 13:35:46 | INFO | -2026-06-18 13:35:46 | INFO | Database connections established -2026-06-18 13:35:46 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:35:47 | INFO | Found 836 MIDs -2026-06-18 13:35:48 | INFO | ================================================================================ -2026-06-18 13:35:48 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:35:48 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- -2026-06-18 13:35:48 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:35:48 | INFO | Fetching data for 836 MIDs -2026-06-18 13:35:49 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:35:49 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:35:49 | INFO | Fetched 1885 rows -2026-06-18 13:35:49 | INFO | Deleting from additional_visibility -2026-06-18 13:35:49 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:35:49 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:35:49 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:35:49 | INFO | ================================================================================ -2026-06-18 13:35:49 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:35:49 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- -2026-06-18 13:35:49 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:35:49 | INFO | Fetching data for 836 MIDs -2026-06-18 13:36:18 | INFO | Fetched 779 rows from SQL Server -2026-06-18 13:36:18 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! -2026-06-18 13:36:18 | INFO | Fetched 779 rows -2026-06-18 13:36:18 | INFO | Creating table Coverage -2026-06-18 13:36:18 | INFO | Table ready: Coverage -2026-06-18 13:36:19 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:36:19 | INFO | Coverage: inserted 779 rows into ClickHouse -2026-06-18 13:36:19 | INFO | Coverage loaded successfully (779 rows) -2026-06-18 13:36:19 | INFO | ================================================================================ -2026-06-18 13:36:19 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:36:19 | INFO | Fetching Data from sql server for table-:Survey--------------------------- -2026-06-18 13:36:19 | INFO | Fetching data for 836 MIDs -2026-06-18 13:36:20 | INFO | Fetched 141 rows from SQL Server -2026-06-18 13:36:20 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! -2026-06-18 13:36:20 | INFO | Fetched 141 rows -2026-06-18 13:36:20 | INFO | Creating table Survey -2026-06-18 13:36:20 | INFO | Table ready: Survey -2026-06-18 13:36:20 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:36:20 | INFO | Survey: inserted 141 rows into ClickHouse -2026-06-18 13:36:20 | INFO | Survey loaded successfully (141 rows) -2026-06-18 13:36:20 | INFO | ================================================================================ -2026-06-18 13:36:20 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-18 13:36:20 | INFO | Fetching Data from sql server for table-:Login--------------------------- -2026-06-18 13:36:20 | INFO | Fetching data for 836 MIDs -2026-06-18 13:36:22 | INFO | Fetched 475 rows from SQL Server -2026-06-18 13:36:22 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! -2026-06-18 13:36:22 | INFO | Fetched 475 rows -2026-06-18 13:36:22 | INFO | Creating table Login -2026-06-18 13:36:23 | INFO | Table ready: Login -2026-06-18 13:36:23 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:36:23 | INFO | Login: inserted 475 rows into ClickHouse -2026-06-18 13:36:23 | INFO | Login loaded successfully (475 rows) -2026-06-18 13:36:23 | INFO | ================================================================================ -2026-06-18 13:36:23 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:36:23 | INFO | Fetching Data from sql server for table-:Stock_Details--------------------------- -2026-06-18 13:36:23 | INFO | Fetching data for 836 MIDs -2026-06-18 13:36:29 | INFO | Fetched 39,313 rows from SQL Server -2026-06-18 13:36:29 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! -2026-06-18 13:36:29 | INFO | Fetched 39313 rows -2026-06-18 13:36:29 | INFO | Creating table Stock_Details -2026-06-18 13:36:30 | INFO | Table ready: Stock_Details -2026-06-18 13:36:30 | INFO | inserting data into clickhouse db from sql server------------------ -2026-06-18 13:36:30 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse -2026-06-18 13:36:30 | INFO | Stock_Details loaded successfully (39313 rows) -2026-06-18 13:36:30 | INFO | ================================================================================ -2026-06-18 13:36:30 | INFO | Pipeline Completed Successfully -2026-06-18 13:36:30 | INFO | ================================================================================ -2026-06-18 13:37:48 | INFO | ================================================================================ -2026-06-18 13:37:48 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:37:48 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:37:48 | INFO | Connecting to databases... -2026-06-18 13:37:49 | INFO | -2026-06-18 13:37:51 | INFO | -2026-06-18 13:37:51 | INFO | Database connections established -2026-06-18 13:37:51 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:37:52 | INFO | Found 836 MIDs -2026-06-18 13:37:53 | INFO | ================================================================================ -2026-06-18 13:37:53 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:37:53 | INFO | Fetching Data from sql server for table-:additional_visibility .............. -2026-06-18 13:37:53 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:37:53 | INFO | Fetching data for 836 MIDs -2026-06-18 13:37:54 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:37:54 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:37:54 | INFO | Fetched 1885 rows -2026-06-18 13:37:54 | INFO | Deleting from additional_visibility -2026-06-18 13:37:54 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:37:54 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:37:54 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:37:54 | INFO | ================================================================================ -2026-06-18 13:37:54 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:37:54 | INFO | Fetching Data from sql server for table-:Coverage .............. -2026-06-18 13:37:54 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:37:54 | INFO | Fetching data for 836 MIDs -2026-06-18 13:38:17 | INFO | Fetched 779 rows from SQL Server -2026-06-18 13:38:17 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! -2026-06-18 13:38:17 | INFO | Fetched 779 rows -2026-06-18 13:38:17 | INFO | Deleting from Coverage -2026-06-18 13:38:17 | ERROR | Failed processing table Coverage -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 79, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - table_name, - ^^^^^^^^^^^ - f"Mid IN ({mids_str})", - ^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'Mid' while processing: 'isZeroOrNull(Mid IN (1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147)), _block_offset, _block_number, _table, _disk_name, _part_granule_offset, _part_offset, _part_uuid, _part_index, is_executed, _part, camera_allow, update_by, update_date, reason_remarks, is_covered, duration_in_minutes, in_time, Unique_Id, _part_data_version, _part_starting_offset, employee_id, coverage_type, storetype_id, _database, _distance, out_time, supervisor_id, _sample_factor, _partition_id, reasonId, store_id, MID, project_id, distance_in_meters, visit_date', required columns: 'Mid' 'reason_remarks' '_block_number' '_disk_name' 'is_covered' '_part_index' '_part' 'is_executed' '_part_granule_offset' '_part_uuid' '_table' '_block_offset' '_part_offset' 'camera_allow' 'update_by' 'update_date' 'duration_in_minutes' 'Unique_Id' 'in_time' '_part_starting_offset' 'coverage_type' 'employee_id' '_part_data_version' 'storetype_id' '_database' '_distance' 'out_time' 'reasonId' '_partition_id' '_sample_factor' 'supervisor_id' 'store_id' 'MID' 'project_id' 'distance_in_meters' 'visit_date', maybe you meant: 'MID', 'reason_remarks', 'is_covered', 'is_executed', 'camera_allow', 'update_by', 'update_date', 'duration_in_minutes', 'Unique_Id', 'in_time', 'coverage_type', 'employee_id', 'storetype_id', 'out_time', 'reasonId', 'supervisor_id', 'store_id', 'project_id', 'distance_in_meters' or 'visit_date'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-18 13:43:13 | INFO | ================================================================================ -2026-06-18 13:43:13 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:43:13 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:43:13 | INFO | Connecting to databases... -2026-06-18 13:43:14 | INFO | -2026-06-18 13:43:16 | INFO | -2026-06-18 13:43:16 | INFO | Database connections established -2026-06-18 13:43:16 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:43:17 | INFO | Found 836 MIDs -2026-06-18 13:43:18 | INFO | ================================================================================ -2026-06-18 13:43:18 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:43:18 | INFO | Fetching Data from sql server for table-: additional_visibility .............. -2026-06-18 13:43:18 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:43:18 | INFO | Fetching data for 836 MIDs -2026-06-18 13:43:19 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:43:19 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:43:19 | INFO | Fetched 1885 rows -2026-06-18 13:43:19 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ -2026-06-18 13:43:19 | ERROR | Failed processing table additional_visibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 76, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - table_name, - ^^^^^^^^^^^ - f"Mid IN ({mids_str}) or MID IN ({mids_str})", - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 43, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull((Mid IN (1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135)) OR (MID IN (1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135))), _block_offset, _block_number, _table, _database, _distance, _disk_name, _part_granule_offset, _partition_id, _sample_factor, brand_id, _part_uuid, _part_index, _part, created_by, storetype_id, created_date, camera_allowed, image_url, display_id, Remarks, is_present, _part_data_version, _part_starting_offset, visit_date, Mid, channel_id, store_id, chain_id, _part_offset, emp_id, project_id', required columns: 'Mid' 'MID' '_part_granule_offset' '_block_number' '_table' '_block_offset' '_part_uuid' 'brand_id' '_database' '_distance' 'Remarks' 'is_present' '_disk_name' '_partition_id' '_sample_factor' 'display_id' '_part_index' '_part' 'created_by' 'storetype_id' 'created_date' 'camera_allowed' 'image_url' '_part_data_version' '_part_starting_offset' 'visit_date' 'channel_id' 'store_id' 'chain_id' '_part_offset' 'emp_id' 'project_id', maybe you meant: 'Mid', 'brand_id', 'Remarks', 'is_present', 'display_id', 'created_by', 'storetype_id', 'created_date', 'camera_allowed', 'image_url', 'visit_date', 'channel_id', 'store_id', 'chain_id', 'emp_id' or 'project_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-18 13:46:45 | INFO | ================================================================================ -2026-06-18 13:46:45 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:46:45 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:46:45 | INFO | Connecting to databases... -2026-06-18 13:46:45 | INFO | -2026-06-18 13:46:48 | INFO | -2026-06-18 13:46:48 | INFO | Database connections established -2026-06-18 13:46:48 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:46:49 | INFO | Found 836 MIDs -2026-06-18 13:46:49 | INFO | ================================================================================ -2026-06-18 13:46:49 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:46:49 | INFO | Fetching Data from sql server for table-: additional_visibility .............. -2026-06-18 13:46:49 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:46:49 | INFO | Fetching data for 836 MIDs -2026-06-18 13:46:53 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:46:53 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:46:53 | INFO | Fetched 1885 rows -2026-06-18 13:46:53 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ -2026-06-18 13:46:53 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:46:54 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:46:54 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:46:54 | INFO | ================================================================================ -2026-06-18 13:46:54 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:46:54 | INFO | Fetching Data from sql server for table-: Coverage .............. -2026-06-18 13:46:54 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:46:54 | INFO | Fetching data for 836 MIDs -2026-06-18 13:47:11 | INFO | Fetched 779 rows from SQL Server -2026-06-18 13:47:11 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! -2026-06-18 13:47:11 | INFO | Fetched 779 rows -2026-06-18 13:47:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ -2026-06-18 13:47:11 | ERROR | Failed processing table Coverage -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 76, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - table_name, - ^^^^^^^^^^^ - f"Mid IN ({mids_str})", - ^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 43, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'Mid' while processing: 'isZeroOrNull(Mid IN (1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146)), _block_offset, _block_number, _table, _disk_name, _part_granule_offset, _part_offset, _part_uuid, _part_index, is_executed, _part, camera_allow, update_by, update_date, reason_remarks, is_covered, duration_in_minutes, in_time, Unique_Id, _part_data_version, _part_starting_offset, employee_id, coverage_type, storetype_id, _database, _distance, out_time, supervisor_id, _sample_factor, _partition_id, reasonId, store_id, MID, project_id, distance_in_meters, visit_date', required columns: 'Mid' 'reason_remarks' '_block_number' '_disk_name' 'is_covered' '_part_index' '_part' 'is_executed' '_part_granule_offset' '_part_uuid' '_table' '_block_offset' '_part_offset' 'camera_allow' 'update_by' 'update_date' 'duration_in_minutes' 'Unique_Id' 'in_time' '_part_starting_offset' 'coverage_type' 'employee_id' '_part_data_version' 'storetype_id' '_database' '_distance' 'out_time' 'reasonId' '_partition_id' '_sample_factor' 'supervisor_id' 'store_id' 'MID' 'project_id' 'distance_in_meters' 'visit_date', maybe you meant: 'MID', 'reason_remarks', 'is_covered', 'is_executed', 'camera_allow', 'update_by', 'update_date', 'duration_in_minutes', 'Unique_Id', 'in_time', 'coverage_type', 'employee_id', 'storetype_id', 'out_time', 'reasonId', 'supervisor_id', 'store_id', 'project_id', 'distance_in_meters' or 'visit_date'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-18 13:50:13 | INFO | ================================================================================ -2026-06-18 13:50:13 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:50:13 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:50:13 | INFO | Connecting to databases... -2026-06-18 13:50:14 | INFO | -2026-06-18 13:50:15 | INFO | -2026-06-18 13:50:16 | INFO | Database connections established -2026-06-18 13:50:16 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:50:17 | INFO | Found 836 MIDs -2026-06-18 13:50:19 | INFO | ================================================================================ -2026-06-18 13:50:19 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:50:19 | INFO | Fetching Data from sql server for table-: additional_visibility .............. -2026-06-18 13:50:19 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:50:19 | INFO | Fetching data for 836 MIDs -2026-06-18 13:50:22 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:50:22 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:50:22 | INFO | Fetched 1885 rows -2026-06-18 13:50:22 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ -2026-06-18 13:50:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:50:22 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:50:22 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:50:22 | INFO | ================================================================================ -2026-06-18 13:50:22 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:50:22 | INFO | Fetching Data from sql server for table-: Coverage .............. -2026-06-18 13:50:22 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:50:22 | INFO | Fetching data for 836 MIDs -2026-06-18 13:50:49 | INFO | Fetched 779 rows from SQL Server -2026-06-18 13:50:49 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! -2026-06-18 13:50:49 | INFO | Fetched 779 rows -2026-06-18 13:50:50 | INFO | Creating table Coverage -2026-06-18 13:50:50 | INFO | Table ready: Coverage -2026-06-18 13:50:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:50:50 | INFO | Coverage: inserted 779 rows into ClickHouse -2026-06-18 13:50:50 | INFO | Coverage loaded successfully (779 rows) -2026-06-18 13:50:50 | INFO | ================================================================================ -2026-06-18 13:50:50 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:50:50 | INFO | Fetching Data from sql server for table-: Survey .............. -2026-06-18 13:50:50 | INFO | Fetching data for 836 MIDs -2026-06-18 13:50:51 | INFO | Fetched 141 rows from SQL Server -2026-06-18 13:50:51 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! -2026-06-18 13:50:51 | INFO | Fetched 141 rows -2026-06-18 13:50:51 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ -2026-06-18 13:50:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:50:51 | INFO | Survey: inserted 141 rows into ClickHouse -2026-06-18 13:50:51 | INFO | Survey loaded successfully (141 rows) -2026-06-18 13:50:51 | INFO | ================================================================================ -2026-06-18 13:50:51 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-18 13:50:51 | INFO | Fetching Data from sql server for table-: Login .............. -2026-06-18 13:50:51 | INFO | Fetching data for 836 MIDs -2026-06-18 13:50:56 | INFO | Fetched 475 rows from SQL Server -2026-06-18 13:50:56 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! -2026-06-18 13:50:56 | INFO | Fetched 475 rows -2026-06-18 13:50:56 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ -2026-06-18 13:50:56 | INFO | No delete logic required for Login -2026-06-18 13:50:56 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:50:56 | INFO | Login: inserted 475 rows into ClickHouse -2026-06-18 13:50:56 | INFO | Login loaded successfully (475 rows) -2026-06-18 13:50:56 | INFO | ================================================================================ -2026-06-18 13:50:56 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:50:56 | INFO | Fetching Data from sql server for table-: Stock_Details .............. -2026-06-18 13:50:56 | INFO | Fetching data for 836 MIDs -2026-06-18 13:51:15 | INFO | Fetched 39,313 rows from SQL Server -2026-06-18 13:51:15 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! -2026-06-18 13:51:15 | INFO | Fetched 39313 rows -2026-06-18 13:51:15 | INFO | Creating table Stock_Details -2026-06-18 13:51:15 | INFO | Table ready: Stock_Details -2026-06-18 13:51:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:51:16 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse -2026-06-18 13:51:16 | INFO | Stock_Details loaded successfully (39313 rows) -2026-06-18 13:51:16 | INFO | ================================================================================ -2026-06-18 13:51:16 | INFO | Pipeline Completed Successfully -2026-06-18 13:51:16 | INFO | ================================================================================ -2026-06-18 13:51:30 | INFO | ================================================================================ -2026-06-18 13:51:30 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:51:30 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:51:30 | INFO | Connecting to databases... -2026-06-18 13:51:31 | INFO | -2026-06-18 13:51:33 | INFO | -2026-06-18 13:51:33 | INFO | Database connections established -2026-06-18 13:51:33 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:51:34 | INFO | Found 836 MIDs -2026-06-18 13:51:35 | INFO | ================================================================================ -2026-06-18 13:51:35 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:51:35 | INFO | Fetching Data from sql server for table-: additional_visibility .............. -2026-06-18 13:51:35 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:51:35 | INFO | Fetching data for 836 MIDs -2026-06-18 13:51:36 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:51:36 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:51:36 | INFO | Fetched 1885 rows -2026-06-18 13:51:36 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ -2026-06-18 13:51:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:51:36 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:51:36 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:51:36 | INFO | ================================================================================ -2026-06-18 13:51:36 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:51:36 | INFO | Fetching Data from sql server for table-: Coverage .............. -2026-06-18 13:51:36 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:51:36 | INFO | Fetching data for 836 MIDs -2026-06-18 13:51:57 | INFO | Fetched 779 rows from SQL Server -2026-06-18 13:51:57 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! -2026-06-18 13:51:57 | INFO | Fetched 779 rows -2026-06-18 13:51:57 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ -2026-06-18 13:51:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:51:57 | INFO | Coverage: inserted 779 rows into ClickHouse -2026-06-18 13:51:57 | INFO | Coverage loaded successfully (779 rows) -2026-06-18 13:51:57 | INFO | ================================================================================ -2026-06-18 13:51:57 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:51:57 | INFO | Fetching Data from sql server for table-: Survey .............. -2026-06-18 13:51:57 | INFO | Fetching data for 836 MIDs -2026-06-18 13:51:58 | INFO | Fetched 141 rows from SQL Server -2026-06-18 13:51:58 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! -2026-06-18 13:51:58 | INFO | Fetched 141 rows -2026-06-18 13:51:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ -2026-06-18 13:51:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:51:58 | INFO | Survey: inserted 141 rows into ClickHouse -2026-06-18 13:51:58 | INFO | Survey loaded successfully (141 rows) -2026-06-18 13:51:58 | INFO | ================================================================================ -2026-06-18 13:51:58 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-18 13:51:58 | INFO | Fetching Data from sql server for table-: Login .............. -2026-06-18 13:51:58 | INFO | Fetching data for 836 MIDs -2026-06-18 13:52:00 | INFO | Fetched 475 rows from SQL Server -2026-06-18 13:52:00 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! -2026-06-18 13:52:00 | INFO | Fetched 475 rows -2026-06-18 13:52:00 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ -2026-06-18 13:52:00 | INFO | No delete logic required for Login -2026-06-18 13:52:00 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:52:00 | INFO | Login: inserted 475 rows into ClickHouse -2026-06-18 13:52:00 | INFO | Login loaded successfully (475 rows) -2026-06-18 13:52:00 | INFO | ================================================================================ -2026-06-18 13:52:00 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:52:00 | INFO | Fetching Data from sql server for table-: Stock_Details .............. -2026-06-18 13:52:00 | INFO | Fetching data for 836 MIDs -2026-06-18 13:52:05 | INFO | Fetched 39,313 rows from SQL Server -2026-06-18 13:52:05 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! -2026-06-18 13:52:05 | INFO | Fetched 39313 rows -2026-06-18 13:52:05 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ -2026-06-18 13:52:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:52:05 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse -2026-06-18 13:52:05 | INFO | Stock_Details loaded successfully (39313 rows) -2026-06-18 13:52:05 | INFO | ================================================================================ -2026-06-18 13:52:05 | INFO | Pipeline Completed Successfully -2026-06-18 13:52:05 | INFO | ================================================================================ -2026-06-18 13:52:12 | INFO | ================================================================================ -2026-06-18 13:52:12 | INFO | Hello from data-move Python data pipeline! -2026-06-18 13:52:12 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 13:52:12 | INFO | Connecting to databases... -2026-06-18 13:52:13 | INFO | -2026-06-18 13:52:15 | INFO | -2026-06-18 13:52:16 | INFO | Database connections established -2026-06-18 13:52:16 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 13:52:16 | INFO | Found 836 MIDs -2026-06-18 13:52:17 | INFO | ================================================================================ -2026-06-18 13:52:17 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:52:17 | INFO | Fetching Data from sql server for table-: additional_visibility .............. -2026-06-18 13:52:17 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:52:17 | INFO | Fetching data for 836 MIDs -2026-06-18 13:52:17 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 13:52:17 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 13:52:17 | INFO | Fetched 1885 rows -2026-06-18 13:52:17 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ -2026-06-18 13:52:17 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:52:17 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 13:52:17 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 13:52:17 | INFO | ================================================================================ -2026-06-18 13:52:17 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:52:17 | INFO | Fetching Data from sql server for table-: Coverage .............. -2026-06-18 13:52:17 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 13:52:17 | INFO | Fetching data for 836 MIDs -2026-06-18 13:52:29 | INFO | Fetched 779 rows from SQL Server -2026-06-18 13:52:29 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! -2026-06-18 13:52:29 | INFO | Fetched 779 rows -2026-06-18 13:52:29 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ -2026-06-18 13:52:30 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:52:30 | INFO | Coverage: inserted 779 rows into ClickHouse -2026-06-18 13:52:30 | INFO | Coverage loaded successfully (779 rows) -2026-06-18 13:52:30 | INFO | ================================================================================ -2026-06-18 13:52:30 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:52:30 | INFO | Fetching Data from sql server for table-: Survey .............. -2026-06-18 13:52:30 | INFO | Fetching data for 836 MIDs -2026-06-18 13:52:31 | INFO | Fetched 141 rows from SQL Server -2026-06-18 13:52:31 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! -2026-06-18 13:52:31 | INFO | Fetched 141 rows -2026-06-18 13:52:31 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ -2026-06-18 13:52:31 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:52:31 | INFO | Survey: inserted 141 rows into ClickHouse -2026-06-18 13:52:31 | INFO | Survey loaded successfully (141 rows) -2026-06-18 13:52:31 | INFO | ================================================================================ -2026-06-18 13:52:31 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-18 13:52:31 | INFO | Fetching Data from sql server for table-: Login .............. -2026-06-18 13:52:31 | INFO | Fetching data for 836 MIDs -2026-06-18 13:52:33 | INFO | Fetched 475 rows from SQL Server -2026-06-18 13:52:33 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! -2026-06-18 13:52:33 | INFO | Fetched 475 rows -2026-06-18 13:52:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ -2026-06-18 13:52:33 | INFO | No delete logic required for Login -2026-06-18 13:52:33 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:52:33 | INFO | Login: inserted 475 rows into ClickHouse -2026-06-18 13:52:33 | INFO | Login loaded successfully (475 rows) -2026-06-18 13:52:33 | INFO | ================================================================================ -2026-06-18 13:52:33 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 13:52:33 | INFO | Fetching Data from sql server for table-: Stock_Details .............. -2026-06-18 13:52:33 | INFO | Fetching data for 836 MIDs -2026-06-18 13:52:43 | INFO | Fetched 39,313 rows from SQL Server -2026-06-18 13:52:43 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! -2026-06-18 13:52:43 | INFO | Fetched 39313 rows -2026-06-18 13:52:43 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ -2026-06-18 13:52:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 13:52:45 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse -2026-06-18 13:52:45 | INFO | Stock_Details loaded successfully (39313 rows) -2026-06-18 13:52:45 | INFO | ================================================================================ -2026-06-18 13:52:45 | INFO | Pipeline Completed Successfully -2026-06-18 13:52:45 | INFO | ================================================================================ -2026-06-18 15:49:07 | INFO | ================================================================================ -2026-06-18 15:49:07 | INFO | Hello from data-move Python data pipeline! -2026-06-18 15:49:07 | INFO | Pipeline Run Date: 2026-06-17 -2026-06-18 15:49:07 | INFO | Connecting to databases... -2026-06-18 15:49:12 | INFO | -2026-06-18 15:49:13 | INFO | -2026-06-18 15:49:13 | INFO | Database connections established -2026-06-18 15:49:13 | INFO | Collecting MIDs for: 2026-06-17 -2026-06-18 15:49:13 | INFO | Found 836 MIDs -2026-06-18 15:49:14 | INFO | ================================================================================ -2026-06-18 15:49:14 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 15:49:14 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. -2026-06-18 15:49:14 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 15:49:14 | INFO | Fetching data for 836 MIDs -2026-06-18 15:49:19 | INFO | Fetched 3,407 rows from SQL Server -2026-06-18 15:49:19 | INFO | Fetched total row -: 3407 from sql server for table-:SOS_OneApp ...........!!! -2026-06-18 15:49:19 | INFO | Fetched 3407 rows -2026-06-18 15:49:19 | INFO | Creating table SOS_OneApp -2026-06-18 15:49:19 | INFO | Table ready: SOS_OneApp -2026-06-18 15:49:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 15:49:19 | INFO | SOS_OneApp: inserted 3,407 rows into ClickHouse -2026-06-18 15:49:19 | INFO | SOS_OneApp loaded successfully (3407 rows) -2026-06-18 15:49:19 | INFO | ================================================================================ -2026-06-18 15:49:19 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 15:49:19 | INFO | Fetching Data from sql server for table-: additional_visibility .............. -2026-06-18 15:49:19 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 15:49:19 | INFO | Fetching data for 836 MIDs -2026-06-18 15:49:20 | INFO | Fetched 1,885 rows from SQL Server -2026-06-18 15:49:20 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! -2026-06-18 15:49:20 | INFO | Fetched 1885 rows -2026-06-18 15:49:20 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ -2026-06-18 15:49:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 15:49:21 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse -2026-06-18 15:49:21 | INFO | additional_visibility loaded successfully (1885 rows) -2026-06-18 15:49:21 | INFO | ================================================================================ -2026-06-18 15:49:21 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 15:49:21 | INFO | Fetching Data from sql server for table-: Coverage .............. -2026-06-18 15:49:21 | INFO | Start Fetching data for these 836 MIDs -2026-06-18 15:49:21 | INFO | Fetching data for 836 MIDs -2026-06-18 15:49:33 | INFO | Fetched 779 rows from SQL Server -2026-06-18 15:49:33 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! -2026-06-18 15:49:33 | INFO | Fetched 779 rows -2026-06-18 15:49:34 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ -2026-06-18 15:49:34 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 15:49:35 | INFO | Coverage: inserted 779 rows into ClickHouse -2026-06-18 15:49:35 | INFO | Coverage loaded successfully (779 rows) -2026-06-18 15:49:35 | INFO | ================================================================================ -2026-06-18 15:49:35 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 15:49:35 | INFO | Fetching Data from sql server for table-: Survey .............. -2026-06-18 15:49:35 | INFO | Fetching data for 836 MIDs -2026-06-18 15:49:35 | INFO | Fetched 141 rows from SQL Server -2026-06-18 15:49:35 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! -2026-06-18 15:49:35 | INFO | Fetched 141 rows -2026-06-18 15:49:36 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ -2026-06-18 15:49:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 15:49:37 | INFO | Survey: inserted 141 rows into ClickHouse -2026-06-18 15:49:37 | INFO | Survey loaded successfully (141 rows) -2026-06-18 15:49:37 | INFO | ================================================================================ -2026-06-18 15:49:37 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-18 15:49:37 | INFO | Fetching Data from sql server for table-: Login .............. -2026-06-18 15:49:37 | INFO | Fetching data for 836 MIDs -2026-06-18 15:49:39 | INFO | Fetched 475 rows from SQL Server -2026-06-18 15:49:39 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! -2026-06-18 15:49:39 | INFO | Fetched 475 rows -2026-06-18 15:49:39 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ -2026-06-18 15:49:39 | INFO | No delete logic required for Login -2026-06-18 15:49:39 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 15:49:39 | INFO | Login: inserted 475 rows into ClickHouse -2026-06-18 15:49:39 | INFO | Login loaded successfully (475 rows) -2026-06-18 15:49:39 | INFO | ================================================================================ -2026-06-18 15:49:39 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-18 15:49:39 | INFO | Fetching Data from sql server for table-: Stock_Details .............. -2026-06-18 15:49:39 | INFO | Fetching data for 836 MIDs -2026-06-18 15:50:23 | INFO | Fetched 39,313 rows from SQL Server -2026-06-18 15:50:23 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! -2026-06-18 15:50:23 | INFO | Fetched 39313 rows -2026-06-18 15:50:23 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ -2026-06-18 15:50:24 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-18 15:50:24 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse -2026-06-18 15:50:24 | INFO | Stock_Details loaded successfully (39313 rows) -2026-06-18 15:50:24 | INFO | ================================================================================ -2026-06-18 15:50:24 | INFO | Pipeline Completed Successfully -2026-06-18 15:50:24 | INFO | ================================================================================ diff --git a/logs/etl_20260619.log b/logs/etl_20260619.log deleted file mode 100644 index 3fee27f..0000000 --- a/logs/etl_20260619.log +++ /dev/null @@ -1,1451 +0,0 @@ -2026-06-19 10:14:16 | INFO | ================================================================================ -2026-06-19 10:14:16 | INFO | Hello from data-move Python data pipeline! -2026-06-19 10:14:16 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 10:14:16 | INFO | Connecting to databases... -2026-06-19 10:14:18 | INFO | -2026-06-19 10:14:22 | INFO | -2026-06-19 10:14:23 | INFO | Database connections established -2026-06-19 10:14:23 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 10:14:24 | INFO | Found 859 MIDs -2026-06-19 10:14:25 | INFO | ================================================================================ -2026-06-19 10:14:25 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT -2026-06-19 10:14:25 | INFO | Fetching Data from sql server for table-: Attendance .............. -2026-06-19 10:14:25 | ERROR | Failed processing table Attendance -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 550, in fetch_Attendance - sql = sql_template.format( - star_date=start_date.strftime("%Y-%m-%d"), - run_date=run_date.strftime("%Y-%m-%d") - ) -KeyError: 'start_date' -2026-06-19 10:14:55 | INFO | ================================================================================ -2026-06-19 10:14:55 | INFO | Hello from data-move Python data pipeline! -2026-06-19 10:14:55 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 10:14:55 | INFO | Connecting to databases... -2026-06-19 10:14:56 | INFO | -2026-06-19 10:14:57 | INFO | -2026-06-19 10:14:58 | INFO | Database connections established -2026-06-19 10:14:58 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 10:14:58 | INFO | Found 859 MIDs -2026-06-19 10:14:58 | INFO | ================================================================================ -2026-06-19 10:14:58 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT -2026-06-19 10:14:58 | INFO | Fetching Data from sql server for table-: Attendance .............. -2026-06-19 10:14:58 | ERROR | Failed processing table Attendance -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\fact.py", line 550, in fetch_Attendance - sql = sql_template.format( - star_date=start_date.strftime("%Y-%m-%d"), - run_date=run_date.strftime("%Y-%m-%d") - ) -KeyError: 'start_date' -2026-06-19 10:15:38 | INFO | ================================================================================ -2026-06-19 10:15:38 | INFO | Hello from data-move Python data pipeline! -2026-06-19 10:15:38 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 10:15:38 | INFO | Connecting to databases... -2026-06-19 10:15:39 | INFO | -2026-06-19 10:15:41 | INFO | -2026-06-19 10:15:41 | INFO | Database connections established -2026-06-19 10:15:41 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 10:15:42 | INFO | Found 859 MIDs -2026-06-19 10:15:42 | INFO | ================================================================================ -2026-06-19 10:15:42 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT -2026-06-19 10:15:42 | INFO | Fetching Data from sql server for table-: Attendance .............. -2026-06-19 10:15:42 | INFO | Fetching Attendance data from 2026-06-03 to 2026-06-18 -2026-06-19 10:15:51 | INFO | Fetched 121,124 attendance rows for 540 employees -2026-06-19 10:15:51 | INFO | Fetched total row -: 121124 from sql server for table-:Attendance ...........!!! -2026-06-19 10:15:51 | INFO | Fetched 121124 rows -2026-06-19 10:15:51 | INFO | Creating table Attendance -2026-06-19 10:15:51 | INFO | Table ready: Attendance -2026-06-19 10:15:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 10:15:53 | INFO | Attendance: inserted 121,124 rows into ClickHouse -2026-06-19 10:15:53 | INFO | Attendance loaded successfully (121124 rows) -2026-06-19 10:15:53 | INFO | ================================================================================ -2026-06-19 10:15:53 | INFO | Pipeline Completed Successfully -2026-06-19 10:15:53 | INFO | ================================================================================ -2026-06-19 10:15:59 | INFO | ================================================================================ -2026-06-19 10:15:59 | INFO | Hello from data-move Python data pipeline! -2026-06-19 10:15:59 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 10:15:59 | INFO | Connecting to databases... -2026-06-19 10:16:00 | INFO | -2026-06-19 10:16:01 | INFO | -2026-06-19 10:16:02 | INFO | Database connections established -2026-06-19 10:16:02 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 10:16:02 | INFO | Found 859 MIDs -2026-06-19 10:16:02 | INFO | ================================================================================ -2026-06-19 10:16:02 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT -2026-06-19 10:16:02 | INFO | Fetching Data from sql server for table-: Attendance .............. -2026-06-19 10:16:02 | INFO | Fetching Attendance data from 2026-06-03 to 2026-06-18 -2026-06-19 10:16:17 | INFO | Fetched 121,124 attendance rows for 540 employees -2026-06-19 10:16:17 | INFO | Fetched total row -: 121124 from sql server for table-:Attendance ...........!!! -2026-06-19 10:16:17 | INFO | Fetched 121124 rows -2026-06-19 10:16:18 | INFO | Truncated table Attendance -2026-06-19 10:16:18 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 10:16:19 | INFO | Attendance: inserted 121,124 rows into ClickHouse -2026-06-19 10:16:19 | INFO | Attendance loaded successfully (121124 rows) -2026-06-19 10:16:19 | INFO | ================================================================================ -2026-06-19 10:16:19 | INFO | Pipeline Completed Successfully -2026-06-19 10:16:19 | INFO | ================================================================================ -2026-06-19 11:17:09 | INFO | ================================================================================ -2026-06-19 11:17:09 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:17:09 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:17:09 | INFO | Connecting to databases... -2026-06-19 11:17:11 | INFO | -2026-06-19 11:17:12 | INFO | -2026-06-19 11:17:13 | INFO | Database connections established -2026-06-19 11:17:13 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:17:14 | INFO | Found 859 MIDs -2026-06-19 11:17:14 | INFO | ================================================================================ -2026-06-19 11:17:14 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:17:14 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:17:14 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:17:14 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:17:14 | INFO | Fetching data for 859 MIDs -2026-06-19 11:17:16 | INFO | Fetched 21,439 rows from SQL Server -2026-06-19 11:17:16 | INFO | Fetched total row -: 21439 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:17:16 | INFO | Fetched 21439 rows -2026-06-19 11:17:16 | INFO | Creating table Journey_Plan -2026-06-19 11:17:16 | INFO | Table ready: Journey_Plan -2026-06-19 11:17:16 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 11:17:17 | INFO | Journey_Plan: inserted 21,439 rows into ClickHouse -2026-06-19 11:17:17 | INFO | Journey_Plan loaded successfully (21439 rows) -2026-06-19 11:17:17 | INFO | ================================================================================ -2026-06-19 11:17:17 | INFO | Pipeline Completed Successfully -2026-06-19 11:17:17 | INFO | ================================================================================ -2026-06-19 11:17:27 | INFO | ================================================================================ -2026-06-19 11:17:27 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:17:27 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:17:27 | INFO | Connecting to databases... -2026-06-19 11:17:28 | INFO | -2026-06-19 11:17:30 | INFO | -2026-06-19 11:17:30 | INFO | Database connections established -2026-06-19 11:17:30 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:17:31 | INFO | Found 859 MIDs -2026-06-19 11:17:31 | INFO | ================================================================================ -2026-06-19 11:17:31 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:17:31 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:17:31 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:17:31 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:17:31 | INFO | Fetching data for 859 MIDs -2026-06-19 11:17:33 | INFO | Fetched 21,439 rows from SQL Server -2026-06-19 11:17:33 | INFO | Fetched total row -: 21439 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:17:33 | INFO | Fetched 21439 rows -2026-06-19 11:17:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ -2026-06-19 11:17:33 | ERROR | Failed processing table Journey_Plan -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<4 lines>... - """, - ^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((toMonth(visit_date) = 6) AND (toYear(visit_date) = 2026)). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) -2026-06-19 11:21:40 | INFO | ================================================================================ -2026-06-19 11:21:40 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:21:40 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:21:40 | INFO | Connecting to databases... -2026-06-19 11:21:41 | INFO | -2026-06-19 11:21:43 | INFO | -2026-06-19 11:21:43 | INFO | Database connections established -2026-06-19 11:21:43 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:21:44 | INFO | Found 859 MIDs -2026-06-19 11:21:44 | INFO | ================================================================================ -2026-06-19 11:21:44 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:21:44 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:21:44 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:21:44 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:21:44 | INFO | Fetching data for 859 MIDs -2026-06-19 11:21:46 | INFO | Fetched 21,440 rows from SQL Server -2026-06-19 11:21:46 | INFO | Fetched total row -: 21440 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:21:46 | INFO | Fetched 21440 rows -2026-06-19 11:21:46 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ -2026-06-19 11:21:46 | ERROR | Failed processing table Journey_Plan -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<5 lines>... - """, - ^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'Visit_Date' while processing: 'isZeroOrNull((project_id = 40148) AND (toMonth(Visit_Date) = toMonth(toDate('2026-06-18'))) AND (toYear(Visit_Date) = toYear(toDate('2026-06-18')))), _block_offset, _part_granule_offset, _disk_name, _sample_factor, _partition_id, _part_index, _part, UpdateDate, _block_number, CreateDate, CreateBy, _part_offset, _part_starting_offset, employee_id, _part_data_version, process_id, _database, _distance, UpdateBy, _table, _part_uuid, visit_date, store_id, project_id', required columns: 'project_id' 'Visit_Date' '_block_offset' 'UpdateBy' '_table' '_part_uuid' '_part_granule_offset' '_part_index' '_part' '_sample_factor' '_partition_id' '_disk_name' 'UpdateDate' '_block_number' 'CreateDate' 'CreateBy' '_part_data_version' '_part_starting_offset' 'employee_id' '_database' '_distance' '_part_offset' 'visit_date' 'process_id' 'store_id', maybe you meant: 'project_id', 'visit_date', 'UpdateBy', 'UpdateDate', 'CreateDate', 'CreateBy', 'employee_id', 'process_id' or 'store_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) -2026-06-19 11:22:52 | INFO | ================================================================================ -2026-06-19 11:22:52 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:22:52 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:22:52 | INFO | Connecting to databases... -2026-06-19 11:22:53 | INFO | -2026-06-19 11:22:55 | INFO | -2026-06-19 11:22:56 | INFO | Database connections established -2026-06-19 11:22:56 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:22:56 | INFO | Found 859 MIDs -2026-06-19 11:22:57 | INFO | ================================================================================ -2026-06-19 11:22:57 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:22:57 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:22:57 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:22:57 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:22:57 | INFO | Fetching data for 859 MIDs -2026-06-19 11:22:58 | INFO | Fetched 21,441 rows from SQL Server -2026-06-19 11:22:58 | INFO | Fetched total row -: 21441 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:22:58 | INFO | Fetched 21441 rows -2026-06-19 11:22:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ -2026-06-19 11:22:58 | ERROR | Failed processing table Journey_Plan -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<5 lines>... - """, - ^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((project_id = 40148) AND (toMonth(visit_date) = toMonth(toDate('2026-06-18'))) AND (toYear(visit_date) = toYear(toDate('2026-06-18')))). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) -2026-06-19 11:25:44 | INFO | ================================================================================ -2026-06-19 11:25:44 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:25:44 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:25:44 | INFO | Connecting to databases... -2026-06-19 11:25:46 | INFO | -2026-06-19 11:25:47 | INFO | -2026-06-19 11:25:48 | INFO | Database connections established -2026-06-19 11:25:48 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:25:48 | INFO | Found 859 MIDs -2026-06-19 11:25:49 | INFO | ================================================================================ -2026-06-19 11:25:49 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:25:49 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:25:49 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:25:49 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:25:49 | INFO | Fetching data for 859 MIDs -2026-06-19 11:25:50 | INFO | Fetched 21,441 rows from SQL Server -2026-06-19 11:25:50 | INFO | Fetched total row -: 21441 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:25:50 | INFO | Fetched 21441 rows -2026-06-19 11:25:50 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ -2026-06-19 11:25:51 | ERROR | Failed processing table Journey_Plan -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<5 lines>... - """, - ^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((project_id = 40148) AND (toMonth(visit_date) = toMonth(toDate('2026-06-18'))) AND (toYear(visit_date) = toYear(toDate('2026-06-18')))). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) -2026-06-19 11:27:56 | INFO | ================================================================================ -2026-06-19 11:27:56 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:27:56 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:27:56 | INFO | Connecting to databases... -2026-06-19 11:27:57 | INFO | -2026-06-19 11:27:59 | INFO | -2026-06-19 11:28:00 | INFO | Database connections established -2026-06-19 11:28:00 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:28:00 | INFO | Found 859 MIDs -2026-06-19 11:28:00 | INFO | ================================================================================ -2026-06-19 11:28:00 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:28:00 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:28:00 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:28:00 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:28:00 | INFO | Fetching data for 859 MIDs -2026-06-19 11:28:02 | INFO | Fetched 21,441 rows from SQL Server -2026-06-19 11:28:02 | INFO | Fetched total row -: 21441 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:28:02 | INFO | Fetched 21441 rows -2026-06-19 11:28:02 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ -2026-06-19 11:28:02 | ERROR | Failed processing table Journey_Plan -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<5 lines>... - """, - ^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((project_id = 40148) AND (toMonth(visit_date) = toMonth(toDate('2026-06-18'))) AND (toYear(visit_date) = toYear(toDate('2026-06-18')))). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) -2026-06-19 11:28:09 | INFO | ================================================================================ -2026-06-19 11:28:09 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:28:09 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:28:09 | INFO | Connecting to databases... -2026-06-19 11:28:10 | INFO | -2026-06-19 11:28:11 | INFO | -2026-06-19 11:28:13 | INFO | Database connections established -2026-06-19 11:28:13 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:28:13 | INFO | Found 859 MIDs -2026-06-19 11:28:13 | INFO | ================================================================================ -2026-06-19 11:28:13 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:28:13 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:28:13 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:28:13 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:28:13 | INFO | Fetching data for 859 MIDs -2026-06-19 11:28:14 | INFO | Fetched 21,441 rows from SQL Server -2026-06-19 11:28:14 | INFO | Fetched total row -: 21441 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:28:14 | INFO | Fetched 21441 rows -2026-06-19 11:28:15 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ -2026-06-19 11:28:15 | ERROR | Failed processing table Journey_Plan -Traceback (most recent call last): - File "D:\data_move\main2.py", line 243, in main - delete_existing_data( - ~~~~~~~~~~~~~~~~~~~~^ - client=client, - ^^^^^^^^^^^^^^ - ...<3 lines>... - emp_visit_df=emp_visit_df, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data - delete_rows( - ~~~~~~~~~~~^ - client, - ^^^^^^^ - ...<5 lines>... - """, - ^^^^ - ) - ^ - File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows - client.command(query) - ~~~~~~~~~~~~~~^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command - response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((project_id = 40148) AND (toMonth(visit_date) = toMonth(toDate('2026-06-18'))) AND (toYear(visit_date) = toYear(toDate('2026-06-18')))). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) -2026-06-19 11:33:41 | INFO | ================================================================================ -2026-06-19 11:33:41 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:33:41 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:33:41 | INFO | Connecting to databases... -2026-06-19 11:33:43 | INFO | -2026-06-19 11:33:44 | INFO | -2026-06-19 11:33:44 | INFO | Database connections established -2026-06-19 11:33:44 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:33:45 | INFO | Found 859 MIDs -2026-06-19 11:33:46 | INFO | ================================================================================ -2026-06-19 11:33:46 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:33:46 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:33:46 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:33:46 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:33:46 | INFO | Fetching data for 859 MIDs -2026-06-19 11:33:48 | INFO | Fetched 21,442 rows from SQL Server -2026-06-19 11:33:48 | INFO | Fetched total row -: 21442 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:33:48 | INFO | Fetched 21442 rows -2026-06-19 11:33:48 | INFO | Creating table Journey_Plan -2026-06-19 11:33:48 | INFO | Table ready: Journey_Plan -2026-06-19 11:33:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 11:33:49 | INFO | Journey_Plan: inserted 21,442 rows into ClickHouse -2026-06-19 11:33:49 | INFO | Journey_Plan loaded successfully (21442 rows) -2026-06-19 11:33:49 | INFO | ================================================================================ -2026-06-19 11:33:49 | INFO | Pipeline Completed Successfully -2026-06-19 11:33:49 | INFO | ================================================================================ -2026-06-19 11:33:53 | INFO | ================================================================================ -2026-06-19 11:33:53 | INFO | Hello from data-move Python data pipeline! -2026-06-19 11:33:53 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 11:33:53 | INFO | Connecting to databases... -2026-06-19 11:33:55 | INFO | -2026-06-19 11:33:56 | INFO | -2026-06-19 11:33:56 | INFO | Database connections established -2026-06-19 11:33:56 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 11:33:57 | INFO | Found 859 MIDs -2026-06-19 11:33:58 | INFO | ================================================================================ -2026-06-19 11:33:58 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 11:33:58 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. -2026-06-19 11:33:58 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 11:33:58 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 11:33:58 | INFO | Fetching data for 859 MIDs -2026-06-19 11:33:59 | INFO | Fetched 21,442 rows from SQL Server -2026-06-19 11:33:59 | INFO | Fetched total row -: 21442 from sql server for table-:Journey_Plan ...........!!! -2026-06-19 11:33:59 | INFO | Fetched 21442 rows -2026-06-19 11:33:59 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ -2026-06-19 11:33:59 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 11:34:00 | INFO | Journey_Plan: inserted 21,442 rows into ClickHouse -2026-06-19 11:34:00 | INFO | Journey_Plan loaded successfully (21442 rows) -2026-06-19 11:34:00 | INFO | ================================================================================ -2026-06-19 11:34:00 | INFO | Pipeline Completed Successfully -2026-06-19 11:34:00 | INFO | ================================================================================ -2026-06-19 12:02:14 | INFO | ================================================================================ -2026-06-19 12:02:14 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:02:14 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:02:14 | INFO | Connecting to databases... -2026-06-19 12:02:15 | INFO | -2026-06-19 12:02:17 | INFO | -2026-06-19 12:02:17 | INFO | Database connections established -2026-06-19 12:02:17 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:02:18 | INFO | Found 859 MIDs -2026-06-19 12:02:19 | INFO | ================================================================================ -2026-06-19 12:02:19 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-19 12:02:19 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. -2026-06-19 12:02:19 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 12:02:19 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 12:02:19 | INFO | Fetching data for 859 MIDs -2026-06-19 12:02:23 | INFO | Fetched 937 rows from SQL Server -2026-06-19 12:02:23 | INFO | Fetched total row -: 937 from sql server for table-:PaidVisibility ...........!!! -2026-06-19 12:02:23 | INFO | Fetched 937 rows -2026-06-19 12:02:23 | INFO | Creating table PaidVisibility -2026-06-19 12:02:23 | INFO | Table ready: PaidVisibility -2026-06-19 12:02:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 12:02:24 | INFO | PaidVisibility: inserted 937 rows into ClickHouse -2026-06-19 12:02:24 | INFO | PaidVisibility loaded successfully (937 rows) -2026-06-19 12:02:24 | INFO | ================================================================================ -2026-06-19 12:02:24 | INFO | Pipeline Completed Successfully -2026-06-19 12:02:24 | INFO | ================================================================================ -2026-06-19 12:03:50 | INFO | ================================================================================ -2026-06-19 12:03:50 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:03:50 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:03:50 | INFO | Connecting to databases... -2026-06-19 12:03:58 | INFO | -2026-06-19 12:04:00 | INFO | -2026-06-19 12:04:01 | INFO | Database connections established -2026-06-19 12:04:01 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:04:01 | INFO | Found 859 MIDs -2026-06-19 12:04:02 | INFO | ================================================================================ -2026-06-19 12:04:02 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-19 12:04:02 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. -2026-06-19 12:04:02 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 12:04:02 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 12:04:02 | INFO | Fetching data for 859 MIDs -2026-06-19 12:04:05 | INFO | Fetched 937 rows from SQL Server -2026-06-19 12:04:05 | INFO | Fetched total row -: 937 from sql server for table-:PaidVisibility ...........!!! -2026-06-19 12:04:05 | INFO | Fetched 937 rows -2026-06-19 12:04:05 | INFO | Creating table PaidVisibility -2026-06-19 12:04:05 | INFO | Table ready: PaidVisibility -2026-06-19 12:04:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 12:04:05 | INFO | PaidVisibility: inserted 937 rows into ClickHouse -2026-06-19 12:04:05 | INFO | PaidVisibility loaded successfully (937 rows) -2026-06-19 12:04:05 | INFO | ================================================================================ -2026-06-19 12:04:05 | INFO | Pipeline Completed Successfully -2026-06-19 12:04:05 | INFO | ================================================================================ -2026-06-19 12:04:52 | INFO | ================================================================================ -2026-06-19 12:04:52 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:04:52 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:04:52 | INFO | Connecting to databases... -2026-06-19 12:04:53 | INFO | -2026-06-19 12:04:54 | INFO | -2026-06-19 12:04:55 | INFO | Database connections established -2026-06-19 12:04:55 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:04:56 | INFO | Found 859 MIDs -2026-06-19 12:04:57 | INFO | ================================================================================ -2026-06-19 12:04:57 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT -2026-06-19 12:04:57 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. -2026-06-19 12:04:57 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 12:04:57 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 12:04:57 | INFO | Fetching data for 859 MIDs -2026-06-19 12:04:59 | INFO | Fetched 937 rows from SQL Server -2026-06-19 12:04:59 | INFO | Fetched total row -: 937 from sql server for table-:PaidVisibility ...........!!! -2026-06-19 12:04:59 | INFO | Fetched 937 rows -2026-06-19 12:04:59 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ -2026-06-19 12:04:59 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 12:04:59 | INFO | PaidVisibility: inserted 937 rows into ClickHouse -2026-06-19 12:04:59 | INFO | PaidVisibility loaded successfully (937 rows) -2026-06-19 12:04:59 | INFO | ================================================================================ -2026-06-19 12:04:59 | INFO | Pipeline Completed Successfully -2026-06-19 12:04:59 | INFO | ================================================================================ -2026-06-19 12:21:12 | INFO | ================================================================================ -2026-06-19 12:21:12 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:21:12 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:21:12 | INFO | Connecting to databases... -2026-06-19 12:21:14 | INFO | -2026-06-19 12:21:15 | INFO | -2026-06-19 12:21:16 | INFO | Database connections established -2026-06-19 12:21:16 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:21:16 | INFO | Found 859 MIDs -2026-06-19 12:21:17 | INFO | ================================================================================ -2026-06-19 12:21:17 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 12:21:17 | INFO | Fetching Data from sql server for table-: Web_Logins .............. -2026-06-19 12:21:17 | ERROR | Failed processing table Web_Logins -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 77, in get_dataframe - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Web_Logins' -2026-06-19 12:22:20 | INFO | ================================================================================ -2026-06-19 12:22:20 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:22:20 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:22:20 | INFO | Connecting to databases... -2026-06-19 12:22:22 | INFO | -2026-06-19 12:22:24 | INFO | -2026-06-19 12:22:25 | INFO | Database connections established -2026-06-19 12:22:25 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:22:25 | INFO | Found 859 MIDs -2026-06-19 12:22:26 | INFO | ================================================================================ -2026-06-19 12:22:26 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 12:22:26 | INFO | Fetching Data from sql server for table-: Web_Logins .............. -2026-06-19 12:22:26 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 12:22:26 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 12:22:26 | INFO | Fetching data for 859 MIDs -2026-06-19 12:22:27 | INFO | Fetched 136 rows from SQL Server -2026-06-19 12:22:27 | INFO | Fetched total row -: 136 from sql server for table-:Web_Logins ...........!!! -2026-06-19 12:22:27 | INFO | Fetched 136 rows -2026-06-19 12:22:27 | INFO | Creating table Web_Logins -2026-06-19 12:22:27 | INFO | Table ready: Web_Logins -2026-06-19 12:22:27 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 12:22:27 | INFO | Web_Logins: inserted 136 rows into ClickHouse -2026-06-19 12:22:27 | INFO | Web_Logins loaded successfully (136 rows) -2026-06-19 12:22:27 | INFO | ================================================================================ -2026-06-19 12:22:27 | INFO | Pipeline Completed Successfully -2026-06-19 12:22:27 | INFO | ================================================================================ -2026-06-19 12:26:29 | INFO | ================================================================================ -2026-06-19 12:26:29 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:26:29 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:26:29 | INFO | Connecting to databases... -2026-06-19 12:26:31 | INFO | -2026-06-19 12:26:33 | INFO | -2026-06-19 12:26:34 | INFO | Database connections established -2026-06-19 12:26:34 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:26:34 | INFO | Found 859 MIDs -2026-06-19 12:26:35 | INFO | ================================================================================ -2026-06-19 12:26:35 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 12:26:35 | INFO | Fetching Data from sql server for table-: Web_Logins .............. -2026-06-19 12:26:35 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 12:26:35 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 12:26:35 | INFO | Fetching data for 859 MIDs -2026-06-19 12:26:36 | INFO | Fetched 136 rows from SQL Server -2026-06-19 12:26:36 | INFO | Fetched total row -: 136 from sql server for table-:Web_Logins ...........!!! -2026-06-19 12:26:36 | INFO | Fetched 136 rows -2026-06-19 12:26:36 | INFO | Creating table Web_Logins -2026-06-19 12:26:36 | INFO | Table ready: Web_Logins -2026-06-19 12:26:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 12:26:36 | INFO | Web_Logins: inserted 136 rows into ClickHouse -2026-06-19 12:26:36 | INFO | Web_Logins loaded successfully (136 rows) -2026-06-19 12:26:36 | INFO | ================================================================================ -2026-06-19 12:26:36 | INFO | Pipeline Completed Successfully -2026-06-19 12:26:36 | INFO | ================================================================================ -2026-06-19 12:26:54 | INFO | ================================================================================ -2026-06-19 12:26:54 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:26:54 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:26:54 | INFO | Connecting to databases... -2026-06-19 12:26:55 | INFO | -2026-06-19 12:26:56 | INFO | -2026-06-19 12:26:57 | INFO | Database connections established -2026-06-19 12:26:57 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:26:57 | INFO | Found 859 MIDs -2026-06-19 12:26:57 | INFO | ================================================================================ -2026-06-19 12:26:57 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 12:26:57 | INFO | Fetching Data from sql server for table-: Web_Logins .............. -2026-06-19 12:26:57 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 12:26:57 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 12:26:57 | INFO | Fetching data for 859 MIDs -2026-06-19 12:26:58 | INFO | Fetched 136 rows from SQL Server -2026-06-19 12:26:58 | INFO | Fetched total row -: 136 from sql server for table-:Web_Logins ...........!!! -2026-06-19 12:26:58 | INFO | Fetched 136 rows -2026-06-19 12:26:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ -2026-06-19 12:26:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 12:26:58 | INFO | Web_Logins: inserted 136 rows into ClickHouse -2026-06-19 12:26:58 | INFO | Web_Logins loaded successfully (136 rows) -2026-06-19 12:26:58 | INFO | ================================================================================ -2026-06-19 12:26:58 | INFO | Pipeline Completed Successfully -2026-06-19 12:26:58 | INFO | ================================================================================ -2026-06-19 12:27:05 | INFO | ================================================================================ -2026-06-19 12:27:05 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:27:05 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:27:05 | INFO | Connecting to databases... -2026-06-19 12:27:07 | INFO | -2026-06-19 12:27:09 | INFO | -2026-06-19 12:27:10 | INFO | Database connections established -2026-06-19 12:27:10 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:27:10 | INFO | Found 859 MIDs -2026-06-19 12:27:10 | INFO | ================================================================================ -2026-06-19 12:27:10 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT -2026-06-19 12:27:10 | INFO | Fetching Data from sql server for table-: Web_Logins .............. -2026-06-19 12:27:10 | INFO | Start Fetching data for these 859 MIDs -2026-06-19 12:27:10 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 -2026-06-19 12:27:10 | INFO | Fetching data for 859 MIDs -2026-06-19 12:27:11 | INFO | Fetched 136 rows from SQL Server -2026-06-19 12:27:11 | INFO | Fetched total row -: 136 from sql server for table-:Web_Logins ...........!!! -2026-06-19 12:27:11 | INFO | Fetched 136 rows -2026-06-19 12:27:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ -2026-06-19 12:27:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 12:27:11 | INFO | Web_Logins: inserted 136 rows into ClickHouse -2026-06-19 12:27:11 | INFO | Web_Logins loaded successfully (136 rows) -2026-06-19 12:27:11 | INFO | ================================================================================ -2026-06-19 12:27:11 | INFO | Pipeline Completed Successfully -2026-06-19 12:27:11 | INFO | ================================================================================ -2026-06-19 12:53:21 | INFO | ================================================================================ -2026-06-19 12:53:21 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:53:21 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:53:21 | INFO | Connecting to databases... -2026-06-19 12:53:22 | INFO | -2026-06-19 12:53:24 | INFO | -2026-06-19 12:53:24 | INFO | Database connections established -2026-06-19 12:53:24 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:53:25 | INFO | Found 859 MIDs -2026-06-19 12:53:25 | INFO | ================================================================================ -2026-06-19 12:53:25 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-19 12:53:25 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 12:53:25 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - return fn(sql_engine, table_name , table_type, mids, run_date) -TypeError: fetch_Store_Master() takes 1 positional argument but 5 were given -2026-06-19 12:57:22 | INFO | ================================================================================ -2026-06-19 12:57:22 | INFO | Hello from data-move Python data pipeline! -2026-06-19 12:57:22 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 12:57:22 | INFO | Connecting to databases... -2026-06-19 12:57:24 | INFO | -2026-06-19 12:57:27 | INFO | -2026-06-19 12:57:27 | INFO | Database connections established -2026-06-19 12:57:27 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 12:57:28 | INFO | Found 859 MIDs -2026-06-19 12:57:29 | INFO | ================================================================================ -2026-06-19 12:57:29 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-19 12:57:29 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 12:57:29 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 77, in get_dataframe - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:00:35 | INFO | ================================================================================ -2026-06-19 13:00:35 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:00:35 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:00:35 | INFO | Connecting to databases... -2026-06-19 13:00:36 | INFO | -2026-06-19 13:00:38 | INFO | -2026-06-19 13:00:39 | INFO | Database connections established -2026-06-19 13:00:39 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:00:39 | INFO | Found 859 MIDs -2026-06-19 13:00:41 | INFO | ================================================================================ -2026-06-19 13:00:41 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-19 13:00:41 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:00:41 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 77, in get_dataframe - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:01:45 | INFO | ================================================================================ -2026-06-19 13:01:45 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:01:45 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:01:45 | INFO | Connecting to databases... -2026-06-19 13:01:46 | INFO | -2026-06-19 13:01:48 | INFO | -2026-06-19 13:01:49 | INFO | Database connections established -2026-06-19 13:01:49 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:01:50 | INFO | Found 859 MIDs -2026-06-19 13:01:51 | INFO | ================================================================================ -2026-06-19 13:01:51 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-19 13:01:51 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:01:51 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 77, in get_dataframe - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:03:00 | INFO | ================================================================================ -2026-06-19 13:03:00 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:03:00 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:03:00 | INFO | Connecting to databases... -2026-06-19 13:03:01 | INFO | -2026-06-19 13:03:03 | INFO | -2026-06-19 13:03:03 | INFO | Database connections established -2026-06-19 13:03:03 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:03:04 | INFO | Found 859 MIDs -2026-06-19 13:03:05 | INFO | ================================================================================ -2026-06-19 13:03:05 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:03:05 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:03:05 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 182, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:03:18 | INFO | ================================================================================ -2026-06-19 13:03:18 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:03:18 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:03:18 | INFO | Connecting to databases... -2026-06-19 13:03:19 | INFO | -2026-06-19 13:03:21 | INFO | -2026-06-19 13:03:22 | INFO | Database connections established -2026-06-19 13:03:22 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:03:22 | INFO | Found 859 MIDs -2026-06-19 13:03:23 | INFO | ================================================================================ -2026-06-19 13:03:23 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:03:23 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:03:23 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 182, in main - df = get_dataframe( - sql_engine, - ...<5 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:07:20 | INFO | ================================================================================ -2026-06-19 13:07:20 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:07:20 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:07:20 | INFO | Connecting to databases... -2026-06-19 13:07:22 | INFO | -2026-06-19 13:07:23 | INFO | -2026-06-19 13:07:24 | INFO | Database connections established -2026-06-19 13:07:24 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:07:25 | INFO | Found 859 MIDs -2026-06-19 13:07:26 | INFO | ================================================================================ -2026-06-19 13:07:26 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:07:26 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:07:26 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 182, in main - df = get_dataframe( - sql_engine, - ...<4 lines>... - run_date=run_date, - ) - File "D:\data_move\main2.py", line 80, in get_dataframe - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:13:46 | INFO | ================================================================================ -2026-06-19 13:13:46 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:13:46 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:13:46 | INFO | Connecting to databases... -2026-06-19 13:13:47 | INFO | -2026-06-19 13:13:48 | INFO | -2026-06-19 13:13:49 | INFO | Database connections established -2026-06-19 13:13:49 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:13:49 | INFO | Found 859 MIDs -2026-06-19 13:13:50 | INFO | ================================================================================ -2026-06-19 13:13:50 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:13:50 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:13:50 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 184, in main - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:17:37 | INFO | ================================================================================ -2026-06-19 13:17:37 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:17:37 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:17:37 | INFO | Connecting to databases... -2026-06-19 13:17:38 | INFO | -2026-06-19 13:17:39 | INFO | -2026-06-19 13:17:40 | INFO | Database connections established -2026-06-19 13:17:40 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:17:40 | INFO | Found 859 MIDs -2026-06-19 13:17:41 | INFO | ================================================================================ -2026-06-19 13:17:41 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:17:41 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:17:41 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 184, in main - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:18:26 | INFO | ================================================================================ -2026-06-19 13:18:26 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:18:26 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:18:26 | INFO | Connecting to databases... -2026-06-19 13:18:27 | INFO | -2026-06-19 13:18:29 | INFO | -2026-06-19 13:18:29 | INFO | Database connections established -2026-06-19 13:18:29 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:18:30 | INFO | Found 859 MIDs -2026-06-19 13:18:30 | INFO | ================================================================================ -2026-06-19 13:18:30 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:18:30 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:18:30 | INFO | ================================================================================ -2026-06-19 13:18:30 | INFO | Pipeline Completed Successfully -2026-06-19 13:18:30 | INFO | ================================================================================ -2026-06-19 13:19:45 | INFO | ================================================================================ -2026-06-19 13:19:45 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:19:45 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:19:45 | INFO | Connecting to databases... -2026-06-19 13:19:46 | INFO | -2026-06-19 13:19:48 | INFO | -2026-06-19 13:19:48 | INFO | Database connections established -2026-06-19 13:19:48 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:19:49 | INFO | Found 859 MIDs -2026-06-19 13:19:49 | INFO | ================================================================================ -2026-06-19 13:19:49 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:19:49 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:19:49 | INFO | ================================================================================ -2026-06-19 13:19:49 | INFO | Pipeline Completed Successfully -2026-06-19 13:19:49 | INFO | ================================================================================ -2026-06-19 13:21:52 | INFO | ================================================================================ -2026-06-19 13:21:52 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:21:52 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:21:52 | INFO | Connecting to databases... -2026-06-19 13:21:54 | INFO | -2026-06-19 13:21:56 | INFO | -2026-06-19 13:21:56 | INFO | Database connections established -2026-06-19 13:21:56 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:21:57 | INFO | Found 859 MIDs -2026-06-19 13:21:57 | INFO | ================================================================================ -2026-06-19 13:21:57 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:21:57 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:21:57 | INFO | ================================================================================ -2026-06-19 13:21:57 | INFO | Pipeline Completed Successfully -2026-06-19 13:21:57 | INFO | ================================================================================ -2026-06-19 13:24:49 | INFO | ================================================================================ -2026-06-19 13:24:49 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:24:49 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:24:49 | INFO | Connecting to databases... -2026-06-19 13:24:50 | INFO | -2026-06-19 13:24:51 | INFO | -2026-06-19 13:24:52 | INFO | Database connections established -2026-06-19 13:24:52 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:24:52 | INFO | Found 859 MIDs -2026-06-19 13:24:53 | INFO | ================================================================================ -2026-06-19 13:24:53 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:24:53 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:24:53 | INFO | ================================================================================ -2026-06-19 13:24:53 | INFO | Pipeline Completed Successfully -2026-06-19 13:24:53 | INFO | ================================================================================ -2026-06-19 13:26:17 | INFO | ================================================================================ -2026-06-19 13:26:17 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:26:17 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:26:17 | INFO | Connecting to databases... -2026-06-19 13:26:18 | INFO | -2026-06-19 13:26:20 | INFO | -2026-06-19 13:26:21 | INFO | Database connections established -2026-06-19 13:26:21 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:26:21 | INFO | Found 859 MIDs -2026-06-19 13:26:21 | INFO | ================================================================================ -2026-06-19 13:26:21 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:26:21 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:26:21 | INFO | ================================================================================ -2026-06-19 13:26:21 | INFO | Pipeline Completed Successfully -2026-06-19 13:26:21 | INFO | ================================================================================ -2026-06-19 13:27:00 | INFO | ================================================================================ -2026-06-19 13:27:00 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:27:00 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:27:00 | INFO | Connecting to databases... -2026-06-19 13:27:01 | INFO | -2026-06-19 13:27:03 | INFO | -2026-06-19 13:27:03 | INFO | Database connections established -2026-06-19 13:27:03 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:27:04 | INFO | Found 859 MIDs -2026-06-19 13:27:04 | INFO | ================================================================================ -2026-06-19 13:27:04 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:27:04 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:27:04 | INFO | ================================================================================ -2026-06-19 13:27:04 | INFO | Pipeline Completed Successfully -2026-06-19 13:27:04 | INFO | ================================================================================ -2026-06-19 13:28:58 | INFO | ================================================================================ -2026-06-19 13:28:58 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:28:58 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:28:58 | INFO | Connecting to databases... -2026-06-19 13:28:59 | INFO | -2026-06-19 13:29:01 | INFO | -2026-06-19 13:29:01 | INFO | Database connections established -2026-06-19 13:29:01 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:29:02 | INFO | Found 859 MIDs -2026-06-19 13:29:02 | INFO | ================================================================================ -2026-06-19 13:29:02 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:29:02 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:29:02 | INFO | Fetching data from sql server for Master table...... -2026-06-19 13:29:02 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 188, in main - df = fetch_data(sql_engine ,table_name,table_type) - File "D:\data_move\src\dim.py", line 38, in fetch_data - with open(sql_file, "r", encoding="utf-8") as f: - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'src\\sql\\dimension\\Store_Master.sql' -2026-06-19 13:29:53 | INFO | ================================================================================ -2026-06-19 13:29:53 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:29:53 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:29:53 | INFO | Connecting to databases... -2026-06-19 13:29:55 | INFO | -2026-06-19 13:29:56 | INFO | -2026-06-19 13:29:57 | INFO | Database connections established -2026-06-19 13:29:57 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:29:57 | INFO | Found 859 MIDs -2026-06-19 13:29:57 | INFO | ================================================================================ -2026-06-19 13:29:57 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:29:57 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:29:57 | INFO | Fetching data from sql server for Master table...... -2026-06-19 13:29:57 | INFO | Fetching in progress .... -2026-06-19 13:30:00 | INFO | Fetched 5,991 rows from SQL Server -2026-06-19 13:30:00 | INFO | Fetched total row -: 5991 from sql server for table-:Store_Master ...........!!! -2026-06-19 13:30:00 | INFO | Fetched 5991 rows -2026-06-19 13:30:00 | INFO | Creating table Store_Master -2026-06-19 13:30:01 | INFO | Table ready: Store_Master -2026-06-19 13:30:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 13:30:01 | INFO | Store_Master: inserted 5,991 rows into ClickHouse -2026-06-19 13:30:01 | INFO | Store_Master loaded successfully (5991 rows) -2026-06-19 13:30:01 | INFO | ================================================================================ -2026-06-19 13:30:01 | INFO | Pipeline Completed Successfully -2026-06-19 13:30:01 | INFO | ================================================================================ -2026-06-19 13:30:48 | INFO | ================================================================================ -2026-06-19 13:30:48 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:30:48 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:30:48 | INFO | Connecting to databases... -2026-06-19 13:30:49 | INFO | -2026-06-19 13:30:50 | INFO | -2026-06-19 13:30:51 | INFO | Database connections established -2026-06-19 13:30:51 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:30:51 | INFO | Found 859 MIDs -2026-06-19 13:30:52 | INFO | ================================================================================ -2026-06-19 13:30:52 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:30:52 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:30:52 | ERROR | Failed processing table Store_Master -Traceback (most recent call last): - File "D:\data_move\main2.py", line 181, in main - fn = globals()[fn_name] - ~~~~~~~~~^^^^^^^^^ -KeyError: 'fetch_Store_Master' -2026-06-19 13:32:15 | INFO | ================================================================================ -2026-06-19 13:32:15 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:32:15 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:32:15 | INFO | Connecting to databases... -2026-06-19 13:32:16 | INFO | -2026-06-19 13:32:18 | INFO | -2026-06-19 13:32:19 | INFO | Database connections established -2026-06-19 13:32:19 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:32:19 | INFO | Found 859 MIDs -2026-06-19 13:32:20 | INFO | ================================================================================ -2026-06-19 13:32:20 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:32:20 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:32:20 | INFO | Fetching data from sql server for Master table...... -2026-06-19 13:32:20 | INFO | Fetching in progress .... -2026-06-19 13:32:26 | INFO | Fetched 5,991 rows from SQL Server -2026-06-19 13:32:26 | INFO | Fetched total row -: 5991 from sql server for table-:Store_Master ...........!!! -2026-06-19 13:32:26 | INFO | Fetched 5991 rows -2026-06-19 13:32:26 | INFO | Truncated table Store_Master -2026-06-19 13:32:26 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 13:32:29 | INFO | Store_Master: inserted 5,991 rows into ClickHouse -2026-06-19 13:32:29 | INFO | Store_Master loaded successfully (5991 rows) -2026-06-19 13:32:29 | INFO | ================================================================================ -2026-06-19 13:32:29 | INFO | Pipeline Completed Successfully -2026-06-19 13:32:29 | INFO | ================================================================================ -2026-06-19 13:32:34 | INFO | ================================================================================ -2026-06-19 13:32:34 | INFO | Hello from data-move Python data pipeline! -2026-06-19 13:32:34 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 13:32:34 | INFO | Connecting to databases... -2026-06-19 13:32:35 | INFO | -2026-06-19 13:32:37 | INFO | -2026-06-19 13:32:38 | INFO | Database connections established -2026-06-19 13:32:38 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 13:32:38 | INFO | Found 859 MIDs -2026-06-19 13:32:38 | INFO | ================================================================================ -2026-06-19 13:32:38 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 13:32:38 | INFO | Fetching Data from sql server for table-: Store_Master .............. -2026-06-19 13:32:38 | INFO | Fetching data from sql server for Master table...... -2026-06-19 13:32:38 | INFO | Fetching in progress .... -2026-06-19 13:32:43 | INFO | Fetched 5,991 rows from SQL Server -2026-06-19 13:32:43 | INFO | Fetched total row -: 5991 from sql server for table-:Store_Master ...........!!! -2026-06-19 13:32:43 | INFO | Fetched 5991 rows -2026-06-19 13:32:43 | INFO | Truncated table Store_Master -2026-06-19 13:32:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 13:32:44 | INFO | Store_Master: inserted 5,991 rows into ClickHouse -2026-06-19 13:32:44 | INFO | Store_Master loaded successfully (5991 rows) -2026-06-19 13:32:44 | INFO | ================================================================================ -2026-06-19 13:32:44 | INFO | Pipeline Completed Successfully -2026-06-19 13:32:44 | INFO | ================================================================================ -2026-06-19 15:28:51 | INFO | ================================================================================ -2026-06-19 15:28:51 | INFO | Hello from data-move Python data pipeline! -2026-06-19 15:28:51 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 15:28:51 | INFO | Connecting to databases... -2026-06-19 15:28:53 | INFO | -2026-06-19 15:28:54 | INFO | -2026-06-19 15:28:55 | INFO | Database connections established -2026-06-19 15:28:55 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 15:28:55 | INFO | Found 859 MIDs -2026-06-19 15:28:56 | INFO | ================================================================================ -2026-06-19 15:28:56 | INFO | Processing Table: coverage_remarks | Table type is -: DIMENSION | Based on -reason_id and operation is used -DELETE+INSERT -2026-06-19 15:28:56 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. -2026-06-19 15:28:57 | WARNING | Table 'coverage_remarks' does not exist. -2026-06-19 15:28:57 | INFO | Fetching data from sql server for Master table...... -2026-06-19 15:28:57 | INFO | Fetching in progress .... -2026-06-19 15:28:57 | INFO | Fetched 29 rows from SQL Server -2026-06-19 15:28:57 | INFO | Fetched 29 rows from SQL Server -2026-06-19 15:28:57 | INFO | Fetched total row -: 29 from sql server for table-:coverage_remarks ...........!!! -2026-06-19 15:28:57 | INFO | Fetched 29 rows -2026-06-19 15:28:57 | INFO | Creating table coverage_remarks -2026-06-19 15:28:57 | INFO | Table ready: coverage_remarks -2026-06-19 15:28:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 15:28:57 | INFO | coverage_remarks: inserted 29 rows into ClickHouse -2026-06-19 15:28:57 | INFO | coverage_remarks loaded successfully (29 rows) -2026-06-19 15:28:57 | INFO | ================================================================================ -2026-06-19 15:28:57 | INFO | Pipeline Completed Successfully -2026-06-19 15:28:57 | INFO | ================================================================================ -2026-06-19 15:29:04 | INFO | ================================================================================ -2026-06-19 15:29:04 | INFO | Hello from data-move Python data pipeline! -2026-06-19 15:29:04 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 15:29:04 | INFO | Connecting to databases... -2026-06-19 15:29:05 | INFO | -2026-06-19 15:29:07 | INFO | -2026-06-19 15:29:07 | INFO | Database connections established -2026-06-19 15:29:07 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 15:29:08 | INFO | Found 859 MIDs -2026-06-19 15:29:08 | INFO | ================================================================================ -2026-06-19 15:29:08 | INFO | Processing Table: coverage_remarks | Table type is -: DIMENSION | Based on -reason_id and operation is used -DELETE+INSERT -2026-06-19 15:29:08 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. -2026-06-19 15:29:09 | INFO | Fetching data from sql server for Master table...... -2026-06-19 15:29:09 | INFO | Fetching in progress .... -2026-06-19 15:29:09 | INFO | Fetched 0 rows from SQL Server -2026-06-19 15:29:09 | INFO | Fetched 0 rows from SQL Server -2026-06-19 15:29:09 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! -2026-06-19 15:29:09 | WARNING | coverage_remarks returned no rows -2026-06-19 15:29:09 | INFO | ================================================================================ -2026-06-19 15:29:09 | INFO | Pipeline Completed Successfully -2026-06-19 15:29:09 | INFO | ================================================================================ -2026-06-19 15:51:47 | INFO | ================================================================================ -2026-06-19 15:51:47 | INFO | Hello from data-move Python data pipeline! -2026-06-19 15:51:47 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 15:51:47 | INFO | Connecting to databases... -2026-06-19 15:51:49 | INFO | -2026-06-19 15:51:51 | INFO | -2026-06-19 15:51:52 | INFO | Database connections established -2026-06-19 15:51:52 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 15:51:53 | INFO | Found 859 MIDs -2026-06-19 15:51:54 | INFO | ================================================================================ -2026-06-19 15:51:54 | INFO | Processing Table: SKU_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 15:51:54 | INFO | Fetching Data from sql server for table-: SKU_Master .............. -2026-06-19 15:51:54 | INFO | Fetching data from sql server for Master table...... -2026-06-19 15:51:54 | INFO | Fetching in progress .... -2026-06-19 15:51:55 | INFO | Fetched 160 rows from SQL Server -2026-06-19 15:51:55 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! -2026-06-19 15:51:55 | INFO | Fetched 160 rows -2026-06-19 15:51:55 | INFO | Creating table SKU_Master -2026-06-19 15:51:55 | INFO | Table ready: SKU_Master -2026-06-19 15:51:55 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 15:51:55 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-19 15:51:55 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-19 15:51:55 | INFO | ================================================================================ -2026-06-19 15:51:55 | INFO | Pipeline Completed Successfully -2026-06-19 15:51:55 | INFO | ================================================================================ -2026-06-19 15:52:22 | INFO | ================================================================================ -2026-06-19 15:52:22 | INFO | Hello from data-move Python data pipeline! -2026-06-19 15:52:22 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 15:52:22 | INFO | Connecting to databases... -2026-06-19 15:52:23 | INFO | -2026-06-19 15:52:25 | INFO | -2026-06-19 15:52:28 | INFO | Database connections established -2026-06-19 15:52:28 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 15:52:28 | INFO | Found 859 MIDs -2026-06-19 15:52:29 | INFO | ================================================================================ -2026-06-19 15:52:29 | INFO | Processing Table: SKU_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 15:52:29 | INFO | Fetching Data from sql server for table-: SKU_Master .............. -2026-06-19 15:52:29 | INFO | Fetching data from sql server for Master table...... -2026-06-19 15:52:29 | INFO | Fetching in progress .... -2026-06-19 15:52:29 | INFO | Fetched 160 rows from SQL Server -2026-06-19 15:52:29 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! -2026-06-19 15:52:29 | INFO | Fetched 160 rows -2026-06-19 15:52:30 | INFO | Truncated table SKU_Master -2026-06-19 15:52:30 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 15:52:30 | INFO | SKU_Master: inserted 160 rows into ClickHouse -2026-06-19 15:52:30 | INFO | SKU_Master loaded successfully (160 rows) -2026-06-19 15:52:30 | INFO | ================================================================================ -2026-06-19 15:52:30 | INFO | Pipeline Completed Successfully -2026-06-19 15:52:30 | INFO | ================================================================================ -2026-06-19 15:56:07 | INFO | ================================================================================ -2026-06-19 15:56:07 | INFO | Hello from data-move Python data pipeline! -2026-06-19 15:56:07 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 15:56:07 | INFO | Connecting to databases... -2026-06-19 15:56:08 | INFO | -2026-06-19 15:56:09 | INFO | -2026-06-19 15:56:10 | INFO | Database connections established -2026-06-19 15:56:10 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 15:56:11 | INFO | Found 859 MIDs -2026-06-19 15:56:12 | INFO | ================================================================================ -2026-06-19 15:56:12 | INFO | Processing Table: display_master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 15:56:12 | INFO | Fetching Data from sql server for table-: display_master .............. -2026-06-19 15:56:12 | INFO | Fetching data from sql server for Master table...... -2026-06-19 15:56:12 | INFO | Fetching in progress .... -2026-06-19 15:56:12 | INFO | Fetched 135 rows from SQL Server -2026-06-19 15:56:12 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! -2026-06-19 15:56:12 | INFO | Fetched 135 rows -2026-06-19 15:56:12 | INFO | Creating table display_master -2026-06-19 15:56:12 | INFO | Table ready: display_master -2026-06-19 15:56:12 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 15:56:12 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-19 15:56:12 | INFO | display_master loaded successfully (135 rows) -2026-06-19 15:56:12 | INFO | ================================================================================ -2026-06-19 15:56:12 | INFO | Pipeline Completed Successfully -2026-06-19 15:56:12 | INFO | ================================================================================ -2026-06-19 15:56:19 | INFO | ================================================================================ -2026-06-19 15:56:19 | INFO | Hello from data-move Python data pipeline! -2026-06-19 15:56:19 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 15:56:19 | INFO | Connecting to databases... -2026-06-19 15:56:20 | INFO | -2026-06-19 15:56:22 | INFO | -2026-06-19 15:56:23 | INFO | Database connections established -2026-06-19 15:56:23 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 15:56:24 | INFO | Found 859 MIDs -2026-06-19 15:56:24 | INFO | ================================================================================ -2026-06-19 15:56:24 | INFO | Processing Table: display_master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 15:56:24 | INFO | Fetching Data from sql server for table-: display_master .............. -2026-06-19 15:56:24 | INFO | Fetching data from sql server for Master table...... -2026-06-19 15:56:24 | INFO | Fetching in progress .... -2026-06-19 15:56:24 | INFO | Fetched 135 rows from SQL Server -2026-06-19 15:56:24 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! -2026-06-19 15:56:24 | INFO | Fetched 135 rows -2026-06-19 15:56:25 | INFO | Truncated table display_master -2026-06-19 15:56:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 15:56:25 | INFO | display_master: inserted 135 rows into ClickHouse -2026-06-19 15:56:25 | INFO | display_master loaded successfully (135 rows) -2026-06-19 15:56:25 | INFO | ================================================================================ -2026-06-19 15:56:25 | INFO | Pipeline Completed Successfully -2026-06-19 15:56:25 | INFO | ================================================================================ -2026-06-19 16:00:13 | INFO | ================================================================================ -2026-06-19 16:00:13 | INFO | Hello from data-move Python data pipeline! -2026-06-19 16:00:13 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 16:00:13 | INFO | Connecting to databases... -2026-06-19 16:00:15 | INFO | -2026-06-19 16:00:17 | INFO | -2026-06-19 16:00:18 | INFO | Database connections established -2026-06-19 16:00:18 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 16:00:19 | INFO | Found 859 MIDs -2026-06-19 16:00:20 | INFO | ================================================================================ -2026-06-19 16:00:20 | INFO | Processing Table: Employee_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 16:00:20 | INFO | Fetching Data from sql server for table-: Employee_Master .............. -2026-06-19 16:00:20 | INFO | Fetching data from sql server for Master table...... -2026-06-19 16:00:20 | INFO | Fetching in progress .... -2026-06-19 16:00:23 | INFO | Fetched 2,287 rows from SQL Server -2026-06-19 16:00:23 | INFO | Fetched total row -: 2287 from sql server for table-:Employee_Master ...........!!! -2026-06-19 16:00:23 | INFO | Fetched 2287 rows -2026-06-19 16:00:23 | INFO | Creating table Employee_Master -2026-06-19 16:00:23 | INFO | Table ready: Employee_Master -2026-06-19 16:00:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 16:00:24 | INFO | Employee_Master: inserted 2,287 rows into ClickHouse -2026-06-19 16:00:24 | INFO | Employee_Master loaded successfully (2287 rows) -2026-06-19 16:00:24 | INFO | ================================================================================ -2026-06-19 16:00:24 | INFO | Pipeline Completed Successfully -2026-06-19 16:00:24 | INFO | ================================================================================ -2026-06-19 16:00:33 | INFO | ================================================================================ -2026-06-19 16:00:33 | INFO | Hello from data-move Python data pipeline! -2026-06-19 16:00:33 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 16:00:33 | INFO | Connecting to databases... -2026-06-19 16:00:34 | INFO | -2026-06-19 16:00:36 | INFO | -2026-06-19 16:00:36 | INFO | Database connections established -2026-06-19 16:00:36 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 16:00:37 | INFO | Found 859 MIDs -2026-06-19 16:00:37 | INFO | ================================================================================ -2026-06-19 16:00:37 | INFO | Processing Table: Employee_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT -2026-06-19 16:00:37 | INFO | Fetching Data from sql server for table-: Employee_Master .............. -2026-06-19 16:00:37 | INFO | Fetching data from sql server for Master table...... -2026-06-19 16:00:37 | INFO | Fetching in progress .... -2026-06-19 16:00:38 | INFO | Fetched 2,287 rows from SQL Server -2026-06-19 16:00:38 | INFO | Fetched total row -: 2287 from sql server for table-:Employee_Master ...........!!! -2026-06-19 16:00:38 | INFO | Fetched 2287 rows -2026-06-19 16:00:38 | INFO | Truncated table Employee_Master -2026-06-19 16:00:38 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-19 16:00:39 | INFO | Employee_Master: inserted 2,287 rows into ClickHouse -2026-06-19 16:00:39 | INFO | Employee_Master loaded successfully (2287 rows) -2026-06-19 16:00:39 | INFO | ================================================================================ -2026-06-19 16:00:39 | INFO | Pipeline Completed Successfully -2026-06-19 16:00:39 | INFO | ================================================================================ -2026-06-19 18:19:29 | INFO | ================================================================================ -2026-06-19 18:19:29 | INFO | Hello from data-move Python data pipeline! -2026-06-19 18:19:29 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 18:19:29 | INFO | Connecting to databases... -2026-06-19 18:19:30 | INFO | -2026-06-19 18:19:31 | INFO | -2026-06-19 18:19:31 | INFO | Database connections established -2026-06-19 18:19:31 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 18:19:31 | INFO | Found 859 MIDs -2026-06-19 18:19:32 | INFO | ================================================================================ -2026-06-19 18:19:32 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT -2026-06-19 18:19:32 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. -2026-06-19 18:19:32 | ERROR | Failed processing table mapping_storevisibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 159, in main - df=fn(sql_engine, table_name , table_type, mids, run_date) -TypeError: fetch_mapping_storevisibility() takes 2 positional arguments but 5 were given -2026-06-19 18:20:42 | INFO | ================================================================================ -2026-06-19 18:20:42 | INFO | Hello from data-move Python data pipeline! -2026-06-19 18:20:42 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 18:20:42 | INFO | Connecting to databases... -2026-06-19 18:20:42 | INFO | -2026-06-19 18:20:43 | INFO | -2026-06-19 18:20:44 | INFO | Database connections established -2026-06-19 18:20:44 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 18:20:44 | INFO | Found 859 MIDs -2026-06-19 18:20:44 | INFO | ================================================================================ -2026-06-19 18:20:44 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT -2026-06-19 18:20:44 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. -2026-06-19 18:20:44 | ERROR | Failed processing table mapping_storevisibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 159, in main - df=fn(sql_engine, table_name , table_type, mids, run_date) -TypeError: fetch_mapping_storevisibility() takes 2 positional arguments but 5 were given -2026-06-19 18:25:10 | INFO | ================================================================================ -2026-06-19 18:25:10 | INFO | Hello from data-move Python data pipeline! -2026-06-19 18:25:10 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 18:25:10 | INFO | Connecting to databases... -2026-06-19 18:25:11 | INFO | -2026-06-19 18:25:12 | INFO | -2026-06-19 18:25:13 | INFO | Database connections established -2026-06-19 18:25:13 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 18:25:13 | INFO | Found 859 MIDs -2026-06-19 18:25:13 | INFO | ================================================================================ -2026-06-19 18:25:13 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT -2026-06-19 18:25:13 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. -2026-06-19 18:25:13 | ERROR | Failed processing table mapping_storevisibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 159, in main - df=fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\bridge.py", line 100, in fetch_mapping_storevisibility - store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks") - File "D:\data_move\src\bridge.py", line 61, in get_reason_ids_mapping_storevisibility - arrow_table = client.query_arrow(query) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 736, in query_arrow - self.raw_query( - ~~~~~~~~~~~~~~^ - query, - ^^^^^^ - ...<4 lines>... - transport_settings=transport_settings, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 661, in raw_query - return self._raw_request(body, params, fields=fields, headers=transport_settings, retries=self.query_retries).data - ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 60, server response: Code: 60. DB::Exception: Unknown table expression identifier 'mapping_storevisibility' in scope SELECT DISTINCT StoreId FROM mapping_storevisibility WHERE (toDate(Fromdate) <= toDate('2026-06-20')) AND (toDate(Todate) >= toDate('2026-06-20')) AND (project_Id = '40148'). (UNKNOWN_TABLE) (for url http://172.188.12.194:8123) -2026-06-19 18:25:58 | INFO | ================================================================================ -2026-06-19 18:25:58 | INFO | Hello from data-move Python data pipeline! -2026-06-19 18:25:58 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 18:25:58 | INFO | Connecting to databases... -2026-06-19 18:25:59 | INFO | -2026-06-19 18:26:00 | INFO | -2026-06-19 18:26:01 | INFO | Database connections established -2026-06-19 18:26:01 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 18:26:01 | INFO | Found 859 MIDs -2026-06-19 18:26:01 | INFO | ================================================================================ -2026-06-19 18:26:01 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT -2026-06-19 18:26:01 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. -2026-06-19 18:26:01 | ERROR | Failed processing table mapping_storevisibility -Traceback (most recent call last): - File "D:\data_move\main2.py", line 159, in main - df=fn(sql_engine, table_name , table_type, mids, run_date) - File "D:\data_move\src\bridge.py", line 100, in fetch_mapping_storevisibility - store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks") - File "D:\data_move\src\bridge.py", line 61, in get_reason_ids_mapping_storevisibility - arrow_table = client.query_arrow(query) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 736, in query_arrow - self.raw_query( - ~~~~~~~~~~~~~~^ - query, - ^^^^^^ - ...<4 lines>... - transport_settings=transport_settings, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ) - ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 661, in raw_query - return self._raw_request(body, params, fields=fields, headers=transport_settings, retries=self.query_retries).data - ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request - self._error_handler(response) - ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler - raise err_type(err_str, code=code, name=name) from None -clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 60, server response: Code: 60. DB::Exception: Unknown table expression identifier 'mapping_storevisibility' in scope SELECT DISTINCT StoreId FROM mapping_storevisibility WHERE (toDate(Fromdate) <= toDate('2026-06-20')) AND (toDate(Todate) >= toDate('2026-06-20')) AND (project_Id = '40148'). (UNKNOWN_TABLE) (for url http://172.188.12.194:8123) -2026-06-19 18:27:24 | INFO | ================================================================================ -2026-06-19 18:27:24 | INFO | Hello from data-move Python data pipeline! -2026-06-19 18:27:24 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 18:27:24 | INFO | Connecting to databases... -2026-06-19 18:27:25 | INFO | -2026-06-19 18:27:26 | INFO | -2026-06-19 18:27:26 | INFO | Database connections established -2026-06-19 18:27:26 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 18:27:27 | INFO | Found 859 MIDs -2026-06-19 18:27:27 | INFO | ================================================================================ -2026-06-19 18:27:27 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT -2026-06-19 18:27:27 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. -2026-06-19 18:27:27 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids -2026-06-19 18:27:27 | INFO | Fetching data from sql server for Master table...... -2026-06-19 18:27:27 | INFO | Fetching in progress .... -2026-06-19 18:27:27 | INFO | Fetched 0 rows from SQL Server -2026-06-19 18:27:27 | INFO | Fetched 0 rows from SQL Server -2026-06-19 18:27:27 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! -2026-06-19 18:27:27 | WARNING | mapping_storevisibility returned no rows -2026-06-19 18:27:27 | INFO | ================================================================================ -2026-06-19 18:27:27 | INFO | Pipeline Completed Successfully -2026-06-19 18:27:27 | INFO | ================================================================================ -2026-06-19 18:27:32 | INFO | ================================================================================ -2026-06-19 18:27:32 | INFO | Hello from data-move Python data pipeline! -2026-06-19 18:27:32 | INFO | Pipeline Run Date: 2026-06-18 -2026-06-19 18:27:32 | INFO | Connecting to databases... -2026-06-19 18:27:33 | INFO | -2026-06-19 18:27:34 | INFO | -2026-06-19 18:27:35 | INFO | Database connections established -2026-06-19 18:27:35 | INFO | Collecting MIDs for: 2026-06-18 -2026-06-19 18:27:35 | INFO | Found 859 MIDs -2026-06-19 18:27:35 | INFO | ================================================================================ -2026-06-19 18:27:35 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT -2026-06-19 18:27:35 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. -2026-06-19 18:27:35 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids -2026-06-19 18:27:35 | INFO | Fetching data from sql server for Master table...... -2026-06-19 18:27:35 | INFO | Fetching in progress .... -2026-06-19 18:27:35 | INFO | Fetched 0 rows from SQL Server -2026-06-19 18:27:35 | INFO | Fetched 0 rows from SQL Server -2026-06-19 18:27:35 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! -2026-06-19 18:27:35 | WARNING | mapping_storevisibility returned no rows -2026-06-19 18:27:35 | INFO | ================================================================================ -2026-06-19 18:27:35 | INFO | Pipeline Completed Successfully -2026-06-19 18:27:35 | INFO | ================================================================================ diff --git a/logs/etl_20260622.log b/logs/etl_20260622.log index 169b564..9e8ea89 100644 --- a/logs/etl_20260622.log +++ b/logs/etl_20260622.log @@ -1,42 +1,115 @@ -2026-06-22 10:05:56 | INFO | ================================================================================ -2026-06-22 10:05:56 | INFO | Hello from data-move Python data pipeline! -2026-06-22 10:05:56 | INFO | Pipeline Run Date: 2026-06-21 -2026-06-22 10:05:56 | INFO | Connecting to databases... -2026-06-22 10:05:57 | INFO | -2026-06-22 10:06:00 | INFO | -2026-06-22 10:06:03 | INFO | Database connections established -2026-06-22 10:06:03 | INFO | Collecting MIDs for: 2026-06-21 -2026-06-22 10:06:05 | INFO | Found 569 MIDs -2026-06-22 10:06:07 | INFO | ================================================================================ -2026-06-22 10:06:07 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 10:06:07 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. -2026-06-22 10:06:07 | INFO | Fetching data from sql server for Master table...... -2026-06-22 10:06:07 | INFO | Fetching in progress .... -2026-06-22 10:06:31 | ERROR | Failed processing table Master_VisibilityReason +2026-06-22 12:12:58 | INFO | ================================================================================ +2026-06-22 12:12:58 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:12:58 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:12:58 | INFO | Connecting to databases... +2026-06-22 12:12:59 | INFO | +2026-06-22 12:13:03 | INFO | +2026-06-22 12:13:03 | INFO | Database connections established +2026-06-22 12:13:03 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:13:04 | INFO | Found 569 MIDs +2026-06-22 12:13:05 | INFO | ================================================================================ +2026-06-22 12:13:05 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:13:05 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 12:13:05 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:13:05 | INFO | Fetching in progress .... +2026-06-22 12:13:05 | INFO | Fetched 17 rows from SQL Server +2026-06-22 12:13:05 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 12:13:05 | INFO | Fetched 17 rows +2026-06-22 12:13:05 | INFO | Truncated table Master_VisibilityReason +2026-06-22 12:13:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:13:05 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 12:13:05 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 12:13:05 | INFO | ================================================================================ +2026-06-22 12:13:05 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:13:05 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 12:13:05 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:13:05 | INFO | Fetching in progress .... +2026-06-22 12:13:05 | INFO | Fetched 861 rows from SQL Server +2026-06-22 12:13:05 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 12:13:05 | INFO | Fetched 861 rows +2026-06-22 12:13:05 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 12:13:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:13:06 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 12:13:06 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 12:13:06 | INFO | ================================================================================ +2026-06-22 12:13:06 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:13:06 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 12:13:06 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:13:06 | INFO | Fetching in progress .... +2026-06-22 12:13:06 | INFO | Fetched 33 rows from SQL Server +2026-06-22 12:13:06 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 12:13:06 | INFO | Fetched 33 rows +2026-06-22 12:13:06 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 12:13:06 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:13:06 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 12:13:06 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 12:13:06 | INFO | ================================================================================ +2026-06-22 12:13:06 | INFO | Pipeline Completed Successfully +2026-06-22 12:13:06 | INFO | ================================================================================ +2026-06-22 12:16:47 | INFO | ================================================================================ +2026-06-22 12:16:47 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:16:47 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:16:47 | INFO | Connecting to databases... +2026-06-22 12:16:48 | INFO | +2026-06-22 12:16:50 | INFO | +2026-06-22 12:16:52 | INFO | Database connections established +2026-06-22 12:16:52 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:16:52 | INFO | Found 569 MIDs +2026-06-22 12:16:53 | INFO | ================================================================================ +2026-06-22 12:16:53 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:16:53 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:16:53 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:16:53 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:16:53 | INFO | Fetching data for 569 MIDs +2026-06-22 12:16:57 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:16:57 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:16:57 | INFO | Fetched 1907 rows +2026-06-22 12:16:57 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:16:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:16:58 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:16:58 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:16:58 | INFO | ================================================================================ +2026-06-22 12:16:58 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:16:58 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:16:58 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:17:14 | INFO | Fetched 291,134 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:17:14 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:17:14 | INFO | Fetched 0 matched empids fetched for OQAD +2026-06-22 12:17:14 | INFO | Exists: True +2026-06-22 12:17:14 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 12:17:14 | INFO | Fetching data for 0 EMPIDs +2026-06-22 12:17:14 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 12:17:14 | ERROR | Failed processing table OQaD Traceback (most recent call last): - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context self.dialect.do_execute( ~~~~~~~~~~~~~~~~~~~~~~~^ cursor, str_statement, effective_parameters, context ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute cursor.execute(statement, parameters) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)') +pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ')'. (102) (SQLExecDirectW)") The above exception was the direct cause of the following exception: Traceback (most recent call last): - File "D:\data_move\main2.py", line 161, in main - df = fetch_data(sql_engine ,table_name,table_type) - File "D:\data_move\src\dim.py", line 45, in fetch_data + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 285, in fetch_OQaD + df=fetch_data( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\z\data_move\src\fact.py", line 277, in fetch_data df = pl.read_database( query=sql, - connection=engine + connection=engine, ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database return cx.execute( ~~~~~~~~~~^ query=query, @@ -45,263 +118,811 @@ Traceback (most recent call last): ^^^^^^^^^^^^^^^^^^^^^^^^ ).to_polars( ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute result = cursor_execute(query, **options) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute return meth( self, distilled_parameters, execution_options or NO_OPTIONS, ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection return connection._execute_clauseelement( ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ self, distilled_params, execution_options ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement ret = self._execute_context( dialect, ...<8 lines>... cache_hit=cache_hit, ) - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context return self._exec_single_context( ~~~~~~~~~~~~~~~~~~~~~~~~~^ dialect, context, statement, parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context self._handle_dbapi_exception( ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ e, str_statement, effective_parameters, cursor, context ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception raise sqlalchemy_exception.with_traceback(exc_info[2]) from e - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context self.dialect.do_execute( ~~~~~~~~~~~~~~~~~~~~~~~^ cursor, str_statement, effective_parameters, context ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ - File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute cursor.execute(statement, parameters) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ -sqlalchemy.exc.OperationalError: (pyodbc.OperationalError) ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)') -[SQL: with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby) - as ( +sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ')'. (102) (SQLExecDirectW)") +[SQL: WITH MID_TABLE_COV1 AS +( + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= 2026-06-21 + AND CreateDate < DATEADD(DAY,1,'2026-06-21') -select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius' -FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason + UNION ALL + + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= 2026-06-21 + AND UpdateDate < DATEADD(DAY,1, '2026-06-21') +), + + +QUIZ AS +( + SELECT DISTINCT + E.EmpId, + E.EmpName, + E.SupervisorId, + E.SupervisorName, + E.DesignationName, + E.CityName, + E.StateName, + E.RegionName, + CAST(DQ.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + DQ.AnswerId, + QC.QuestionCategoryId, + QC.QuestionCategory + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + WHERE E.EmpName NOT LIKE '%TEST%' + AND E.RightId = 6 + AND ( + E.ResignDate IS NULL + OR CAST(E.ResignDate AS DATE) >= '2026-06-21' + ) + AND EXISTS + ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE) + ) +) + +SELECT + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId,0) AS answer_id, + ISNULL(QA.Answer,'') AS answer, + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + GETDATE() AS update_date, + 'SP-Pius' AS update_by +FROM QUIZ Q +INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM + ON Q.QuestionId = QM.QuestionId +LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId +where Q.EmpId not in () + + +] +(Background on this error at: https://sqlalche.me/e/20/f405) +2026-06-22 12:27:42 | INFO | ================================================================================ +2026-06-22 12:27:42 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:27:42 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:27:42 | INFO | Connecting to databases... +2026-06-22 12:27:43 | INFO | +2026-06-22 12:27:44 | INFO | +2026-06-22 12:27:44 | INFO | Database connections established +2026-06-22 12:27:44 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:27:45 | INFO | Found 569 MIDs +2026-06-22 12:27:45 | INFO | ================================================================================ +2026-06-22 12:27:45 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:27:45 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:27:45 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:27:45 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:27:45 | INFO | Fetching data for 569 MIDs +2026-06-22 12:27:50 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:27:50 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:27:50 | INFO | Fetched 1907 rows +2026-06-22 12:27:50 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:27:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:27:52 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:27:52 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:27:52 | INFO | ================================================================================ +2026-06-22 12:27:52 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:27:52 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:27:53 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:28:12 | INFO | Fetched 291,145 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:28:12 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:28:12 | WARNING | OQaD Matched df in OQaD returned no rows +2026-06-22 12:28:12 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 257, in fetch_OQaD + log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") + ^^^^^^ +UnboundLocalError: cannot access local variable 'empids' where it is not associated with a value +2026-06-22 12:28:18 | INFO | ================================================================================ +2026-06-22 12:28:18 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:28:18 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:28:18 | INFO | Connecting to databases... +2026-06-22 12:28:19 | INFO | +2026-06-22 12:28:19 | INFO | +2026-06-22 12:28:20 | INFO | Database connections established +2026-06-22 12:28:20 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:28:20 | INFO | Found 569 MIDs +2026-06-22 12:28:20 | INFO | ================================================================================ +2026-06-22 12:28:20 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:28:20 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:28:20 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:28:20 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:28:20 | INFO | Fetching data for 569 MIDs +2026-06-22 12:28:25 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:28:25 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:28:25 | INFO | Fetched 1907 rows +2026-06-22 12:28:25 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:28:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:28:25 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:28:25 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:28:25 | INFO | ================================================================================ +2026-06-22 12:28:25 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:28:25 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:28:25 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:28:47 | INFO | Fetched 291,145 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:28:47 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:28:47 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 248, in fetch_OQaD + log.warning( + ~~~~~~~~~~~^ + "%s Matched df in OQaD returned no rows", + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + empids=[0] + ^^^^^^^^^^ ) -select * from Master_VisibilityReason] -(Background on this error at: https://sqlalche.me/e/20/e3q8) -2026-06-22 10:10:52 | INFO | ================================================================================ -2026-06-22 10:10:52 | INFO | Hello from data-move Python data pipeline! -2026-06-22 10:10:52 | INFO | Pipeline Run Date: 2026-06-21 -2026-06-22 10:10:53 | INFO | Connecting to databases... -2026-06-22 10:10:53 | INFO | -2026-06-22 10:10:55 | INFO | -2026-06-22 10:10:56 | INFO | Database connections established -2026-06-22 10:10:56 | INFO | Collecting MIDs for: 2026-06-21 -2026-06-22 10:10:56 | INFO | Found 569 MIDs -2026-06-22 10:10:57 | INFO | ================================================================================ -2026-06-22 10:10:57 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 10:10:57 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. -2026-06-22 10:10:57 | INFO | Fetching data from sql server for Master table...... -2026-06-22 10:10:57 | INFO | Fetching in progress .... -2026-06-22 10:10:57 | INFO | Fetched 17 rows from SQL Server -2026-06-22 10:10:57 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! -2026-06-22 10:10:57 | INFO | Fetched 17 rows -2026-06-22 10:10:57 | INFO | Creating table Master_VisibilityReason -2026-06-22 10:10:57 | INFO | Table ready: Master_VisibilityReason -2026-06-22 10:10:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 10:10:58 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-22 10:10:58 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-22 10:10:58 | INFO | ================================================================================ -2026-06-22 10:10:58 | INFO | Pipeline Completed Successfully -2026-06-22 10:10:58 | INFO | ================================================================================ -2026-06-22 10:11:04 | INFO | ================================================================================ -2026-06-22 10:11:04 | INFO | Hello from data-move Python data pipeline! -2026-06-22 10:11:04 | INFO | Pipeline Run Date: 2026-06-21 -2026-06-22 10:11:04 | INFO | Connecting to databases... -2026-06-22 10:11:05 | INFO | -2026-06-22 10:11:06 | INFO | -2026-06-22 10:11:07 | INFO | Database connections established -2026-06-22 10:11:07 | INFO | Collecting MIDs for: 2026-06-21 -2026-06-22 10:11:07 | INFO | Found 569 MIDs -2026-06-22 10:11:08 | INFO | ================================================================================ -2026-06-22 10:11:08 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 10:11:08 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. -2026-06-22 10:11:08 | INFO | Fetching data from sql server for Master table...... -2026-06-22 10:11:08 | INFO | Fetching in progress .... -2026-06-22 10:11:08 | INFO | Fetched 17 rows from SQL Server -2026-06-22 10:11:08 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! -2026-06-22 10:11:08 | INFO | Fetched 17 rows -2026-06-22 10:11:08 | INFO | Truncated table Master_VisibilityReason -2026-06-22 10:11:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 10:11:08 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-22 10:11:08 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-22 10:11:08 | INFO | ================================================================================ -2026-06-22 10:11:08 | INFO | Pipeline Completed Successfully -2026-06-22 10:11:08 | INFO | ================================================================================ -2026-06-22 11:07:54 | INFO | ================================================================================ -2026-06-22 11:07:54 | INFO | Hello from data-move Python data pipeline! -2026-06-22 11:07:54 | INFO | Pipeline Run Date: 2026-06-21 -2026-06-22 11:07:54 | INFO | Connecting to databases... -2026-06-22 11:07:56 | INFO | -2026-06-22 11:08:00 | INFO | -2026-06-22 11:08:01 | INFO | Database connections established -2026-06-22 11:08:01 | INFO | Collecting MIDs for: 2026-06-21 -2026-06-22 11:08:02 | INFO | Found 569 MIDs -2026-06-22 11:08:42 | INFO | ================================================================================ -2026-06-22 11:08:42 | INFO | Hello from data-move Python data pipeline! -2026-06-22 11:08:42 | INFO | Pipeline Run Date: 2026-06-21 -2026-06-22 11:08:42 | INFO | Connecting to databases... -2026-06-22 11:08:44 | INFO | -2026-06-22 11:08:46 | INFO | -2026-06-22 11:08:47 | INFO | Database connections established -2026-06-22 11:08:47 | INFO | Collecting MIDs for: 2026-06-21 -2026-06-22 11:08:48 | INFO | Found 569 MIDs -2026-06-22 11:08:48 | INFO | ================================================================================ -2026-06-22 11:08:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 11:08:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. -2026-06-22 11:08:48 | INFO | Fetching data from sql server for Master table...... -2026-06-22 11:08:48 | INFO | Fetching in progress .... -2026-06-22 11:08:48 | INFO | Fetched 17 rows from SQL Server -2026-06-22 11:08:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! -2026-06-22 11:08:48 | INFO | Fetched 17 rows -2026-06-22 11:08:48 | INFO | Truncated table Master_VisibilityReason -2026-06-22 11:08:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 11:08:49 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-22 11:08:49 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-22 11:08:49 | INFO | ================================================================================ -2026-06-22 11:08:49 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. -2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table...... -2026-06-22 11:08:49 | INFO | Fetching in progress .... -2026-06-22 11:08:49 | INFO | Fetched 861 rows from SQL Server -2026-06-22 11:08:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! -2026-06-22 11:08:49 | INFO | Fetched 861 rows -2026-06-22 11:08:49 | INFO | Creating table Master_VisibilityDefinition -2026-06-22 11:08:49 | INFO | Table ready: Master_VisibilityDefinition -2026-06-22 11:08:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-22 11:08:49 | INFO | ================================================================================ -2026-06-22 11:08:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. -2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table...... -2026-06-22 11:08:49 | INFO | Fetching in progress .... -2026-06-22 11:08:49 | INFO | Fetched 33 rows from SQL Server -2026-06-22 11:08:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! -2026-06-22 11:08:49 | INFO | Fetched 33 rows -2026-06-22 11:08:50 | INFO | Creating table Master_Salesterritorylayer -2026-06-22 11:08:50 | INFO | Table ready: Master_Salesterritorylayer -2026-06-22 11:08:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-22 11:08:50 | INFO | ================================================================================ -2026-06-22 11:08:50 | INFO | Pipeline Completed Successfully -2026-06-22 11:08:50 | INFO | ================================================================================ -2026-06-22 11:08:56 | INFO | ================================================================================ -2026-06-22 11:08:56 | INFO | Hello from data-move Python data pipeline! -2026-06-22 11:08:56 | INFO | Pipeline Run Date: 2026-06-21 -2026-06-22 11:08:56 | INFO | Connecting to databases... -2026-06-22 11:08:57 | INFO | -2026-06-22 11:08:59 | INFO | -2026-06-22 11:09:00 | INFO | Database connections established -2026-06-22 11:09:00 | INFO | Collecting MIDs for: 2026-06-21 -2026-06-22 11:09:00 | INFO | Found 569 MIDs -2026-06-22 11:09:00 | INFO | ================================================================================ -2026-06-22 11:09:00 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 11:09:00 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. -2026-06-22 11:09:00 | INFO | Fetching data from sql server for Master table...... -2026-06-22 11:09:00 | INFO | Fetching in progress .... -2026-06-22 11:09:01 | INFO | Fetched 17 rows from SQL Server -2026-06-22 11:09:01 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! -2026-06-22 11:09:01 | INFO | Fetched 17 rows -2026-06-22 11:09:01 | INFO | Truncated table Master_VisibilityReason -2026-06-22 11:09:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 11:09:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-22 11:09:01 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-22 11:09:01 | INFO | ================================================================================ -2026-06-22 11:09:01 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 11:09:01 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. -2026-06-22 11:09:01 | INFO | Fetching data from sql server for Master table...... -2026-06-22 11:09:01 | INFO | Fetching in progress .... -2026-06-22 11:09:02 | INFO | Fetched 861 rows from SQL Server -2026-06-22 11:09:02 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! -2026-06-22 11:09:02 | INFO | Fetched 861 rows -2026-06-22 11:09:02 | INFO | Truncated table Master_VisibilityDefinition -2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-22 11:09:02 | INFO | ================================================================================ -2026-06-22 11:09:02 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 11:09:02 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. -2026-06-22 11:09:02 | INFO | Fetching data from sql server for Master table...... -2026-06-22 11:09:02 | INFO | Fetching in progress .... -2026-06-22 11:09:02 | INFO | Fetched 33 rows from SQL Server -2026-06-22 11:09:02 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! -2026-06-22 11:09:02 | INFO | Fetched 33 rows -2026-06-22 11:09:02 | INFO | Truncated table Master_Salesterritorylayer -2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-22 11:09:03 | INFO | ================================================================================ -2026-06-22 11:09:03 | INFO | Pipeline Completed Successfully -2026-06-22 11:09:03 | INFO | ================================================================================ -2026-06-22 12:04:40 | INFO | ================================================================================ -2026-06-22 12:04:40 | INFO | Hello from data-move Python data pipeline! -2026-06-22 12:04:40 | INFO | Pipeline Run Date: 2026-06-21 -2026-06-22 12:04:40 | INFO | Connecting to databases... -2026-06-22 12:04:43 | INFO | -2026-06-22 12:04:44 | INFO | -2026-06-22 12:04:45 | INFO | Database connections established -2026-06-22 12:04:45 | INFO | Collecting MIDs for: 2026-06-21 -2026-06-22 12:04:46 | INFO | Found 569 MIDs -2026-06-22 12:04:48 | INFO | ================================================================================ -2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. -2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table...... -2026-06-22 12:04:48 | INFO | Fetching in progress .... -2026-06-22 12:04:48 | INFO | Fetched 17 rows from SQL Server -2026-06-22 12:04:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! -2026-06-22 12:04:48 | INFO | Fetched 17 rows -2026-06-22 12:04:48 | INFO | Truncated table Master_VisibilityReason -2026-06-22 12:04:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 12:04:48 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse -2026-06-22 12:04:48 | INFO | Master_VisibilityReason loaded successfully (17 rows) -2026-06-22 12:04:48 | INFO | ================================================================================ -2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. -2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table...... -2026-06-22 12:04:48 | INFO | Fetching in progress .... -2026-06-22 12:04:49 | INFO | Fetched 861 rows from SQL Server -2026-06-22 12:04:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! -2026-06-22 12:04:49 | INFO | Fetched 861 rows -2026-06-22 12:04:49 | INFO | Truncated table Master_VisibilityDefinition -2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse -2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) -2026-06-22 12:04:49 | INFO | ================================================================================ -2026-06-22 12:04:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT -2026-06-22 12:04:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. -2026-06-22 12:04:49 | INFO | Fetching data from sql server for Master table...... -2026-06-22 12:04:49 | INFO | Fetching in progress .... -2026-06-22 12:04:49 | INFO | Fetched 33 rows from SQL Server -2026-06-22 12:04:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! -2026-06-22 12:04:49 | INFO | Fetched 33 rows -2026-06-22 12:04:49 | INFO | Truncated table Master_Salesterritorylayer -2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ -2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse -2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) -2026-06-22 12:04:50 | INFO | ================================================================================ -2026-06-22 12:04:50 | INFO | Pipeline Completed Successfully -2026-06-22 12:04:50 | INFO | ================================================================================ + ^ + File "C:\Users\ankitm\AppData\Roaming\uv\python\cpython-3.14-windows-x86_64-none\Lib\logging\__init__.py", line 1532, in warning + self._log(WARNING, msg, args, **kwargs) + ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: Logger._log() got an unexpected keyword argument 'empids' +2026-06-22 12:29:11 | INFO | ================================================================================ +2026-06-22 12:29:11 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:29:11 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:29:11 | INFO | Connecting to databases... +2026-06-22 12:29:12 | INFO | +2026-06-22 12:29:13 | INFO | +2026-06-22 12:29:13 | INFO | Database connections established +2026-06-22 12:29:13 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:29:14 | INFO | Found 569 MIDs +2026-06-22 12:29:14 | INFO | ================================================================================ +2026-06-22 12:29:14 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:29:14 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:29:14 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:29:14 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:29:14 | INFO | Fetching data for 569 MIDs +2026-06-22 12:29:15 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:29:15 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:29:15 | INFO | Fetched 1907 rows +2026-06-22 12:29:15 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:29:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:29:15 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:29:15 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:29:15 | INFO | ================================================================================ +2026-06-22 12:29:15 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:29:15 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:29:16 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:29:27 | INFO | Fetched 291,145 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:29:27 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:29:27 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 248, in fetch_OQaD + log.warning( + ~~~~~~~~~~~^ + "%s Matched df in OQaD returned no rows", + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + empids=[0] + ^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Roaming\uv\python\cpython-3.14-windows-x86_64-none\Lib\logging\__init__.py", line 1532, in warning + self._log(WARNING, msg, args, **kwargs) + ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: Logger._log() got an unexpected keyword argument 'empids' +2026-06-22 12:30:00 | INFO | ================================================================================ +2026-06-22 12:30:00 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:30:00 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:30:00 | INFO | Connecting to databases... +2026-06-22 12:30:02 | INFO | +2026-06-22 12:30:03 | INFO | +2026-06-22 12:30:03 | INFO | Database connections established +2026-06-22 12:30:03 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:30:04 | INFO | Found 569 MIDs +2026-06-22 12:30:04 | INFO | ================================================================================ +2026-06-22 12:30:04 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:30:04 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:30:04 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:30:04 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:30:04 | INFO | Fetching data for 569 MIDs +2026-06-22 12:30:06 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:30:06 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:30:06 | INFO | Fetched 1907 rows +2026-06-22 12:30:06 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:30:06 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:30:06 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:30:06 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:30:06 | INFO | ================================================================================ +2026-06-22 12:30:06 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:30:06 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:30:07 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:30:24 | INFO | Fetched 291,147 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:30:24 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:30:24 | WARNING | OQaD Matched df in OQaD returned no rows +2026-06-22 12:30:24 | INFO | Fetched 1 matched empids fetched for OQAD +2026-06-22 12:30:24 | INFO | Exists: True +2026-06-22 12:30:24 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 12:30:24 | INFO | Fetching data for 1 EMPIDs +2026-06-22 12:30:24 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 12:32:08 | INFO | fn name is fetch_OQad ------Fetched 290730 rows +2026-06-22 12:32:08 | INFO | Fetched 290,730 rows from SQL Server +2026-06-22 12:32:08 | INFO | Fetched total row -: 290730 from sql server for table-:OQaD ...........!!! +2026-06-22 12:32:08 | INFO | Fetched 290730 rows +2026-06-22 12:32:08 | INFO | Creating table OQaD +2026-06-22 12:32:08 | INFO | Table ready: OQaD +2026-06-22 12:32:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:32:12 | INFO | OQaD: inserted 290,730 rows into ClickHouse +2026-06-22 12:32:12 | INFO | OQaD loaded successfully (290730 rows) +2026-06-22 12:32:12 | INFO | ================================================================================ +2026-06-22 12:32:12 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:32:12 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 12:32:12 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:32:12 | INFO | Fetching data for 569 MIDs +2026-06-22 12:32:13 | INFO | Fetched 165 rows from SQL Server +2026-06-22 12:32:13 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 12:32:13 | INFO | Fetched 165 rows +2026-06-22 12:32:13 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 12:32:13 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:32:13 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 12:32:13 | INFO | Survey loaded successfully (165 rows) +2026-06-22 12:32:13 | INFO | ================================================================================ +2026-06-22 12:32:13 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:32:13 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 12:32:13 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:32:13 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:32:13 | INFO | Fetching data for 569 MIDs +2026-06-22 12:32:14 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 12:32:14 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 12:32:14 | INFO | Fetched 1579 rows +2026-06-22 12:32:14 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 12:32:14 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:32:14 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 12:32:14 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 12:32:14 | INFO | ================================================================================ +2026-06-22 12:32:14 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:32:14 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 12:32:14 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:32:14 | INFO | Fetching data for 569 MIDs +2026-06-22 12:32:33 | INFO | Fetched 512 rows from SQL Server +2026-06-22 12:32:33 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 12:32:33 | INFO | Fetched 512 rows +2026-06-22 12:32:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 12:32:33 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:32:33 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 12:32:33 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 12:32:33 | INFO | ================================================================================ +2026-06-22 12:32:33 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:32:33 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 12:32:33 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:32:33 | ERROR | Failed processing table Login +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 443, in fetch_Login + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) +KeyError: 'start_date' +2026-06-22 12:43:20 | INFO | ================================================================================ +2026-06-22 12:43:20 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:43:20 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:43:20 | INFO | Connecting to databases... +2026-06-22 12:43:21 | INFO | +2026-06-22 12:43:22 | INFO | +2026-06-22 12:43:23 | INFO | Database connections established +2026-06-22 12:43:23 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:43:23 | INFO | Found 569 MIDs +2026-06-22 12:43:23 | INFO | ================================================================================ +2026-06-22 12:43:23 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:43:23 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:43:23 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:43:23 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:43:23 | INFO | Fetching data for 569 MIDs +2026-06-22 12:43:25 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:43:25 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:43:25 | INFO | Fetched 1907 rows +2026-06-22 12:43:25 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:43:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:43:26 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:43:26 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:43:26 | INFO | ================================================================================ +2026-06-22 12:43:26 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:43:26 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:43:26 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:43:57 | INFO | Fetched 291,158 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:43:59 | INFO | Fetched 290,722 matched empids fetched for OQAD +2026-06-22 12:43:59 | INFO | Exists: True +2026-06-22 12:43:59 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 12:43:59 | INFO | Fetching data for 290,722 EMPIDs +2026-06-22 12:43:59 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 12:44:40 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-22 12:44:40 | INFO | Fetched 0 rows from SQL Server +2026-06-22 12:44:40 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-22 12:44:40 | WARNING | OQaD returned no rows +2026-06-22 12:44:40 | INFO | ================================================================================ +2026-06-22 12:44:40 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:44:40 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 12:44:40 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:44:40 | INFO | Fetching data for 569 MIDs +2026-06-22 12:44:40 | INFO | Fetched 165 rows from SQL Server +2026-06-22 12:44:40 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 12:44:40 | INFO | Fetched 165 rows +2026-06-22 12:44:40 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 12:44:40 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:44:41 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 12:44:41 | INFO | Survey loaded successfully (165 rows) +2026-06-22 12:44:41 | INFO | ================================================================================ +2026-06-22 12:44:41 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:44:41 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 12:44:41 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:44:41 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:44:41 | INFO | Fetching data for 569 MIDs +2026-06-22 12:44:41 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 12:44:41 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 12:44:41 | INFO | Fetched 1579 rows +2026-06-22 12:44:41 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 12:44:41 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:44:42 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 12:44:42 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 12:44:42 | INFO | ================================================================================ +2026-06-22 12:44:42 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:44:42 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 12:44:42 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:44:42 | INFO | Fetching data for 569 MIDs +2026-06-22 12:44:58 | INFO | Fetched 512 rows from SQL Server +2026-06-22 12:44:58 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 12:44:58 | INFO | Fetched 512 rows +2026-06-22 12:44:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 12:44:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:44:59 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 12:44:59 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 12:44:59 | INFO | ================================================================================ +2026-06-22 12:44:59 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:44:59 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 12:44:59 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:44:59 | INFO | Fetching data for 569 MIDs +2026-06-22 12:45:02 | INFO | Fetched 454 rows from SQL Server +2026-06-22 12:45:02 | INFO | Fetched total row -: 454 from sql server for table-:Login ...........!!! +2026-06-22 12:45:02 | INFO | Fetched 454 rows +2026-06-22 12:45:02 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-22 12:45:02 | INFO | No delete logic required for Login +2026-06-22 12:45:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:45:03 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-22 12:45:03 | INFO | Login loaded successfully (454 rows) +2026-06-22 12:45:03 | INFO | ================================================================================ +2026-06-22 12:45:03 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:45:03 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-22 12:45:03 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:45:03 | INFO | Fetching data for 569 MIDs +2026-06-22 12:45:19 | INFO | Fetched 27,359 rows from SQL Server +2026-06-22 12:45:19 | INFO | Fetched total row -: 27359 from sql server for table-:Stock_Details ...........!!! +2026-06-22 12:45:19 | INFO | Fetched 27359 rows +2026-06-22 12:45:19 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-22 12:45:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:45:19 | INFO | Stock_Details: inserted 27,359 rows into ClickHouse +2026-06-22 12:45:19 | INFO | Stock_Details loaded successfully (27359 rows) +2026-06-22 12:45:19 | INFO | ================================================================================ +2026-06-22 12:45:19 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-22 12:45:19 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-22 12:45:19 | INFO | Fetching Attendance data from 2026-06-06 to 2026-06-21 +2026-06-22 12:47:52 | INFO | Fetched 121,639 attendance rows for 544 employees +2026-06-22 12:47:52 | INFO | Fetched total row -: 121639 from sql server for table-:Attendance ...........!!! +2026-06-22 12:47:52 | INFO | Fetched 121639 rows +2026-06-22 12:47:52 | INFO | Truncated table Attendance +2026-06-22 12:47:52 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:47:54 | INFO | Attendance: inserted 121,639 rows into ClickHouse +2026-06-22 12:47:54 | INFO | Attendance loaded successfully (121639 rows) +2026-06-22 12:47:54 | INFO | ================================================================================ +2026-06-22 12:47:54 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:47:54 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-22 12:47:54 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:47:54 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:47:54 | INFO | Fetching data for 569 MIDs +2026-06-22 12:48:07 | INFO | Fetched 21,621 rows from SQL Server +2026-06-22 12:48:07 | INFO | Fetched total row -: 21621 from sql server for table-:Journey_Plan ...........!!! +2026-06-22 12:48:07 | INFO | Fetched 21621 rows +2026-06-22 12:48:07 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-22 12:48:07 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:48:08 | INFO | Journey_Plan: inserted 21,621 rows into ClickHouse +2026-06-22 12:48:08 | INFO | Journey_Plan loaded successfully (21621 rows) +2026-06-22 12:48:08 | INFO | ================================================================================ +2026-06-22 12:48:08 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:48:08 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-22 12:48:08 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:48:08 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:48:08 | INFO | Fetching data for 569 MIDs +2026-06-22 12:48:08 | INFO | Fetched 12 rows from SQL Server +2026-06-22 12:48:08 | INFO | Fetched total row -: 12 from sql server for table-:Web_Logins ...........!!! +2026-06-22 12:48:08 | INFO | Fetched 12 rows +2026-06-22 12:48:08 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-22 12:48:09 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:48:09 | INFO | Web_Logins: inserted 12 rows into ClickHouse +2026-06-22 12:48:09 | INFO | Web_Logins loaded successfully (12 rows) +2026-06-22 12:48:09 | INFO | ================================================================================ +2026-06-22 12:48:09 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:48:09 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 12:48:09 | ERROR | Failed processing table Promotion +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 158, in main + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Promotion' +2026-06-22 13:04:30 | INFO | ================================================================================ +2026-06-22 13:04:30 | INFO | Hello from data-move Python data pipeline! +2026-06-22 13:04:30 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 13:04:30 | INFO | Connecting to databases... +2026-06-22 13:04:30 | INFO | +2026-06-22 13:04:31 | INFO | +2026-06-22 13:04:32 | INFO | Database connections established +2026-06-22 13:04:32 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 13:04:32 | INFO | Found 569 MIDs +2026-06-22 13:04:32 | INFO | ================================================================================ +2026-06-22 13:04:32 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:04:32 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 13:04:32 | INFO | Fetching data for 569 MIDs +2026-06-22 13:04:36 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 13:04:36 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 13:04:36 | INFO | Fetched 4223 rows +2026-06-22 13:04:36 | INFO | Creating table Promotion +2026-06-22 13:04:36 | INFO | Table ready: Promotion +2026-06-22 13:04:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:04:36 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 13:04:36 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 13:04:36 | INFO | ================================================================================ +2026-06-22 13:04:36 | INFO | Pipeline Completed Successfully +2026-06-22 13:04:36 | INFO | ================================================================================ +2026-06-22 13:04:41 | INFO | ================================================================================ +2026-06-22 13:04:41 | INFO | Hello from data-move Python data pipeline! +2026-06-22 13:04:41 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 13:04:41 | INFO | Connecting to databases... +2026-06-22 13:04:42 | INFO | +2026-06-22 13:04:43 | INFO | +2026-06-22 13:04:44 | INFO | Database connections established +2026-06-22 13:04:44 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 13:04:44 | INFO | Found 569 MIDs +2026-06-22 13:04:44 | INFO | ================================================================================ +2026-06-22 13:04:44 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:04:44 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 13:04:44 | INFO | Fetching data for 569 MIDs +2026-06-22 13:04:54 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 13:04:54 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 13:04:54 | INFO | Fetched 4223 rows +2026-06-22 13:04:54 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-22 13:04:54 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:04:55 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 13:04:55 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 13:04:55 | INFO | ================================================================================ +2026-06-22 13:04:55 | INFO | Pipeline Completed Successfully +2026-06-22 13:04:55 | INFO | ================================================================================ +2026-06-22 13:05:14 | INFO | ================================================================================ +2026-06-22 13:05:14 | INFO | Hello from data-move Python data pipeline! +2026-06-22 13:05:14 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 13:05:14 | INFO | Connecting to databases... +2026-06-22 13:05:15 | INFO | +2026-06-22 13:05:16 | INFO | +2026-06-22 13:05:17 | INFO | Database connections established +2026-06-22 13:05:17 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 13:05:17 | INFO | Found 569 MIDs +2026-06-22 13:05:17 | INFO | ================================================================================ +2026-06-22 13:05:17 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:05:17 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 13:05:17 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:05:17 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:05:17 | INFO | Fetching data for 569 MIDs +2026-06-22 13:05:19 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 13:05:19 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 13:05:19 | INFO | Fetched 1907 rows +2026-06-22 13:05:19 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 13:05:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:05:20 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 13:05:20 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 13:05:20 | INFO | ================================================================================ +2026-06-22 13:05:20 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:05:20 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 13:05:20 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 13:05:32 | INFO | Fetched 291,165 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 13:05:34 | INFO | Fetched 290,722 matched empids fetched for OQAD +2026-06-22 13:05:34 | INFO | Exists: True +2026-06-22 13:05:34 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 13:05:34 | INFO | Fetching data for 290,722 EMPIDs +2026-06-22 13:05:34 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 13:06:10 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-22 13:06:10 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:06:10 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-22 13:06:10 | WARNING | OQaD returned no rows +2026-06-22 13:06:10 | INFO | ================================================================================ +2026-06-22 13:06:10 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:10 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 13:06:10 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:10 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:10 | INFO | Fetched 165 rows from SQL Server +2026-06-22 13:06:10 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 13:06:10 | INFO | Fetched 165 rows +2026-06-22 13:06:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 13:06:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:11 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 13:06:11 | INFO | Survey loaded successfully (165 rows) +2026-06-22 13:06:11 | INFO | ================================================================================ +2026-06-22 13:06:11 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:11 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 13:06:11 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:06:11 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:11 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:11 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 13:06:11 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 13:06:11 | INFO | Fetched 1579 rows +2026-06-22 13:06:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 13:06:12 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:12 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 13:06:12 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 13:06:12 | INFO | ================================================================================ +2026-06-22 13:06:12 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:12 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 13:06:12 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:12 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:27 | INFO | Fetched 512 rows from SQL Server +2026-06-22 13:06:27 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 13:06:27 | INFO | Fetched 512 rows +2026-06-22 13:06:27 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 13:06:27 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:28 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 13:06:28 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 13:06:28 | INFO | ================================================================================ +2026-06-22 13:06:28 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:06:28 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 13:06:28 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:28 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:29 | INFO | Fetched 454 rows from SQL Server +2026-06-22 13:06:29 | INFO | Fetched total row -: 454 from sql server for table-:Login ...........!!! +2026-06-22 13:06:29 | INFO | Fetched 454 rows +2026-06-22 13:06:29 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-22 13:06:29 | INFO | No delete logic required for Login +2026-06-22 13:06:29 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:30 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-22 13:06:30 | INFO | Login loaded successfully (454 rows) +2026-06-22 13:06:30 | INFO | ================================================================================ +2026-06-22 13:06:30 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:30 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-22 13:06:30 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:30 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:35 | INFO | Fetched 27,359 rows from SQL Server +2026-06-22 13:06:35 | INFO | Fetched total row -: 27359 from sql server for table-:Stock_Details ...........!!! +2026-06-22 13:06:35 | INFO | Fetched 27359 rows +2026-06-22 13:06:35 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-22 13:06:35 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:35 | INFO | Stock_Details: inserted 27,359 rows into ClickHouse +2026-06-22 13:06:35 | INFO | Stock_Details loaded successfully (27359 rows) +2026-06-22 13:06:35 | INFO | ================================================================================ +2026-06-22 13:06:35 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-22 13:06:35 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-22 13:06:35 | INFO | Fetching Attendance data from 2026-06-06 to 2026-06-21 +2026-06-22 13:06:51 | INFO | Fetched 121,639 attendance rows for 544 employees +2026-06-22 13:06:51 | INFO | Fetched total row -: 121639 from sql server for table-:Attendance ...........!!! +2026-06-22 13:06:51 | INFO | Fetched 121639 rows +2026-06-22 13:06:51 | INFO | Truncated table Attendance +2026-06-22 13:06:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:56 | INFO | Attendance: inserted 121,639 rows into ClickHouse +2026-06-22 13:06:56 | INFO | Attendance loaded successfully (121639 rows) +2026-06-22 13:06:56 | INFO | ================================================================================ +2026-06-22 13:06:56 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:06:56 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-22 13:06:56 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:06:56 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:56 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:57 | INFO | Fetched 21,625 rows from SQL Server +2026-06-22 13:06:57 | INFO | Fetched total row -: 21625 from sql server for table-:Journey_Plan ...........!!! +2026-06-22 13:06:57 | INFO | Fetched 21625 rows +2026-06-22 13:06:57 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-22 13:06:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:57 | INFO | Journey_Plan: inserted 21,625 rows into ClickHouse +2026-06-22 13:06:57 | INFO | Journey_Plan loaded successfully (21625 rows) +2026-06-22 13:06:57 | INFO | ================================================================================ +2026-06-22 13:06:57 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:06:57 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-22 13:06:57 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:06:57 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:57 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:58 | INFO | Fetched 12 rows from SQL Server +2026-06-22 13:06:58 | INFO | Fetched total row -: 12 from sql server for table-:Web_Logins ...........!!! +2026-06-22 13:06:58 | INFO | Fetched 12 rows +2026-06-22 13:06:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-22 13:06:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:58 | INFO | Web_Logins: inserted 12 rows into ClickHouse +2026-06-22 13:06:58 | INFO | Web_Logins loaded successfully (12 rows) +2026-06-22 13:06:58 | INFO | ================================================================================ +2026-06-22 13:06:58 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:58 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 13:06:58 | INFO | Fetching data for 569 MIDs +2026-06-22 13:07:00 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 13:07:00 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 13:07:00 | INFO | Fetched 4223 rows +2026-06-22 13:07:00 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-22 13:07:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:01 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 13:07:01 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 13:07:01 | INFO | ================================================================================ +2026-06-22 13:07:01 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:07:01 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-22 13:07:01 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:07:01 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:07:01 | INFO | Fetching data for 569 MIDs +2026-06-22 13:07:02 | INFO | Fetched 1,284 rows from SQL Server +2026-06-22 13:07:02 | INFO | Fetched total row -: 1284 from sql server for table-:PaidVisibility ...........!!! +2026-06-22 13:07:02 | INFO | Fetched 1284 rows +2026-06-22 13:07:02 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-22 13:07:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:03 | INFO | PaidVisibility: inserted 1,284 rows into ClickHouse +2026-06-22 13:07:03 | INFO | PaidVisibility loaded successfully (1284 rows) +2026-06-22 13:07:03 | INFO | ================================================================================ +2026-06-22 13:07:03 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:07:03 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-22 13:07:03 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:03 | INFO | Fetching in progress .... +2026-06-22 13:07:05 | INFO | Fetched 5,998 rows from SQL Server +2026-06-22 13:07:05 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-22 13:07:05 | INFO | Fetched 5998 rows +2026-06-22 13:07:05 | INFO | Truncated table Store_Master +2026-06-22 13:07:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:07 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-22 13:07:07 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-22 13:07:07 | INFO | ================================================================================ +2026-06-22 13:07:07 | INFO | Processing Table: SKU_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:07:07 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-22 13:07:07 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:07 | INFO | Fetching in progress .... +2026-06-22 13:07:07 | INFO | Fetched 160 rows from SQL Server +2026-06-22 13:07:07 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-22 13:07:07 | INFO | Fetched 160 rows +2026-06-22 13:07:07 | INFO | Truncated table SKU_Master +2026-06-22 13:07:07 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:07 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-22 13:07:07 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-22 13:07:07 | INFO | ================================================================================ +2026-06-22 13:07:07 | INFO | Processing Table: display_master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:07:07 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-22 13:07:07 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:07 | INFO | Fetching in progress .... +2026-06-22 13:07:07 | INFO | Fetched 135 rows from SQL Server +2026-06-22 13:07:07 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-22 13:07:07 | INFO | Fetched 135 rows +2026-06-22 13:07:07 | INFO | Truncated table display_master +2026-06-22 13:07:07 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:07 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-22 13:07:07 | INFO | display_master loaded successfully (135 rows) +2026-06-22 13:07:07 | INFO | ================================================================================ +2026-06-22 13:07:07 | INFO | Processing Table: Employee_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:07:07 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-22 13:07:07 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:07 | INFO | Fetching in progress .... +2026-06-22 13:07:08 | INFO | Fetched 2,288 rows from SQL Server +2026-06-22 13:07:08 | INFO | Fetched total row -: 2288 from sql server for table-:Employee_Master ...........!!! +2026-06-22 13:07:08 | INFO | Fetched 2288 rows +2026-06-22 13:07:08 | INFO | Truncated table Employee_Master +2026-06-22 13:07:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:08 | INFO | Employee_Master: inserted 2,288 rows into ClickHouse +2026-06-22 13:07:08 | INFO | Employee_Master loaded successfully (2288 rows) +2026-06-22 13:07:08 | INFO | ================================================================================ +2026-06-22 13:07:08 | INFO | Processing Table: coverage_remarks | Table type is -: DIMENSION | Based on -reason_id and operation is used -DELETE+INSERT +2026-06-22 13:07:08 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-22 13:07:09 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:09 | INFO | Fetching in progress .... +2026-06-22 13:07:09 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:07:09 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:07:09 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-22 13:07:09 | WARNING | coverage_remarks returned no rows +2026-06-22 13:07:09 | INFO | ================================================================================ +2026-06-22 13:07:09 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-22 13:07:09 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-22 13:07:10 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-22 13:07:10 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:10 | INFO | Fetching in progress .... +2026-06-22 13:07:10 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:07:10 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:07:10 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-22 13:07:10 | WARNING | mapping_storevisibility returned no rows +2026-06-22 13:07:10 | INFO | ================================================================================ +2026-06-22 13:07:10 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:07:10 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 13:07:10 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:10 | INFO | Fetching in progress .... +2026-06-22 13:07:10 | INFO | Fetched 17 rows from SQL Server +2026-06-22 13:07:10 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 13:07:10 | INFO | Fetched 17 rows +2026-06-22 13:07:11 | INFO | Truncated table Master_VisibilityReason +2026-06-22 13:07:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:11 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 13:07:11 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 13:07:11 | INFO | ================================================================================ +2026-06-22 13:07:11 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:07:11 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 13:07:11 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:11 | INFO | Fetching in progress .... +2026-06-22 13:07:11 | INFO | Fetched 861 rows from SQL Server +2026-06-22 13:07:11 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 13:07:11 | INFO | Fetched 861 rows +2026-06-22 13:07:11 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 13:07:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:11 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 13:07:11 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 13:07:11 | INFO | ================================================================================ +2026-06-22 13:07:11 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:07:11 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 13:07:11 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:11 | INFO | Fetching in progress .... +2026-06-22 13:07:11 | INFO | Fetched 33 rows from SQL Server +2026-06-22 13:07:11 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 13:07:11 | INFO | Fetched 33 rows +2026-06-22 13:07:11 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 13:07:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:12 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 13:07:12 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 13:07:12 | INFO | ================================================================================ +2026-06-22 13:07:12 | INFO | Pipeline Completed Successfully +2026-06-22 13:07:12 | INFO | ================================================================================ diff --git a/main.py b/main.py deleted file mode 100644 index 13343eb..0000000 --- a/main.py +++ /dev/null @@ -1,276 +0,0 @@ -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "polars>=0.20.0", -# "pyarrow>=18.0.0", -# "sqlalchemy>=2.0.0", -# "pyodbc>=5.0.0", -# "clickhouse-connect>=0.7.0", -# "clickhouse-sqlalchemy>=0.3.2", -# "pyyaml>=6.0.3", -# "python-dotenv>=1.0.0", -# ] -# /// - - -from __future__ import annotations - -import sys -from datetime import date, datetime, timedelta - -import polars as pl -import yaml - - -from log import log - -from clickhouse_task.create_table import create_clickhouse_table -from clickhouse_task.delete_task import ( - delete_existing_data, - truncate_table, -) - -from clickhouse_task.load_table import load_to_clickhouse - -from db_con.connection import ( - build_sql_server_engine, - build_clickhouse_engine, - get_clickhouse_client, -) - -from mids import ( - MID_TABLE_COV, - MID_TABLE_COV1, -) - -from masters.dimensions import * -from masters.bridge import * -from kpi.facts import * - - -# ========================================================== -# Helpers -# ========================================================== - -def table_exists( - client, - table_name: str, -) -> bool: - - return bool( - client.command( - f"EXISTS TABLE {table_name}" - ) - ) - - -def get_dataframe( - fn_name: str, - fetch_by: str, - sql_engine, - mids, - run_date, -): - - fn = globals()[fn_name] - - if fetch_by == "mids": - return fn(sql_engine, mids) - - if fetch_by == "run_date": - return fn(sql_engine, run_date) - - return fn(sql_engine) - - -# ========================================================== -# Main -# ========================================================== - -def main(): - - log.info("=" * 80) - log.info("Hello from data-move Python data pipeline!") - - # ------------------------------------------------------ - # Run Date - # ------------------------------------------------------ - - if len(sys.argv) > 1: - run_date = datetime.strptime( - sys.argv[1], - "%Y-%m-%d", - ).date() - else: - run_date = date.today() - timedelta(days=1) - - log.info( - "Pipeline Run Date: %s", - run_date, - ) - - # ------------------------------------------------------ - # Connections - # ------------------------------------------------------ - - log.info( - "Connecting to databases..." - ) - - sql_engine = build_sql_server_engine() - clickhouse_engine = build_clickhouse_engine() - client = get_clickhouse_client() - - log.info( - "Database connections established" - ) - - # ------------------------------------------------------ - # Delete Keys - # ------------------------------------------------------ - - mids = MID_TABLE_COV( - sql_engine, - run_date, - ) - - emp_visit_df = MID_TABLE_COV1( - sql_engine, - run_date, - ) - - # ------------------------------------------------------ - # Config - # ------------------------------------------------------ - - with open( - "t.yml", - "r", - ) as file: - - config = yaml.safe_load(file) - - # ------------------------------------------------------ - # Process Tables - # ------------------------------------------------------ - - for table in config["tables"]: - - table_name = table["name"] - operation = table["operation"] - fetch_by = table["fetch_by"] - table_type=table["type"] - - log.info("=" * 80) - log.info( - "Processing Table: %s", - table_name, - ) - - try: - - # ------------------------------------------ - # Fetch Data - # ------------------------------------------ - - fn_name = f"fetch_{table_name}" - - df = get_dataframe( - fn_name=fn_name, - fetch_by=fetch_by, - sql_engine=sql_engine, - mids=mids, - run_date=run_date, - ) - - if df.is_empty(): - - log.warning( - "%s returned no rows", - table_name, - ) - - continue - - log.info( - "Fetched %s rows", - len(df), - ) - - # ------------------------------------------ - # Create Table If Missing - # ------------------------------------------ - - exists = table_exists( - client, - table_name, - ) - - if not exists: - - log.info( - "Creating table %s", - table_name, - ) - - create_clickhouse_table( - df=df, - table_name=table_name, - clickhouse_engine=clickhouse_engine, - ) - - # ------------------------------------------ - # Existing Table Logic - # ------------------------------------------ - - else: - - if operation == "DELETE+INSERT": - - truncate_table( - client, - table_name, - ) - - else: - - delete_existing_data( - client=client, - table_name=table_name, - run_date=run_date, - mids=mids, - emp_visit_df=emp_visit_df, - ) - - # ------------------------------------------ - # Load Data - # ------------------------------------------ - - load_to_clickhouse( - client=client, - table_name=table_name, - df=df, - ) - - log.info( - "%s loaded successfully (%s rows)", - table_name, - len(df), - ) - - except Exception: - - log.exception( - "Failed processing table %s", - table_name, - ) - - raise - - log.info("=" * 80) - log.info("Pipeline Completed Successfully") - log.info("=" * 80) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/main2.py b/main2.py index 54ad901..c3f079b 100644 --- a/main2.py +++ b/main2.py @@ -126,7 +126,7 @@ def main(): # ------------------------------------------------------ with open( - "y.yml", + "t.yml", "r", ) as file: diff --git a/masters/bridge.py b/masters/bridge.py deleted file mode 100644 index 3d6722b..0000000 --- a/masters/bridge.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -# import pyarrow -import sys -import logging -from datetime import date, timedelta -import polars as pl -from sqlalchemy import create_engine, text -from sqlalchemy.engine import Engine, URL -import clickhouse_connect -from dotenv import load_dotenv - -from log import log -from clickhouse_task.create_table import * -from db_con.connection import * -from mids import * - - - - - - - - - -def fetch_mapping_storevisibility( - engine: Engine, - run_date: date -) -> pl.DataFrame: - - sql = f""" - SELECT DISTINCT - 40148 AS project_id, - Z.StoreId AS store_id, - Z.VisibilityDefinitionId AS visibility_definition_id, - Z.FromDate AS from_date, - Z.ToDate AS to_date - - FROM OneApp_KelloggsMT.dbo.Mapping_StoreVisibility Z - - WHERE CAST(Z.FromDate AS DATE) <= '{run_date}' - AND CAST(Z.ToDate AS DATE) >= '{run_date}' - - AND Z.VisibilityDefinitionId IN - ( - SELECT DISTINCT VisibilityDefinitionId - FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition - WHERE MenuId = 22 - ) - """ - - log.info( - f"Fetching Mapping Store Visibility for {run_date}" - ) - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Mapping Store Visibility records" - ) - - return df \ No newline at end of file diff --git a/masters/dimensions.py b/masters/dimensions.py deleted file mode 100644 index 9f63bf8..0000000 --- a/masters/dimensions.py +++ /dev/null @@ -1,515 +0,0 @@ -import os -# import pyarrow -import sys -import logging -from datetime import date, timedelta -import polars as pl -from sqlalchemy import create_engine, text -from sqlalchemy.engine import Engine, URL -import clickhouse_connect -from dotenv import load_dotenv - -from log import log -from clickhouse_task.create_table import * -from db_con.connection import * -from mids import * - - - -#PROJECT_ID = 40148 -p=40148 - - - - - - - - - -def fetch_Store_Master(engine: Engine) -> pl.DataFrame: - sql = """ - SELECT - - RegionName AS region, - StateId AS state_id, - StateName AS state, - CityId AS city_id, - CityName AS city, - CityCode AS cpm_city_id, - - ChannelId AS channel_id, - ChannelName AS channel, - - DistributorId AS distributor_id, - Distributor AS distributor_name, - - ChainId AS keyaccount_id, - ChainName AS keyaccount, - - StoreUniqueCode AS insight_store_id, - StoreCode AS client_store_code, - - Latitude AS latitude, - Longitude AS longitude, - - StoreCategoryId AS store_category_id, - StoreCategory AS store_category, - - StoreTypeId AS store_type_id, - StoreType AS store_type, - - StoreClassId AS store_classification_id, - StoreClass AS store_classification, - - StLayerFourId, - - StoreId AS store_id, - StoreName AS store_name, - Address AS address - - FROM OneApp_KelloggsMT.dbo.vw_storedetail - """ - - log.info("Fetching Store Master data") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} stores" - ) - - return df - - - - -def fetch_SKU_Master(engine: Engine) -> pl.DataFrame: - - sql = """ - SELECT - CM.CategoryId AS category_id, - CM.CategoryCode AS category_code, - CM.CategoryName AS category_name, - - SCA.SubCategoryId AS sub_category_id, - SCA.SubCategoryCode AS sub_category_code, - SCA.SubCategoryName AS sub_category_name, - - BR.BrandId AS brand_id, - BR.BrandCode AS brand_code, - BR.BrandName AS brand_name, - - SB.SubBrandId AS sub_brand_id, - SB.SubBrandCode AS sub_brand_code, - SB.SubBrandName AS sub_brand_name, - - P.ProductId AS product_id, - P.ProductName AS product_name, - P.ProductCode AS product_code, - - P.MRP AS mrp, - - FL.FlavourId AS flavour_id, - FL.Flavour AS flavour, - - P.Grammage AS grammage, - P.ProductSequence AS product_sequence, - P.CaseSize AS case_size, - - MC.Company AS company_name, - MC.IsCompetitor AS is_competitor, - - P.PTR AS ptr - - FROM OneApp_KelloggsMT.dbo.Master_Product P - - RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Flavour FL - ON P.FlavourId = FL.FlavourId - - RIGHT JOIN OneApp_KelloggsMT.dbo.Master_SubBrand SB - ON P.SubBrandId = SB.SubBrandId - - RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Brand BR - ON SB.BrandId = BR.BrandId - - RIGHT JOIN OneApp_KelloggsMT.dbo.Master_SubCategory SCA - ON BR.SubCategoryId = SCA.SubCategoryId - - RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Category CM - ON SCA.CategoryId = CM.CategoryId - - RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Company MC - ON MC.CompanyId = BR.CompanyId - """ - - log.info("Fetching SKU Master data") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} SKU Master rows" - ) - - return df - - - - -def fetch_display_master(engine: Engine) -> pl.DataFrame: - """ - Fetch Display Master data. - Source: Master_Display - Target: display_master - """ - - sql = """ - SELECT - DisplayId AS display_id, - DisplayCode AS display_code, - DisplayName AS display_name, - DisplayRefImage AS display_ref_url - FROM OneApp_KelloggsMT.dbo.Master_Display - """ - - log.info("Fetching Display Master data") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Display Master records" - ) - - return df - - - -def fetch_Employee_Master(engine: Engine) -> pl.DataFrame: - """ - Fetch Employee Master data. - Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position - Target: Employee_Master - """ - - sql = """ - SELECT - RegionId AS region_id, - RegionName AS region, - - StateId AS state_id, - StateName AS state, - - CityId AS city_id, - CityName AS city, - - A.EmpId AS employee_id, - EmpName AS employee_name, - Gender AS gender, - - A.DesignationId AS designation_id, - DesignationName AS designation, - - SupervisorId AS manager_id, - SupervisorName AS manager_name, - - JoinDate AS employee_joining_date, - ResignDate AS employee_resign_date, - - C.PositionCode AS position_code, - - EmpCode AS employee_legacy_code, - - RIGHTNAME AS employee_role, - - CASE - WHEN RIGHTNAME IN ('Client', 'Client HO') - THEN 'NON CPM' - ELSE 'CPM' - END AS employee_type - - FROM OneApp_KelloggsMT.dbo.vw_Employee_Detail A - - LEFT JOIN - ( - SELECT DISTINCT - PositionId, - EmpId - FROM OneApp_KelloggsMT.dbo.Mapping_PositionUser - WHERE DATEDIFF(DAY, FromDate, GETDATE()) >= 0 - AND DATEDIFF(DAY, ToDate, GETDATE()) <= 0 - ) B - ON A.EmpId = B.EmpId - - LEFT JOIN - ( - SELECT - PositionId, - PositionCode - FROM OneApp_KelloggsMT.dbo.Master_Position - ) C - ON B.PositionId = C.PositionId - """ - - log.info("Fetching Employee Master data") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Employee Master records" - ) - - return df - - - - - -def fetch_Employee_Master(engine: Engine) -> pl.DataFrame: - """ - Fetch Employee Master data. - Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position - Target: Employee_Master - """ - - sql = """ - SELECT - RegionId AS region_id, - RegionName AS region, - - StateId AS state_id, - StateName AS state, - - CityId AS city_id, - CityName AS city, - - A.EmpId AS employee_id, - EmpName AS employee_name, - Gender AS gender, - - A.DesignationId AS designation_id, - DesignationName AS designation, - - SupervisorId AS manager_id, - SupervisorName AS manager_name, - - JoinDate AS employee_joining_date, - ResignDate AS employee_resign_date, - - C.PositionCode AS position_code, - - EmpCode AS employee_legacy_code, - - RIGHTNAME AS employee_role, - - CASE - WHEN RIGHTNAME IN ('Client', 'Client HO') - THEN 'NON CPM' - ELSE 'CPM' - END AS employee_type - - FROM OneApp_KelloggsMT.dbo.vw_Employee_Detail A - - LEFT JOIN - ( - SELECT DISTINCT - PositionId, - EmpId - FROM OneApp_KelloggsMT.dbo.Mapping_PositionUser - WHERE DATEDIFF(DAY, FromDate, GETDATE()) >= 0 - AND DATEDIFF(DAY, ToDate, GETDATE()) <= 0 - ) B - ON A.EmpId = B.EmpId - - LEFT JOIN - ( - SELECT - PositionId, - PositionCode - FROM OneApp_KelloggsMT.dbo.Master_Position - ) C - ON B.PositionId = C.PositionId - """ - - log.info("Fetching Employee Master data") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Employee Master records" - ) - - return df - - - -def fetch_Master_VisibilityReason(engine: Engine) -> pl.DataFrame: - """ - Source: - Master_VisibilityReason - - Target: - Master_VisibilityReason - """ - - sql = """ - SELECT DISTINCT - 40148 AS project_id, - MenuId AS menu_id, - VisibilityReasonId AS reason_id, - VisibilityReason AS reason - FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason - """ - - log.info("Fetching Master Visibility Reason data") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Master Visibility Reason records" - ) - - return df - - - -def fetch_Master_VisibilityDefinition(engine: Engine) -> pl.DataFrame: - """ - Source: - OneApp_KelloggsMT.dbo.Master_VisibilityDefinition - - Target: - Master_VisibilityDefinition - """ - - sql = """ - SELECT DISTINCT - 40148 AS project_id, - VisibilityDefinitionId AS visibility_definition_id, - VisibilityDefinitionName AS visibility_definition_name, - GETDATE() AS create_date, - 'SP-Pius' AS create_by - FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition - """ - - log.info("Fetching Master Visibility Definition data") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Master Visibility Definition records" - ) - - return df - - - - -def fetch_Master_Salesterritorylayer( - engine: Engine -) -> pl.DataFrame: - """ - Source: - Master_SalesTerritoryLayerOne - Master_SalesTerritoryLayerTwo - Master_SalesTerritoryLayerThree - Master_SalesTerritoryLayerFour - - Target: - Master_Salesterritorylayer - """ - - sql = """ - SELECT DISTINCT - 40148 AS project_id, - - D.StLayerOneId AS st_layer_one_id, - D.StLayerOneName AS st_layer_one_name, - - C.StLayerTwoId AS st_layer_two_id, - C.StLayerTwoName AS st_layer_two_name, - - B.StLayerThreeId AS st_layer_three_id, - B.StLayerThreeName AS st_layer_three_name, - - A.StLayerFourId AS st_layer_four_id, - A.StLayerFourName AS st_layer_four_name, - - GETDATE() AS create_date, - 'SP-Pius' AS create_by - - FROM OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerFour A - - INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerThree B - ON A.StLayerThreeId = B.StLayerThreeId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerTwo C - ON B.StLayerTwoId = C.StLayerTwoId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerOne D - ON C.StLayerOneId = D.StLayerOneId - """ - - log.info("Fetching Master Sales Territory Layer data") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Master Sales Territory Layer records" - ) - - return df - - -def fetch_coverage_remarks(engine: Engine ) -> pl.DataFrame: - """ - Source: - OneApp_KelloggsMT.dbo.Master_NonWorkingReason - - Target: - coverage_remarks - """ - - sql = """ - SELECT - 40148 AS project_id, - ReasonId AS reason_id, - Reason AS reason_remarks - FROM OneApp_KelloggsMT.dbo.Master_NonWorkingReason - """ - - log.info("Fetching Coverage Remarks") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Coverage Remark records" - ) - - return df diff --git a/src/fact.py b/src/fact.py index cee6fae..dcf2f21 100644 --- a/src/fact.py +++ b/src/fact.py @@ -242,7 +242,18 @@ def fetch_OQaD( on=["empid", "visitdate"], how="inner", ) - empids=matched["empid"].to_list() + + if matched.is_empty(): + + empids=[0] + log.warning( + "%s Matched df in OQaD returned no rows", + table_name, + ) + + else: + empids=matched["empid"].to_list() + log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") @@ -418,9 +429,9 @@ def fetch_Login( engine: Engine, - if not mids: - log.warning("No MIDs — nothing to fetch.") - return pl.DataFrame() + # if not mids: + # log.warning("No MIDs — nothing to fetch.") + # return pl.DataFrame() mid_list = ",".join(str(mid) for mid in mids) log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") @@ -430,7 +441,7 @@ def fetch_Login( engine: Engine, sql_template = f.read() sql = sql_template.format( - mid_list=mid_list, + # mid_list=mid_list, run_date=run_date.strftime("%Y-%m-%d") ) log.info(f"Fetching data for {len(mids):,} MIDs") @@ -673,4 +684,40 @@ def fetch_Web_Logins( engine: Engine, ) log.info(f"Fetched {len(df):,} rows from SQL Server") + return df + + +def fetch_Promotion( + engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + mid_list = ",".join(str(mid) for mid in mids) + + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info(f"Fetched {len(df):,} rows from SQL Server") + return df \ No newline at end of file diff --git a/src/sql/fact/Login.sql b/src/sql/fact/Login.sql index 9660fe7..7fd83dd 100644 --- a/src/sql/fact/Login.sql +++ b/src/sql/fact/Login.sql @@ -12,7 +12,7 @@ with employee_detail AS INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId WHERE em.RightId IN (6) - AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{start_date}' AS DATE)) + AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{run_date}' AS DATE)) AND em.EmployeeName NOT LIKE '%test%' ), Login ( @@ -71,7 +71,7 @@ Login ( ) AS [Last Store Out Time] FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId - WHERE CAST(ud.LoginDate AS DATE) = CAST('{start_date}' AS DATE) -- fixed: direct date comparison + WHERE CAST(ud.LoginDate AS DATE) = CAST('{run_date}' AS DATE) -- fixed: direct date comparison ) AS T1 WHERE T1.col = 1 ) AS T ON Em.[Employee code] = T.[Employee Code] diff --git a/src/sql/fact/Promotion.sql b/src/sql/fact/Promotion.sql index 05b5d8e..4775b9f 100644 --- a/src/sql/fact/Promotion.sql +++ b/src/sql/fact/Promotion.sql @@ -1,10 +1,11 @@ -with Promotion(Mid, +with Promotion (Mid, project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,promo_definition_id,promo_definition_name, promotion_deatils,promotion_deatils_id,promotion_value_name, present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by) - as ( - Select - sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId, + as ( + +Select +sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId, MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category' when isnull(TS.PromoTable,'')='Master_SubCategory' then 'SubCategory' when isnull(TS.PromoTable,'')='Master_Brand' then 'Brand' @@ -15,14 +16,14 @@ with Promotion(Mid, when isnull(TS.PromoTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.PromoValue=a.SubCategoryId ) when isnull(TS.PromoTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.PromoValue=a.BrandId ) when isnull(TS.PromoTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.PromoValue=a.SubBrandId ) end as - [PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present, - case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question, - ISNULL(tpq.PromoAnswerName,'') AS Answer , +[PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present, + case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question, + ISNULL(tpq.PromoAnswerName,'') AS Answer , case when isnull(SHI.PromoImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage1 end as Image1 ,case when isnull(SHI.PromoImage2,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage2 end as Image2 ,GETDATE(),'SP-Pius' - - from + +from OneApp_KelloggsMT.dbo.T_Promotion ts Inner join OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join @@ -36,8 +37,7 @@ with Promotion(Mid, OneApp_KelloggsMT.dbo.Master_PromotionReason mnp on ts.PromoReasonId=mnp.PromoReasonId left join OneApp_KelloggsMT.dbo.t_promotionquestion tpq on ts.PId=tpq.PId left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId - + Where 1=1 and Em.EmpName not like 'test%' - AND sc.MID in ({mid_list}) - ) -select * from Promotion \ No newline at end of file +AND sc.MID in ({mid_list}) ) +select * from Promotion \ No newline at end of file diff --git a/y.yml b/y.yml index e3cf9f8..737c148 100644 --- a/y.yml +++ b/y.yml @@ -85,20 +85,25 @@ tables: # operation: ONLY_INSERT # fetch_by: run_date - - name: Master_VisibilityReason - type: DIMENSION - operation: DELETE+INSERT - fetch_by: none + # - name: Master_VisibilityReason + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: none - - name: Master_VisibilityDefinition - type: DIMENSION - operation: DELETE+INSERT - fetch_by: none + # - name: Master_VisibilityDefinition + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: none - - name: Master_Salesterritorylayer - type: DIMENSION - operation: DELETE+INSERT - fetch_by: none + # - name: Master_Salesterritorylayer + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: none + + - name: Promotion + type: FACT + operation: INSERT + fetch_by: mids